fluencelabs/lalrpop
1
0
Fork 0
mirror of https://github.com/fluencelabs/lalrpop synced 2025-03-16 17:00:53 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #212 from nikomatsakis/fix-lane-table-unit-test

Browse Source 
Fix lane table unit test
This commit is contained in:
Niko Matsakis 2017-04-03 11:22:01 -04:00 committed by GitHub
commit 5cb728e5bb
15 changed files with 76 additions and 1137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -34,3 +34,4 @@ lalrpop-test/src/sub_table.rs
lalrpop-test/src/unit.rs
lalrpop-test/src/use_super.rs
lalrpop-test/src/no_clone_tok.rs
lalrpop-test/src/match_section.rs

3
.travis.yml
View File

@ -4,4 +4,5 @@ rust:
- beta
- nightly
script:
- sh ./test.sh
- cargo test --all
- LALRPOP_LANE_TABLE=enabled cargo test --all

4
lalrpop-test/src/main.rs
View File

@ -401,6 +401,10 @@ fn issue_55_test1() {
#[test]
fn unit_test1() {
assert!(unit::parse_Expr("3 + 4 * 5").is_ok());
}
#[test]
fn unit_test2() {
assert!(unit::parse_Expr("3 + +").is_err());
}

1087
lalrpop-test/src/match_section.rs
View File

File diff suppressed because it is too large Load Diff

3
lalrpop/src/grammar/repr.rs
View File

@ -28,6 +28,9 @@ pub struct Grammar {
// algorithm user requested for this parser
pub algorithm: Algorithm,
// true if the grammar mentions the `!` terminal anywhere
pub uses_error_recovery: bool,
// these are the nonterminals that were declared to be public; the
// key is the user's name for the symbol, the value is the
// artificial symbol we introduce, which will always have a single

19
lalrpop/src/lr1/build/mod.rs
View File

@ -24,18 +24,21 @@ fn build_lr1_states_legacy<'grammar>(grammar: &'grammar Grammar, start: Nontermi
type ConstructionFunction<'grammar> = fn(&'grammar Grammar, NonterminalString) -> LR1Result<'grammar> ;
fn use_lane_table() -> bool {
match env::var("LALRPOP_LANE_TABLE") {
Ok(ref s) => s == "enabled",
_ => false
}
}
pub fn build_lr1_states<'grammar>(grammar: &'grammar Grammar,
start: NonterminalString)
-> LR1Result<'grammar>
{
let (method_name, method_fn) = match env::var("LALRPOP_LANE_TABLE") {
Ok(ref s) if s == "enabled" => {
("lane", build_lane_table_states as ConstructionFunction)
}
_ => {
("legacy", build_lr1_states_legacy as ConstructionFunction)
}
let (method_name, method_fn) = if use_lane_table() {
("lane", build_lane_table_states as ConstructionFunction)
} else {
("legacy", build_lr1_states_legacy as ConstructionFunction)
};
profile! {

7
lalrpop/src/lr1/build/test.rs
View File

@ -10,7 +10,7 @@ use lr1::lookahead::TokenSet;
use lr1::tls::Lr1Tls;
use tls::Tls;
use super::{LR, build_lr0_states, build_lr1_states};
use super::{LR, use_lane_table, build_lr0_states, build_lr1_states};
fn nt(t: &str) -> NonterminalString {
NonterminalString(intern(t))
@ -35,7 +35,7 @@ fn random_test<'g>(grammar: &Grammar,
macro_rules! tokens {
($($x:expr),*) => {
vec![$(TerminalString::quoted(intern($x))),*].into_iter()
vec![$(TerminalString::quoted(intern($x))),*]
}
}
@ -128,7 +128,8 @@ grammar;
// for now, just test that process does not result in an error
// and yields expected number of states.
let states = build_lr1_states(&grammar, nt("S")).unwrap();
assert_eq!(states.len(), 16);
println!("{:#?}", states);
assert_eq!(states.len(), if use_lane_table() { 9 } else { 16 });
// execute it on some sample inputs.
let tree = interpret(&states, tokens!["N", "-", "(", "N", "-", "N", ")"]).unwrap();

2
lalrpop/src/lr1/build_lalr/test.rs
View File

@ -12,7 +12,7 @@ fn nt(t: &str) -> NonterminalString {
macro_rules! tokens {
($($x:expr),*) => {
vec![$(TerminalString::quoted(intern($x))),*].into_iter()
vec![$(TerminalString::quoted(intern($x))),*]
}
}

25
lalrpop/src/lr1/codegen/parse_table.rs
View File

@ -501,7 +501,7 @@ impl<'ascent, 'grammar, W: Write> CodeGenerator<'ascent, 'grammar, W, TableDrive
// Error.
rust!(self.out, "}} else {{");
if self.uses_error_recovery() {
if self.grammar.uses_error_recovery {
let prefix = self.prefix;
try!(self.unrecognized_token_error(&format!("Some({}lookahead.clone())", prefix)));
rust!(self.out, "let mut {}dropped_tokens = Vec::new();", self.prefix);
@ -621,7 +621,7 @@ impl<'ascent, 'grammar, W: Write> CodeGenerator<'ascent, 'grammar, W, TableDrive
// EOF error recovery
try!(self.unrecognized_token_error("None"));
if self.uses_error_recovery() {
if self.grammar.uses_error_recovery {
let extra_test = format!("&& {}EOF_ACTION[({}error_state as usize - 1)] != 0 ",
self.prefix,
self.prefix);
@ -1096,15 +1096,6 @@ impl<'ascent, 'grammar, W: Write> CodeGenerator<'ascent, 'grammar, W, TableDrive
format!("({},{},{})", loc_type, self.symbol_type(), loc_type)
}
fn uses_error_recovery(&self) -> bool {
self.states.iter().any(|state| {
state.shifts.contains_key(&TerminalString::Error) ||
state.reductions
.iter()
.any(|&(ref t, _)| t.contains(Token::Terminal(TerminalString::Error)))
})
}
fn unrecognized_token_error(&mut self, token: &str) -> io::Result<()> {
rust!(self.out, "let {}state = *{}states.last().unwrap() as usize;",
self.prefix,
@ -1127,9 +1118,15 @@ impl<'ascent, 'grammar, W: Write> CodeGenerator<'ascent, 'grammar, W, TableDrive
self.prefix);
rust!(self.out, "const {}TERMINAL: &'static [&'static str] = &[", self.prefix);
// Subtract one to exlude the error terminal
for &terminal in &self.grammar.terminals.all[..self.grammar.terminals.all.len() - 1] {
// Three # should hopefully be enough to prevent any reasonable terminal from escaping the literal
let all_terminals = if self.grammar.uses_error_recovery {
// Subtract one to exlude the error terminal
&self.grammar.terminals.all[..self.grammar.terminals.all.len() - 1]
} else {
&self.grammar.terminals.all
};
for &terminal in all_terminals {
// Three # should hopefully be enough to prevent any
// reasonable terminal from escaping the literal
rust!(self.out, "r###\"{}\"###,", terminal);
}
rust!(self.out, "];");

10
lalrpop/src/lr1/interpret.rs
View File

@ -11,12 +11,12 @@ use util::Sep;
pub type InterpretError<'grammar, L> = (&'grammar State<'grammar, L>, Token);
/// Feed in the given tokens and then EOF, returning the final parse tree that is reduced.
pub fn interpret<'grammar, TOKENS, L>(states: &'grammar [State<'grammar, L>],
tokens: TOKENS)
-> Result<ParseTree, InterpretError<'grammar, L>>
where TOKENS: IntoIterator<Item = TerminalString>,
L: LookaheadInterpret
pub fn interpret<'grammar, L>(states: &'grammar [State<'grammar, L>],
tokens: Vec<TerminalString>)
-> Result<ParseTree, InterpretError<'grammar, L>>
where L: LookaheadInterpret
{
println!("interpret(tokens={:?})", tokens);
let mut m = Machine::new(states);
m.execute(tokens.into_iter())
}

15
lalrpop/src/lr1/lane_table/construct/mod.rs
View File

@ -22,22 +22,22 @@ use self::state_set::StateSet;
pub struct LaneTableConstruct<'grammar> {
grammar: &'grammar Grammar,
first_sets: FirstSets,
start: NonterminalString,
start_nt: NonterminalString,
}
impl<'grammar> LaneTableConstruct<'grammar> {
pub fn new(grammar: &'grammar Grammar, start: NonterminalString) -> Self {
pub fn new(grammar: &'grammar Grammar, start_nt: NonterminalString) -> Self {
let first_sets = FirstSets::new(grammar);
Self {
grammar: grammar,
start: start,
start_nt: start_nt,
first_sets: first_sets,
}
}
pub fn construct(self) -> Result<Vec<LR1State<'grammar>>, LR1TableConstructionError<'grammar>> {
let TableConstructionError { states, conflicts: _ } = {
match build::build_lr0_states(self.grammar, self.start) {
match build::build_lr0_states(self.grammar, self.start_nt) {
// This is the easy (and very rare...) case.
Ok(lr0) => return Ok(self.promote_lr0_states(lr0)),
Err(err) => err,
@ -78,6 +78,7 @@ impl<'grammar> LaneTableConstruct<'grammar> {
/// states in the README.
fn promote_lr0_states(&self, lr0: Vec<LR0State<'grammar>>) -> Vec<LR1State<'grammar>> {
let all = TokenSet::all();
debug!("promote_lr0_states: all={:?}", all);
lr0.into_iter()
.map(|s| {
let items = s.items
@ -110,11 +111,16 @@ impl<'grammar> LaneTableConstruct<'grammar> {
states: &mut Vec<LR1State<'grammar>>,
inconsistent_state: StateIndex)
-> Result<(), StateIndex> {
debug!("resolve_inconsistencies(inconsistent_state={:?}/{:#?}",
inconsistent_state, states[inconsistent_state.0]);
let actions = super::conflicting_actions(&states[inconsistent_state.0]);
if actions.is_empty() {
return Ok(());
}
debug!("resolve_inconsistencies: conflicting_actions={:?}", actions);
let table = self.build_lane_table(states, inconsistent_state, &actions);
// Consider first the "LALR" case, where the lookaheads for each
@ -178,6 +184,7 @@ impl<'grammar> LaneTableConstruct<'grammar> {
-> LaneTable<'grammar> {
let state_graph = StateGraph::new(states);
let mut tracer = LaneTracer::new(self.grammar,
self.start_nt,
states,
&self.first_sets,
&state_graph,

16
lalrpop/src/lr1/lane_table/lane/mod.rs
View File

@ -15,10 +15,12 @@ pub struct LaneTracer<'trace, 'grammar: 'trace, L: Lookahead + 'trace> {
first_sets: &'trace FirstSets,
state_graph: &'trace StateGraph,
table: LaneTable<'grammar>,
start_nt: NonterminalString,
}
impl<'trace, 'grammar, L: Lookahead> LaneTracer<'trace, 'grammar, L> {
pub fn new(grammar: &'grammar Grammar,
start_nt: NonterminalString,
states: &'trace [State<'grammar, L>],
first_sets: &'trace FirstSets,
state_graph: &'trace StateGraph,
@ -28,6 +30,7 @@ impl<'trace, 'grammar, L: Lookahead> LaneTracer<'trace, 'grammar, L> {
states: states,
first_sets: first_sets,
state_graph: state_graph,
start_nt: start_nt,
table: LaneTable::new(grammar, conflicts),
}
}
@ -112,8 +115,19 @@ impl<'trace, 'grammar, L: Lookahead> LaneTracer<'trace, 'grammar, L> {
let state_items = &self.states[state.0].items.vec;
let nonterminal = item.production.nonterminal;
if nonterminal == self.start_nt {
// as a special case, if the `X` above is the special, synthetic
// start-terminal, then the only thing that comes afterwards is EOF.
self.table.add_lookahead(state, conflict, &TokenSet::eof());
}
// NB: Under the normal LR terms, the start nonterminal will
// only have one production like `X' = X`, in which case this
// loop is useless, but sometimes in tests we don't observe
// that restriction, so do it anyway.
for pred_item in state_items.iter()
.filter(|i| i.can_shift_nonterminal(nonterminal)) {
.filter(|i| i.can_shift_nonterminal(nonterminal))
{
let symbol_sets = pred_item.symbol_sets();
let mut first = self.first_sets.first0(symbol_sets.suffix);
let derives_epsilon = first.take_eof();

3
lalrpop/src/lr1/lane_table/test.rs
View File

@ -15,7 +15,7 @@ use super::table::*;
macro_rules! tokens {
($($x:expr),*) => {
vec![$(TerminalString::quoted(intern($x))),*].into_iter()
vec![$(TerminalString::quoted(intern($x))),*]
}
}
@ -117,6 +117,7 @@ fn build_table<'grammar>(grammar: &'grammar Grammar,
let first_sets = FirstSets::new(&grammar);
let state_graph = StateGraph::new(&lr0_err.states);
let mut tracer = LaneTracer::new(&grammar,
nt("G"),
&lr0_err.states,
&first_sets,
&state_graph,

7
lalrpop/src/normalize/lower/mod.rs
View File

@ -164,7 +164,11 @@ impl<'s> LowerState<'s> {
let mut all_terminals: Vec<_> = self.conversions
.iter()
.map(|c| c.0)
.chain(Some(TerminalString::Error))
.chain(if self.uses_error_recovery {
Some(TerminalString::Error)
} else {
None
})
.collect();
all_terminals.sort();
@ -174,6 +178,7 @@ impl<'s> LowerState<'s> {
.collect();
Ok(r::Grammar {
uses_error_recovery: self.uses_error_recovery,
prefix: self.prefix,
start_nonterminals: start_symbols,
uses: uses,

11
test.sh
View File

@ -1,11 +0,0 @@
#!/usr/bin/env bash
EXIT_STATUS=0
(cd lalrpop-util && cargo test) || EXIT_STATUS=$?
(cd lalrpop-intern && cargo test) || EXIT_STATUS=$?
(cd lalrpop && cargo test) || EXIT_STATUS=$?
(cd lalrpop-test && cargo test) || EXIT_STATUS=$?
(cd doc/calculator && cargo test) || EXIT_STATUS=$?
(cd doc/whitespace && cargo test) || EXIT_STATUS=$?
(cd doc/pascal/lalrpop && cargo test) || EXIT_STATUS=$?
exit $EXIT_STATUS