fluencelabs/lalrpop
1
0
Fork 0
mirror of https://github.com/fluencelabs/lalrpop synced 2025-03-16 17:00:53 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #50 from nikomatsakis/new-snapshot

Browse Source 
Move to a new snapshot
This commit is contained in:
Niko Matsakis 2016-01-22 15:48:28 -05:00
commit 8fafb028f5
37 changed files with 31856 additions and 30458 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lalrpop-snap/Cargo.toml
View File

@ -14,7 +14,7 @@ rand = "0.3"
itertools = "0.3"
term = "0.2"
unicode-xid = "0.0.2"
rusty-peg = "0.3"
petgraph = "0.1.11"
[dependencies.lalrpop-util]
path = "../lalrpop-util"

206
lalrpop-snap/src/build/action.rs Normal file
View File

@ -0,0 +1,206 @@
//! Code for generating action code.
use grammar::repr as r;
use rust::RustWrite;
use std::io::{self, Write};
pub fn emit_action_code<W:Write>(grammar: &r::Grammar,
rust: &mut RustWrite<W>)
-> io::Result<()>
{
for (i, defn) in grammar.action_fn_defns.iter().enumerate() {
rust!(rust, "");
match defn.kind {
r::ActionFnDefnKind::User(ref data) =>
try!(emit_user_action_code(grammar, rust, i, defn, data)),
r::ActionFnDefnKind::Lookaround(ref variant) =>
try!(emit_lookaround_action_code(grammar, rust, i, defn, variant)),
r::ActionFnDefnKind::Inline(ref data) =>
try!(emit_inline_action_code(grammar, rust, i, defn, data)),
}
}
Ok(())
}
fn ret_type_string(grammar: &r::Grammar,
defn: &r::ActionFnDefn)
-> String
{
if defn.fallible {
format!("Result<{},{}ParseError<{},{},{}>>",
defn.ret_type,
grammar.prefix,
grammar.types.terminal_loc_type(),
grammar.types.terminal_token_type(),
grammar.types.error_type())
} else {
format!("{}", defn.ret_type)
}
}
fn emit_user_action_code<W:Write>(grammar: &r::Grammar,
rust: &mut RustWrite<W>,
index: usize,
defn: &r::ActionFnDefn,
data: &r::UserActionFnDefn)
-> io::Result<()>
{
let ret_type = ret_type_string(grammar, defn);
let lookarounds = vec![
format!("{}lookbehind: &Option<{}>", grammar.prefix, grammar.types.terminal_loc_type()),
format!("{}lookahead: &Option<{}>", grammar.prefix, grammar.types.triple_type())];
try!(rust.write_pub_fn_header(
grammar,
format!("{}action{}", grammar.prefix, index),
vec![],
data.arg_patterns.iter()
.zip(data.arg_types.iter())
.map(|(p, t)| format!("{}: {}", p, t))
.chain(lookarounds)
.collect(),
ret_type,
vec![]));
rust!(rust, "{{");
rust!(rust, "{}", data.code);
rust!(rust, "}}");
Ok(())
}
fn emit_lookaround_action_code<W:Write>(grammar: &r::Grammar,
rust: &mut RustWrite<W>,
index: usize,
_defn: &r::ActionFnDefn,
data: &r::LookaroundActionFnDefn)
-> io::Result<()>
{
try!(rust.write_pub_fn_header(
grammar,
format!("{}action{}", grammar.prefix, index),
vec![],
vec![format!("{}lookbehind: &Option<{}>", grammar.prefix, grammar.types.terminal_loc_type()),
format!("{}lookahead: &Option<{}>", grammar.prefix, grammar.types.triple_type())],
format!("{}", grammar.types.terminal_loc_type()),
vec![]));
rust!(rust, "{{");
match *data {
r::LookaroundActionFnDefn::Lookahead => {
// take the lookahead, if any; otherwise, we are
// at EOF, so taker the lookbehind (end of last
// pushed token); if that is missing too, then
// supply default.
rust!(rust, "{}lookahead.as_ref()\
.map(|o| ::std::clone::Clone::clone(&o.0))\
.or_else(|| ::std::clone::Clone::clone(&{}lookbehind))\
.unwrap_or_default()",
grammar.prefix, grammar.prefix);
}
r::LookaroundActionFnDefn::Lookbehind => {
// take lookbehind or supply default
rust!(rust, "::std::clone::Clone::clone(&{}lookbehind).unwrap_or_default()",
grammar.prefix);
}
}
rust!(rust, "}}");
Ok(())
}
fn emit_inline_action_code<W:Write>(grammar: &r::Grammar,
rust: &mut RustWrite<W>,
index: usize,
defn: &r::ActionFnDefn,
data: &r::InlineActionFnDefn)
-> io::Result<()>
{
let ret_type = ret_type_string(grammar, defn);
let arg_types: Vec<_> =
data.symbols.iter()
.flat_map(|sym| match *sym {
r::InlinedSymbol::Original(s) => vec![s],
r::InlinedSymbol::Inlined(_, ref syms) => syms.clone(),
})
.map(|s| s.ty(&grammar.types))
.collect();
let arguments: Vec<_> =
arg_types.iter()
.enumerate()
.map(|(i, t)| format!("{}{}: {}", grammar.prefix, i, t))
.chain(vec![
format!("{}lookbehind: &Option<{}>",
grammar.prefix,
grammar.types.terminal_loc_type()),
format!("{}lookahead: &Option<{}>",
grammar.prefix,
grammar.types.triple_type())])
.collect();
try!(rust.write_pub_fn_header(
grammar,
format!("{}action{}", grammar.prefix, index),
vec![],
arguments,
ret_type,
vec![]));
rust!(rust, "{{");
// create temporaries for the inlined things
let mut arg_counter = 0;
let mut temp_counter = 0;
for symbol in &data.symbols {
match *symbol {
r::InlinedSymbol::Original(_) => {
arg_counter += 1;
}
r::InlinedSymbol::Inlined(inlined_action, ref syms) => {
rust!(rust, "let {}temp{} = {}action{}(",
grammar.prefix, temp_counter,
grammar.prefix, inlined_action.index());
for parameter in &grammar.parameters {
rust!(rust, "{},", parameter.name);
}
for _ in syms {
rust!(rust, "{}{},", grammar.prefix, arg_counter);
arg_counter += 1;
}
rust!(rust, "{}lookbehind,", grammar.prefix);
rust!(rust, "{}lookahead,", grammar.prefix);
rust!(rust, ");");
temp_counter += 1;
}
}
}
rust!(rust, "{}action{}(", grammar.prefix, data.action.index());
for parameter in &grammar.parameters {
rust!(rust, "{},", parameter.name);
}
let mut arg_counter = 0;
let mut temp_counter = 0;
for symbol in &data.symbols {
match *symbol {
r::InlinedSymbol::Original(_) => {
rust!(rust, "{}{},", grammar.prefix, arg_counter);
arg_counter += 1;
}
r::InlinedSymbol::Inlined(_, ref syms) => {
rust!(rust, "{}temp{},", grammar.prefix, temp_counter);
temp_counter += 1;
arg_counter += syms.len();
}
}
}
rust!(rust, "{}lookbehind,", grammar.prefix);
rust!(rust, "{}lookahead,", grammar.prefix);
rust!(rust, ")");
rust!(rust, "}}");
Ok(())
}

43
lalrpop-snap/src/build/mod.rs
View File

@ -11,19 +11,21 @@ use rust::RustWrite;
use tok;
use self::filetext::FileText;
use std::env::current_dir;
use std::fs;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::process::exit;
mod action;
mod filetext;
pub fn process_root() -> io::Result<()> {
process_dir("src", false)
process_dir(try!(current_dir()), false)
}
pub fn process_root_unconditionally() -> io::Result<()> {
process_dir("src", true)
process_dir(try!(current_dir()), true)
}
fn process_dir<P:AsRef<Path>>(root_dir: P, force_build: bool) -> io::Result<()> {
@ -274,48 +276,13 @@ fn emit_recursive_ascent(output_path: &Path, grammar: &r::Grammar) -> io::Result
try!(intern_token::compile(&grammar, intern_token, &mut rust));
}
try!(emit_action_code(grammar, &mut rust));
try!(action::emit_action_code(grammar, &mut rust));
try!(emit_to_triple_trait(grammar, &mut rust));
Ok(())
}
fn emit_action_code<W:Write>(grammar: &r::Grammar,
rust: &mut RustWrite<W>)
-> io::Result<()>
{
for (i, defn) in grammar.action_fn_defns.iter().enumerate() {
rust!(rust, "");
let ret_type = if defn.fallible {
format!("Result<{},{}ParseError<{},{},{}>>",
defn.ret_type,
grammar.prefix,
grammar.types.terminal_loc_type(),
grammar.types.terminal_token_type(),
grammar.types.error_type())
} else {
format!("{}", defn.ret_type)
};
try!(rust.write_pub_fn_header(
grammar,
format!("{}action{}", grammar.prefix, i),
vec![],
defn.arg_patterns.iter()
.zip(defn.arg_types.iter())
.map(|(p, t)| format!("{}: {}", p, t))
.collect(),
ret_type,
vec![]));
rust!(rust, "{{");
rust!(rust, "{}", defn.code);
rust!(rust, "}}");
}
Ok(())
}
fn emit_to_triple_trait<W:Write>(grammar: &r::Grammar,
rust: &mut RustWrite<W>)
-> io::Result<()>

2
lalrpop-snap/src/generate.rs
View File

@ -37,7 +37,7 @@ impl<'grammar> Generator<'grammar> {
}
self.depth += 1;
let productions = &self.grammar.productions[&nt];
let productions = self.grammar.productions_for(nt);
let index: usize = self.rng.gen_range(0, productions.len());
let production = &productions[index];
let trees: Option<Vec<_>> = production.symbols.iter()

15
lalrpop-snap/src/grammar/consts.rs Normal file
View File

@ -0,0 +1,15 @@
/// Recognized associated type for the token location
pub const LOCATION: &'static str = "Location";
/// Recognized associated type for custom errors
pub const ERROR: &'static str = "Error";
/// The lifetime parameter injected when we do not have an external token enum
pub const INPUT_LIFETIME: &'static str = "'input";
/// The parameter injected when we do not have an external token enum
pub const INPUT_PARAMETER: &'static str = "input";
/// The annotation to request inlining.
pub const INLINE: &'static str = "inline";

1
lalrpop-snap/src/grammar/mod.rs
View File

@ -1,5 +1,6 @@
//! The grammar definition.
pub mod consts;
pub mod parse_tree;
pub mod pattern;
pub mod repr;

23
lalrpop-snap/src/grammar/parse_tree.rs
View File

@ -14,6 +14,8 @@ use util::Sep;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Grammar {
// see field `prefix` in `grammar::repr::Grammar`
pub prefix: String,
pub span: Span,
pub type_parameters: Vec<TypeParameter>,
pub parameters: Vec<Parameter>,
@ -62,12 +64,6 @@ pub struct AssociatedType {
pub type_ref: TypeRef,
}
/// Recognized associated type for the token location
pub const LOCATION: &'static str = "Location";
/// Recognized associated type for custom errors
pub const ERROR: &'static str = "Error";
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct EnumToken {
pub type_name: TypeRef,
@ -115,18 +111,12 @@ pub enum TypeRef {
OfSymbol(SymbolKind),
}
/// The lifetime parameter injected when we do not have an external token enum
pub const INPUT_LIFETIME: &'static str = "'input";
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum TypeParameter {
Lifetime(InternedString),
Id(InternedString),
}
/// The parameter injected when we do not have an external token enum
pub const INPUT_PARAMETER: &'static str = "input";
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Parameter {
pub name: InternedString,
@ -605,12 +595,3 @@ impl Path {
}
}
}
impl ActionKind {
pub fn as_user(&self) -> Option<&String> {
match *self {
ActionKind::User(ref s) => Some(s),
_ => None
}
}
}

108
lalrpop-snap/src/grammar/repr.rs
View File

@ -10,7 +10,8 @@ use std::fmt::{Debug, Display, Formatter, Error};
use util::{map, Map, Sep};
// These concepts we re-use wholesale
pub use grammar::parse_tree::{InternToken,
pub use grammar::parse_tree::{Annotation,
InternToken,
NonterminalString,
Path,
Span,
@ -50,12 +51,20 @@ pub struct Grammar {
// the grammar proper:
pub action_fn_defns: Vec<ActionFnDefn>,
pub productions: Map<NonterminalString, Vec<Production>>,
pub nonterminals: Map<NonterminalString, NonterminalData>,
pub token_span: Span,
pub conversions: Map<TerminalString, Pattern<TypeRepr>>,
pub types: Types,
}
#[derive(Clone, Debug)]
pub struct NonterminalData {
pub name: NonterminalString,
pub span: Span,
pub annotations: Vec<Annotation>,
pub productions: Vec<Production>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Algorithm {
LR1,
@ -74,7 +83,7 @@ pub struct Production {
// handy to have it
pub nonterminal: NonterminalString,
pub symbols: Vec<Symbol>,
pub action: ActionKind,
pub action: ActionFn,
pub span: Span,
}
@ -84,22 +93,56 @@ pub enum Symbol {
Terminal(TerminalString),
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
pub enum ActionKind {
// execute code provided by the user
Call(ActionFn),
TryCall(ActionFn),
#[derive(Clone, PartialEq, Eq)]
pub struct ActionFnDefn {
pub fallible: bool,
pub ret_type: TypeRepr,
pub kind: ActionFnDefnKind,
}
#[derive(Clone, PartialEq, Eq)]
pub enum ActionFnDefnKind {
User(UserActionFnDefn),
Inline(InlineActionFnDefn),
Lookaround(LookaroundActionFnDefn),
}
/// An action fn written by a user.
#[derive(Clone, PartialEq, Eq)]
pub struct UserActionFnDefn {
pub arg_patterns: Vec<InternedString>,
pub arg_types: Vec<TypeRepr>,
pub code: String,
}
/// An action fn generated by the inlining pass. If we were
/// inlining `A = B C D` (with action 44) into `X = Y A Z` (with
/// action 22), this would look something like:
///
/// ```
/// fn __action66(__0: Y, __1: B, __2: C, __3: D, __4: Z) {
/// __action22(__0, __action44(__1, __2, __3), __4)
/// }
/// ```
#[derive(Clone, PartialEq, Eq)]
pub struct InlineActionFnDefn {
/// in the example above, this would be `action22`
pub action: ActionFn,
/// in the example above, this would be `Y, {action44: B, C, D}, Z`
pub symbols: Vec<InlinedSymbol>
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum LookaroundActionFnDefn {
Lookahead,
Lookbehind,
}
#[derive(Clone, PartialEq, Eq)]
pub struct ActionFnDefn {
pub arg_patterns: Vec<InternedString>,
pub arg_types: Vec<TypeRepr>,
pub ret_type: TypeRepr,
pub fallible: bool,
pub code: String,
pub enum InlinedSymbol {
Original(Symbol),
Inlined(ActionFn, Vec<Symbol>),
}
#[derive(Clone, PartialEq, Eq)]
@ -346,6 +389,16 @@ impl Debug for ActionFnDefn {
impl ActionFnDefn {
fn to_fn_string(&self, name: &str) -> String {
match self.kind {
ActionFnDefnKind::User(ref data) => data.to_fn_string(self, name),
ActionFnDefnKind::Inline(ref data) => data.to_fn_string(name),
ActionFnDefnKind::Lookaround(ref data) => format!("{:?}", data),
}
}
}
impl UserActionFnDefn {
fn to_fn_string(&self, defn: &ActionFnDefn, name: &str) -> String {
let arg_strings: Vec<String> =
self.arg_patterns
.iter()
@ -354,7 +407,24 @@ impl ActionFnDefn {
.collect();
format!("fn {}({}) -> {} {{ {} }}",
name, Sep(", ", &arg_strings), self.ret_type, self.code)
name, Sep(", ", &arg_strings), defn.ret_type, self.code)
}
}
impl InlineActionFnDefn {
fn to_fn_string(&self, name: &str) -> String {
let arg_strings: Vec<String> =
self.symbols
.iter()
.map(|inline_sym| match *inline_sym {
InlinedSymbol::Original(s) =>
format!("{}", s),
InlinedSymbol::Inlined(a, ref s) =>
format!("{:?}({})", a, Sep(", ", s)),
})
.collect();
format!("fn {}(..) {{ {:?}({}) }}", name, self.action, Sep(", ", &arg_strings))
}
}
@ -364,8 +434,8 @@ impl Grammar {
}
pub fn productions_for(&self, nonterminal: NonterminalString) -> &[Production] {
match self.productions.get(&nonterminal) {
Some(v) => &v[..],
match self.nonterminals.get(&nonterminal) {
Some(v) => &v.productions[..],
None => &[], // this...probably shouldn't happen actually?
}
}
@ -377,6 +447,10 @@ impl Grammar {
}
result
}
pub fn action_is_fallible(&self, f: ActionFn) -> bool {
self.action_fn_defns[f.index()].fallible
}
}
impl Algorithm {

6
lalrpop-snap/src/lexer/dfa/mod.rs
View File

@ -124,7 +124,7 @@ impl<'nfa> DFABuilder<'nfa> {
.iter()
.all(|&item| self.nfa(item).is_rejecting_state(item.nfa_state));
let kind = if all_rejects {
let kind = if all_rejects || item_set.items.is_empty() {
Kind::Reject
} else if all_accepts.len() == 0 {
Kind::Neither
@ -160,7 +160,7 @@ impl<'nfa> DFABuilder<'nfa> {
test_edges.sort();
// Consider what there is some cahracter that doesn't meet
// Consider what there is some character that doesn't meet
// any of the tests. In this case, we can just ignore all
// the test edges for each of the items and just union all
// the "other" edges -- because if it were one of those
@ -171,7 +171,7 @@ impl<'nfa> DFABuilder<'nfa> {
.collect();
// we never know the full set
assert!(!other_transitions.is_empty());
assert!(item_set.items.is_empty() || !other_transitions.is_empty());
let other_edge = kernel_set.add_state(self.transitive_closure(other_transitions));

11
lalrpop-snap/src/lexer/dfa/test.rs
View File

@ -38,3 +38,14 @@ fn ambiguous_regex() {
assert!(dfa(&[(r#"class"#, P0),
(r#"[a-zA-Z_][a-zA-Z0-9_]*"#, P0)]).is_err());
}
#[test]
fn issue_32() {
assert!(dfa(&[(r#"."#, P0)]).is_ok());
}
#[test]
fn issue_35() {
assert!(dfa(&[(r#".*"#, P0),
(r"[-+]?[0-9]*\.?[0-9]+", P0)]).is_err());
}

4
lalrpop-snap/src/lexer/nfa/mod.rs
View File

@ -31,7 +31,7 @@ pub struct Other;
/// edges by enumerating subsequent edges in the vectors until you
/// find one with a different `from` value.
#[derive(Debug)]
struct State {
pub struct State {
kind: StateKind,
first_noop_edge: usize,
first_test_edge: usize,
@ -52,7 +52,7 @@ pub struct NFAStateIndex(usize);
/// now we just ensure this during construction, but one could easily
/// sort).
#[derive(Debug)]
struct Edges {
pub struct Edges {
noop_edges: Vec<Edge<Noop>>,
// edges where we are testing the character in some way; for any

4
lalrpop-snap/src/lexer/re/mod.rs
View File

@ -103,7 +103,7 @@ impl<'str> RegexParser<'str> {
alternatives.push(try!(self.alternative()));
match self.lookahead {
Some((_, '|')) => { continue; }
Some((_, '|')) => { self.bump(); continue; }
_ => { break; }
}
}
@ -124,7 +124,7 @@ impl<'str> RegexParser<'str> {
'[' => { elems.push(try!(self.range(index))); }
']' => { break; }
'|' => { break; }
'.' => { elems.push(Elem::Any); }
'.' => { self.bump(); elems.push(Elem::Any); }
_ => { self.bump(); elems.push(Elem::Test(Test::Char(c))); }
}
}

4
lalrpop-snap/src/lexer/re/test.rs
View File

@ -20,3 +20,7 @@ fn parse_neg() {
fn parse_unclosed_group() {
parse_regex(r"(123").unwrap_err();
}
#[test]
fn alt_oom() {
parse_regex(r"(%%|[^%])+").unwrap();
}

6
lalrpop-snap/src/lib.rs
View File

@ -11,12 +11,14 @@
extern crate diff;
extern crate lalrpop_intern as intern;
extern crate lalrpop_util;
extern crate rand;
extern crate petgraph;
extern crate regex;
extern crate term;
extern crate itertools;
extern crate unicode_xid;
#[cfg(test)]
extern crate rand;
// rust exports a macro that others use, so hoist it early.
#[macro_use]
mod rust;

70
lalrpop-snap/src/lr1/ascent.rs
View File

@ -2,8 +2,7 @@
//!
//! [recursive ascent]: https://en.wikipedia.org/wiki/Recursive_ascent_parser
use grammar::repr::{ActionKind,
Grammar,
use grammar::repr::{Grammar,
NonterminalString,
Symbol,
TerminalString, TypeParameter, TypeRepr, Types};
@ -142,7 +141,7 @@ impl<'ascent,'grammar,W:Write> RecursiveAscent<'ascent,'grammar,W> {
// making different enums per state, but this would mean we
// have to unwrap and rewrap as we pass up the stack, which
// seems silly
for &nt in self.grammar.productions.keys() {
for &nt in self.grammar.nonterminals.keys() {
rust!(self.out, "{}({}),", Escape(nt), self.types.nonterminal_type(nt));
}
@ -360,51 +359,32 @@ impl<'ascent,'grammar,W:Write> RecursiveAscent<'ascent,'grammar,W> {
rust!(self.out, "let {} = {}.take().unwrap();", sym, sym);
}
let transfered_syms = transfer_syms.len();
let mut args = transfer_syms;
args.push(format!("&{}lookbehind", self.prefix));
args.push(format!("&{}lookahead", self.prefix));
// invoke the action code
match production.action {
ActionKind::Call(action_fn) => {
rust!(self.out, "let {}nt = super::{}action{}({}{});",
self.prefix,
self.prefix,
action_fn.index(),
self.grammar.user_parameter_refs(),
Sep(", ", &transfer_syms))
}
ActionKind::TryCall(action_fn) => {
rust!(self.out, "let {}nt = try!(super::{}action{}({}{}));",
self.prefix,
self.prefix,
action_fn.index(),
self.grammar.user_parameter_refs(),
Sep(", ", &transfer_syms))
}
ActionKind::Lookahead => {
// take the lookahead, if any; otherwise, we are
// at EOF, so taker the lookbehind (end of last
// pushed token); if that is missing too, then
// supply default.
rust!(self.out,
"let {}nt = \
{}lookahead.as_ref()\
.map(|o| ::std::clone::Clone::clone(&o.0))\
.or_else(|| ::std::clone::Clone::clone(&{}lookbehind))\
.unwrap_or_default();",
self.prefix, self.prefix, self.prefix);
}
ActionKind::Lookbehind => {
// take lookbehind or supply default.
rust!(self.out,
"let {}nt = ::std::clone::Clone::clone(&{}lookbehind)\
.unwrap_or_default();",
self.prefix, self.prefix);
}
let is_fallible = self.grammar.action_is_fallible(production.action);
if is_fallible {
rust!(self.out, "let {}nt = try!(super::{}action{}({}{}));",
self.prefix,
self.prefix,
production.action.index(),
self.grammar.user_parameter_refs(),
Sep(", ", &args))
} else {
rust!(self.out, "let {}nt = super::{}action{}({}{});",
self.prefix,
self.prefix,
production.action.index(),
self.grammar.user_parameter_refs(),
Sep(", ", &args))
}
// wrap up the result along with the (unused) lookahead
if !transfer_syms.is_empty() {
if transfered_syms != 0 {
// if we popped anything off of the stack, then this frame is done
rust!(self.out, "return Ok(({}lookbehind, {}lookahead, {}Nonterminal::{}({}nt)));",
self.prefix, self.prefix, self.prefix,
@ -455,7 +435,7 @@ impl<'ascent,'grammar,W:Write> RecursiveAscent<'ascent,'grammar,W> {
// errors are not possible in the goto phase; a missing entry
// indicates parse successfully completed, so just bail out
if this_state.gotos.len() != self.grammar.productions.keys().len() {
if this_state.gotos.len() != self.grammar.nonterminals.keys().len() {
rust!(self.out, "_ => {{");
rust!(self.out, "return Ok(({}lookbehind, {}lookahead, {}nt));",
self.prefix, self.prefix, self.prefix);

12
lalrpop-snap/src/lr1/core/test.rs
View File

@ -55,9 +55,8 @@ grammar;
"#);
let items = items(&grammar, "A", 0, EOF);
expect_debug(items.vec, r#"[
() = (*) ["C"],
A = (*) B "C" [EOF],
B = (*) () ["C"],
B = (*) ["C"],
B = (*) "D" ["C"]
]"#);
}
@ -79,19 +78,16 @@ C: Option<u32> = {
"#);
expect_debug(items(&grammar, "A", 0, EOF).vec, r#"[
() = (*) [EOF],
() = (*) ["C1"],
A = (*) B C [EOF],
B = (*) () [EOF],
B = (*) () ["C1"],
B = (*) [EOF],
B = (*) ["C1"],
B = (*) "B1" [EOF],
B = (*) "B1" ["C1"]
]"#);
expect_debug(items(&grammar, "A", 1, EOF).vec, r#"[
() = (*) [EOF],
A = B (*) C [EOF],
C = (*) () [EOF],
C = (*) [EOF],
C = (*) "C1" [EOF]
]"#);
}

2
lalrpop-snap/src/lr1/first/mod.rs
View File

@ -20,7 +20,7 @@ impl FirstSets {
let mut changed = true;
while changed {
changed = false;
for production in grammar.productions.values().flat_map(|p| p.iter()) {
for production in grammar.nonterminals.values().flat_map(|p| &p.productions) {
let nt = production.nonterminal;
let lookahead = this.first(&production.symbols, Lookahead::EOF);
let first_set = this.map.entry(nt).or_insert_with(|| set());

2
lalrpop-snap/src/lr1/mod.rs
View File

@ -40,7 +40,7 @@ struct Items<'grammar> {
struct StateIndex(usize);
#[derive(Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)]
enum Lookahead {
pub enum Lookahead {
EOF,
Terminal(TerminalString),
}

116
lalrpop-snap/src/normalize/inline/graph/mod.rs Normal file
View File

@ -0,0 +1,116 @@
#![allow(dead_code)]
use intern::intern;
use normalize::{NormError, NormResult};
use petgraph::graph::{Graph, NodeIndex};
use grammar::consts::INLINE;
use grammar::repr::*;
use util::{map, Map};
#[cfg(test)]
mod test;
/// Computes the proper order to inline the various nonterminals in
/// `grammar`. Reports an error if there is an inline
/// cycle. Otherwise, yields an ordering such that we inline X before
/// Y if Y references X. I actually think it doesn't matter what
/// order we do the inlining, really, but this order seems better
/// somehow. :) (That is, inline into something before we inline it.)
pub fn inline_order(grammar: &Grammar) -> NormResult<Vec<NonterminalString>> {
let mut graph = NonterminalGraph::new(grammar);
graph.create_nodes();
graph.add_edges();
graph.inline_order()
}
struct NonterminalGraph<'grammar> {
grammar: &'grammar Grammar,
graph: Graph<NonterminalString, ()>,
nonterminal_map: Map<NonterminalString, NodeIndex>,
}
#[derive(Copy, Clone)]
enum WalkState {
NotVisited,
Visiting,
Visited,
}
impl<'grammar> NonterminalGraph<'grammar> {
fn new(grammar: &'grammar Grammar) -> NonterminalGraph<'grammar> {
NonterminalGraph {
grammar: grammar,
graph: Graph::new(),
nonterminal_map: map(),
}
}
fn create_nodes(&mut self) {
let inline = intern(INLINE);
for (&name, data) in &self.grammar.nonterminals {
if data.annotations.iter().any(|a| a.id == inline) {
let index = self.graph.add_node(name);
self.nonterminal_map.insert(name, index);
}
}
}
fn add_edges(&mut self) {
for production in self.grammar.nonterminals.values().flat_map(|d| &d.productions) {
let from_index = match self.nonterminal_map.get(&production.nonterminal) {
Some(&index) => index,
None => continue, // this is not an inlined nonterminal
};
for &symbol in &production.symbols {
match symbol {
Symbol::Nonterminal(to) => {
if let Some(&to_index) = self.nonterminal_map.get(&to) {
self.graph.add_edge(from_index, to_index, ());
}
}
Symbol::Terminal(_) => { }
}
}
}
}
fn inline_order(&self) -> NormResult<Vec<NonterminalString>> {
let mut states = vec![WalkState::NotVisited; self.graph.node_count()];
let mut result = vec![];
for node in self.nonterminal_map.values().cloned() {
try!(self.walk(&mut states, &mut result, node));
}
Ok(result)
}
fn walk(&self,
states: &mut Vec<WalkState>,
result: &mut Vec<NonterminalString>,
source: NodeIndex)
-> NormResult<()>
{
let nt = *self.graph.node_weight(source).unwrap();
match states[source.index()] {
WalkState::NotVisited => {
states[source.index()] = WalkState::Visiting;
for target in self.graph.neighbors(source) {
try!(self.walk(states, result, target));
}
states[source.index()] = WalkState::Visited;
result.push(nt);
Ok(())
}
WalkState::Visited => {
Ok(())
}
WalkState::Visiting => {
return_err!(
self.grammar.nonterminals[&nt].span,
"cyclic inline directive: `{}` would have to be inlined into itself", nt);
}
}
}
}

45
lalrpop-snap/src/normalize/inline/graph/test.rs Normal file
View File

@ -0,0 +1,45 @@
use intern::intern;
use grammar::repr::NonterminalString;
use normalize::lower_helper;
use parser;
use super::inline_order;
#[test]
fn test_inline_self_cycle() {
let grammar = parser::parse_grammar(r#"
grammar;
extern { }
#[inline] A: () = A;
"#).unwrap();
let grammar = lower_helper(grammar, true).unwrap();
assert!(inline_order(&grammar).is_err());
}
#[test]
fn test_inline_cycle_3() {
let grammar = parser::parse_grammar(r#"
grammar;
extern { }
#[inline] A: () = B;
#[inline] B: () = C;
#[inline] C: () = A;
"#).unwrap();
let grammar = lower_helper(grammar, true).unwrap();
assert!(inline_order(&grammar).is_err());
}
#[test]
fn test_inline_order() {
// because C references A, we inline A first.
let grammar = parser::parse_grammar(r#"
grammar;
extern { }
#[inline] A: () = B;
B: () = C;
#[inline] C: () = A;
"#).unwrap();
let grammar = lower_helper(grammar, true).unwrap();
let a = NonterminalString(intern("A"));
let c = NonterminalString(intern("C"));
assert_eq!(inline_order(&grammar).unwrap(), vec![a, c]);
}

152
lalrpop-snap/src/normalize/inline/mod.rs Normal file
View File

@ -0,0 +1,152 @@
/*!
* Inlining of nonterminals
*/
use grammar::repr::*;
use normalize::NormResult;
mod graph;
#[cfg(test)]
mod test;
pub fn inline(mut grammar: Grammar) -> NormResult<Grammar> {
let order = try!(graph::inline_order(&grammar));
for nt in order {
inline_nt(&mut grammar, nt);
}
Ok(grammar)
}
fn inline_nt(grammar: &mut Grammar, inline_nt: NonterminalString) {
let inline_productions: Vec<_> = grammar.productions_for(inline_nt).iter().cloned().collect();
for (_, data) in &mut grammar.nonterminals {
let mut new_productions = vec![];
let mut new_action_fn_defns = vec![];
for into_production in &data.productions {
if !into_production.symbols.contains(&Symbol::Nonterminal(inline_nt)) {
new_productions.push(into_production.clone());
continue;
}
let mut inliner = Inliner {
action_fn_defns: &grammar.action_fn_defns,
inline_nonterminal: inline_nt,
into_production: into_production,
inline_fallible: 0,
inline_productions: &inline_productions,
new_symbols: vec![],
new_productions: &mut new_productions,
new_action_fn_defns: &mut new_action_fn_defns,
};
inliner.inline(&into_production.symbols);
}
data.productions = new_productions;
grammar.action_fn_defns.extend(new_action_fn_defns);
}
}
struct Inliner<'a> {
/// Action fn defns
action_fn_defns: &'a [ActionFnDefn],
/// The nonterminal `A` being inlined
inline_nonterminal: NonterminalString,
/// The full set of productions `A = B C D | E F G` for the
/// nonterminal `A` being inlined
inline_productions: &'a [Production],
/// Number of actions that we have inlined for `A` so far which
/// have been fallible. IOW, if we are inlining `A` into `X = Y A
/// A Z`, and in the first instance of `A` we used a fallible
/// action, but the second we used an infallible one, count would
/// be 1.
inline_fallible: u32,
/// The `X = Y A Z` being inlined into
into_production: &'a Production,
/// The list of symbols we building up for the new production.
/// For example, this would (eventually) contain `Y B C D Z`,
/// given our running example.
new_symbols: Vec<InlinedSymbol>,
/// The output vector of all productions for `X` that we have created
new_productions: &'a mut Vec<Production>,
/// Vector of all action fn defns from the grammar.
new_action_fn_defns: &'a mut Vec<ActionFnDefn>,
}
impl<'a> Inliner<'a> {
fn inline(&mut self, into_symbols: &[Symbol]) {
if into_symbols.is_empty() {
// create an action fn for the result of inlining
let into_action = self.into_production.action;
let into_fallible = self.action_fn_defns[into_action.index()].fallible;
let into_ret_type = self.action_fn_defns[into_action.index()].ret_type.clone();
let inline_fallible = self.inline_fallible != 0;
let index = self.action_fn_defns.len() + self.new_action_fn_defns.len();
let action_fn = ActionFn::new(index);
let inline_defn = InlineActionFnDefn {
action: into_action,
symbols: self.new_symbols.clone()
};
self.new_action_fn_defns.push(ActionFnDefn {
fallible: into_fallible || inline_fallible,
ret_type: into_ret_type,
kind: ActionFnDefnKind::Inline(inline_defn),
});
let prod_symbols: Vec<Symbol> =
self.new_symbols.iter()
.flat_map(|sym| match *sym {
InlinedSymbol::Original(s) => vec![s],
InlinedSymbol::Inlined(_, ref s) => s.clone(),
})
.collect();
self.new_productions.push(Production {
nonterminal: self.into_production.nonterminal,
span: self.into_production.span,
symbols: prod_symbols,
action: action_fn,
});
} else {
let next_symbol = into_symbols[0];
match next_symbol {
Symbol::Nonterminal(n) if n == self.inline_nonterminal => {
// Replace the current symbol with each of the
// `inline_productions` in turn.
for inline_production in self.inline_productions {
// If this production is fallible, increment
// count of fallible actions.
let inline_action = inline_production.action;
let fallible = self.action_fn_defns[inline_action.index()].fallible;
self.inline_fallible += fallible as u32;
// Push the symbols of the production inline.
self.new_symbols.push(
InlinedSymbol::Inlined(
inline_production.action,
inline_production.symbols.clone()));
// Inline remaining symbols:
self.inline(&into_symbols[1..]);
// Reset state after we have inlined remaining symbols:
self.new_symbols.pop();
self.inline_fallible -= fallible as u32;
}
}
_ => {
self.new_symbols.push(InlinedSymbol::Original(next_symbol));
self.inline(&into_symbols[1..]);
self.new_symbols.pop();
}
}
}
}
}

94
lalrpop-snap/src/normalize/inline/test.rs Normal file
View File

@ -0,0 +1,94 @@
use grammar::parse_tree::NonterminalString;
use grammar::repr::Grammar;
use intern::intern;
use normalize::{self, NormResult};
use parser;
use super::inline;
fn inlined_grammar(text: &str) -> NormResult<Grammar> {
let g = parser::parse_grammar(text).unwrap();
let g = normalize::lower_helper(g, true).unwrap();
inline(g)
}
#[test]
fn sri() {
// This grammar gets a shift-reduce conflict because if the input
// is "&" (*) "L", then we see two possibilities, and we must decide
// between them:
//
// "&" (*) "L" E
// | | |
// +-------+--|
// |
// E
//
// or
//
// "&" (*) "L"
// | |
// | OPT_L E
// | | |
// +---+---+----+
// |
// E
//
// to some extent this may be a false conflict, in that inlined
// rules would address it, but it's an interesting one for
// producing a useful error message.
let grammar = inlined_grammar(r#"
grammar;
E: () = {
"L",
"&" OPT_L E
};
#[inline] OPT_L: () = {
(),
"L"
};
"#).unwrap();
let nt = NonterminalString(intern("E"));
// After inlining, we expect:
//
// E = "L"
// E = "&" E
// E = "&" "L" E
//
// Note that the `()` also gets inlined.
let e_productions = grammar.productions_for(nt);
assert_eq!(e_productions.len(), 3);
assert_eq!(format!("{:?}", e_productions[0].symbols), r#"["L"]"#);
assert_eq!(format!("{:?}", e_productions[1].symbols), r#"["&", E]"#);
assert_eq!(format!("{:?}", e_productions[2].symbols), r#"["&", "L", E]"#);
}
#[test]
fn issue_55() {
let grammar = inlined_grammar(r#"
grammar;
pub E: () = {
"X" "{" <a:AT*> <e:ET> <b:AT*> "}" => ()
};
AT: () = {
"type" ";" => ()
};
ET: () = {
"enum" "{" "}" => ()
};
"#).unwrap();
let nt = NonterminalString(intern("E"));
// The problem in issue #55 was that we would inline both `AT*`
// the same way, so we ended up with `E = X { ET }` and `E = X {
// AT+ ET AT+ }` but not `E = X { AT+ ET }` or `E = X { ET AT+ }`.
assert!(grammar.productions_for(nt).len() == 4);
}

167
lalrpop-snap/src/normalize/lower/mod.rs
View File

@ -5,15 +5,13 @@
use intern::{self, intern, InternedString};
use normalize::NormResult;
use normalize::norm_util::{self, Symbols};
use grammar::consts::INPUT_LIFETIME;
use grammar::pattern::{Pattern, PatternKind};
use grammar::parse_tree as pt;
use grammar::parse_tree::{InternToken, NonterminalString, TerminalString};
use grammar::repr as r;
use util::{map, Map};
#[cfg(test)]
mod test;
pub fn lower(grammar: pt::Grammar, types: r::Types) -> NormResult<r::Grammar> {
let state = LowerState::new(types, &grammar);
state.lower(grammar)
@ -22,7 +20,7 @@ pub fn lower(grammar: pt::Grammar, types: r::Types) -> NormResult<r::Grammar> {
struct LowerState {
prefix: String,
action_fn_defns: Vec<r::ActionFnDefn>,
productions: Vec<r::Production>,
nonterminals: Map<NonterminalString, r::NonterminalData>,
conversions: Vec<(TerminalString, Pattern<r::TypeRepr>)>,
intern_token: Option<InternToken>,
types: r::Types,
@ -31,9 +29,9 @@ struct LowerState {
impl LowerState {
fn new(types: r::Types, grammar: &pt::Grammar) -> LowerState {
LowerState {
prefix: find_prefix(&grammar),
prefix: grammar.prefix.clone(),
action_fn_defns: vec![],
productions: vec![],
nonterminals: map(),
conversions: vec![],
types: types,
intern_token: None,
@ -56,7 +54,7 @@ impl LowerState {
token_span = Some(grammar.span);
let span = grammar.span;
let input_str = r::TypeRepr::Ref {
lifetime: Some(intern(pt::INPUT_LIFETIME)),
lifetime: Some(intern(INPUT_LIFETIME)),
mutable: false,
referent: Box::new(r::TypeRepr::Nominal(r::NominalTypeRepr {
path: r::Path::str(),
@ -99,28 +97,33 @@ impl LowerState {
}
pt::GrammarItem::Nonterminal(nt) => {
for alt in nt.alternatives {
let nt_type = self.types.nonterminal_type(nt.name).clone();
let symbols = self.symbols(&alt.expr.symbols);
let action = self.action_kind(nt_type, &alt.expr, &symbols, alt.action);
let production = r::Production {
nonterminal: nt.name,
span: alt.span,
symbols: symbols,
action: action,
};
self.productions.push(production);
}
let nt_name = nt.name;
let productions: Vec<_> =
nt.alternatives
.into_iter()
.map(|alt| {
let nt_type = self.types.nonterminal_type(nt_name).clone();
let symbols = self.symbols(&alt.expr.symbols);
let action = self.action_kind(nt_type, &alt.expr,
&symbols, alt.action);
r::Production {
nonterminal: nt_name,
span: alt.span,
symbols: symbols,
action: action,
}
})
.collect();
self.nonterminals.insert(nt_name, r::NonterminalData {
name: nt_name,
annotations: nt.annotations,
span: nt.span,
productions: productions
});
}
}
}
let mut productions = map();
for production in self.productions {
let mut vec = productions.entry(production.nonterminal).or_insert(vec![]);
vec.push(production);
}
let parameters =
grammar.parameters.iter()
.map(|p| r::Parameter { name: p.name, ty: p.ty.type_repr() })
@ -137,7 +140,7 @@ impl LowerState {
start_nonterminals: start_symbols,
uses: uses,
action_fn_defns: self.action_fn_defns,
productions: productions,
nonterminals: self.nonterminals,
conversions: self.conversions.into_iter().collect(),
types: self.types,
token_span: token_span.unwrap(),
@ -172,12 +175,20 @@ impl LowerState {
};
let symbols = vec![r::Symbol::Nonterminal(nt.name)];
let action_fn = self.action_fn(nt_type, false, &expr, &symbols, None);
self.productions.push(r::Production {
let production = r::Production {
nonterminal: fake_name,
symbols: symbols,
action: r::ActionKind::Call(action_fn),
action: action_fn,
span: nt.span
});
};
self.nonterminals.insert(
fake_name,
r::NonterminalData {
name: fake_name,
annotations: vec![],
span: nt.span,
productions: vec![production]
});
(nt.name, fake_name)
})
.collect()
@ -188,28 +199,42 @@ impl LowerState {
expr: &pt::ExprSymbol,
symbols: &[r::Symbol],
action: Option<pt::ActionKind>)
-> r::ActionKind
-> r::ActionFn
{
match action {
Some(pt::ActionKind::Lookahead) =>
r::ActionKind::Lookahead,
self.lookahead_action_fn(),
Some(pt::ActionKind::Lookbehind) =>
r::ActionKind::Lookbehind,
Some(pt::ActionKind::User(string)) => {
let action_fn = self.action_fn(nt_type, false, &expr, &symbols, Some(string));
r::ActionKind::Call(action_fn)
}
Some(pt::ActionKind::Fallible(string)) => {
let action_fn = self.action_fn(nt_type, true, &expr, &symbols, Some(string));
r::ActionKind::TryCall(action_fn)
}
None => {
let action_fn = self.action_fn(nt_type, false, &expr, &symbols, None);
r::ActionKind::Call(action_fn)
}
self.lookbehind_action_fn(),
Some(pt::ActionKind::User(string)) =>
self.action_fn(nt_type, false, &expr, &symbols, Some(string)),
Some(pt::ActionKind::Fallible(string)) =>
self.action_fn(nt_type, true, &expr, &symbols, Some(string)),
None =>
self.action_fn(nt_type, false, &expr, &symbols, None),
}
}
fn lookahead_action_fn(&mut self) -> r::ActionFn {
let action_fn_defn = r::ActionFnDefn {
fallible: false,
ret_type: self.types.terminal_loc_type(),
kind: r::ActionFnDefnKind::Lookaround(r::LookaroundActionFnDefn::Lookahead),
};
self.add_action_fn(action_fn_defn)
}
fn lookbehind_action_fn(&mut self) -> r::ActionFn {
let action_fn_defn = r::ActionFnDefn {
fallible: false,
ret_type: self.types.terminal_loc_type(),
kind: r::ActionFnDefnKind::Lookaround(r::LookaroundActionFnDefn::Lookbehind),
};
self.add_action_fn(action_fn_defn)
}
fn action_fn(&mut self,
nt_type: r::TypeRepr,
fallible: bool,
@ -240,11 +265,13 @@ impl LowerState {
symbols.len());
r::ActionFnDefn {
arg_patterns: arg_patterns,
arg_types: arg_types,
ret_type: nt_type,
fallible: fallible,
code: action
ret_type: nt_type,
kind: r::ActionFnDefnKind::User(r::UserActionFnDefn {
arg_patterns: arg_patterns,
arg_types: arg_types,
code: action,
}),
}
}
Symbols::Anon(indices) => {
@ -260,18 +287,23 @@ impl LowerState {
});
let action = action.replace("<>", &name_str);
r::ActionFnDefn {
arg_patterns: arg_patterns,
arg_types: arg_types,
ret_type: nt_type,
fallible: fallible,
code: action
ret_type: nt_type,
kind: r::ActionFnDefnKind::User(r::UserActionFnDefn {
arg_patterns: arg_patterns,
arg_types: arg_types,
code: action,
}),
}
}
};
self.add_action_fn(action_fn_defn)
}
fn add_action_fn(&mut self, action_fn_defn: r::ActionFnDefn) -> r::ActionFn {
let index = r::ActionFn::new(self.action_fn_defns.len());
self.action_fn_defns.push(action_fn_defn);
index
}
@ -325,32 +357,3 @@ fn patterns<I>(mut chosen: I, num_args: usize) -> Vec<InternedString>
result
}
// Find a unique prefix like `__` or `___` that doesn't appear
// anywhere in any action strings, nonterminal names, etc. Obviously
// this is stricter than needed, since the action string might be like
// `print("__1")`, in which case we'll detect a false conflict (or it
// might contain a variable named `__1x`, etc). But so what.
fn find_prefix(grammar: &pt::Grammar) -> String {
let mut prefix = format!("__");
while
grammar.items
.iter()
.filter_map(|i| i.as_nonterminal())
.flat_map(|nt| nt.alternatives.iter())
.filter_map(|alt| alt.action.as_ref())
.filter_map(|action| action.as_user())
.any(|s| s.contains(&prefix))
||
grammar.items
.iter()
.filter_map(|i| i.as_nonterminal())
.any(|nt| nt.name.0.starts_with(&prefix))
{
prefix.push('_');
}
prefix
}

66
lalrpop-snap/src/normalize/lower/test.rs
View File

@ -1,66 +0,0 @@
use grammar::repr::{Grammar, Production};
use normalize::normalize_without_validating;
use parser;
use test_util::expect_debug;
fn flat_productions(grammar: &Grammar) -> Vec<Production> {
let mut productions: Vec<_> =
grammar.productions.values()
.flat_map(|prods| prods.iter().cloned())
.collect();
// sort by the action fn index just to get a consistent ordering
productions.sort_by(|k1, k2| {
Ord::cmp(&k1.action, &k2.action)
});
productions
}
#[test]
fn test_comma() {
let grammar = parser::parse_grammar(r#"
grammar;
extern { enum Tok { "," => .., "Id" => .. } }
Comma<E>: Vec<E> =
<v:(<E> ",")*> <e:E?> =>
v.into_iter().chain(e.into_iter()).collect();
Ids = Comma<"Id">;
"#).unwrap();
let actual = normalize_without_validating(grammar).unwrap();
expect_debug(flat_productions(&actual),
r#"[
Ids = Comma<"Id"> => Call(ActionFn(0));,
Comma<"Id"> = (<"Id"> ",")*, "Id"? => Call(ActionFn(1));,
"Id"? = "Id" => Call(ActionFn(2));,
"Id"? = => Call(ActionFn(3));,
(<"Id"> ",")* = => Call(ActionFn(4));,
(<"Id"> ",")* = (<"Id"> ",")*, (<"Id"> ",") => Call(ActionFn(5));,
(<"Id"> ",") = "Id", "," => Call(ActionFn(6));
]"#);
expect_debug(&actual.action_fn_defns,
r#"[
fn _(__0: Vec<Tok>) -> Vec<Tok> { (__0) },
fn _(v: ::std::vec::Vec<Tok>, e: ::std::option::Option<Tok>) -> Vec<Tok> { v.into_iter().chain(e.into_iter()).collect() },
fn _(__0: Tok) -> ::std::option::Option<Tok> { Some(__0) },
fn _() -> ::std::option::Option<Tok> { None },
fn _() -> ::std::vec::Vec<Tok> { vec![] },
fn _(v: ::std::vec::Vec<Tok>, e: Tok) -> ::std::vec::Vec<Tok> { { let mut v = v; v.push(e); v } },
fn _(__0: Tok, _: Tok) -> Tok { (__0) }
]"#);
}
#[test]
fn test_intern_token() {
let grammar = parser::parse_grammar(r#"
grammar;
extern { }
A = ",";
"#).unwrap();
normalize_without_validating(grammar).unwrap();
}

32
lalrpop-snap/src/normalize/macro_expand/mod.rs
View File

@ -1,6 +1,7 @@
use std::collections::{HashMap, HashSet};
use intern::{intern, read, InternedString};
use grammar::parse_tree::{ActionKind, Alternative,
use grammar::consts::INLINE;
use grammar::parse_tree::{ActionKind, Alternative, Annotation,
Condition, ConditionOp,
ExprSymbol,
Grammar, GrammarItem,
@ -361,7 +362,7 @@ impl MacroExpander {
public: false,
span: span,
name: name,
annotations: vec![],
annotations: inline(span),
args: vec![],
type_decl: Some(ty_ref),
alternatives: vec![Alternative { span: span,
@ -386,11 +387,16 @@ impl MacroExpander {
let path = Path::vec();
let ty_ref = TypeRef::Nominal { path: path, types: vec![base_symbol_ty] };
let plus_repeat = Box::new(RepeatSymbol {
op: RepeatOp::Plus,
symbol: repeat.symbol.clone()
});
Ok(GrammarItem::Nonterminal(NonterminalData {
public: false,
span: span,
name: name,
annotations: vec![],
annotations: inline(span),
args: vec![],
type_decl: Some(ty_ref),
alternatives: vec![
@ -402,7 +408,7 @@ impl MacroExpander {
action: action("vec![]")
},
// X* = <v:X+> <e:X>
// X* = <v:X+>
Alternative {
span: span,
expr: ExprSymbol {
@ -413,15 +419,10 @@ impl MacroExpander {
v,
Box::new(
Symbol::new(span,
SymbolKind::Nonterminal(name))))),
Symbol::new(
span,
SymbolKind::Name(
e,
Box::new(repeat.symbol.clone())))]
SymbolKind::Repeat(plus_repeat)))))],
},
condition: None,
action: action("{ let mut v = v; v.push(e); v }")
action: action("v"),
}],
}))
}
@ -473,7 +474,7 @@ impl MacroExpander {
public: false,
span: span,
name: name,
annotations: vec![],
annotations: inline(span),
args: vec![],
type_decl: Some(ty_ref),
alternatives: vec![
@ -527,3 +528,10 @@ fn maybe_tuple(v: Vec<TypeRef>) -> TypeRef {
fn action(s: &str) -> Option<ActionKind> {
Some(ActionKind::User(s.to_string()))
}
fn inline(span: Span) -> Vec<Annotation> {
vec![Annotation {
id_span: span,
id: intern(INLINE),
}]
}

14
lalrpop-snap/src/normalize/macro_expand/test.rs
View File

@ -23,17 +23,27 @@ grammar;
`Comma<"Id">`: Vec<#"Id"#> =
<v:`(<"Id"> ",")*`> <e:`"Id"?`> => v.into_iter().chain(e.into_iter()).collect();
#[inline]
`"Id"?`: ::std::option::Option<#"Id"#> = {
"Id" => Some(<>),
=> None
};
#[inline]
`(<"Id"> ",")*`: ::std::vec::Vec<#`(<"Id"> ",")`#> = {
=> vec![],
<v:`(<"Id"> ",")*`> <e:`(<"Id"> ",")`> => { let mut v = v; v.push(e); v }
<v:`(<"Id"> ",")+`> => v,
};
`(<"Id"> ",")`: #"Id"# = <"Id"> "," => (<>);
#[inline]
`(<"Id"> ",")`: #"Id"# = {
<"Id"> "," => (<>),
};
`(<"Id"> ",")+`: ::std::vec::Vec<#`(<"Id"> ",")`#> = {
`(<"Id"> ",")` => vec![<>],
<v:`(<"Id"> ",")+`> <e:`(<"Id"> ",")`> => { let mut v = v; v.push(e); v },
};
"##).unwrap();
compare(actual, expected);

8
lalrpop-snap/src/normalize/mod.rs
View File

@ -35,6 +35,11 @@ pub fn normalize_without_validating(grammar: pt::Grammar) -> NormResult<r::Gramm
}
fn normalize_helper(grammar: pt::Grammar, validate: bool) -> NormResult<r::Grammar> {
let grammar = try!(lower_helper(grammar, validate));
inline::inline(grammar)
}
fn lower_helper(grammar: pt::Grammar, validate: bool) -> NormResult<r::Grammar> {
if validate { try!(prevalidate::validate(&grammar)); }
let grammar = try!(resolve::resolve(grammar));
let grammar = try!(macro_expand::expand_macros(grammar));
@ -80,6 +85,9 @@ mod tyinfer;
// Lowers the parse tree to the repr notation.
mod lower;
// Inline nonterminals that have requested it.
mod inline;
///////////////////////////////////////////////////////////////////////////
// Shared routines

19
lalrpop-snap/src/normalize/prevalidate/mod.rs
View File

@ -3,6 +3,7 @@
use super::{NormResult, NormError};
use super::norm_util::{self, Symbols};
use grammar::consts::*;
use grammar::parse_tree::*;
use grammar::repr;
use intern::{intern, InternedString};
@ -73,10 +74,22 @@ impl<'grammar> Validator<'grammar> {
}
}
GrammarItem::Nonterminal(ref data) => {
let inline_annotation = intern(INLINE);
let known_annotations = vec![inline_annotation];
let mut found_annotations = set();
for annotation in &data.annotations {
return_err!(annotation.id_span,
"unrecognized annotation `{}`",
annotation.id);
if !known_annotations.contains(&annotation.id) {
return_err!(annotation.id_span,
"unrecognized annotation `{}`",
annotation.id);
} else if !found_annotations.insert(annotation.id) {
return_err!(annotation.id_span,
"duplicate annotation `{}`",
annotation.id);
} else if annotation.id == inline_annotation && data.public {
return_err!(annotation.id_span,
"public items cannot be marked #[inline]");
}
}
for alternative in &data.alternatives {

16
lalrpop-snap/src/normalize/prevalidate/test.rs
View File

@ -54,3 +54,19 @@ fn unrecognized_annotation() {
r#"grammar; #[foo] Term = ();"#,
r#" ~~~ "#);
}
#[test]
fn duplicate_annotation() {
check_err(
r#"duplicate annotation `inline`"#,
r#"grammar; #[inline] #[inline] Term = ();"#,
r#" ~~~~~~ "#);
}
#[test]
fn pub_inline_annotation() {
check_err(
r#"public items cannot be marked #\[inline\]"#,
r#"grammar; #[inline] pub Term = ();"#,
r#" ~~~~~~ "#);
}

3
lalrpop-snap/src/normalize/token_check/mod.rs
View File

@ -10,6 +10,7 @@ use super::{NormResult, NormError};
use intern::{self, intern};
use lexer::re;
use lexer::dfa::{self, Precedence};
use grammar::consts::*;
use grammar::parse_tree::*;
use util::{Set};
use util::{map, Map};
@ -233,7 +234,7 @@ pub fn construct(grammar: &mut Grammar, literals_map: Map<TerminalLiteral, Span>
}
}
grammar.type_parameters.push(TypeParameter::Lifetime(input_lifetime));
grammar.type_parameters.insert(0, TypeParameter::Lifetime(input_lifetime));
let parameter = Parameter {
name: input_parameter,

4
lalrpop-snap/src/normalize/tyinfer/mod.rs
View File

@ -2,11 +2,9 @@ use super::{NormResult, NormError};
use super::norm_util::{self, AlternativeAction, Symbols};
use std::collections::{HashMap};
use grammar::consts::{ERROR, INPUT_LIFETIME, LOCATION};
use grammar::parse_tree::{ActionKind, Alternative,
ERROR,
Grammar,
INPUT_LIFETIME,
LOCATION,
NonterminalData, NonterminalString,
Path,
Span,

60999
lalrpop-snap/src/parser/lrgrammar.rs
View File

File diff suppressed because it is too large Load Diff

9
lalrpop-snap/src/parser/mod.rs
View File

@ -12,7 +12,14 @@ pub fn parse_grammar<'input>(input: &'input str)
-> Result<Grammar, ParseError<'input>>
{
let tokenizer = tok::Tokenizer::new(input, 0);
lrgrammar::parse_Grammar(input, tokenizer)
let mut grammar = try!(lrgrammar::parse_Grammar(input, tokenizer));
// find a unique prefix that does not appear anywhere in the input
while input.contains(&grammar.prefix) {
grammar.prefix.push('_');
}
Ok(grammar)
}
fn parse_pattern<'input>(input: &'input str, offset: usize)

3
lalrpop-snap/src/test_util.rs
View File

@ -60,5 +60,6 @@ pub fn check_norm_err(expected_err: &str,
assert!(start_index <= end_index);
assert_eq!(err.span, pt::Span(start_index, end_index));
assert!(expected_err.is_match(&err.message),
"unexpected error text `{}`, did not match `{}`", err.message, expected_err);
"unexpected error text `{}`, which did not match regular expression `{}`",
err.message, expected_err);
}

19
lalrpop-snap/src/tok/mod.rs
View File

@ -379,8 +379,25 @@ impl<'input> Tokenizer<'input> {
// for a suitable terminator: `,`, `;`, `]`, `}`, or `)`.
let mut balance = 0; // number of unclosed `(` etc
loop {
println!("code: balance={:?} self.lookahead={:?}", balance, self.lookahead);
if let Some((idx, c)) = self.lookahead {
if open_delims.find(c).is_some() {
if c == '"' {
self.bump();
try!(self.string_literal(idx)); // discard the produced token
continue;
} else if c == 'r' {
self.bump();
if let Some((idx, '#')) = self.lookahead {
try!(self.regex_literal(idx));
}
continue;
} else if c == '/' {
self.bump();
if let Some((_, '/')) = self.lookahead {
self.take_until(|c| c == '\n');
}
continue;
} else if open_delims.find(c).is_some() {
balance += 1;
} else if balance > 0 {
if close_delims.find(c).is_some() {

26
lalrpop-snap/src/tok/test.rs
View File

@ -45,6 +45,32 @@ fn code1() {
]);
}
#[test]
fn code_paren() { // Issue #25
test(r#"=> a("(", c),"#, vec![
(r#"~~~~~~~~~~~~ "#, EqualsGreaterThanCode(r#" a("(", c)"#)),
(r#" ~"#, Comma),
]);
}
#[test]
fn code_regex_paren() { // Issue #25
test(r###"=> a(r##"("#""##, c),"###, vec![
(r###"~~~~~~~~~~~~~~~~~~~~ "###, EqualsGreaterThanCode(r###" a(r##"("#""##, c)"###)),
(r###" ~"###, Comma),
]);
}
#[test]
fn code_comment_eol() {
test("=> a(// (
),", vec![
("~~~~~~~~~
~,", EqualsGreaterThanCode(" a(// (\n)")),
("=> a(// (
)~", Comma)]);
}
#[test]
fn code2() {
test("=>? a(b, c),", vec![

1
lalrpop/src/tok/mod.rs
View File

@ -379,7 +379,6 @@ impl<'input> Tokenizer<'input> {
// for a suitable terminator: `,`, `;`, `]`, `}`, or `)`.
let mut balance = 0; // number of unclosed `(` etc
loop {
println!("code: balance={:?} self.lookahead={:?}", balance, self.lookahead);
if let Some((idx, c)) = self.lookahead {
if c == '"' {
self.bump();