mirror of
https://github.com/fluencelabs/lalrpop
synced 2025-03-16 17:00:53 +00:00
Merge pull request #50 from nikomatsakis/new-snapshot
Move to a new snapshot
This commit is contained in:
commit
8fafb028f5
@ -14,7 +14,7 @@ rand = "0.3"
|
||||
itertools = "0.3"
|
||||
term = "0.2"
|
||||
unicode-xid = "0.0.2"
|
||||
rusty-peg = "0.3"
|
||||
petgraph = "0.1.11"
|
||||
|
||||
[dependencies.lalrpop-util]
|
||||
path = "../lalrpop-util"
|
||||
|
206
lalrpop-snap/src/build/action.rs
Normal file
206
lalrpop-snap/src/build/action.rs
Normal file
@ -0,0 +1,206 @@
|
||||
//! Code for generating action code.
|
||||
|
||||
use grammar::repr as r;
|
||||
use rust::RustWrite;
|
||||
use std::io::{self, Write};
|
||||
|
||||
pub fn emit_action_code<W:Write>(grammar: &r::Grammar,
|
||||
rust: &mut RustWrite<W>)
|
||||
-> io::Result<()>
|
||||
{
|
||||
for (i, defn) in grammar.action_fn_defns.iter().enumerate() {
|
||||
rust!(rust, "");
|
||||
|
||||
match defn.kind {
|
||||
r::ActionFnDefnKind::User(ref data) =>
|
||||
try!(emit_user_action_code(grammar, rust, i, defn, data)),
|
||||
r::ActionFnDefnKind::Lookaround(ref variant) =>
|
||||
try!(emit_lookaround_action_code(grammar, rust, i, defn, variant)),
|
||||
r::ActionFnDefnKind::Inline(ref data) =>
|
||||
try!(emit_inline_action_code(grammar, rust, i, defn, data)),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ret_type_string(grammar: &r::Grammar,
|
||||
defn: &r::ActionFnDefn)
|
||||
-> String
|
||||
{
|
||||
if defn.fallible {
|
||||
format!("Result<{},{}ParseError<{},{},{}>>",
|
||||
defn.ret_type,
|
||||
grammar.prefix,
|
||||
grammar.types.terminal_loc_type(),
|
||||
grammar.types.terminal_token_type(),
|
||||
grammar.types.error_type())
|
||||
} else {
|
||||
format!("{}", defn.ret_type)
|
||||
}
|
||||
}
|
||||
|
||||
fn emit_user_action_code<W:Write>(grammar: &r::Grammar,
|
||||
rust: &mut RustWrite<W>,
|
||||
index: usize,
|
||||
defn: &r::ActionFnDefn,
|
||||
data: &r::UserActionFnDefn)
|
||||
-> io::Result<()>
|
||||
{
|
||||
let ret_type = ret_type_string(grammar, defn);
|
||||
|
||||
let lookarounds = vec![
|
||||
format!("{}lookbehind: &Option<{}>", grammar.prefix, grammar.types.terminal_loc_type()),
|
||||
format!("{}lookahead: &Option<{}>", grammar.prefix, grammar.types.triple_type())];
|
||||
|
||||
try!(rust.write_pub_fn_header(
|
||||
grammar,
|
||||
format!("{}action{}", grammar.prefix, index),
|
||||
vec![],
|
||||
data.arg_patterns.iter()
|
||||
.zip(data.arg_types.iter())
|
||||
.map(|(p, t)| format!("{}: {}", p, t))
|
||||
.chain(lookarounds)
|
||||
.collect(),
|
||||
ret_type,
|
||||
vec![]));
|
||||
rust!(rust, "{{");
|
||||
rust!(rust, "{}", data.code);
|
||||
rust!(rust, "}}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn emit_lookaround_action_code<W:Write>(grammar: &r::Grammar,
|
||||
rust: &mut RustWrite<W>,
|
||||
index: usize,
|
||||
_defn: &r::ActionFnDefn,
|
||||
data: &r::LookaroundActionFnDefn)
|
||||
-> io::Result<()>
|
||||
{
|
||||
try!(rust.write_pub_fn_header(
|
||||
grammar,
|
||||
format!("{}action{}", grammar.prefix, index),
|
||||
vec![],
|
||||
vec![format!("{}lookbehind: &Option<{}>", grammar.prefix, grammar.types.terminal_loc_type()),
|
||||
format!("{}lookahead: &Option<{}>", grammar.prefix, grammar.types.triple_type())],
|
||||
format!("{}", grammar.types.terminal_loc_type()),
|
||||
vec![]));
|
||||
|
||||
rust!(rust, "{{");
|
||||
match *data {
|
||||
r::LookaroundActionFnDefn::Lookahead => {
|
||||
// take the lookahead, if any; otherwise, we are
|
||||
// at EOF, so taker the lookbehind (end of last
|
||||
// pushed token); if that is missing too, then
|
||||
// supply default.
|
||||
rust!(rust, "{}lookahead.as_ref()\
|
||||
.map(|o| ::std::clone::Clone::clone(&o.0))\
|
||||
.or_else(|| ::std::clone::Clone::clone(&{}lookbehind))\
|
||||
.unwrap_or_default()",
|
||||
grammar.prefix, grammar.prefix);
|
||||
}
|
||||
r::LookaroundActionFnDefn::Lookbehind => {
|
||||
// take lookbehind or supply default
|
||||
rust!(rust, "::std::clone::Clone::clone(&{}lookbehind).unwrap_or_default()",
|
||||
grammar.prefix);
|
||||
}
|
||||
}
|
||||
rust!(rust, "}}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn emit_inline_action_code<W:Write>(grammar: &r::Grammar,
|
||||
rust: &mut RustWrite<W>,
|
||||
index: usize,
|
||||
defn: &r::ActionFnDefn,
|
||||
data: &r::InlineActionFnDefn)
|
||||
-> io::Result<()>
|
||||
{
|
||||
let ret_type = ret_type_string(grammar, defn);
|
||||
|
||||
let arg_types: Vec<_> =
|
||||
data.symbols.iter()
|
||||
.flat_map(|sym| match *sym {
|
||||
r::InlinedSymbol::Original(s) => vec![s],
|
||||
r::InlinedSymbol::Inlined(_, ref syms) => syms.clone(),
|
||||
})
|
||||
.map(|s| s.ty(&grammar.types))
|
||||
.collect();
|
||||
|
||||
let arguments: Vec<_> =
|
||||
arg_types.iter()
|
||||
.enumerate()
|
||||
.map(|(i, t)| format!("{}{}: {}", grammar.prefix, i, t))
|
||||
.chain(vec![
|
||||
format!("{}lookbehind: &Option<{}>",
|
||||
grammar.prefix,
|
||||
grammar.types.terminal_loc_type()),
|
||||
format!("{}lookahead: &Option<{}>",
|
||||
grammar.prefix,
|
||||
grammar.types.triple_type())])
|
||||
.collect();
|
||||
|
||||
try!(rust.write_pub_fn_header(
|
||||
grammar,
|
||||
format!("{}action{}", grammar.prefix, index),
|
||||
vec![],
|
||||
arguments,
|
||||
ret_type,
|
||||
vec![]));
|
||||
rust!(rust, "{{");
|
||||
|
||||
// create temporaries for the inlined things
|
||||
let mut arg_counter = 0;
|
||||
let mut temp_counter = 0;
|
||||
for symbol in &data.symbols {
|
||||
match *symbol {
|
||||
r::InlinedSymbol::Original(_) => {
|
||||
arg_counter += 1;
|
||||
}
|
||||
r::InlinedSymbol::Inlined(inlined_action, ref syms) => {
|
||||
rust!(rust, "let {}temp{} = {}action{}(",
|
||||
grammar.prefix, temp_counter,
|
||||
grammar.prefix, inlined_action.index());
|
||||
for parameter in &grammar.parameters {
|
||||
rust!(rust, "{},", parameter.name);
|
||||
}
|
||||
for _ in syms {
|
||||
rust!(rust, "{}{},", grammar.prefix, arg_counter);
|
||||
arg_counter += 1;
|
||||
}
|
||||
rust!(rust, "{}lookbehind,", grammar.prefix);
|
||||
rust!(rust, "{}lookahead,", grammar.prefix);
|
||||
rust!(rust, ");");
|
||||
temp_counter += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rust!(rust, "{}action{}(", grammar.prefix, data.action.index());
|
||||
for parameter in &grammar.parameters {
|
||||
rust!(rust, "{},", parameter.name);
|
||||
}
|
||||
let mut arg_counter = 0;
|
||||
let mut temp_counter = 0;
|
||||
for symbol in &data.symbols {
|
||||
match *symbol {
|
||||
r::InlinedSymbol::Original(_) => {
|
||||
rust!(rust, "{}{},", grammar.prefix, arg_counter);
|
||||
arg_counter += 1;
|
||||
}
|
||||
r::InlinedSymbol::Inlined(_, ref syms) => {
|
||||
rust!(rust, "{}temp{},", grammar.prefix, temp_counter);
|
||||
temp_counter += 1;
|
||||
arg_counter += syms.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
rust!(rust, "{}lookbehind,", grammar.prefix);
|
||||
rust!(rust, "{}lookahead,", grammar.prefix);
|
||||
rust!(rust, ")");
|
||||
|
||||
rust!(rust, "}}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
@ -11,19 +11,21 @@ use rust::RustWrite;
|
||||
use tok;
|
||||
use self::filetext::FileText;
|
||||
|
||||
use std::env::current_dir;
|
||||
use std::fs;
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::exit;
|
||||
|
||||
mod action;
|
||||
mod filetext;
|
||||
|
||||
pub fn process_root() -> io::Result<()> {
|
||||
process_dir("src", false)
|
||||
process_dir(try!(current_dir()), false)
|
||||
}
|
||||
|
||||
pub fn process_root_unconditionally() -> io::Result<()> {
|
||||
process_dir("src", true)
|
||||
process_dir(try!(current_dir()), true)
|
||||
}
|
||||
|
||||
fn process_dir<P:AsRef<Path>>(root_dir: P, force_build: bool) -> io::Result<()> {
|
||||
@ -274,48 +276,13 @@ fn emit_recursive_ascent(output_path: &Path, grammar: &r::Grammar) -> io::Result
|
||||
try!(intern_token::compile(&grammar, intern_token, &mut rust));
|
||||
}
|
||||
|
||||
try!(emit_action_code(grammar, &mut rust));
|
||||
try!(action::emit_action_code(grammar, &mut rust));
|
||||
|
||||
try!(emit_to_triple_trait(grammar, &mut rust));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn emit_action_code<W:Write>(grammar: &r::Grammar,
|
||||
rust: &mut RustWrite<W>)
|
||||
-> io::Result<()>
|
||||
{
|
||||
for (i, defn) in grammar.action_fn_defns.iter().enumerate() {
|
||||
rust!(rust, "");
|
||||
|
||||
let ret_type = if defn.fallible {
|
||||
format!("Result<{},{}ParseError<{},{},{}>>",
|
||||
defn.ret_type,
|
||||
grammar.prefix,
|
||||
grammar.types.terminal_loc_type(),
|
||||
grammar.types.terminal_token_type(),
|
||||
grammar.types.error_type())
|
||||
} else {
|
||||
format!("{}", defn.ret_type)
|
||||
};
|
||||
|
||||
try!(rust.write_pub_fn_header(
|
||||
grammar,
|
||||
format!("{}action{}", grammar.prefix, i),
|
||||
vec![],
|
||||
defn.arg_patterns.iter()
|
||||
.zip(defn.arg_types.iter())
|
||||
.map(|(p, t)| format!("{}: {}", p, t))
|
||||
.collect(),
|
||||
ret_type,
|
||||
vec![]));
|
||||
rust!(rust, "{{");
|
||||
rust!(rust, "{}", defn.code);
|
||||
rust!(rust, "}}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn emit_to_triple_trait<W:Write>(grammar: &r::Grammar,
|
||||
rust: &mut RustWrite<W>)
|
||||
-> io::Result<()>
|
||||
|
@ -37,7 +37,7 @@ impl<'grammar> Generator<'grammar> {
|
||||
}
|
||||
|
||||
self.depth += 1;
|
||||
let productions = &self.grammar.productions[&nt];
|
||||
let productions = self.grammar.productions_for(nt);
|
||||
let index: usize = self.rng.gen_range(0, productions.len());
|
||||
let production = &productions[index];
|
||||
let trees: Option<Vec<_>> = production.symbols.iter()
|
||||
|
15
lalrpop-snap/src/grammar/consts.rs
Normal file
15
lalrpop-snap/src/grammar/consts.rs
Normal file
@ -0,0 +1,15 @@
|
||||
/// Recognized associated type for the token location
|
||||
pub const LOCATION: &'static str = "Location";
|
||||
|
||||
/// Recognized associated type for custom errors
|
||||
pub const ERROR: &'static str = "Error";
|
||||
|
||||
/// The lifetime parameter injected when we do not have an external token enum
|
||||
pub const INPUT_LIFETIME: &'static str = "'input";
|
||||
|
||||
/// The parameter injected when we do not have an external token enum
|
||||
pub const INPUT_PARAMETER: &'static str = "input";
|
||||
|
||||
/// The annotation to request inlining.
|
||||
pub const INLINE: &'static str = "inline";
|
||||
|
@ -1,5 +1,6 @@
|
||||
//! The grammar definition.
|
||||
|
||||
pub mod consts;
|
||||
pub mod parse_tree;
|
||||
pub mod pattern;
|
||||
pub mod repr;
|
||||
|
@ -14,6 +14,8 @@ use util::Sep;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Grammar {
|
||||
// see field `prefix` in `grammar::repr::Grammar`
|
||||
pub prefix: String,
|
||||
pub span: Span,
|
||||
pub type_parameters: Vec<TypeParameter>,
|
||||
pub parameters: Vec<Parameter>,
|
||||
@ -62,12 +64,6 @@ pub struct AssociatedType {
|
||||
pub type_ref: TypeRef,
|
||||
}
|
||||
|
||||
/// Recognized associated type for the token location
|
||||
pub const LOCATION: &'static str = "Location";
|
||||
|
||||
/// Recognized associated type for custom errors
|
||||
pub const ERROR: &'static str = "Error";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct EnumToken {
|
||||
pub type_name: TypeRef,
|
||||
@ -115,18 +111,12 @@ pub enum TypeRef {
|
||||
OfSymbol(SymbolKind),
|
||||
}
|
||||
|
||||
/// The lifetime parameter injected when we do not have an external token enum
|
||||
pub const INPUT_LIFETIME: &'static str = "'input";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum TypeParameter {
|
||||
Lifetime(InternedString),
|
||||
Id(InternedString),
|
||||
}
|
||||
|
||||
/// The parameter injected when we do not have an external token enum
|
||||
pub const INPUT_PARAMETER: &'static str = "input";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Parameter {
|
||||
pub name: InternedString,
|
||||
@ -605,12 +595,3 @@ impl Path {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ActionKind {
|
||||
pub fn as_user(&self) -> Option<&String> {
|
||||
match *self {
|
||||
ActionKind::User(ref s) => Some(s),
|
||||
_ => None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -10,7 +10,8 @@ use std::fmt::{Debug, Display, Formatter, Error};
|
||||
use util::{map, Map, Sep};
|
||||
|
||||
// These concepts we re-use wholesale
|
||||
pub use grammar::parse_tree::{InternToken,
|
||||
pub use grammar::parse_tree::{Annotation,
|
||||
InternToken,
|
||||
NonterminalString,
|
||||
Path,
|
||||
Span,
|
||||
@ -50,12 +51,20 @@ pub struct Grammar {
|
||||
// the grammar proper:
|
||||
|
||||
pub action_fn_defns: Vec<ActionFnDefn>,
|
||||
pub productions: Map<NonterminalString, Vec<Production>>,
|
||||
pub nonterminals: Map<NonterminalString, NonterminalData>,
|
||||
pub token_span: Span,
|
||||
pub conversions: Map<TerminalString, Pattern<TypeRepr>>,
|
||||
pub types: Types,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct NonterminalData {
|
||||
pub name: NonterminalString,
|
||||
pub span: Span,
|
||||
pub annotations: Vec<Annotation>,
|
||||
pub productions: Vec<Production>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Algorithm {
|
||||
LR1,
|
||||
@ -74,7 +83,7 @@ pub struct Production {
|
||||
// handy to have it
|
||||
pub nonterminal: NonterminalString,
|
||||
pub symbols: Vec<Symbol>,
|
||||
pub action: ActionKind,
|
||||
pub action: ActionFn,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
@ -84,22 +93,56 @@ pub enum Symbol {
|
||||
Terminal(TerminalString),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub enum ActionKind {
|
||||
// execute code provided by the user
|
||||
Call(ActionFn),
|
||||
TryCall(ActionFn),
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ActionFnDefn {
|
||||
pub fallible: bool,
|
||||
pub ret_type: TypeRepr,
|
||||
pub kind: ActionFnDefnKind,
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub enum ActionFnDefnKind {
|
||||
User(UserActionFnDefn),
|
||||
Inline(InlineActionFnDefn),
|
||||
Lookaround(LookaroundActionFnDefn),
|
||||
}
|
||||
|
||||
/// An action fn written by a user.
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct UserActionFnDefn {
|
||||
pub arg_patterns: Vec<InternedString>,
|
||||
pub arg_types: Vec<TypeRepr>,
|
||||
pub code: String,
|
||||
}
|
||||
|
||||
/// An action fn generated by the inlining pass. If we were
|
||||
/// inlining `A = B C D` (with action 44) into `X = Y A Z` (with
|
||||
/// action 22), this would look something like:
|
||||
///
|
||||
/// ```
|
||||
/// fn __action66(__0: Y, __1: B, __2: C, __3: D, __4: Z) {
|
||||
/// __action22(__0, __action44(__1, __2, __3), __4)
|
||||
/// }
|
||||
/// ```
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct InlineActionFnDefn {
|
||||
/// in the example above, this would be `action22`
|
||||
pub action: ActionFn,
|
||||
|
||||
/// in the example above, this would be `Y, {action44: B, C, D}, Z`
|
||||
pub symbols: Vec<InlinedSymbol>
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum LookaroundActionFnDefn {
|
||||
Lookahead,
|
||||
Lookbehind,
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ActionFnDefn {
|
||||
pub arg_patterns: Vec<InternedString>,
|
||||
pub arg_types: Vec<TypeRepr>,
|
||||
pub ret_type: TypeRepr,
|
||||
pub fallible: bool,
|
||||
pub code: String,
|
||||
pub enum InlinedSymbol {
|
||||
Original(Symbol),
|
||||
Inlined(ActionFn, Vec<Symbol>),
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
@ -346,6 +389,16 @@ impl Debug for ActionFnDefn {
|
||||
|
||||
impl ActionFnDefn {
|
||||
fn to_fn_string(&self, name: &str) -> String {
|
||||
match self.kind {
|
||||
ActionFnDefnKind::User(ref data) => data.to_fn_string(self, name),
|
||||
ActionFnDefnKind::Inline(ref data) => data.to_fn_string(name),
|
||||
ActionFnDefnKind::Lookaround(ref data) => format!("{:?}", data),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl UserActionFnDefn {
|
||||
fn to_fn_string(&self, defn: &ActionFnDefn, name: &str) -> String {
|
||||
let arg_strings: Vec<String> =
|
||||
self.arg_patterns
|
||||
.iter()
|
||||
@ -354,7 +407,24 @@ impl ActionFnDefn {
|
||||
.collect();
|
||||
|
||||
format!("fn {}({}) -> {} {{ {} }}",
|
||||
name, Sep(", ", &arg_strings), self.ret_type, self.code)
|
||||
name, Sep(", ", &arg_strings), defn.ret_type, self.code)
|
||||
}
|
||||
}
|
||||
|
||||
impl InlineActionFnDefn {
|
||||
fn to_fn_string(&self, name: &str) -> String {
|
||||
let arg_strings: Vec<String> =
|
||||
self.symbols
|
||||
.iter()
|
||||
.map(|inline_sym| match *inline_sym {
|
||||
InlinedSymbol::Original(s) =>
|
||||
format!("{}", s),
|
||||
InlinedSymbol::Inlined(a, ref s) =>
|
||||
format!("{:?}({})", a, Sep(", ", s)),
|
||||
})
|
||||
.collect();
|
||||
|
||||
format!("fn {}(..) {{ {:?}({}) }}", name, self.action, Sep(", ", &arg_strings))
|
||||
}
|
||||
}
|
||||
|
||||
@ -364,8 +434,8 @@ impl Grammar {
|
||||
}
|
||||
|
||||
pub fn productions_for(&self, nonterminal: NonterminalString) -> &[Production] {
|
||||
match self.productions.get(&nonterminal) {
|
||||
Some(v) => &v[..],
|
||||
match self.nonterminals.get(&nonterminal) {
|
||||
Some(v) => &v.productions[..],
|
||||
None => &[], // this...probably shouldn't happen actually?
|
||||
}
|
||||
}
|
||||
@ -377,6 +447,10 @@ impl Grammar {
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn action_is_fallible(&self, f: ActionFn) -> bool {
|
||||
self.action_fn_defns[f.index()].fallible
|
||||
}
|
||||
}
|
||||
|
||||
impl Algorithm {
|
||||
|
@ -124,7 +124,7 @@ impl<'nfa> DFABuilder<'nfa> {
|
||||
.iter()
|
||||
.all(|&item| self.nfa(item).is_rejecting_state(item.nfa_state));
|
||||
|
||||
let kind = if all_rejects {
|
||||
let kind = if all_rejects || item_set.items.is_empty() {
|
||||
Kind::Reject
|
||||
} else if all_accepts.len() == 0 {
|
||||
Kind::Neither
|
||||
@ -160,7 +160,7 @@ impl<'nfa> DFABuilder<'nfa> {
|
||||
|
||||
test_edges.sort();
|
||||
|
||||
// Consider what there is some cahracter that doesn't meet
|
||||
// Consider what there is some character that doesn't meet
|
||||
// any of the tests. In this case, we can just ignore all
|
||||
// the test edges for each of the items and just union all
|
||||
// the "other" edges -- because if it were one of those
|
||||
@ -171,7 +171,7 @@ impl<'nfa> DFABuilder<'nfa> {
|
||||
.collect();
|
||||
|
||||
// we never know the full set
|
||||
assert!(!other_transitions.is_empty());
|
||||
assert!(item_set.items.is_empty() || !other_transitions.is_empty());
|
||||
|
||||
let other_edge = kernel_set.add_state(self.transitive_closure(other_transitions));
|
||||
|
||||
|
@ -38,3 +38,14 @@ fn ambiguous_regex() {
|
||||
assert!(dfa(&[(r#"class"#, P0),
|
||||
(r#"[a-zA-Z_][a-zA-Z0-9_]*"#, P0)]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn issue_32() {
|
||||
assert!(dfa(&[(r#"."#, P0)]).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn issue_35() {
|
||||
assert!(dfa(&[(r#".*"#, P0),
|
||||
(r"[-+]?[0-9]*\.?[0-9]+", P0)]).is_err());
|
||||
}
|
||||
|
@ -31,7 +31,7 @@ pub struct Other;
|
||||
/// edges by enumerating subsequent edges in the vectors until you
|
||||
/// find one with a different `from` value.
|
||||
#[derive(Debug)]
|
||||
struct State {
|
||||
pub struct State {
|
||||
kind: StateKind,
|
||||
first_noop_edge: usize,
|
||||
first_test_edge: usize,
|
||||
@ -52,7 +52,7 @@ pub struct NFAStateIndex(usize);
|
||||
/// now we just ensure this during construction, but one could easily
|
||||
/// sort).
|
||||
#[derive(Debug)]
|
||||
struct Edges {
|
||||
pub struct Edges {
|
||||
noop_edges: Vec<Edge<Noop>>,
|
||||
|
||||
// edges where we are testing the character in some way; for any
|
||||
|
@ -103,7 +103,7 @@ impl<'str> RegexParser<'str> {
|
||||
alternatives.push(try!(self.alternative()));
|
||||
|
||||
match self.lookahead {
|
||||
Some((_, '|')) => { continue; }
|
||||
Some((_, '|')) => { self.bump(); continue; }
|
||||
_ => { break; }
|
||||
}
|
||||
}
|
||||
@ -124,7 +124,7 @@ impl<'str> RegexParser<'str> {
|
||||
'[' => { elems.push(try!(self.range(index))); }
|
||||
']' => { break; }
|
||||
'|' => { break; }
|
||||
'.' => { elems.push(Elem::Any); }
|
||||
'.' => { self.bump(); elems.push(Elem::Any); }
|
||||
_ => { self.bump(); elems.push(Elem::Test(Test::Char(c))); }
|
||||
}
|
||||
}
|
||||
|
@ -20,3 +20,7 @@ fn parse_neg() {
|
||||
fn parse_unclosed_group() {
|
||||
parse_regex(r"(123").unwrap_err();
|
||||
}
|
||||
#[test]
|
||||
fn alt_oom() {
|
||||
parse_regex(r"(%%|[^%])+").unwrap();
|
||||
}
|
||||
|
@ -11,12 +11,14 @@
|
||||
extern crate diff;
|
||||
extern crate lalrpop_intern as intern;
|
||||
extern crate lalrpop_util;
|
||||
extern crate rand;
|
||||
extern crate petgraph;
|
||||
extern crate regex;
|
||||
extern crate term;
|
||||
extern crate itertools;
|
||||
extern crate unicode_xid;
|
||||
|
||||
#[cfg(test)]
|
||||
extern crate rand;
|
||||
|
||||
// rust exports a macro that others use, so hoist it early.
|
||||
#[macro_use]
|
||||
mod rust;
|
||||
|
@ -2,8 +2,7 @@
|
||||
//!
|
||||
//! [recursive ascent]: https://en.wikipedia.org/wiki/Recursive_ascent_parser
|
||||
|
||||
use grammar::repr::{ActionKind,
|
||||
Grammar,
|
||||
use grammar::repr::{Grammar,
|
||||
NonterminalString,
|
||||
Symbol,
|
||||
TerminalString, TypeParameter, TypeRepr, Types};
|
||||
@ -142,7 +141,7 @@ impl<'ascent,'grammar,W:Write> RecursiveAscent<'ascent,'grammar,W> {
|
||||
// making different enums per state, but this would mean we
|
||||
// have to unwrap and rewrap as we pass up the stack, which
|
||||
// seems silly
|
||||
for &nt in self.grammar.productions.keys() {
|
||||
for &nt in self.grammar.nonterminals.keys() {
|
||||
rust!(self.out, "{}({}),", Escape(nt), self.types.nonterminal_type(nt));
|
||||
}
|
||||
|
||||
@ -360,51 +359,32 @@ impl<'ascent,'grammar,W:Write> RecursiveAscent<'ascent,'grammar,W> {
|
||||
rust!(self.out, "let {} = {}.take().unwrap();", sym, sym);
|
||||
}
|
||||
|
||||
let transfered_syms = transfer_syms.len();
|
||||
|
||||
let mut args = transfer_syms;
|
||||
args.push(format!("&{}lookbehind", self.prefix));
|
||||
args.push(format!("&{}lookahead", self.prefix));
|
||||
|
||||
// invoke the action code
|
||||
match production.action {
|
||||
ActionKind::Call(action_fn) => {
|
||||
rust!(self.out, "let {}nt = super::{}action{}({}{});",
|
||||
self.prefix,
|
||||
self.prefix,
|
||||
action_fn.index(),
|
||||
self.grammar.user_parameter_refs(),
|
||||
Sep(", ", &transfer_syms))
|
||||
}
|
||||
|
||||
ActionKind::TryCall(action_fn) => {
|
||||
rust!(self.out, "let {}nt = try!(super::{}action{}({}{}));",
|
||||
self.prefix,
|
||||
self.prefix,
|
||||
action_fn.index(),
|
||||
self.grammar.user_parameter_refs(),
|
||||
Sep(", ", &transfer_syms))
|
||||
}
|
||||
|
||||
ActionKind::Lookahead => {
|
||||
// take the lookahead, if any; otherwise, we are
|
||||
// at EOF, so taker the lookbehind (end of last
|
||||
// pushed token); if that is missing too, then
|
||||
// supply default.
|
||||
rust!(self.out,
|
||||
"let {}nt = \
|
||||
{}lookahead.as_ref()\
|
||||
.map(|o| ::std::clone::Clone::clone(&o.0))\
|
||||
.or_else(|| ::std::clone::Clone::clone(&{}lookbehind))\
|
||||
.unwrap_or_default();",
|
||||
self.prefix, self.prefix, self.prefix);
|
||||
}
|
||||
|
||||
ActionKind::Lookbehind => {
|
||||
// take lookbehind or supply default.
|
||||
rust!(self.out,
|
||||
"let {}nt = ::std::clone::Clone::clone(&{}lookbehind)\
|
||||
.unwrap_or_default();",
|
||||
self.prefix, self.prefix);
|
||||
}
|
||||
let is_fallible = self.grammar.action_is_fallible(production.action);
|
||||
if is_fallible {
|
||||
rust!(self.out, "let {}nt = try!(super::{}action{}({}{}));",
|
||||
self.prefix,
|
||||
self.prefix,
|
||||
production.action.index(),
|
||||
self.grammar.user_parameter_refs(),
|
||||
Sep(", ", &args))
|
||||
} else {
|
||||
rust!(self.out, "let {}nt = super::{}action{}({}{});",
|
||||
self.prefix,
|
||||
self.prefix,
|
||||
production.action.index(),
|
||||
self.grammar.user_parameter_refs(),
|
||||
Sep(", ", &args))
|
||||
}
|
||||
|
||||
// wrap up the result along with the (unused) lookahead
|
||||
if !transfer_syms.is_empty() {
|
||||
if transfered_syms != 0 {
|
||||
// if we popped anything off of the stack, then this frame is done
|
||||
rust!(self.out, "return Ok(({}lookbehind, {}lookahead, {}Nonterminal::{}({}nt)));",
|
||||
self.prefix, self.prefix, self.prefix,
|
||||
@ -455,7 +435,7 @@ impl<'ascent,'grammar,W:Write> RecursiveAscent<'ascent,'grammar,W> {
|
||||
|
||||
// errors are not possible in the goto phase; a missing entry
|
||||
// indicates parse successfully completed, so just bail out
|
||||
if this_state.gotos.len() != self.grammar.productions.keys().len() {
|
||||
if this_state.gotos.len() != self.grammar.nonterminals.keys().len() {
|
||||
rust!(self.out, "_ => {{");
|
||||
rust!(self.out, "return Ok(({}lookbehind, {}lookahead, {}nt));",
|
||||
self.prefix, self.prefix, self.prefix);
|
||||
|
@ -55,9 +55,8 @@ grammar;
|
||||
"#);
|
||||
let items = items(&grammar, "A", 0, EOF);
|
||||
expect_debug(items.vec, r#"[
|
||||
() = (*) ["C"],
|
||||
A = (*) B "C" [EOF],
|
||||
B = (*) () ["C"],
|
||||
B = (*) ["C"],
|
||||
B = (*) "D" ["C"]
|
||||
]"#);
|
||||
}
|
||||
@ -79,19 +78,16 @@ C: Option<u32> = {
|
||||
"#);
|
||||
|
||||
expect_debug(items(&grammar, "A", 0, EOF).vec, r#"[
|
||||
() = (*) [EOF],
|
||||
() = (*) ["C1"],
|
||||
A = (*) B C [EOF],
|
||||
B = (*) () [EOF],
|
||||
B = (*) () ["C1"],
|
||||
B = (*) [EOF],
|
||||
B = (*) ["C1"],
|
||||
B = (*) "B1" [EOF],
|
||||
B = (*) "B1" ["C1"]
|
||||
]"#);
|
||||
|
||||
expect_debug(items(&grammar, "A", 1, EOF).vec, r#"[
|
||||
() = (*) [EOF],
|
||||
A = B (*) C [EOF],
|
||||
C = (*) () [EOF],
|
||||
C = (*) [EOF],
|
||||
C = (*) "C1" [EOF]
|
||||
]"#);
|
||||
}
|
||||
|
@ -20,7 +20,7 @@ impl FirstSets {
|
||||
let mut changed = true;
|
||||
while changed {
|
||||
changed = false;
|
||||
for production in grammar.productions.values().flat_map(|p| p.iter()) {
|
||||
for production in grammar.nonterminals.values().flat_map(|p| &p.productions) {
|
||||
let nt = production.nonterminal;
|
||||
let lookahead = this.first(&production.symbols, Lookahead::EOF);
|
||||
let first_set = this.map.entry(nt).or_insert_with(|| set());
|
||||
|
@ -40,7 +40,7 @@ struct Items<'grammar> {
|
||||
struct StateIndex(usize);
|
||||
|
||||
#[derive(Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)]
|
||||
enum Lookahead {
|
||||
pub enum Lookahead {
|
||||
EOF,
|
||||
Terminal(TerminalString),
|
||||
}
|
||||
|
116
lalrpop-snap/src/normalize/inline/graph/mod.rs
Normal file
116
lalrpop-snap/src/normalize/inline/graph/mod.rs
Normal file
@ -0,0 +1,116 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use intern::intern;
|
||||
use normalize::{NormError, NormResult};
|
||||
use petgraph::graph::{Graph, NodeIndex};
|
||||
use grammar::consts::INLINE;
|
||||
use grammar::repr::*;
|
||||
use util::{map, Map};
|
||||
|
||||
#[cfg(test)]
|
||||
mod test;
|
||||
|
||||
/// Computes the proper order to inline the various nonterminals in
|
||||
/// `grammar`. Reports an error if there is an inline
|
||||
/// cycle. Otherwise, yields an ordering such that we inline X before
|
||||
/// Y if Y references X. I actually think it doesn't matter what
|
||||
/// order we do the inlining, really, but this order seems better
|
||||
/// somehow. :) (That is, inline into something before we inline it.)
|
||||
pub fn inline_order(grammar: &Grammar) -> NormResult<Vec<NonterminalString>> {
|
||||
let mut graph = NonterminalGraph::new(grammar);
|
||||
graph.create_nodes();
|
||||
graph.add_edges();
|
||||
graph.inline_order()
|
||||
}
|
||||
|
||||
struct NonterminalGraph<'grammar> {
|
||||
grammar: &'grammar Grammar,
|
||||
graph: Graph<NonterminalString, ()>,
|
||||
nonterminal_map: Map<NonterminalString, NodeIndex>,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone)]
|
||||
enum WalkState {
|
||||
NotVisited,
|
||||
Visiting,
|
||||
Visited,
|
||||
}
|
||||
|
||||
impl<'grammar> NonterminalGraph<'grammar> {
|
||||
fn new(grammar: &'grammar Grammar) -> NonterminalGraph<'grammar> {
|
||||
NonterminalGraph {
|
||||
grammar: grammar,
|
||||
graph: Graph::new(),
|
||||
nonterminal_map: map(),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_nodes(&mut self) {
|
||||
let inline = intern(INLINE);
|
||||
for (&name, data) in &self.grammar.nonterminals {
|
||||
if data.annotations.iter().any(|a| a.id == inline) {
|
||||
let index = self.graph.add_node(name);
|
||||
self.nonterminal_map.insert(name, index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn add_edges(&mut self) {
|
||||
for production in self.grammar.nonterminals.values().flat_map(|d| &d.productions) {
|
||||
let from_index = match self.nonterminal_map.get(&production.nonterminal) {
|
||||
Some(&index) => index,
|
||||
None => continue, // this is not an inlined nonterminal
|
||||
};
|
||||
|
||||
for &symbol in &production.symbols {
|
||||
match symbol {
|
||||
Symbol::Nonterminal(to) => {
|
||||
if let Some(&to_index) = self.nonterminal_map.get(&to) {
|
||||
self.graph.add_edge(from_index, to_index, ());
|
||||
}
|
||||
}
|
||||
Symbol::Terminal(_) => { }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn inline_order(&self) -> NormResult<Vec<NonterminalString>> {
|
||||
let mut states = vec![WalkState::NotVisited; self.graph.node_count()];
|
||||
let mut result = vec![];
|
||||
for node in self.nonterminal_map.values().cloned() {
|
||||
try!(self.walk(&mut states, &mut result, node));
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn walk(&self,
|
||||
states: &mut Vec<WalkState>,
|
||||
result: &mut Vec<NonterminalString>,
|
||||
source: NodeIndex)
|
||||
-> NormResult<()>
|
||||
{
|
||||
let nt = *self.graph.node_weight(source).unwrap();
|
||||
|
||||
match states[source.index()] {
|
||||
WalkState::NotVisited => {
|
||||
states[source.index()] = WalkState::Visiting;
|
||||
for target in self.graph.neighbors(source) {
|
||||
try!(self.walk(states, result, target));
|
||||
}
|
||||
states[source.index()] = WalkState::Visited;
|
||||
result.push(nt);
|
||||
Ok(())
|
||||
}
|
||||
WalkState::Visited => {
|
||||
Ok(())
|
||||
}
|
||||
WalkState::Visiting => {
|
||||
return_err!(
|
||||
self.grammar.nonterminals[&nt].span,
|
||||
"cyclic inline directive: `{}` would have to be inlined into itself", nt);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
45
lalrpop-snap/src/normalize/inline/graph/test.rs
Normal file
45
lalrpop-snap/src/normalize/inline/graph/test.rs
Normal file
@ -0,0 +1,45 @@
|
||||
use intern::intern;
|
||||
use grammar::repr::NonterminalString;
|
||||
use normalize::lower_helper;
|
||||
use parser;
|
||||
use super::inline_order;
|
||||
|
||||
#[test]
|
||||
fn test_inline_self_cycle() {
|
||||
let grammar = parser::parse_grammar(r#"
|
||||
grammar;
|
||||
extern { }
|
||||
#[inline] A: () = A;
|
||||
"#).unwrap();
|
||||
let grammar = lower_helper(grammar, true).unwrap();
|
||||
assert!(inline_order(&grammar).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_inline_cycle_3() {
|
||||
let grammar = parser::parse_grammar(r#"
|
||||
grammar;
|
||||
extern { }
|
||||
#[inline] A: () = B;
|
||||
#[inline] B: () = C;
|
||||
#[inline] C: () = A;
|
||||
"#).unwrap();
|
||||
let grammar = lower_helper(grammar, true).unwrap();
|
||||
assert!(inline_order(&grammar).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_inline_order() {
|
||||
// because C references A, we inline A first.
|
||||
let grammar = parser::parse_grammar(r#"
|
||||
grammar;
|
||||
extern { }
|
||||
#[inline] A: () = B;
|
||||
B: () = C;
|
||||
#[inline] C: () = A;
|
||||
"#).unwrap();
|
||||
let grammar = lower_helper(grammar, true).unwrap();
|
||||
let a = NonterminalString(intern("A"));
|
||||
let c = NonterminalString(intern("C"));
|
||||
assert_eq!(inline_order(&grammar).unwrap(), vec![a, c]);
|
||||
}
|
152
lalrpop-snap/src/normalize/inline/mod.rs
Normal file
152
lalrpop-snap/src/normalize/inline/mod.rs
Normal file
@ -0,0 +1,152 @@
|
||||
/*!
|
||||
* Inlining of nonterminals
|
||||
*/
|
||||
|
||||
use grammar::repr::*;
|
||||
use normalize::NormResult;
|
||||
|
||||
mod graph;
|
||||
|
||||
#[cfg(test)]
|
||||
mod test;
|
||||
|
||||
pub fn inline(mut grammar: Grammar) -> NormResult<Grammar> {
|
||||
let order = try!(graph::inline_order(&grammar));
|
||||
for nt in order {
|
||||
inline_nt(&mut grammar, nt);
|
||||
}
|
||||
Ok(grammar)
|
||||
}
|
||||
|
||||
fn inline_nt(grammar: &mut Grammar, inline_nt: NonterminalString) {
|
||||
let inline_productions: Vec<_> = grammar.productions_for(inline_nt).iter().cloned().collect();
|
||||
for (_, data) in &mut grammar.nonterminals {
|
||||
let mut new_productions = vec![];
|
||||
let mut new_action_fn_defns = vec![];
|
||||
|
||||
for into_production in &data.productions {
|
||||
if !into_production.symbols.contains(&Symbol::Nonterminal(inline_nt)) {
|
||||
new_productions.push(into_production.clone());
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut inliner = Inliner {
|
||||
action_fn_defns: &grammar.action_fn_defns,
|
||||
inline_nonterminal: inline_nt,
|
||||
into_production: into_production,
|
||||
inline_fallible: 0,
|
||||
inline_productions: &inline_productions,
|
||||
new_symbols: vec![],
|
||||
new_productions: &mut new_productions,
|
||||
new_action_fn_defns: &mut new_action_fn_defns,
|
||||
};
|
||||
|
||||
inliner.inline(&into_production.symbols);
|
||||
}
|
||||
|
||||
data.productions = new_productions;
|
||||
grammar.action_fn_defns.extend(new_action_fn_defns);
|
||||
}
|
||||
}
|
||||
|
||||
struct Inliner<'a> {
|
||||
/// Action fn defns
|
||||
action_fn_defns: &'a [ActionFnDefn],
|
||||
|
||||
/// The nonterminal `A` being inlined
|
||||
inline_nonterminal: NonterminalString,
|
||||
|
||||
/// The full set of productions `A = B C D | E F G` for the
|
||||
/// nonterminal `A` being inlined
|
||||
inline_productions: &'a [Production],
|
||||
|
||||
/// Number of actions that we have inlined for `A` so far which
|
||||
/// have been fallible. IOW, if we are inlining `A` into `X = Y A
|
||||
/// A Z`, and in the first instance of `A` we used a fallible
|
||||
/// action, but the second we used an infallible one, count would
|
||||
/// be 1.
|
||||
inline_fallible: u32,
|
||||
|
||||
/// The `X = Y A Z` being inlined into
|
||||
into_production: &'a Production,
|
||||
|
||||
/// The list of symbols we building up for the new production.
|
||||
/// For example, this would (eventually) contain `Y B C D Z`,
|
||||
/// given our running example.
|
||||
new_symbols: Vec<InlinedSymbol>,
|
||||
|
||||
/// The output vector of all productions for `X` that we have created
|
||||
new_productions: &'a mut Vec<Production>,
|
||||
|
||||
/// Vector of all action fn defns from the grammar.
|
||||
new_action_fn_defns: &'a mut Vec<ActionFnDefn>,
|
||||
}
|
||||
|
||||
impl<'a> Inliner<'a> {
|
||||
fn inline(&mut self, into_symbols: &[Symbol]) {
|
||||
if into_symbols.is_empty() {
|
||||
// create an action fn for the result of inlining
|
||||
let into_action = self.into_production.action;
|
||||
let into_fallible = self.action_fn_defns[into_action.index()].fallible;
|
||||
let into_ret_type = self.action_fn_defns[into_action.index()].ret_type.clone();
|
||||
let inline_fallible = self.inline_fallible != 0;
|
||||
let index = self.action_fn_defns.len() + self.new_action_fn_defns.len();
|
||||
let action_fn = ActionFn::new(index);
|
||||
let inline_defn = InlineActionFnDefn {
|
||||
action: into_action,
|
||||
symbols: self.new_symbols.clone()
|
||||
};
|
||||
self.new_action_fn_defns.push(ActionFnDefn {
|
||||
fallible: into_fallible || inline_fallible,
|
||||
ret_type: into_ret_type,
|
||||
kind: ActionFnDefnKind::Inline(inline_defn),
|
||||
});
|
||||
let prod_symbols: Vec<Symbol> =
|
||||
self.new_symbols.iter()
|
||||
.flat_map(|sym| match *sym {
|
||||
InlinedSymbol::Original(s) => vec![s],
|
||||
InlinedSymbol::Inlined(_, ref s) => s.clone(),
|
||||
})
|
||||
.collect();
|
||||
self.new_productions.push(Production {
|
||||
nonterminal: self.into_production.nonterminal,
|
||||
span: self.into_production.span,
|
||||
symbols: prod_symbols,
|
||||
action: action_fn,
|
||||
});
|
||||
} else {
|
||||
let next_symbol = into_symbols[0];
|
||||
match next_symbol {
|
||||
Symbol::Nonterminal(n) if n == self.inline_nonterminal => {
|
||||
// Replace the current symbol with each of the
|
||||
// `inline_productions` in turn.
|
||||
for inline_production in self.inline_productions {
|
||||
// If this production is fallible, increment
|
||||
// count of fallible actions.
|
||||
let inline_action = inline_production.action;
|
||||
let fallible = self.action_fn_defns[inline_action.index()].fallible;
|
||||
self.inline_fallible += fallible as u32;
|
||||
|
||||
// Push the symbols of the production inline.
|
||||
self.new_symbols.push(
|
||||
InlinedSymbol::Inlined(
|
||||
inline_production.action,
|
||||
inline_production.symbols.clone()));
|
||||
|
||||
// Inline remaining symbols:
|
||||
self.inline(&into_symbols[1..]);
|
||||
|
||||
// Reset state after we have inlined remaining symbols:
|
||||
self.new_symbols.pop();
|
||||
self.inline_fallible -= fallible as u32;
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
self.new_symbols.push(InlinedSymbol::Original(next_symbol));
|
||||
self.inline(&into_symbols[1..]);
|
||||
self.new_symbols.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
94
lalrpop-snap/src/normalize/inline/test.rs
Normal file
94
lalrpop-snap/src/normalize/inline/test.rs
Normal file
@ -0,0 +1,94 @@
|
||||
use grammar::parse_tree::NonterminalString;
|
||||
use grammar::repr::Grammar;
|
||||
use intern::intern;
|
||||
use normalize::{self, NormResult};
|
||||
use parser;
|
||||
|
||||
use super::inline;
|
||||
|
||||
fn inlined_grammar(text: &str) -> NormResult<Grammar> {
|
||||
let g = parser::parse_grammar(text).unwrap();
|
||||
let g = normalize::lower_helper(g, true).unwrap();
|
||||
inline(g)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sri() {
|
||||
// This grammar gets a shift-reduce conflict because if the input
|
||||
// is "&" (*) "L", then we see two possibilities, and we must decide
|
||||
// between them:
|
||||
//
|
||||
// "&" (*) "L" E
|
||||
// | | |
|
||||
// +-------+--|
|
||||
// |
|
||||
// E
|
||||
//
|
||||
// or
|
||||
//
|
||||
// "&" (*) "L"
|
||||
// | |
|
||||
// | OPT_L E
|
||||
// | | |
|
||||
// +---+---+----+
|
||||
// |
|
||||
// E
|
||||
//
|
||||
// to some extent this may be a false conflict, in that inlined
|
||||
// rules would address it, but it's an interesting one for
|
||||
// producing a useful error message.
|
||||
|
||||
let grammar = inlined_grammar(r#"
|
||||
grammar;
|
||||
|
||||
E: () = {
|
||||
"L",
|
||||
"&" OPT_L E
|
||||
};
|
||||
|
||||
#[inline] OPT_L: () = {
|
||||
(),
|
||||
"L"
|
||||
};
|
||||
"#).unwrap();
|
||||
|
||||
let nt = NonterminalString(intern("E"));
|
||||
|
||||
// After inlining, we expect:
|
||||
//
|
||||
// E = "L"
|
||||
// E = "&" E
|
||||
// E = "&" "L" E
|
||||
//
|
||||
// Note that the `()` also gets inlined.
|
||||
let e_productions = grammar.productions_for(nt);
|
||||
assert_eq!(e_productions.len(), 3);
|
||||
assert_eq!(format!("{:?}", e_productions[0].symbols), r#"["L"]"#);
|
||||
assert_eq!(format!("{:?}", e_productions[1].symbols), r#"["&", E]"#);
|
||||
assert_eq!(format!("{:?}", e_productions[2].symbols), r#"["&", "L", E]"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn issue_55() {
|
||||
let grammar = inlined_grammar(r#"
|
||||
grammar;
|
||||
|
||||
pub E: () = {
|
||||
"X" "{" <a:AT*> <e:ET> <b:AT*> "}" => ()
|
||||
};
|
||||
|
||||
AT: () = {
|
||||
"type" ";" => ()
|
||||
};
|
||||
|
||||
ET: () = {
|
||||
"enum" "{" "}" => ()
|
||||
};
|
||||
"#).unwrap();
|
||||
let nt = NonterminalString(intern("E"));
|
||||
|
||||
// The problem in issue #55 was that we would inline both `AT*`
|
||||
// the same way, so we ended up with `E = X { ET }` and `E = X {
|
||||
// AT+ ET AT+ }` but not `E = X { AT+ ET }` or `E = X { ET AT+ }`.
|
||||
assert!(grammar.productions_for(nt).len() == 4);
|
||||
}
|
@ -5,15 +5,13 @@
|
||||
use intern::{self, intern, InternedString};
|
||||
use normalize::NormResult;
|
||||
use normalize::norm_util::{self, Symbols};
|
||||
use grammar::consts::INPUT_LIFETIME;
|
||||
use grammar::pattern::{Pattern, PatternKind};
|
||||
use grammar::parse_tree as pt;
|
||||
use grammar::parse_tree::{InternToken, NonterminalString, TerminalString};
|
||||
use grammar::repr as r;
|
||||
use util::{map, Map};
|
||||
|
||||
#[cfg(test)]
|
||||
mod test;
|
||||
|
||||
pub fn lower(grammar: pt::Grammar, types: r::Types) -> NormResult<r::Grammar> {
|
||||
let state = LowerState::new(types, &grammar);
|
||||
state.lower(grammar)
|
||||
@ -22,7 +20,7 @@ pub fn lower(grammar: pt::Grammar, types: r::Types) -> NormResult<r::Grammar> {
|
||||
struct LowerState {
|
||||
prefix: String,
|
||||
action_fn_defns: Vec<r::ActionFnDefn>,
|
||||
productions: Vec<r::Production>,
|
||||
nonterminals: Map<NonterminalString, r::NonterminalData>,
|
||||
conversions: Vec<(TerminalString, Pattern<r::TypeRepr>)>,
|
||||
intern_token: Option<InternToken>,
|
||||
types: r::Types,
|
||||
@ -31,9 +29,9 @@ struct LowerState {
|
||||
impl LowerState {
|
||||
fn new(types: r::Types, grammar: &pt::Grammar) -> LowerState {
|
||||
LowerState {
|
||||
prefix: find_prefix(&grammar),
|
||||
prefix: grammar.prefix.clone(),
|
||||
action_fn_defns: vec![],
|
||||
productions: vec![],
|
||||
nonterminals: map(),
|
||||
conversions: vec![],
|
||||
types: types,
|
||||
intern_token: None,
|
||||
@ -56,7 +54,7 @@ impl LowerState {
|
||||
token_span = Some(grammar.span);
|
||||
let span = grammar.span;
|
||||
let input_str = r::TypeRepr::Ref {
|
||||
lifetime: Some(intern(pt::INPUT_LIFETIME)),
|
||||
lifetime: Some(intern(INPUT_LIFETIME)),
|
||||
mutable: false,
|
||||
referent: Box::new(r::TypeRepr::Nominal(r::NominalTypeRepr {
|
||||
path: r::Path::str(),
|
||||
@ -99,28 +97,33 @@ impl LowerState {
|
||||
}
|
||||
|
||||
pt::GrammarItem::Nonterminal(nt) => {
|
||||
for alt in nt.alternatives {
|
||||
let nt_type = self.types.nonterminal_type(nt.name).clone();
|
||||
let symbols = self.symbols(&alt.expr.symbols);
|
||||
let action = self.action_kind(nt_type, &alt.expr, &symbols, alt.action);
|
||||
let production = r::Production {
|
||||
nonterminal: nt.name,
|
||||
span: alt.span,
|
||||
symbols: symbols,
|
||||
action: action,
|
||||
};
|
||||
self.productions.push(production);
|
||||
}
|
||||
let nt_name = nt.name;
|
||||
let productions: Vec<_> =
|
||||
nt.alternatives
|
||||
.into_iter()
|
||||
.map(|alt| {
|
||||
let nt_type = self.types.nonterminal_type(nt_name).clone();
|
||||
let symbols = self.symbols(&alt.expr.symbols);
|
||||
let action = self.action_kind(nt_type, &alt.expr,
|
||||
&symbols, alt.action);
|
||||
r::Production {
|
||||
nonterminal: nt_name,
|
||||
span: alt.span,
|
||||
symbols: symbols,
|
||||
action: action,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
self.nonterminals.insert(nt_name, r::NonterminalData {
|
||||
name: nt_name,
|
||||
annotations: nt.annotations,
|
||||
span: nt.span,
|
||||
productions: productions
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut productions = map();
|
||||
for production in self.productions {
|
||||
let mut vec = productions.entry(production.nonterminal).or_insert(vec![]);
|
||||
vec.push(production);
|
||||
}
|
||||
|
||||
let parameters =
|
||||
grammar.parameters.iter()
|
||||
.map(|p| r::Parameter { name: p.name, ty: p.ty.type_repr() })
|
||||
@ -137,7 +140,7 @@ impl LowerState {
|
||||
start_nonterminals: start_symbols,
|
||||
uses: uses,
|
||||
action_fn_defns: self.action_fn_defns,
|
||||
productions: productions,
|
||||
nonterminals: self.nonterminals,
|
||||
conversions: self.conversions.into_iter().collect(),
|
||||
types: self.types,
|
||||
token_span: token_span.unwrap(),
|
||||
@ -172,12 +175,20 @@ impl LowerState {
|
||||
};
|
||||
let symbols = vec![r::Symbol::Nonterminal(nt.name)];
|
||||
let action_fn = self.action_fn(nt_type, false, &expr, &symbols, None);
|
||||
self.productions.push(r::Production {
|
||||
let production = r::Production {
|
||||
nonterminal: fake_name,
|
||||
symbols: symbols,
|
||||
action: r::ActionKind::Call(action_fn),
|
||||
action: action_fn,
|
||||
span: nt.span
|
||||
});
|
||||
};
|
||||
self.nonterminals.insert(
|
||||
fake_name,
|
||||
r::NonterminalData {
|
||||
name: fake_name,
|
||||
annotations: vec![],
|
||||
span: nt.span,
|
||||
productions: vec![production]
|
||||
});
|
||||
(nt.name, fake_name)
|
||||
})
|
||||
.collect()
|
||||
@ -188,28 +199,42 @@ impl LowerState {
|
||||
expr: &pt::ExprSymbol,
|
||||
symbols: &[r::Symbol],
|
||||
action: Option<pt::ActionKind>)
|
||||
-> r::ActionKind
|
||||
-> r::ActionFn
|
||||
{
|
||||
match action {
|
||||
Some(pt::ActionKind::Lookahead) =>
|
||||
r::ActionKind::Lookahead,
|
||||
self.lookahead_action_fn(),
|
||||
Some(pt::ActionKind::Lookbehind) =>
|
||||
r::ActionKind::Lookbehind,
|
||||
Some(pt::ActionKind::User(string)) => {
|
||||
let action_fn = self.action_fn(nt_type, false, &expr, &symbols, Some(string));
|
||||
r::ActionKind::Call(action_fn)
|
||||
}
|
||||
Some(pt::ActionKind::Fallible(string)) => {
|
||||
let action_fn = self.action_fn(nt_type, true, &expr, &symbols, Some(string));
|
||||
r::ActionKind::TryCall(action_fn)
|
||||
}
|
||||
None => {
|
||||
let action_fn = self.action_fn(nt_type, false, &expr, &symbols, None);
|
||||
r::ActionKind::Call(action_fn)
|
||||
}
|
||||
self.lookbehind_action_fn(),
|
||||
Some(pt::ActionKind::User(string)) =>
|
||||
self.action_fn(nt_type, false, &expr, &symbols, Some(string)),
|
||||
Some(pt::ActionKind::Fallible(string)) =>
|
||||
self.action_fn(nt_type, true, &expr, &symbols, Some(string)),
|
||||
None =>
|
||||
self.action_fn(nt_type, false, &expr, &symbols, None),
|
||||
}
|
||||
}
|
||||
|
||||
fn lookahead_action_fn(&mut self) -> r::ActionFn {
|
||||
let action_fn_defn = r::ActionFnDefn {
|
||||
fallible: false,
|
||||
ret_type: self.types.terminal_loc_type(),
|
||||
kind: r::ActionFnDefnKind::Lookaround(r::LookaroundActionFnDefn::Lookahead),
|
||||
};
|
||||
|
||||
self.add_action_fn(action_fn_defn)
|
||||
}
|
||||
|
||||
fn lookbehind_action_fn(&mut self) -> r::ActionFn {
|
||||
let action_fn_defn = r::ActionFnDefn {
|
||||
fallible: false,
|
||||
ret_type: self.types.terminal_loc_type(),
|
||||
kind: r::ActionFnDefnKind::Lookaround(r::LookaroundActionFnDefn::Lookbehind),
|
||||
};
|
||||
|
||||
self.add_action_fn(action_fn_defn)
|
||||
}
|
||||
|
||||
fn action_fn(&mut self,
|
||||
nt_type: r::TypeRepr,
|
||||
fallible: bool,
|
||||
@ -240,11 +265,13 @@ impl LowerState {
|
||||
symbols.len());
|
||||
|
||||
r::ActionFnDefn {
|
||||
arg_patterns: arg_patterns,
|
||||
arg_types: arg_types,
|
||||
ret_type: nt_type,
|
||||
fallible: fallible,
|
||||
code: action
|
||||
ret_type: nt_type,
|
||||
kind: r::ActionFnDefnKind::User(r::UserActionFnDefn {
|
||||
arg_patterns: arg_patterns,
|
||||
arg_types: arg_types,
|
||||
code: action,
|
||||
}),
|
||||
}
|
||||
}
|
||||
Symbols::Anon(indices) => {
|
||||
@ -260,18 +287,23 @@ impl LowerState {
|
||||
});
|
||||
let action = action.replace("<>", &name_str);
|
||||
r::ActionFnDefn {
|
||||
arg_patterns: arg_patterns,
|
||||
arg_types: arg_types,
|
||||
ret_type: nt_type,
|
||||
fallible: fallible,
|
||||
code: action
|
||||
ret_type: nt_type,
|
||||
kind: r::ActionFnDefnKind::User(r::UserActionFnDefn {
|
||||
arg_patterns: arg_patterns,
|
||||
arg_types: arg_types,
|
||||
code: action,
|
||||
}),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
self.add_action_fn(action_fn_defn)
|
||||
}
|
||||
|
||||
fn add_action_fn(&mut self, action_fn_defn: r::ActionFnDefn) -> r::ActionFn {
|
||||
let index = r::ActionFn::new(self.action_fn_defns.len());
|
||||
self.action_fn_defns.push(action_fn_defn);
|
||||
|
||||
index
|
||||
}
|
||||
|
||||
@ -325,32 +357,3 @@ fn patterns<I>(mut chosen: I, num_args: usize) -> Vec<InternedString>
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
// Find a unique prefix like `__` or `___` that doesn't appear
|
||||
// anywhere in any action strings, nonterminal names, etc. Obviously
|
||||
// this is stricter than needed, since the action string might be like
|
||||
// `print("__1")`, in which case we'll detect a false conflict (or it
|
||||
// might contain a variable named `__1x`, etc). But so what.
|
||||
fn find_prefix(grammar: &pt::Grammar) -> String {
|
||||
let mut prefix = format!("__");
|
||||
|
||||
while
|
||||
grammar.items
|
||||
.iter()
|
||||
.filter_map(|i| i.as_nonterminal())
|
||||
.flat_map(|nt| nt.alternatives.iter())
|
||||
.filter_map(|alt| alt.action.as_ref())
|
||||
.filter_map(|action| action.as_user())
|
||||
.any(|s| s.contains(&prefix))
|
||||
||
|
||||
grammar.items
|
||||
.iter()
|
||||
.filter_map(|i| i.as_nonterminal())
|
||||
.any(|nt| nt.name.0.starts_with(&prefix))
|
||||
{
|
||||
prefix.push('_');
|
||||
}
|
||||
|
||||
prefix
|
||||
}
|
||||
|
||||
|
@ -1,66 +0,0 @@
|
||||
use grammar::repr::{Grammar, Production};
|
||||
use normalize::normalize_without_validating;
|
||||
use parser;
|
||||
use test_util::expect_debug;
|
||||
|
||||
fn flat_productions(grammar: &Grammar) -> Vec<Production> {
|
||||
let mut productions: Vec<_> =
|
||||
grammar.productions.values()
|
||||
.flat_map(|prods| prods.iter().cloned())
|
||||
.collect();
|
||||
|
||||
// sort by the action fn index just to get a consistent ordering
|
||||
productions.sort_by(|k1, k2| {
|
||||
Ord::cmp(&k1.action, &k2.action)
|
||||
});
|
||||
|
||||
productions
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_comma() {
|
||||
let grammar = parser::parse_grammar(r#"
|
||||
grammar;
|
||||
extern { enum Tok { "," => .., "Id" => .. } }
|
||||
|
||||
Comma<E>: Vec<E> =
|
||||
<v:(<E> ",")*> <e:E?> =>
|
||||
v.into_iter().chain(e.into_iter()).collect();
|
||||
|
||||
Ids = Comma<"Id">;
|
||||
"#).unwrap();
|
||||
let actual = normalize_without_validating(grammar).unwrap();
|
||||
|
||||
expect_debug(flat_productions(&actual),
|
||||
r#"[
|
||||
Ids = Comma<"Id"> => Call(ActionFn(0));,
|
||||
Comma<"Id"> = (<"Id"> ",")*, "Id"? => Call(ActionFn(1));,
|
||||
"Id"? = "Id" => Call(ActionFn(2));,
|
||||
"Id"? = => Call(ActionFn(3));,
|
||||
(<"Id"> ",")* = => Call(ActionFn(4));,
|
||||
(<"Id"> ",")* = (<"Id"> ",")*, (<"Id"> ",") => Call(ActionFn(5));,
|
||||
(<"Id"> ",") = "Id", "," => Call(ActionFn(6));
|
||||
]"#);
|
||||
|
||||
expect_debug(&actual.action_fn_defns,
|
||||
r#"[
|
||||
fn _(__0: Vec<Tok>) -> Vec<Tok> { (__0) },
|
||||
fn _(v: ::std::vec::Vec<Tok>, e: ::std::option::Option<Tok>) -> Vec<Tok> { v.into_iter().chain(e.into_iter()).collect() },
|
||||
fn _(__0: Tok) -> ::std::option::Option<Tok> { Some(__0) },
|
||||
fn _() -> ::std::option::Option<Tok> { None },
|
||||
fn _() -> ::std::vec::Vec<Tok> { vec![] },
|
||||
fn _(v: ::std::vec::Vec<Tok>, e: Tok) -> ::std::vec::Vec<Tok> { { let mut v = v; v.push(e); v } },
|
||||
fn _(__0: Tok, _: Tok) -> Tok { (__0) }
|
||||
]"#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_intern_token() {
|
||||
let grammar = parser::parse_grammar(r#"
|
||||
grammar;
|
||||
extern { }
|
||||
|
||||
A = ",";
|
||||
"#).unwrap();
|
||||
normalize_without_validating(grammar).unwrap();
|
||||
}
|
@ -1,6 +1,7 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use intern::{intern, read, InternedString};
|
||||
use grammar::parse_tree::{ActionKind, Alternative,
|
||||
use grammar::consts::INLINE;
|
||||
use grammar::parse_tree::{ActionKind, Alternative, Annotation,
|
||||
Condition, ConditionOp,
|
||||
ExprSymbol,
|
||||
Grammar, GrammarItem,
|
||||
@ -361,7 +362,7 @@ impl MacroExpander {
|
||||
public: false,
|
||||
span: span,
|
||||
name: name,
|
||||
annotations: vec![],
|
||||
annotations: inline(span),
|
||||
args: vec![],
|
||||
type_decl: Some(ty_ref),
|
||||
alternatives: vec![Alternative { span: span,
|
||||
@ -386,11 +387,16 @@ impl MacroExpander {
|
||||
let path = Path::vec();
|
||||
let ty_ref = TypeRef::Nominal { path: path, types: vec![base_symbol_ty] };
|
||||
|
||||
let plus_repeat = Box::new(RepeatSymbol {
|
||||
op: RepeatOp::Plus,
|
||||
symbol: repeat.symbol.clone()
|
||||
});
|
||||
|
||||
Ok(GrammarItem::Nonterminal(NonterminalData {
|
||||
public: false,
|
||||
span: span,
|
||||
name: name,
|
||||
annotations: vec![],
|
||||
annotations: inline(span),
|
||||
args: vec![],
|
||||
type_decl: Some(ty_ref),
|
||||
alternatives: vec![
|
||||
@ -402,7 +408,7 @@ impl MacroExpander {
|
||||
action: action("vec![]")
|
||||
},
|
||||
|
||||
// X* = <v:X+> <e:X>
|
||||
// X* = <v:X+>
|
||||
Alternative {
|
||||
span: span,
|
||||
expr: ExprSymbol {
|
||||
@ -413,15 +419,10 @@ impl MacroExpander {
|
||||
v,
|
||||
Box::new(
|
||||
Symbol::new(span,
|
||||
SymbolKind::Nonterminal(name))))),
|
||||
Symbol::new(
|
||||
span,
|
||||
SymbolKind::Name(
|
||||
e,
|
||||
Box::new(repeat.symbol.clone())))]
|
||||
SymbolKind::Repeat(plus_repeat)))))],
|
||||
},
|
||||
condition: None,
|
||||
action: action("{ let mut v = v; v.push(e); v }")
|
||||
action: action("v"),
|
||||
}],
|
||||
}))
|
||||
}
|
||||
@ -473,7 +474,7 @@ impl MacroExpander {
|
||||
public: false,
|
||||
span: span,
|
||||
name: name,
|
||||
annotations: vec![],
|
||||
annotations: inline(span),
|
||||
args: vec![],
|
||||
type_decl: Some(ty_ref),
|
||||
alternatives: vec![
|
||||
@ -527,3 +528,10 @@ fn maybe_tuple(v: Vec<TypeRef>) -> TypeRef {
|
||||
fn action(s: &str) -> Option<ActionKind> {
|
||||
Some(ActionKind::User(s.to_string()))
|
||||
}
|
||||
|
||||
fn inline(span: Span) -> Vec<Annotation> {
|
||||
vec![Annotation {
|
||||
id_span: span,
|
||||
id: intern(INLINE),
|
||||
}]
|
||||
}
|
||||
|
@ -23,17 +23,27 @@ grammar;
|
||||
`Comma<"Id">`: Vec<#"Id"#> =
|
||||
<v:`(<"Id"> ",")*`> <e:`"Id"?`> => v.into_iter().chain(e.into_iter()).collect();
|
||||
|
||||
#[inline]
|
||||
`"Id"?`: ::std::option::Option<#"Id"#> = {
|
||||
"Id" => Some(<>),
|
||||
=> None
|
||||
};
|
||||
|
||||
#[inline]
|
||||
`(<"Id"> ",")*`: ::std::vec::Vec<#`(<"Id"> ",")`#> = {
|
||||
=> vec![],
|
||||
<v:`(<"Id"> ",")*`> <e:`(<"Id"> ",")`> => { let mut v = v; v.push(e); v }
|
||||
<v:`(<"Id"> ",")+`> => v,
|
||||
};
|
||||
|
||||
`(<"Id"> ",")`: #"Id"# = <"Id"> "," => (<>);
|
||||
#[inline]
|
||||
`(<"Id"> ",")`: #"Id"# = {
|
||||
<"Id"> "," => (<>),
|
||||
};
|
||||
|
||||
`(<"Id"> ",")+`: ::std::vec::Vec<#`(<"Id"> ",")`#> = {
|
||||
`(<"Id"> ",")` => vec![<>],
|
||||
<v:`(<"Id"> ",")+`> <e:`(<"Id"> ",")`> => { let mut v = v; v.push(e); v },
|
||||
};
|
||||
"##).unwrap();
|
||||
|
||||
compare(actual, expected);
|
||||
|
@ -35,6 +35,11 @@ pub fn normalize_without_validating(grammar: pt::Grammar) -> NormResult<r::Gramm
|
||||
}
|
||||
|
||||
fn normalize_helper(grammar: pt::Grammar, validate: bool) -> NormResult<r::Grammar> {
|
||||
let grammar = try!(lower_helper(grammar, validate));
|
||||
inline::inline(grammar)
|
||||
}
|
||||
|
||||
fn lower_helper(grammar: pt::Grammar, validate: bool) -> NormResult<r::Grammar> {
|
||||
if validate { try!(prevalidate::validate(&grammar)); }
|
||||
let grammar = try!(resolve::resolve(grammar));
|
||||
let grammar = try!(macro_expand::expand_macros(grammar));
|
||||
@ -80,6 +85,9 @@ mod tyinfer;
|
||||
// Lowers the parse tree to the repr notation.
|
||||
mod lower;
|
||||
|
||||
// Inline nonterminals that have requested it.
|
||||
mod inline;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// Shared routines
|
||||
|
||||
|
@ -3,6 +3,7 @@
|
||||
use super::{NormResult, NormError};
|
||||
use super::norm_util::{self, Symbols};
|
||||
|
||||
use grammar::consts::*;
|
||||
use grammar::parse_tree::*;
|
||||
use grammar::repr;
|
||||
use intern::{intern, InternedString};
|
||||
@ -73,10 +74,22 @@ impl<'grammar> Validator<'grammar> {
|
||||
}
|
||||
}
|
||||
GrammarItem::Nonterminal(ref data) => {
|
||||
let inline_annotation = intern(INLINE);
|
||||
let known_annotations = vec![inline_annotation];
|
||||
let mut found_annotations = set();
|
||||
for annotation in &data.annotations {
|
||||
return_err!(annotation.id_span,
|
||||
"unrecognized annotation `{}`",
|
||||
annotation.id);
|
||||
if !known_annotations.contains(&annotation.id) {
|
||||
return_err!(annotation.id_span,
|
||||
"unrecognized annotation `{}`",
|
||||
annotation.id);
|
||||
} else if !found_annotations.insert(annotation.id) {
|
||||
return_err!(annotation.id_span,
|
||||
"duplicate annotation `{}`",
|
||||
annotation.id);
|
||||
} else if annotation.id == inline_annotation && data.public {
|
||||
return_err!(annotation.id_span,
|
||||
"public items cannot be marked #[inline]");
|
||||
}
|
||||
}
|
||||
|
||||
for alternative in &data.alternatives {
|
||||
|
@ -54,3 +54,19 @@ fn unrecognized_annotation() {
|
||||
r#"grammar; #[foo] Term = ();"#,
|
||||
r#" ~~~ "#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_annotation() {
|
||||
check_err(
|
||||
r#"duplicate annotation `inline`"#,
|
||||
r#"grammar; #[inline] #[inline] Term = ();"#,
|
||||
r#" ~~~~~~ "#);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pub_inline_annotation() {
|
||||
check_err(
|
||||
r#"public items cannot be marked #\[inline\]"#,
|
||||
r#"grammar; #[inline] pub Term = ();"#,
|
||||
r#" ~~~~~~ "#);
|
||||
}
|
||||
|
@ -10,6 +10,7 @@ use super::{NormResult, NormError};
|
||||
use intern::{self, intern};
|
||||
use lexer::re;
|
||||
use lexer::dfa::{self, Precedence};
|
||||
use grammar::consts::*;
|
||||
use grammar::parse_tree::*;
|
||||
use util::{Set};
|
||||
use util::{map, Map};
|
||||
@ -233,7 +234,7 @@ pub fn construct(grammar: &mut Grammar, literals_map: Map<TerminalLiteral, Span>
|
||||
}
|
||||
}
|
||||
|
||||
grammar.type_parameters.push(TypeParameter::Lifetime(input_lifetime));
|
||||
grammar.type_parameters.insert(0, TypeParameter::Lifetime(input_lifetime));
|
||||
|
||||
let parameter = Parameter {
|
||||
name: input_parameter,
|
||||
|
@ -2,11 +2,9 @@ use super::{NormResult, NormError};
|
||||
use super::norm_util::{self, AlternativeAction, Symbols};
|
||||
|
||||
use std::collections::{HashMap};
|
||||
use grammar::consts::{ERROR, INPUT_LIFETIME, LOCATION};
|
||||
use grammar::parse_tree::{ActionKind, Alternative,
|
||||
ERROR,
|
||||
Grammar,
|
||||
INPUT_LIFETIME,
|
||||
LOCATION,
|
||||
NonterminalData, NonterminalString,
|
||||
Path,
|
||||
Span,
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -12,7 +12,14 @@ pub fn parse_grammar<'input>(input: &'input str)
|
||||
-> Result<Grammar, ParseError<'input>>
|
||||
{
|
||||
let tokenizer = tok::Tokenizer::new(input, 0);
|
||||
lrgrammar::parse_Grammar(input, tokenizer)
|
||||
let mut grammar = try!(lrgrammar::parse_Grammar(input, tokenizer));
|
||||
|
||||
// find a unique prefix that does not appear anywhere in the input
|
||||
while input.contains(&grammar.prefix) {
|
||||
grammar.prefix.push('_');
|
||||
}
|
||||
|
||||
Ok(grammar)
|
||||
}
|
||||
|
||||
fn parse_pattern<'input>(input: &'input str, offset: usize)
|
||||
|
@ -60,5 +60,6 @@ pub fn check_norm_err(expected_err: &str,
|
||||
assert!(start_index <= end_index);
|
||||
assert_eq!(err.span, pt::Span(start_index, end_index));
|
||||
assert!(expected_err.is_match(&err.message),
|
||||
"unexpected error text `{}`, did not match `{}`", err.message, expected_err);
|
||||
"unexpected error text `{}`, which did not match regular expression `{}`",
|
||||
err.message, expected_err);
|
||||
}
|
||||
|
@ -379,8 +379,25 @@ impl<'input> Tokenizer<'input> {
|
||||
// for a suitable terminator: `,`, `;`, `]`, `}`, or `)`.
|
||||
let mut balance = 0; // number of unclosed `(` etc
|
||||
loop {
|
||||
println!("code: balance={:?} self.lookahead={:?}", balance, self.lookahead);
|
||||
if let Some((idx, c)) = self.lookahead {
|
||||
if open_delims.find(c).is_some() {
|
||||
if c == '"' {
|
||||
self.bump();
|
||||
try!(self.string_literal(idx)); // discard the produced token
|
||||
continue;
|
||||
} else if c == 'r' {
|
||||
self.bump();
|
||||
if let Some((idx, '#')) = self.lookahead {
|
||||
try!(self.regex_literal(idx));
|
||||
}
|
||||
continue;
|
||||
} else if c == '/' {
|
||||
self.bump();
|
||||
if let Some((_, '/')) = self.lookahead {
|
||||
self.take_until(|c| c == '\n');
|
||||
}
|
||||
continue;
|
||||
} else if open_delims.find(c).is_some() {
|
||||
balance += 1;
|
||||
} else if balance > 0 {
|
||||
if close_delims.find(c).is_some() {
|
||||
|
@ -45,6 +45,32 @@ fn code1() {
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_paren() { // Issue #25
|
||||
test(r#"=> a("(", c),"#, vec![
|
||||
(r#"~~~~~~~~~~~~ "#, EqualsGreaterThanCode(r#" a("(", c)"#)),
|
||||
(r#" ~"#, Comma),
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_regex_paren() { // Issue #25
|
||||
test(r###"=> a(r##"("#""##, c),"###, vec![
|
||||
(r###"~~~~~~~~~~~~~~~~~~~~ "###, EqualsGreaterThanCode(r###" a(r##"("#""##, c)"###)),
|
||||
(r###" ~"###, Comma),
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_comment_eol() {
|
||||
test("=> a(// (
|
||||
),", vec![
|
||||
("~~~~~~~~~
|
||||
~,", EqualsGreaterThanCode(" a(// (\n)")),
|
||||
("=> a(// (
|
||||
)~", Comma)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code2() {
|
||||
test("=>? a(b, c),", vec![
|
||||
|
@ -379,7 +379,6 @@ impl<'input> Tokenizer<'input> {
|
||||
// for a suitable terminator: `,`, `;`, `]`, `}`, or `)`.
|
||||
let mut balance = 0; // number of unclosed `(` etc
|
||||
loop {
|
||||
println!("code: balance={:?} self.lookahead={:?}", balance, self.lookahead);
|
||||
if let Some((idx, c)) = self.lookahead {
|
||||
if c == '"' {
|
||||
self.bump();
|
||||
|
Loading…
x
Reference in New Issue
Block a user