diff --git a/lalrpop-test/src/main.rs b/lalrpop-test/src/main.rs index b176c96..a9b10a2 100644 --- a/lalrpop-test/src/main.rs +++ b/lalrpop-test/src/main.rs @@ -41,6 +41,10 @@ mod expr_generic; mod generics_issue_104; mod generics_issue_104_lib; +/// Grammar parameterized by `F` with where clause `where F: for<'a> FnMut(&'a +/// str)`. +mod where_clause_with_forall; + /// test of inlining mod inline; @@ -428,6 +432,14 @@ fn generics_issue_104_test1() { assert!(generics_issue_104::parse_Schema::<()>("grammar { foo }").is_ok()); } +#[test] +fn where_clause_with_forall_test1() { + assert!(where_clause_with_forall::parse_Term( + &mut |s: &str| println!("log: {}", s), + "(((((42)))))" + ).is_ok()); +} + #[test] fn test_match_section() { assert!(match_section::parse_Query("SELECT foo").is_ok()); diff --git a/lalrpop-test/src/where_clause_with_forall.lalrpop b/lalrpop-test/src/where_clause_with_forall.lalrpop new file mode 100644 index 0000000..dd5317f --- /dev/null +++ b/lalrpop-test/src/where_clause_with_forall.lalrpop @@ -0,0 +1,21 @@ +use std::str::FromStr; + +grammar(logger: &mut F) where F: for<'a> FnMut(&'a str); + +pub Term: i32 = { + => { + let msg = format!("just parsed {}", n); + logger(&msg); + n + }, + "(" ")" => { + logger("propagating..."); + { + let msg = format!("... {}", t); + logger(&msg); + } + t + }, +}; + +Num: i32 = => i32::from_str(s).unwrap(); diff --git a/lalrpop-test/src/where_clause_with_forall.rs b/lalrpop-test/src/where_clause_with_forall.rs new file mode 100644 index 0000000..418ad3b --- /dev/null +++ b/lalrpop-test/src/where_clause_with_forall.rs @@ -0,0 +1,1003 @@ +// auto-generated: "lalrpop 0.13.1" +use std::str::FromStr; +extern crate lalrpop_util as __lalrpop_util; + +mod __parse__Term { + #![allow(non_snake_case, non_camel_case_types, unused_mut, unused_variables, unused_imports)] + + use std::str::FromStr; + extern crate lalrpop_util as __lalrpop_util; + pub fn parse_Term< + 'input, + F, + >( + logger: &mut F, + input: &'input str, + ) -> Result> where + F: for<'a> FnMut(&'a str), + { + let __ascent = __ascent::parse_Term::( + logger, + input, + ); + let __parse_table = __parse_table::parse_Term::( + logger, + input, + ); + assert_eq!(__ascent, __parse_table); + return __ascent; + } + mod __ascent { + + mod __parse__Term { + #![allow(non_snake_case, non_camel_case_types, unused_mut, unused_variables, unused_imports)] + + use std::str::FromStr; + extern crate lalrpop_util as __lalrpop_util; + pub fn parse_Term< + 'input, + F, + >( + logger: &mut F, + input: &'input str, + ) -> Result> where + F: for<'a> FnMut(&'a str), + { + let mut __tokens = super::super::super::__intern_token::__Matcher::new(input); + let __lookahead = match __tokens.next() { + Some(Ok(v)) => Some(v), + None => None, + Some(Err(e)) => return Err(e), + }; + match try!(__state0(logger, input, &mut __tokens, __lookahead, ::std::marker::PhantomData::<(F)>)) { + (Some(__lookahead), _) => { + Err(__lalrpop_util::ParseError::ExtraToken { token: __lookahead }) + } + (None, __Nonterminal::____Term((_, __nt, _))) => { + Ok(__nt) + } + _ => unreachable!(), + } + } + + #[allow(dead_code)] + pub enum __Nonterminal<> { + Num((usize, i32, usize)), + Term((usize, i32, usize)), + ____Term((usize, i32, usize)), + } + + // State 0 + // AllInputs = [] + // OptionalInputs = [] + // FixedInputs = [] + // WillPushLen = 0 + // WillPush = [] + // WillProduce = None + // + // Num = (*) r#"[0-9]+"# ["(", ")", r#"[0-9]+"#, EOF] + // Term = (*) Num ["(", ")", r#"[0-9]+"#, EOF] + // Term = (*) "(" Term ")" ["(", ")", r#"[0-9]+"#, EOF] + // __Term = (*) Term ["(", ")", r#"[0-9]+"#, EOF] + // + // "(" -> S3 + // r#"[0-9]+"# -> S4 + // + // Num -> S1 + // Term -> S2 + fn __state0< + 'input, + F, + __TOKENS: Iterator>>, + >( + logger: &mut F, + input: &'input str, + __tokens: &mut __TOKENS, + __lookahead: Option<(usize, (usize, &'input str), usize)>, + _: ::std::marker::PhantomData<(F)>, + ) -> Result<(Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>), __lalrpop_util::ParseError> where + F: for<'a> FnMut(&'a str), + { + let mut __result: (Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>); + match __lookahead { + Some((__loc1, (1, __tok0), __loc2)) => { + let __sym0 = (__loc1, (__tok0), __loc2); + __result = try!(__state3(logger, input, __tokens, __sym0, ::std::marker::PhantomData::<(F)>)); + } + Some((__loc1, (0, __tok0), __loc2)) => { + let __sym0 = (__loc1, (__tok0), __loc2); + __result = try!(__state4(logger, input, __tokens, __sym0, ::std::marker::PhantomData::<(F)>)); + } + _ => { + return Err(__lalrpop_util::ParseError::UnrecognizedToken { + token: __lookahead, + expected: vec![ + r###""(""###.to_string(), + r###"r#"[0-9]+"#"###.to_string(), + ] + }); + } + } + loop { + let (__lookahead, __nt) = __result; + match __nt { + __Nonterminal::Num(__sym0) => { + __result = try!(__state1(logger, input, __tokens, __lookahead, __sym0, ::std::marker::PhantomData::<(F)>)); + } + __Nonterminal::Term(__sym0) => { + __result = try!(__state2(logger, input, __tokens, __lookahead, __sym0, ::std::marker::PhantomData::<(F)>)); + } + _ => { + return Ok((__lookahead, __nt)); + } + } + } + } + + // State 1 + // AllInputs = [Num] + // OptionalInputs = [] + // FixedInputs = [Num] + // WillPushLen = 0 + // WillPush = [] + // WillProduce = Some(Term) + // + // Term = Num (*) ["(", ")", r#"[0-9]+"#, EOF] + // + // ["(", ")", r#"[0-9]+"#, EOF] -> Term = Num => ActionFn(1); + // + fn __state1< + 'input, + F, + __TOKENS: Iterator>>, + >( + logger: &mut F, + input: &'input str, + __tokens: &mut __TOKENS, + __lookahead: Option<(usize, (usize, &'input str), usize)>, + __sym0: (usize, i32, usize), + _: ::std::marker::PhantomData<(F)>, + ) -> Result<(Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>), __lalrpop_util::ParseError> where + F: for<'a> FnMut(&'a str), + { + let mut __result: (Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>); + match __lookahead { + Some((_, (1, _), _)) | + Some((_, (2, _), _)) | + Some((_, (0, _), _)) | + None => { + let __start = __sym0.0.clone(); + let __end = __sym0.2.clone(); + let __nt = super::super::super::__action1::(logger, input, __sym0); + let __nt = __Nonterminal::Term(( + __start, + __nt, + __end, + )); + __result = (__lookahead, __nt); + return Ok(__result); + } + _ => { + return Err(__lalrpop_util::ParseError::UnrecognizedToken { + token: __lookahead, + expected: vec![ + r###""(""###.to_string(), + r###"")""###.to_string(), + r###"r#"[0-9]+"#"###.to_string(), + ] + }); + } + } + } + + // State 2 + // AllInputs = [Term] + // OptionalInputs = [] + // FixedInputs = [Term] + // WillPushLen = 0 + // WillPush = [] + // WillProduce = Some(__Term) + // + // __Term = Term (*) ["(", ")", r#"[0-9]+"#, EOF] + // + // ["(", ")", r#"[0-9]+"#, EOF] -> __Term = Term => ActionFn(0); + // + fn __state2< + 'input, + F, + __TOKENS: Iterator>>, + >( + logger: &mut F, + input: &'input str, + __tokens: &mut __TOKENS, + __lookahead: Option<(usize, (usize, &'input str), usize)>, + __sym0: (usize, i32, usize), + _: ::std::marker::PhantomData<(F)>, + ) -> Result<(Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>), __lalrpop_util::ParseError> where + F: for<'a> FnMut(&'a str), + { + let mut __result: (Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>); + match __lookahead { + Some((_, (1, _), _)) | + Some((_, (2, _), _)) | + Some((_, (0, _), _)) | + None => { + let __start = __sym0.0.clone(); + let __end = __sym0.2.clone(); + let __nt = super::super::super::__action0::(logger, input, __sym0); + let __nt = __Nonterminal::____Term(( + __start, + __nt, + __end, + )); + __result = (__lookahead, __nt); + return Ok(__result); + } + _ => { + return Err(__lalrpop_util::ParseError::UnrecognizedToken { + token: __lookahead, + expected: vec![ + r###""(""###.to_string(), + r###"")""###.to_string(), + r###"r#"[0-9]+"#"###.to_string(), + ] + }); + } + } + } + + // State 3 + // AllInputs = ["("] + // OptionalInputs = [] + // FixedInputs = ["("] + // WillPushLen = 2 + // WillPush = [Term, ")"] + // WillProduce = Some(Term) + // + // Num = (*) r#"[0-9]+"# ["(", ")", r#"[0-9]+"#, EOF] + // Term = (*) Num ["(", ")", r#"[0-9]+"#, EOF] + // Term = (*) "(" Term ")" ["(", ")", r#"[0-9]+"#, EOF] + // Term = "(" (*) Term ")" ["(", ")", r#"[0-9]+"#, EOF] + // + // "(" -> S3 + // r#"[0-9]+"# -> S4 + // + // Num -> S1 + // Term -> S5 + fn __state3< + 'input, + F, + __TOKENS: Iterator>>, + >( + logger: &mut F, + input: &'input str, + __tokens: &mut __TOKENS, + __sym0: (usize, &'input str, usize), + _: ::std::marker::PhantomData<(F)>, + ) -> Result<(Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>), __lalrpop_util::ParseError> where + F: for<'a> FnMut(&'a str), + { + let mut __result: (Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>); + let __lookahead = match __tokens.next() { + Some(Ok(v)) => Some(v), + None => None, + Some(Err(e)) => return Err(e), + }; + match __lookahead { + Some((__loc1, (1, __tok0), __loc2)) => { + let __sym1 = (__loc1, (__tok0), __loc2); + __result = try!(__state3(logger, input, __tokens, __sym1, ::std::marker::PhantomData::<(F)>)); + } + Some((__loc1, (0, __tok0), __loc2)) => { + let __sym1 = (__loc1, (__tok0), __loc2); + __result = try!(__state4(logger, input, __tokens, __sym1, ::std::marker::PhantomData::<(F)>)); + } + _ => { + return Err(__lalrpop_util::ParseError::UnrecognizedToken { + token: __lookahead, + expected: vec![ + r###""(""###.to_string(), + r###"r#"[0-9]+"#"###.to_string(), + ] + }); + } + } + loop { + let (__lookahead, __nt) = __result; + match __nt { + __Nonterminal::Num(__sym1) => { + __result = try!(__state1(logger, input, __tokens, __lookahead, __sym1, ::std::marker::PhantomData::<(F)>)); + } + __Nonterminal::Term(__sym1) => { + __result = try!(__state5(logger, input, __tokens, __lookahead, __sym0, __sym1, ::std::marker::PhantomData::<(F)>)); + return Ok(__result); + } + _ => { + return Ok((__lookahead, __nt)); + } + } + } + } + + // State 4 + // AllInputs = [r#"[0-9]+"#] + // OptionalInputs = [] + // FixedInputs = [r#"[0-9]+"#] + // WillPushLen = 0 + // WillPush = [] + // WillProduce = Some(Num) + // + // Num = r#"[0-9]+"# (*) ["(", ")", r#"[0-9]+"#, EOF] + // + // ["(", ")", r#"[0-9]+"#, EOF] -> Num = r#"[0-9]+"# => ActionFn(3); + // + fn __state4< + 'input, + F, + __TOKENS: Iterator>>, + >( + logger: &mut F, + input: &'input str, + __tokens: &mut __TOKENS, + __sym0: (usize, &'input str, usize), + _: ::std::marker::PhantomData<(F)>, + ) -> Result<(Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>), __lalrpop_util::ParseError> where + F: for<'a> FnMut(&'a str), + { + let mut __result: (Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>); + let __lookahead = match __tokens.next() { + Some(Ok(v)) => Some(v), + None => None, + Some(Err(e)) => return Err(e), + }; + match __lookahead { + Some((_, (1, _), _)) | + Some((_, (2, _), _)) | + Some((_, (0, _), _)) | + None => { + let __start = __sym0.0.clone(); + let __end = __sym0.2.clone(); + let __nt = super::super::super::__action3::(logger, input, __sym0); + let __nt = __Nonterminal::Num(( + __start, + __nt, + __end, + )); + __result = (__lookahead, __nt); + return Ok(__result); + } + _ => { + return Err(__lalrpop_util::ParseError::UnrecognizedToken { + token: __lookahead, + expected: vec![ + r###""(""###.to_string(), + r###"")""###.to_string(), + r###"r#"[0-9]+"#"###.to_string(), + ] + }); + } + } + } + + // State 5 + // AllInputs = ["(", Term] + // OptionalInputs = [] + // FixedInputs = ["(", Term] + // WillPushLen = 1 + // WillPush = [")"] + // WillProduce = Some(Term) + // + // Term = "(" Term (*) ")" ["(", ")", r#"[0-9]+"#, EOF] + // + // ")" -> S6 + // + fn __state5< + 'input, + F, + __TOKENS: Iterator>>, + >( + logger: &mut F, + input: &'input str, + __tokens: &mut __TOKENS, + __lookahead: Option<(usize, (usize, &'input str), usize)>, + __sym0: (usize, &'input str, usize), + __sym1: (usize, i32, usize), + _: ::std::marker::PhantomData<(F)>, + ) -> Result<(Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>), __lalrpop_util::ParseError> where + F: for<'a> FnMut(&'a str), + { + let mut __result: (Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>); + match __lookahead { + Some((__loc1, (2, __tok0), __loc2)) => { + let __sym2 = (__loc1, (__tok0), __loc2); + __result = try!(__state6(logger, input, __tokens, __sym0, __sym1, __sym2, ::std::marker::PhantomData::<(F)>)); + return Ok(__result); + } + _ => { + return Err(__lalrpop_util::ParseError::UnrecognizedToken { + token: __lookahead, + expected: vec![ + r###"")""###.to_string(), + ] + }); + } + } + } + + // State 6 + // AllInputs = ["(", Term, ")"] + // OptionalInputs = [] + // FixedInputs = ["(", Term, ")"] + // WillPushLen = 0 + // WillPush = [] + // WillProduce = Some(Term) + // + // Term = "(" Term ")" (*) ["(", ")", r#"[0-9]+"#, EOF] + // + // ["(", ")", r#"[0-9]+"#, EOF] -> Term = "(", Term, ")" => ActionFn(2); + // + fn __state6< + 'input, + F, + __TOKENS: Iterator>>, + >( + logger: &mut F, + input: &'input str, + __tokens: &mut __TOKENS, + __sym0: (usize, &'input str, usize), + __sym1: (usize, i32, usize), + __sym2: (usize, &'input str, usize), + _: ::std::marker::PhantomData<(F)>, + ) -> Result<(Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>), __lalrpop_util::ParseError> where + F: for<'a> FnMut(&'a str), + { + let mut __result: (Option<(usize, (usize, &'input str), usize)>, __Nonterminal<>); + let __lookahead = match __tokens.next() { + Some(Ok(v)) => Some(v), + None => None, + Some(Err(e)) => return Err(e), + }; + match __lookahead { + Some((_, (1, _), _)) | + Some((_, (2, _), _)) | + Some((_, (0, _), _)) | + None => { + let __start = __sym0.0.clone(); + let __end = __sym2.2.clone(); + let __nt = super::super::super::__action2::(logger, input, __sym0, __sym1, __sym2); + let __nt = __Nonterminal::Term(( + __start, + __nt, + __end, + )); + __result = (__lookahead, __nt); + return Ok(__result); + } + _ => { + return Err(__lalrpop_util::ParseError::UnrecognizedToken { + token: __lookahead, + expected: vec![ + r###""(""###.to_string(), + r###"")""###.to_string(), + r###"r#"[0-9]+"#"###.to_string(), + ] + }); + } + } + } + } + pub use self::__parse__Term::parse_Term; + } + mod __parse_table { + + mod __parse__Term { + #![allow(non_snake_case, non_camel_case_types, unused_mut, unused_variables, unused_imports)] + + use std::str::FromStr; + extern crate lalrpop_util as __lalrpop_util; + #[allow(dead_code)] + pub enum __Symbol<'input> { + Term_22_28_22(&'input str), + Term_22_29_22(&'input str), + Termr_23_22_5b0_2d9_5d_2b_22_23(&'input str), + NtNum(i32), + NtTerm(i32), + Nt____Term(i32), + } + const __ACTION: &'static [i32] = &[ + // State 0 + // Num = (*) r#"[0-9]+"# ["(", ")", r#"[0-9]+"#, EOF] + // Term = (*) Num ["(", ")", r#"[0-9]+"#, EOF] + // Term = (*) "(" Term ")" ["(", ")", r#"[0-9]+"#, EOF] + // __Term = (*) Term ["(", ")", r#"[0-9]+"#, EOF] + 4, // on "(", goto 3 + 0, // on ")", error + 5, // on r#"[0-9]+"#, goto 4 + + // State 1 + // Term = Num (*) ["(", ")", r#"[0-9]+"#, EOF] + -2, // on "(", reduce `Term = Num => ActionFn(1);` + -2, // on ")", reduce `Term = Num => ActionFn(1);` + -2, // on r#"[0-9]+"#, reduce `Term = Num => ActionFn(1);` + + // State 2 + // __Term = Term (*) ["(", ")", r#"[0-9]+"#, EOF] + -4, // on "(", reduce `__Term = Term => ActionFn(0);` + -4, // on ")", reduce `__Term = Term => ActionFn(0);` + -4, // on r#"[0-9]+"#, reduce `__Term = Term => ActionFn(0);` + + // State 3 + // Num = (*) r#"[0-9]+"# ["(", ")", r#"[0-9]+"#, EOF] + // Term = (*) Num ["(", ")", r#"[0-9]+"#, EOF] + // Term = (*) "(" Term ")" ["(", ")", r#"[0-9]+"#, EOF] + // Term = "(" (*) Term ")" ["(", ")", r#"[0-9]+"#, EOF] + 4, // on "(", goto 3 + 0, // on ")", error + 5, // on r#"[0-9]+"#, goto 4 + + // State 4 + // Num = r#"[0-9]+"# (*) ["(", ")", r#"[0-9]+"#, EOF] + -1, // on "(", reduce `Num = r#"[0-9]+"# => ActionFn(3);` + -1, // on ")", reduce `Num = r#"[0-9]+"# => ActionFn(3);` + -1, // on r#"[0-9]+"#, reduce `Num = r#"[0-9]+"# => ActionFn(3);` + + // State 5 + // Term = "(" Term (*) ")" ["(", ")", r#"[0-9]+"#, EOF] + 0, // on "(", error + 7, // on ")", goto 6 + 0, // on r#"[0-9]+"#, error + + // State 6 + // Term = "(" Term ")" (*) ["(", ")", r#"[0-9]+"#, EOF] + -3, // on "(", reduce `Term = "(", Term, ")" => ActionFn(2);` + -3, // on ")", reduce `Term = "(", Term, ")" => ActionFn(2);` + -3, // on r#"[0-9]+"#, reduce `Term = "(", Term, ")" => ActionFn(2);` + + ]; + const __EOF_ACTION: &'static [i32] = &[ + 0, // on EOF, error + + -2, // on EOF, reduce `Term = Num => ActionFn(1);` + + -4, // on EOF, reduce `__Term = Term => ActionFn(0);` + + 0, // on EOF, error + + -1, // on EOF, reduce `Num = r#"[0-9]+"# => ActionFn(3);` + + 0, // on EOF, error + + -3, // on EOF, reduce `Term = "(", Term, ")" => ActionFn(2);` + + ]; + const __GOTO: &'static [i32] = &[ + // State 0 + 2, // on Num, goto 1 + 3, // on Term, goto 2 + 0, // on __Term, error + + // State 1 + 0, // on Num, error + 0, // on Term, error + 0, // on __Term, error + + // State 2 + 0, // on Num, error + 0, // on Term, error + 0, // on __Term, error + + // State 3 + 2, // on Num, goto 1 + 6, // on Term, goto 5 + 0, // on __Term, error + + // State 4 + 0, // on Num, error + 0, // on Term, error + 0, // on __Term, error + + // State 5 + 0, // on Num, error + 0, // on Term, error + 0, // on __Term, error + + // State 6 + 0, // on Num, error + 0, // on Term, error + 0, // on __Term, error + + ]; + fn __expected_tokens(__state: usize) -> Vec<::std::string::String> { + const __TERMINAL: &'static [&'static str] = &[ + r###""(""###, + r###"")""###, + r###"r#"[0-9]+"#"###, + ]; + __ACTION[(__state * 3)..].iter().zip(__TERMINAL).filter_map(|(&state, terminal)| { + if state == 0 { + None + } else { + Some(terminal.to_string()) + } + }).collect() + } + pub fn parse_Term< + 'input, + F, + >( + logger: &mut F, + input: &'input str, + ) -> Result> where + F: for<'a> FnMut(&'a str), + { + let mut __tokens = super::super::super::__intern_token::__Matcher::new(input); + let mut __states = vec![0_i32]; + let mut __symbols = vec![]; + let mut __integer; + let mut __lookahead; + let mut __last_location = Default::default(); + '__shift: loop { + __lookahead = match __tokens.next() { + Some(Ok(v)) => v, + None => break '__shift, + Some(Err(e)) => return Err(e), + }; + __last_location = __lookahead.2.clone(); + __integer = match __lookahead.1 { + (1, _) if true => 0, + (2, _) if true => 1, + (0, _) if true => 2, + _ => { + let __state = *__states.last().unwrap() as usize; + let __error = __lalrpop_util::ParseError::UnrecognizedToken { + token: Some(__lookahead), + expected: __expected_tokens(__state), + }; + return Err(__error); + } + }; + '__inner: loop { + let __state = *__states.last().unwrap() as usize; + let __action = __ACTION[__state * 3 + __integer]; + if __action > 0 { + let __symbol = match __integer { + 0 => match __lookahead.1 { + (1, __tok0) => __Symbol::Term_22_28_22((__tok0)), + _ => unreachable!(), + }, + 1 => match __lookahead.1 { + (2, __tok0) => __Symbol::Term_22_29_22((__tok0)), + _ => unreachable!(), + }, + 2 => match __lookahead.1 { + (0, __tok0) => __Symbol::Termr_23_22_5b0_2d9_5d_2b_22_23((__tok0)), + _ => unreachable!(), + }, + _ => unreachable!(), + }; + __states.push(__action - 1); + __symbols.push((__lookahead.0, __symbol, __lookahead.2)); + continue '__shift; + } else if __action < 0 { + if let Some(_) = __reduce(logger, input, __action, Some(&__lookahead.0), &mut __states, &mut __symbols, ::std::marker::PhantomData::<(F)>) { + return Err(__lalrpop_util::ParseError::ExtraToken { token: __lookahead }); + } + } else { + let __state = *__states.last().unwrap() as usize; + let __error = __lalrpop_util::ParseError::UnrecognizedToken { + token: Some(__lookahead), + expected: __expected_tokens(__state), + }; + return Err(__error) + } + } + } + loop { + let __state = *__states.last().unwrap() as usize; + let __action = __EOF_ACTION[__state]; + if __action < 0 { + if let Some(r) = __reduce(logger, input, __action, None, &mut __states, &mut __symbols, ::std::marker::PhantomData::<(F)>) { + return r; + } + } else { + let __state = *__states.last().unwrap() as usize; + let __error = __lalrpop_util::ParseError::UnrecognizedToken { + token: None, + expected: __expected_tokens(__state), + }; + return Err(__error); + } + } + } + pub fn __reduce< + 'input, + F, + >( + logger: &mut F, + input: &'input str, + __action: i32, + __lookahead_start: Option<&usize>, + __states: &mut ::std::vec::Vec, + __symbols: &mut ::std::vec::Vec<(usize,__Symbol<'input>,usize)>, + _: ::std::marker::PhantomData<(F)>, + ) -> Option>> where + F: for<'a> FnMut(&'a str), + { + let __nonterminal = match -__action { + 1 => { + // Num = r#"[0-9]+"# => ActionFn(3); + let __sym0 = __pop_Termr_23_22_5b0_2d9_5d_2b_22_23(__symbols); + let __start = __sym0.0.clone(); + let __end = __sym0.2.clone(); + let __nt = super::super::super::__action3::(logger, input, __sym0); + let __states_len = __states.len(); + __states.truncate(__states_len - 1); + __symbols.push((__start, __Symbol::NtNum(__nt), __end)); + 0 + } + 2 => { + // Term = Num => ActionFn(1); + let __sym0 = __pop_NtNum(__symbols); + let __start = __sym0.0.clone(); + let __end = __sym0.2.clone(); + let __nt = super::super::super::__action1::(logger, input, __sym0); + let __states_len = __states.len(); + __states.truncate(__states_len - 1); + __symbols.push((__start, __Symbol::NtTerm(__nt), __end)); + 1 + } + 3 => { + // Term = "(", Term, ")" => ActionFn(2); + let __sym2 = __pop_Term_22_29_22(__symbols); + let __sym1 = __pop_NtTerm(__symbols); + let __sym0 = __pop_Term_22_28_22(__symbols); + let __start = __sym0.0.clone(); + let __end = __sym2.2.clone(); + let __nt = super::super::super::__action2::(logger, input, __sym0, __sym1, __sym2); + let __states_len = __states.len(); + __states.truncate(__states_len - 3); + __symbols.push((__start, __Symbol::NtTerm(__nt), __end)); + 1 + } + 4 => { + // __Term = Term => ActionFn(0); + let __sym0 = __pop_NtTerm(__symbols); + let __start = __sym0.0.clone(); + let __end = __sym0.2.clone(); + let __nt = super::super::super::__action0::(logger, input, __sym0); + return Some(Ok(__nt)); + } + _ => panic!("invalid action code {}", __action) + }; + let __state = *__states.last().unwrap() as usize; + let __next_state = __GOTO[__state * 3 + __nonterminal] - 1; + __states.push(__next_state); + None + } + fn __pop_Term_22_28_22< + 'input, + >( + __symbols: &mut ::std::vec::Vec<(usize,__Symbol<'input>,usize)> + ) -> (usize, &'input str, usize) { + match __symbols.pop().unwrap() { + (__l, __Symbol::Term_22_28_22(__v), __r) => (__l, __v, __r), + _ => panic!("symbol type mismatch") + } + } + fn __pop_Term_22_29_22< + 'input, + >( + __symbols: &mut ::std::vec::Vec<(usize,__Symbol<'input>,usize)> + ) -> (usize, &'input str, usize) { + match __symbols.pop().unwrap() { + (__l, __Symbol::Term_22_29_22(__v), __r) => (__l, __v, __r), + _ => panic!("symbol type mismatch") + } + } + fn __pop_Termr_23_22_5b0_2d9_5d_2b_22_23< + 'input, + >( + __symbols: &mut ::std::vec::Vec<(usize,__Symbol<'input>,usize)> + ) -> (usize, &'input str, usize) { + match __symbols.pop().unwrap() { + (__l, __Symbol::Termr_23_22_5b0_2d9_5d_2b_22_23(__v), __r) => (__l, __v, __r), + _ => panic!("symbol type mismatch") + } + } + fn __pop_NtNum< + 'input, + >( + __symbols: &mut ::std::vec::Vec<(usize,__Symbol<'input>,usize)> + ) -> (usize, i32, usize) { + match __symbols.pop().unwrap() { + (__l, __Symbol::NtNum(__v), __r) => (__l, __v, __r), + _ => panic!("symbol type mismatch") + } + } + fn __pop_NtTerm< + 'input, + >( + __symbols: &mut ::std::vec::Vec<(usize,__Symbol<'input>,usize)> + ) -> (usize, i32, usize) { + match __symbols.pop().unwrap() { + (__l, __Symbol::NtTerm(__v), __r) => (__l, __v, __r), + _ => panic!("symbol type mismatch") + } + } + fn __pop_Nt____Term< + 'input, + >( + __symbols: &mut ::std::vec::Vec<(usize,__Symbol<'input>,usize)> + ) -> (usize, i32, usize) { + match __symbols.pop().unwrap() { + (__l, __Symbol::Nt____Term(__v), __r) => (__l, __v, __r), + _ => panic!("symbol type mismatch") + } + } + } + pub use self::__parse__Term::parse_Term; + } +} +pub use self::__parse__Term::parse_Term; +mod __intern_token { + #![allow(unused_imports)] + use std::str::FromStr; + extern crate lalrpop_util as __lalrpop_util; + extern crate regex as __regex; + pub struct __Matcher<'input> { + text: &'input str, + consumed: usize, + regex_set: __regex::RegexSet, + regex_vec: Vec<__regex::Regex>, + } + + impl<'input> __Matcher<'input> { + pub fn new(s: &'input str) -> __Matcher<'input> { + let __strs: &[&str] = &[ + "^(?u:[0-9])+", + "^(?u:\\()", + "^(?u:\\))", + ]; + let __regex_set = __regex::RegexSet::new(__strs).unwrap(); + let __regex_vec = vec![ + __regex::Regex::new("^(?u:[0-9])+").unwrap(), + __regex::Regex::new("^(?u:\\()").unwrap(), + __regex::Regex::new("^(?u:\\))").unwrap(), + ]; + __Matcher { + text: s, + consumed: 0, + regex_set: __regex_set, + regex_vec: __regex_vec, + } + } + } + + impl<'input> Iterator for __Matcher<'input> { + type Item = Result<(usize, (usize, &'input str), usize), __lalrpop_util::ParseError>; + + fn next(&mut self) -> Option { + let __text = self.text.trim_left(); + let __whitespace = self.text.len() - __text.len(); + let __start_offset = self.consumed + __whitespace; + if __text.is_empty() { + self.text = __text; + self.consumed = __start_offset; + None + } else { + let __matches = self.regex_set.matches(__text); + if !__matches.matched_any() { + Some(Err(__lalrpop_util::ParseError::InvalidToken { + location: __start_offset, + })) + } else { + let mut __longest_match = 0; + let mut __index = 0; + for __i in 0 .. 3 { + if __matches.matched(__i) { + let __match = self.regex_vec[__i].find(__text).unwrap(); + let __len = __match.end(); + if __len >= __longest_match { + __longest_match = __len; + __index = __i; + } + } + } + let __result = &__text[..__longest_match]; + let __remaining = &__text[__longest_match..]; + let __end_offset = __start_offset + __longest_match; + self.text = __remaining; + self.consumed = __end_offset; + Some(Ok((__start_offset, (__index, __result), __end_offset))) + } + } + } + } +} + +#[allow(unused_variables)] +fn __action0< + 'input, + F, +>( + logger: &mut F, + input: &'input str, + (_, __0, _): (usize, i32, usize), +) -> i32 where + F: for<'a> FnMut(&'a str), +{ + (__0) +} + +#[allow(unused_variables)] +fn __action1< + 'input, + F, +>( + logger: &mut F, + input: &'input str, + (_, n, _): (usize, i32, usize), +) -> i32 where + F: for<'a> FnMut(&'a str), +{ + { + let msg = format!("just parsed {}", n); + logger(&msg); + n + } +} + +#[allow(unused_variables)] +fn __action2< + 'input, + F, +>( + logger: &mut F, + input: &'input str, + (_, _, _): (usize, &'input str, usize), + (_, t, _): (usize, i32, usize), + (_, _, _): (usize, &'input str, usize), +) -> i32 where + F: for<'a> FnMut(&'a str), +{ + { + logger("propagating..."); + { + let msg = format!("... {}", t); + logger(&msg); + } + t + } +} + +#[allow(unused_variables)] +fn __action3< + 'input, + F, +>( + logger: &mut F, + input: &'input str, + (_, s, _): (usize, &'input str, usize), +) -> i32 where + F: for<'a> FnMut(&'a str), +{ + i32::from_str(s).unwrap() +} + +pub trait __ToTriple<'input, F, > { + type Error; + fn to_triple(value: Self) -> Result<(usize,(usize, &'input str),usize),Self::Error>; +} + +impl<'input, F, > __ToTriple<'input, F, > for (usize, (usize, &'input str), usize) { + type Error = (); + fn to_triple(value: Self) -> Result<(usize,(usize, &'input str),usize),()> { + Ok(value) + } +} +impl<'input, F, > __ToTriple<'input, F, > for Result<(usize, (usize, &'input str), usize),()> { + type Error = (); + fn to_triple(value: Self) -> Result<(usize,(usize, &'input str),usize),()> { + value + } +} diff --git a/lalrpop/src/grammar/parse_tree.rs b/lalrpop/src/grammar/parse_tree.rs index dc82374..df668a2 100644 --- a/lalrpop/src/grammar/parse_tree.rs +++ b/lalrpop/src/grammar/parse_tree.rs @@ -23,7 +23,7 @@ pub struct Grammar { pub span: Span, pub type_parameters: Vec, pub parameters: Vec, - pub where_clauses: Vec, + pub where_clauses: Vec>, pub items: Vec, pub annotations: Vec, pub module_attributes: Vec, @@ -222,6 +222,105 @@ pub enum TypeRef { OfSymbol(SymbolKind), } +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum WhereClause { + // 'a: 'b + 'c + Lifetime { + lifetime: InternedString, + bounds: Vec, + }, + // where for<'a> &'a T: Debug + Into + Type { + forall: Option>, + ty: T, + bounds: Vec>, + } +} + +impl WhereClause { + pub fn map(&self, mut f: F) -> WhereClause + where F: FnMut(&T) -> U + { + match *self { + WhereClause::Lifetime { lifetime, ref bounds } => WhereClause::Lifetime { + lifetime: lifetime, + bounds: bounds.clone(), + }, + WhereClause::Type { ref forall, ref ty, ref bounds } => WhereClause::Type { + forall: forall.clone(), + ty: f(ty), + bounds: bounds.iter().map(|b| b.map(&mut f)).collect(), + }, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TypeBound { + // The `'a` in `T: 'a`. + Lifetime(InternedString), + // `for<'a> FnMut(&'a usize)` + Fn { + forall: Option>, + path: Path, + parameters: Vec, + ret: Option, + }, + // `some::Trait` or `some::Trait` or `some::Trait` + // or `for<'a> Trait<'a, T>` + Trait { + forall: Option>, + path: Path, + parameters: Vec>, + } +} + +impl TypeBound { + pub fn map(&self, mut f: F) -> TypeBound + where F: FnMut(&T) -> U + { + match *self { + TypeBound::Lifetime(l) => TypeBound::Lifetime(l), + TypeBound::Fn { ref forall, ref path, ref parameters, ref ret } => TypeBound::Fn { + forall: forall.clone(), + path: path.clone(), + parameters: parameters.iter().map(&mut f).collect(), + ret: ret.as_ref().map(f), + }, + TypeBound::Trait { ref forall, ref path, ref parameters } => TypeBound::Trait { + forall: forall.clone(), + path: path.clone(), + parameters: parameters.iter().map(|p| p.map(&mut f)).collect(), + } + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TypeBoundParameter { + // 'a + Lifetime(InternedString), + // `T` or `'a` + TypeParameter(T), + // `Item = T` + Associated(InternedString, T), +} + +impl TypeBoundParameter { + pub fn map(&self, mut f: F) -> TypeBoundParameter + where F: FnMut(&T) -> U + { + match *self { + TypeBoundParameter::Lifetime(l) => + TypeBoundParameter::Lifetime(l), + TypeBoundParameter::TypeParameter(ref t) => + TypeBoundParameter::TypeParameter(f(t)), + TypeBoundParameter::Associated(id, ref t) => + TypeBoundParameter::Associated(id, f(t)), + } + } +} + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum TypeParameter { Lifetime(InternedString), @@ -342,7 +441,7 @@ pub enum SymbolKind { // @R Lookbehind, - + Error } @@ -547,6 +646,123 @@ impl Symbol { } } +impl Display for WhereClause { + fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> { + match *self { + WhereClause::Lifetime { lifetime, ref bounds } => { + write!(fmt, "{}:", lifetime)?; + for (i, b) in bounds.iter().enumerate() { + if i != 0 { + write!(fmt, " +")?; + } + write!(fmt, " {}", b)?; + } + Ok(()) + } + WhereClause::Type { ref forall, ref ty, ref bounds } => { + if let Some(ref forall) = *forall { + write!(fmt, "for<")?; + for (i, l) in forall.iter().enumerate() { + if i != 0 { + write!(fmt, ", ")?; + } + write!(fmt, "{}", l)?; + } + write!(fmt, "> ")?; + } + + write!(fmt, "{}: ", ty)?; + for (i, b) in bounds.iter().enumerate() { + if i != 0 { + write!(fmt, " +")?; + } + write!(fmt, " {}", b)?; + } + Ok(()) + } + } + } +} + +impl Display for TypeBound { + fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> { + match *self { + TypeBound::Lifetime(l) => { + write!(fmt, "{}", l) + } + TypeBound::Fn { ref forall, ref path, ref parameters, ref ret } => { + if let Some(ref forall) = *forall { + write!(fmt, "for<")?; + for (i, l) in forall.iter().enumerate() { + if i != 0 { + write!(fmt, ", ")?; + } + write!(fmt, "{}", l)?; + } + write!(fmt, "> ")?; + } + + write!(fmt, "{}(", path)?; + for (i, p) in parameters.iter().enumerate() { + if i != 0 { + write!(fmt, ", ")?; + } + write!(fmt, "{}", p)?; + } + write!(fmt, ")")?; + + if let Some(ref ret) = *ret { + write!(fmt, " -> {}", ret)?; + } + + Ok(()) + } + TypeBound::Trait { ref forall, ref path, ref parameters } => { + if let Some(ref forall) = *forall { + write!(fmt, "for<")?; + for (i, l) in forall.iter().enumerate() { + if i != 0 { + write!(fmt, ", ")?; + } + write!(fmt, "{}", l)?; + } + write!(fmt, "> ")?; + } + + write!(fmt, "{}", path)?; + if parameters.is_empty() { + return Ok(()); + } + + write!(fmt, "<")?; + for (i, p) in parameters.iter().enumerate() { + if i != 0 { + write!(fmt, ", ")?; + } + write!(fmt, "{}", p)?; + } + write!(fmt, ">") + } + } + } +} + +impl Display for TypeBoundParameter { + fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> { + match *self { + TypeBoundParameter::Lifetime(l) => { + write!(fmt, "{}", l) + } + TypeBoundParameter::TypeParameter(ref t) => { + write!(fmt, "{}", t) + } + TypeBoundParameter::Associated(id, ref t) => { + write!(fmt, "{} = {}", id, t) + } + } + } +} + impl Display for TerminalString { fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> { match *self { @@ -554,7 +770,7 @@ impl Display for TerminalString { write!(fmt, "{}", s), TerminalString::Bare(s) => write!(fmt, "{}", s), - TerminalString::Error => + TerminalString::Error => write!(fmt, "error"), } } diff --git a/lalrpop/src/grammar/repr.rs b/lalrpop/src/grammar/repr.rs index 62bdb1a..86578c7 100644 --- a/lalrpop/src/grammar/repr.rs +++ b/lalrpop/src/grammar/repr.rs @@ -17,7 +17,7 @@ pub use grammar::parse_tree::{Annotation, NonterminalString, Path, Span, - TerminalLiteral, TerminalString, TypeParameter}; + TerminalLiteral, TerminalString, TypeParameter, WhereClause}; #[derive(Clone, Debug)] pub struct Grammar { @@ -47,7 +47,7 @@ pub struct Grammar { pub parameters: Vec, // where clauses declared on the grammar, like `grammar where T: Sized` - pub where_clauses: Vec, + pub where_clauses: Vec>, // optional tokenizer DFA; this is only needed if the user did not supply // an extern token declaration diff --git a/lalrpop/src/normalize/lower/mod.rs b/lalrpop/src/normalize/lower/mod.rs index cbbbce8..997f0fd 100644 --- a/lalrpop/src/normalize/lower/mod.rs +++ b/lalrpop/src/normalize/lower/mod.rs @@ -151,6 +151,11 @@ impl<'s> LowerState<'s> { }) .collect(); + let where_clauses = grammar.where_clauses + .iter() + .map(|wc| wc.map(pt::TypeRef::type_repr)) + .collect(); + let mut algorithm = r::Algorithm::default(); // FIXME Error recovery only works for parse tables so temporarily only generate parse tables for @@ -189,7 +194,7 @@ impl<'s> LowerState<'s> { token_span: token_span.unwrap(), type_parameters: grammar.type_parameters, parameters: parameters, - where_clauses: grammar.where_clauses, + where_clauses: where_clauses, algorithm: algorithm, intern_token: self.intern_token, terminals: r::TerminalSet { @@ -324,7 +329,7 @@ impl<'s> LowerState<'s> { match norm_util::check_between_braces(&action) { norm_util::Presence::None => { action - } + } norm_util::Presence::Normal => { let name_str : String = intern::read(|interner| { let name_strs: Vec<_> = names.iter().map(|&(_,name,_)| interner.data(name)).collect(); diff --git a/lalrpop/src/parser/lrgrammar.lalrpop b/lalrpop/src/parser/lrgrammar.lalrpop index 9851fdf..070569e 100644 --- a/lalrpop/src/parser/lrgrammar.lalrpop +++ b/lalrpop/src/parser/lrgrammar.lalrpop @@ -14,21 +14,14 @@ pub Grammar: Grammar = "grammar" - + ";" => { - let where_clauses = - where_clauses.iter() - .flat_map(|wc| wc.iter()) - .map(|s| strip(s)) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) - .collect(); Grammar { prefix: format!("__"), // adjusted by `parse_grammar` span: Span(lo, hi), type_parameters: tps.unwrap_or(vec![]), parameters: parameters.unwrap_or(vec![]), - where_clauses: where_clauses, + where_clauses: where_clauses.unwrap_or(vec![]), items: uses.into_iter().chain(items).collect(), annotations: annotations, module_attributes: module_attributes} @@ -42,6 +35,48 @@ TypeParameter: TypeParameter = { => TypeParameter::Id(l) }; +pub GrammarWhereClauses: Vec> = + "where" >; + +GrammarWhereClause: WhereClause = { + ":" > => + WhereClause::Lifetime { lifetime: l, bounds: bounds }, + ":" => + WhereClause::Type { forall: f, ty: ty, bounds: bounds } +}; + +ForAll: Vec = + "for" "<" > ">"; + +TypeBounds: Vec> = + >; + +TypeBound: TypeBound = { + => + TypeBound::Lifetime(l), + "(" > ")" " )?> => + TypeBound::Fn { forall: f, path: p, parameters: params, ret: ret }, + > ">")?> => + TypeBound::Trait { forall: f, path: p, parameters: params.unwrap_or(vec![]) } +}; + +TypeBoundParameter: TypeBoundParameter = { + => TypeBoundParameter::Lifetime(l), + => TypeBoundParameter::TypeParameter(ty), + "=" => TypeBoundParameter::Associated(id, ty), +}; + +Plus: Vec = { + "+")*> => match e { + None => v, + Some(e) => { + let mut v = v; + v.push(e); + v + } + } +}; + GrammarParameters: Vec = "(" > ")"; @@ -176,7 +211,7 @@ SymbolKind1: SymbolKind = { "@R" => SymbolKind::Lookbehind, - + "!" => SymbolKind::Error, }; @@ -378,9 +413,10 @@ extern { "mut" => Tok::Mut, "pub" => Tok::Pub, "type" => Tok::Type, + "where" => Tok::Where, + "for" => Tok::For, "!" => Tok::Bang, "use" => Tok::Use(<&'input str>), - "where" => Tok::Where(>), "Escape" => Tok::Escape(<&'input str>), "Id" => Tok::Id(<&'input str>), @@ -412,6 +448,7 @@ extern { "<" => Tok::LessThan, "@L" => Tok::Lookahead, "@R" => Tok::Lookbehind, + "->" => Tok::MinusGreaterThan, "+" => Tok::Plus, "?" => Tok::Question, "}" => Tok::RightBrace, diff --git a/lalrpop/src/parser/mod.rs b/lalrpop/src/parser/mod.rs index 4d8e5a5..1acfd7b 100644 --- a/lalrpop/src/parser/mod.rs +++ b/lalrpop/src/parser/mod.rs @@ -46,3 +46,11 @@ pub fn parse_type_ref<'input>(input: &'input str) let tokenizer = tok::Tokenizer::new(input, 0); lrgrammar::parse_TypeRef(input, tokenizer) } + +#[cfg(test)] +pub fn parse_where_clauses<'input>(input: &'input str) + -> Result>, ParseError<'input>> +{ + let tokenizer = tok::Tokenizer::new(input, 0); + lrgrammar::parse_GrammarWhereClauses(input, tokenizer) +} diff --git a/lalrpop/src/parser/test.rs b/lalrpop/src/parser/test.rs index 72e3120..edfc438 100644 --- a/lalrpop/src/parser/test.rs +++ b/lalrpop/src/parser/test.rs @@ -94,3 +94,59 @@ fn match_complex() { _ => panic!("expected MatchToken, but was: {:?}", first_item) } } + +#[test] +fn where_clauses() { + let clauses = vec![ + "where T: Debug", + "where T: Debug + Display", + "where T: std::ops::Add", + "where T: IntoIterator", + "where T: 'a", + "where 'a: 'b", + "where for<'a> &'a T: Debug", + "where T: for<'a> Flobbles<'a>", + "where T: FnMut(usize)", + "where T: FnMut(usize, bool)", + "where T: FnMut() -> bool", + "where T: for<'a> FnMut(&'a usize)", + "where T: Debug, U: Display", + ]; + + for santa in clauses { + assert!( + parser::parse_where_clauses(santa).is_ok(), + "should parse where clauses: {}", + santa + ); + } +} + +#[test] +fn grammars_with_where_clauses() { + let grammars = vec![ + r###" +grammar where T: StaticMethods; +"###, + + r###" +grammar(methods: &mut T) where T: MutMethods; +"###, + + r###" +grammar<'input, T>(methods: &mut T) where T: 'input + Debug + MutMethods; +"###, + + r###" +grammar(methods: &mut F) where F: for<'a> FnMut(&'a usize) -> bool; +"###, + + r###" +grammar(logger: &mut F) where F: for<'a> FnMut(&'a str); +"###, + ]; + + for g in grammars { + assert!(parser::parse_grammar(g).is_ok()); + } +} diff --git a/lalrpop/src/tok/mod.rs b/lalrpop/src/tok/mod.rs index 5bf6fb6..f2fb8cb 100644 --- a/lalrpop/src/tok/mod.rs +++ b/lalrpop/src/tok/mod.rs @@ -42,11 +42,12 @@ pub enum Tok<'input> { Mut, Pub, Type, + Where, + For, // Special keywords: these are accompanied by a series of // uninterpreted strings representing imports and stuff. Use(&'input str), - Where(Vec<&'input str>), // Identifiers of various kinds: Escape(&'input str), @@ -79,6 +80,7 @@ pub enum Tok<'input> { LessThan, Lookahead, // @L Lookbehind, // @R + MinusGreaterThan, Plus, Question, RightBrace, @@ -111,10 +113,12 @@ const KEYWORDS: &'static [(&'static str, Tok<'static>)] = &[ ("mut", Mut), ("pub", Pub), ("type", Type), + ("where", Where), + ("for", For), ]; /* - * Helper for backtracking. + * Helper for backtracking. */ macro_rules! first { ($this:expr, $action:expr, $fallback:expr) => { @@ -266,7 +270,7 @@ impl<'input> Tokenizer<'input> { } Some((idx0, '#')) => { self.bump(); - first!(self, + first!(self, { self.shebang_attribute(idx0) }, { Ok((idx0, Hash, idx0+1)) }) } @@ -367,6 +371,17 @@ impl<'input> Tokenizer<'input> { } } } + Some((idx0, '-')) => { + match self.bump() { + Some((idx1, '>')) => { + self.bump(); + Some(Ok((idx0, MinusGreaterThan, idx1 + 1))) + } + _ => { + Some(error(UnrecognizedToken, idx0)) + } + } + } Some((idx0, c)) if is_identifier_start(c) => { if c == 'r' { // watch out for r"..." or r#"..."# strings @@ -676,31 +691,6 @@ impl<'input> Tokenizer<'input> { return Ok((start, Tok::Use(code), code_end)); } - if word == "where" { - let mut wcs = vec![]; - let mut wc_start = end; - let mut wc_end; - loop { - // Note: do not include `{` as a delimeter here, as - // that is not legal in the trait/where-clause syntax, - // and in fact signals start of the fn body. But do - // include `<`. - wc_end = try!(self.code(wc_start, "([<", ">])")); - let wc = &self.text[wc_start..wc_end]; - wcs.push(wc); - - // if this ended in a comma, maybe expect another where-clause - if let Some((_, ',')) = self.lookahead { - self.bump(); - wc_start = wc_end + 1; - } else { - break; - } - } - - return Ok((start, Tok::Where(wcs), wc_end)); - } - let tok = // search for a keyword first; if none are found, this is // either a MacroId or an Id, depending on whether there @@ -799,4 +789,3 @@ fn is_identifier_start(c: char) -> bool { fn is_identifier_continue(c: char) -> bool { UnicodeXID::is_xid_continue(c) || c == '_' } - diff --git a/lalrpop/src/tok/test.rs b/lalrpop/src/tok/test.rs index a271979..3ca0a0a 100644 --- a/lalrpop/src/tok/test.rs +++ b/lalrpop/src/tok/test.rs @@ -241,11 +241,60 @@ fn equalsgreaterthancode_nested_function_with_lifetimes() { #[test] fn where_with_lifetimes() { test(r#"where <'a,bar<'b,'c>>,baz;"#, vec![ - (r#"~~~~~~~~~~~~~~~~~~~~~~~~~ "#, Where(vec![" <'a,bar<'b,'c>>", "baz"])), + (r#"~~~~~ "#, Where), + (r#" ~ "#, LessThan), + (r#" ~~ "#, Lifetime("'a")), + (r#" ~ "#, Comma), + (r#" ~~~ "#, MacroId("bar")), + (r#" ~ "#, LessThan), + (r#" ~~ "#, Lifetime("'b")), + (r#" ~ "#, Comma), + (r#" ~~ "#, Lifetime("'c")), + (r#" ~ "#, GreaterThan), + (r#" ~ "#, GreaterThan), + (r#" ~ "#, Comma), + (r#" ~~~ "#, Id("baz")), (r#" ~"#, Semi), ]); } +#[test] +fn forall() { + test(r#"for<'a, 'b, 'c> FnMut"#, vec![ + (r#"~~~ "#, For), + (r#" ~ "#, LessThan), + (r#" ~~ "#, Lifetime("'a")), + (r#" ~ "#, Comma), + (r#" ~~ "#, Lifetime("'b")), + (r#" ~ "#, Comma), + (r#" ~~ "#, Lifetime("'c")), + (r#" ~ "#, GreaterThan), + (r#" ~~~~~"#, Id("FnMut")), + ]); +} + +#[test] +fn where_forall_fnmut_with_return_type() { + test(r#"where F: for<'a> FnMut(&'a T) -> U;"#, vec![ + (r#"~~~~~ "#, Where), + (r#" ~ "#, Id("F")), + (r#" ~ "#, Colon), + (r#" ~~~ "#, For), + (r#" ~ "#, LessThan), + (r#" ~~ "#, Lifetime("'a")), + (r#" ~ "#, GreaterThan), + (r#" ~~~~~ "#, Id("FnMut")), + (r#" ~ "#, LeftParen), + (r#" ~ "#, Ampersand), + (r#" ~~ "#, Lifetime("'a")), + (r#" ~ "#, Id("T")), + (r#" ~ "#, RightParen), + (r#" ~~ "#, MinusGreaterThan), + (r#" ~ "#, Id("U")), + (r#" ~"#, Semi), + ]); +} + #[test] fn equalsgreaterthancode_error_unbalanced() { test_err(r#"=> (,"#, @@ -371,7 +420,14 @@ fn use2() { #[test] fn where1() { test(r#"where ,baz;"#, vec![ - (r#"~~~~~~~~~~~~~~~~~~~ "#, Where(vec![" ", "baz"])), + (r#"~~~~~ "#, Where), + (r#" ~ "#, LessThan), + (r#" ~~~ "#, Id("foo")), + (r#" ~ "#, Comma), + (r#" ~~~ "#, Id("bar")), + (r#" ~ "#, GreaterThan), + (r#" ~ "#, Comma), + (r#" ~~~ "#, Id("baz")), (r#" ~"#, Semi), ]); } @@ -453,5 +509,3 @@ fn char_literals() { (r#" ~~"#, Lifetime("'c")), ]); } - -