fluencelabs/lalrpop
1
0
Fork 0
mirror of https://github.com/fluencelabs/lalrpop synced 2025-03-16 17:00:53 +00:00
Code Issues Projects Releases Wiki Activity

initial take on a tokenizer for LALRPOP

Browse Source
This commit is contained in:
Niko Matsakis 2015-07-19 07:44:13 -04:00
parent 6da930bbf6
commit a40f774ecc
6 changed files with 4117 additions and 3661 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1934
lalrpop-test/src/expr.rs
View File

File diff suppressed because it is too large Load Diff

4910
lalrpop-test/src/expr_arena.rs
View File

File diff suppressed because it is too large Load Diff

220
lalrpop-test/src/loc.rs
View File

@ -24,9 +24,9 @@ mod __parse__Items {
pub enum __Nonterminal<> {
____Items(Vec<(Option<usize>, Option<usize>)>),
_40_3c(::std::option::Option<usize>),
Spanned_3c_22_2b_22_3e((Option<usize>, Option<usize>)),
Items(Vec<(Option<usize>, Option<usize>)>),
_40_3e(::std::option::Option<usize>),
Items(Vec<(Option<usize>, Option<usize>)>),
Spanned_3c_22_2b_22_3e((Option<usize>, Option<usize>)),
}
// State 0
@ -41,9 +41,9 @@ mod __parse__Items {
// Items = (*) Items "-" ["-"]
// __Items = (*) Items [EOF]
//
// "+" -> Reduce(Items = => Call(ActionFn(1));)
// EOF -> Reduce(Items = => Call(ActionFn(1));)
// "-" -> Reduce(Items = => Call(ActionFn(1));)
// "+" -> Reduce(Items = => Call(ActionFn(1));)
//
// Items -> S1
pub fn __state0<
@ -56,6 +56,10 @@ mod __parse__Items {
{
let mut __result: (Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>);
match __lookahead {
Some((_, Tok::Plus(..), _)) => {
let __nt = super::__actions::__action1();
__result = (__lookbehind, __lookahead, __Nonterminal::Items(__nt));
}
None => {
let __nt = super::__actions::__action1();
__result = (__lookbehind, __lookahead, __Nonterminal::Items(__nt));
@ -64,10 +68,6 @@ mod __parse__Items {
let __nt = super::__actions::__action1();
__result = (__lookbehind, __lookahead, __Nonterminal::Items(__nt));
}
Some((_, Tok::Plus(..), _)) => {
let __nt = super::__actions::__action1();
__result = (__lookbehind, __lookahead, __Nonterminal::Items(__nt));
}
_ => {
return Err(__lookahead);
}
@ -99,11 +99,11 @@ mod __parse__Items {
// Spanned<"+"> = (*) @< "+" @> ["-"]
// __Items = Items (*) [EOF]
//
// "+" -> Reduce(@< = => Lookahead;)
// EOF -> Reduce(__Items = Items => Call(ActionFn(0));)
// "-" -> Shift(S4)
// "-" -> Shift(S2)
// "+" -> Reduce(@< = => Lookahead;)
//
// @< -> S2
// @< -> S4
// Spanned<"+"> -> S3
pub fn __state1<
__TOKENS: Iterator<Item=(usize,Tok,usize)>,
@ -120,17 +120,17 @@ mod __parse__Items {
let mut __lookbehind = Some(__loc);
let mut __sym1 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state4(__lookbehind, __lookahead, __tokens, __sym0, __sym1));
}
Some((_, Tok::Plus(..), _)) => {
let __nt = __lookahead.as_ref().map(|o| ::std::clone::Clone::clone(&o.0));
__result = (__lookbehind, __lookahead, __Nonterminal::_40_3c(__nt));
__result = try!(__state2(__lookbehind, __lookahead, __tokens, __sym0, __sym1));
}
None => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action0(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::____Items(__nt)));
}
Some((_, Tok::Plus(..), _)) => {
let __nt = __lookahead.as_ref().map(|o| ::std::clone::Clone::clone(&o.0));
__result = (__lookbehind, __lookahead, __Nonterminal::_40_3c(__nt));
}
_ => {
return Err(__lookahead);
}
@ -140,7 +140,7 @@ mod __parse__Items {
match __nt {
__Nonterminal::_40_3c(__nt) => {
let __sym1 = &mut Some(__nt);
__result = try!(__state2(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state4(__lookbehind, __lookahead, __tokens, __sym1));
}
__Nonterminal::Spanned_3c_22_2b_22_3e(__nt) => {
let __sym1 = &mut Some(__nt);
@ -155,13 +155,103 @@ mod __parse__Items {
}
// State 2
// Items = Items "-" (*) [EOF]
// Items = Items "-" (*) ["+"]
// Items = Items "-" (*) ["-"]
//
// "+" -> Reduce(Items = Items, "-" => Call(ActionFn(3));)
// "-" -> Reduce(Items = Items, "-" => Call(ActionFn(3));)
// EOF -> Reduce(Items = Items, "-" => Call(ActionFn(3));)
//
pub fn __state2<
__TOKENS: Iterator<Item=(usize,Tok,usize)>,
>(
__lookbehind: Option<usize>,
__lookahead: Option<(usize,Tok,usize)>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<Vec<(Option<usize>, Option<usize>)>>,
__sym1: &mut Option<Tok>,
) -> Result<(Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>), Option<(usize,Tok,usize)>>
{
let mut __result: (Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>);
match __lookahead {
Some((_, Tok::Plus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action3(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
Some((_, Tok::Minus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action3(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
None => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action3(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
_ => {
return Err(__lookahead);
}
}
}
// State 3
// Items = Items Spanned<"+"> (*) [EOF]
// Items = Items Spanned<"+"> (*) ["+"]
// Items = Items Spanned<"+"> (*) ["-"]
//
// EOF -> Reduce(Items = Items, Spanned<"+"> => Call(ActionFn(2));)
// "+" -> Reduce(Items = Items, Spanned<"+"> => Call(ActionFn(2));)
// "-" -> Reduce(Items = Items, Spanned<"+"> => Call(ActionFn(2));)
//
pub fn __state3<
__TOKENS: Iterator<Item=(usize,Tok,usize)>,
>(
__lookbehind: Option<usize>,
__lookahead: Option<(usize,Tok,usize)>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<Vec<(Option<usize>, Option<usize>)>>,
__sym1: &mut Option<(Option<usize>, Option<usize>)>,
) -> Result<(Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>), Option<(usize,Tok,usize)>>
{
let mut __result: (Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>);
match __lookahead {
None => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action2(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
Some((_, Tok::Plus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action2(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
Some((_, Tok::Minus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action2(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
_ => {
return Err(__lookahead);
}
}
}
// State 4
// Spanned<"+"> = @< (*) "+" @> [EOF]
// Spanned<"+"> = @< (*) "+" @> ["+"]
// Spanned<"+"> = @< (*) "+" @> ["-"]
//
// "+" -> Shift(S5)
//
pub fn __state2<
pub fn __state4<
__TOKENS: Iterator<Item=(usize,Tok,usize)>,
>(
__lookbehind: Option<usize>,
@ -185,96 +275,6 @@ mod __parse__Items {
return Ok(__result);
}
// State 3
// Items = Items Spanned<"+"> (*) [EOF]
// Items = Items Spanned<"+"> (*) ["+"]
// Items = Items Spanned<"+"> (*) ["-"]
//
// "-" -> Reduce(Items = Items, Spanned<"+"> => Call(ActionFn(2));)
// "+" -> Reduce(Items = Items, Spanned<"+"> => Call(ActionFn(2));)
// EOF -> Reduce(Items = Items, Spanned<"+"> => Call(ActionFn(2));)
//
pub fn __state3<
__TOKENS: Iterator<Item=(usize,Tok,usize)>,
>(
__lookbehind: Option<usize>,
__lookahead: Option<(usize,Tok,usize)>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<Vec<(Option<usize>, Option<usize>)>>,
__sym1: &mut Option<(Option<usize>, Option<usize>)>,
) -> Result<(Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>), Option<(usize,Tok,usize)>>
{
let mut __result: (Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>);
match __lookahead {
Some((_, Tok::Minus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action2(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
Some((_, Tok::Plus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action2(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
None => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action2(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
_ => {
return Err(__lookahead);
}
}
}
// State 4
// Items = Items "-" (*) [EOF]
// Items = Items "-" (*) ["+"]
// Items = Items "-" (*) ["-"]
//
// EOF -> Reduce(Items = Items, "-" => Call(ActionFn(3));)
// "+" -> Reduce(Items = Items, "-" => Call(ActionFn(3));)
// "-" -> Reduce(Items = Items, "-" => Call(ActionFn(3));)
//
pub fn __state4<
__TOKENS: Iterator<Item=(usize,Tok,usize)>,
>(
__lookbehind: Option<usize>,
__lookahead: Option<(usize,Tok,usize)>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<Vec<(Option<usize>, Option<usize>)>>,
__sym1: &mut Option<Tok>,
) -> Result<(Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>), Option<(usize,Tok,usize)>>
{
let mut __result: (Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>);
match __lookahead {
None => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action3(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
Some((_, Tok::Plus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action3(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
Some((_, Tok::Minus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action3(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
_ => {
return Err(__lookahead);
}
}
}
// State 5
// @> = (*) [EOF]
// @> = (*) ["+"]
@ -336,8 +336,8 @@ mod __parse__Items {
// Spanned<"+"> = @< "+" @> (*) ["+"]
// Spanned<"+"> = @< "+" @> (*) ["-"]
//
// EOF -> Reduce(Spanned<"+"> = @<, "+", @> => Call(ActionFn(4));)
// "-" -> Reduce(Spanned<"+"> = @<, "+", @> => Call(ActionFn(4));)
// EOF -> Reduce(Spanned<"+"> = @<, "+", @> => Call(ActionFn(4));)
// "+" -> Reduce(Spanned<"+"> = @<, "+", @> => Call(ActionFn(4));)
//
pub fn __state6<
@ -353,14 +353,14 @@ mod __parse__Items {
{
let mut __result: (Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>);
match __lookahead {
None => {
Some((_, Tok::Minus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __sym2 = __sym2.take().unwrap();
let __nt = super::__actions::__action4(__sym0, __sym1, __sym2);
return Ok((__lookbehind, __lookahead, __Nonterminal::Spanned_3c_22_2b_22_3e(__nt)));
}
Some((_, Tok::Minus(..), _)) => {
None => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __sym2 = __sym2.take().unwrap();

258
lalrpop-test/src/sub.rs
View File

@ -23,9 +23,9 @@ mod __parse__S {
pub enum __Nonterminal<> {
____S(i32),
T(i32),
S(i32),
E(i32),
S(i32),
T(i32),
}
// State 0
@ -40,12 +40,12 @@ mod __parse__S {
// T = (*) "Num" ["-"]
// __S = (*) S [EOF]
//
// "Num" -> Shift(S1)
// "(" -> Shift(S5)
// "Num" -> Shift(S2)
//
// T -> S4
// S -> S2
// E -> S3
// T -> S1
// S -> S3
// E -> S4
pub fn __state0<
__TOKENS: Iterator<Item=Tok>,
>(
@ -56,18 +56,18 @@ mod __parse__S {
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(Tok::Num(__tok0)) => {
let mut __lookbehind = None;
let mut __sym0 = &mut Some((__tok0));
let __lookahead = __tokens.next();
__result = try!(__state1(__lookbehind, __lookahead, __tokens, __sym0));
}
Some(__tok @ Tok::LParen(..)) => {
let mut __lookbehind = None;
let mut __sym0 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state5(__lookbehind, __lookahead, __tokens, __sym0));
}
Some(Tok::Num(__tok0)) => {
let mut __lookbehind = None;
let mut __sym0 = &mut Some((__tok0));
let __lookahead = __tokens.next();
__result = try!(__state2(__lookbehind, __lookahead, __tokens, __sym0));
}
_ => {
return Err(__lookahead);
}
@ -77,15 +77,15 @@ mod __parse__S {
match __nt {
__Nonterminal::T(__nt) => {
let __sym0 = &mut Some(__nt);
__result = try!(__state4(__lookbehind, __lookahead, __tokens, __sym0));
__result = try!(__state1(__lookbehind, __lookahead, __tokens, __sym0));
}
__Nonterminal::S(__nt) => {
let __sym0 = &mut Some(__nt);
__result = try!(__state2(__lookbehind, __lookahead, __tokens, __sym0));
__result = try!(__state3(__lookbehind, __lookahead, __tokens, __sym0));
}
__Nonterminal::E(__nt) => {
let __sym0 = &mut Some(__nt);
__result = try!(__state3(__lookbehind, __lookahead, __tokens, __sym0));
__result = try!(__state4(__lookbehind, __lookahead, __tokens, __sym0));
}
_ => {
return Ok((__lookbehind, __lookahead, __nt));
@ -95,13 +95,47 @@ mod __parse__S {
}
// State 1
// E = T (*) [EOF]
// E = T (*) ["-"]
//
// EOF -> Reduce(E = T => Call(ActionFn(3));)
// "-" -> Reduce(E = T => Call(ActionFn(3));)
//
pub fn __state1<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
__lookahead: Option<Tok>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<i32>,
) -> Result<(Option<()>, Option<Tok>, __Nonterminal<>), Option<Tok>>
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
None => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
Some(Tok::Minus(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
_ => {
return Err(__lookahead);
}
}
}
// State 2
// T = "Num" (*) [EOF]
// T = "Num" (*) ["-"]
//
// EOF -> Reduce(T = "Num" => Call(ActionFn(4));)
// "-" -> Reduce(T = "Num" => Call(ActionFn(4));)
//
pub fn __state1<
pub fn __state2<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
@ -128,12 +162,12 @@ mod __parse__S {
}
}
// State 2
// State 3
// __S = S (*) [EOF]
//
// EOF -> Reduce(__S = S => Call(ActionFn(0));)
//
pub fn __state2<
pub fn __state3<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
@ -155,15 +189,15 @@ mod __parse__S {
}
}
// State 3
// State 4
// E = E (*) "-" T [EOF]
// E = E (*) "-" T ["-"]
// S = E (*) [EOF]
//
// "-" -> Shift(S6)
// EOF -> Reduce(S = E => Call(ActionFn(1));)
// "-" -> Shift(S6)
//
pub fn __state3<
pub fn __state4<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
@ -192,40 +226,6 @@ mod __parse__S {
return Ok(__result);
}
// State 4
// E = T (*) [EOF]
// E = T (*) ["-"]
//
// "-" -> Reduce(E = T => Call(ActionFn(3));)
// EOF -> Reduce(E = T => Call(ActionFn(3));)
//
pub fn __state4<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
__lookahead: Option<Tok>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<i32>,
) -> Result<(Option<()>, Option<Tok>, __Nonterminal<>), Option<Tok>>
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(Tok::Minus(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
None => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
_ => {
return Err(__lookahead);
}
}
}
// State 5
// E = (*) E "-" T [")"]
// E = (*) E "-" T ["-"]
@ -238,11 +238,11 @@ mod __parse__S {
// T = (*) "Num" [")"]
// T = (*) "Num" ["-"]
//
// "Num" -> Shift(S9)
// "(" -> Shift(S10)
// "Num" -> Shift(S8)
// "(" -> Shift(S9)
//
// E -> S8
// T -> S7
// E -> S7
// T -> S10
pub fn __state5<
__TOKENS: Iterator<Item=Tok>,
>(
@ -258,13 +258,13 @@ mod __parse__S {
let mut __lookbehind = None;
let mut __sym1 = &mut Some((__tok0));
let __lookahead = __tokens.next();
__result = try!(__state9(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state8(__lookbehind, __lookahead, __tokens, __sym1));
}
Some(__tok @ Tok::LParen(..)) => {
let mut __lookbehind = None;
let mut __sym1 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state10(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state9(__lookbehind, __lookahead, __tokens, __sym1));
}
_ => {
return Err(__lookahead);
@ -275,11 +275,11 @@ mod __parse__S {
match __nt {
__Nonterminal::E(__nt) => {
let __sym1 = &mut Some(__nt);
__result = try!(__state8(__lookbehind, __lookahead, __tokens, __sym0, __sym1));
__result = try!(__state7(__lookbehind, __lookahead, __tokens, __sym0, __sym1));
}
__Nonterminal::T(__nt) => {
let __sym1 = &mut Some(__nt);
__result = try!(__state7(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state10(__lookbehind, __lookahead, __tokens, __sym1));
}
_ => {
return Ok((__lookbehind, __lookahead, __nt));
@ -297,8 +297,8 @@ mod __parse__S {
// T = (*) "Num" [EOF]
// T = (*) "Num" ["-"]
//
// "Num" -> Shift(S1)
// "(" -> Shift(S5)
// "Num" -> Shift(S2)
//
// T -> S11
pub fn __state6<
@ -313,18 +313,18 @@ mod __parse__S {
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(Tok::Num(__tok0)) => {
let mut __lookbehind = None;
let mut __sym2 = &mut Some((__tok0));
let __lookahead = __tokens.next();
__result = try!(__state1(__lookbehind, __lookahead, __tokens, __sym2));
}
Some(__tok @ Tok::LParen(..)) => {
let mut __lookbehind = None;
let mut __sym2 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state5(__lookbehind, __lookahead, __tokens, __sym2));
}
Some(Tok::Num(__tok0)) => {
let mut __lookbehind = None;
let mut __sym2 = &mut Some((__tok0));
let __lookahead = __tokens.next();
__result = try!(__state2(__lookbehind, __lookahead, __tokens, __sym2));
}
_ => {
return Err(__lookahead);
}
@ -345,49 +345,15 @@ mod __parse__S {
}
// State 7
// E = T (*) [")"]
// E = T (*) ["-"]
//
// "-" -> Reduce(E = T => Call(ActionFn(3));)
// ")" -> Reduce(E = T => Call(ActionFn(3));)
//
pub fn __state7<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
__lookahead: Option<Tok>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<i32>,
) -> Result<(Option<()>, Option<Tok>, __Nonterminal<>), Option<Tok>>
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(Tok::Minus(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
Some(Tok::RParen(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
_ => {
return Err(__lookahead);
}
}
}
// State 8
// E = E (*) "-" T [")"]
// E = E (*) "-" T ["-"]
// T = "(" E (*) ")" [EOF]
// T = "(" E (*) ")" ["-"]
//
// ")" -> Shift(S13)
// "-" -> Shift(S12)
// ")" -> Shift(S13)
//
pub fn __state8<
pub fn __state7<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
@ -399,18 +365,18 @@ mod __parse__S {
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(__tok @ Tok::RParen(..)) => {
let mut __lookbehind = None;
let mut __sym2 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state13(__lookbehind, __lookahead, __tokens, __sym0, __sym1, __sym2));
}
Some(__tok @ Tok::Minus(..)) => {
let mut __lookbehind = None;
let mut __sym2 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state12(__lookbehind, __lookahead, __tokens, __sym1, __sym2));
}
Some(__tok @ Tok::RParen(..)) => {
let mut __lookbehind = None;
let mut __sym2 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state13(__lookbehind, __lookahead, __tokens, __sym0, __sym1, __sym2));
}
_ => {
return Err(__lookahead);
}
@ -418,14 +384,14 @@ mod __parse__S {
return Ok(__result);
}
// State 9
// State 8
// T = "Num" (*) [")"]
// T = "Num" (*) ["-"]
//
// ")" -> Reduce(T = "Num" => Call(ActionFn(4));)
// "-" -> Reduce(T = "Num" => Call(ActionFn(4));)
//
pub fn __state9<
pub fn __state8<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
@ -452,7 +418,7 @@ mod __parse__S {
}
}
// State 10
// State 9
// E = (*) E "-" T [")"]
// E = (*) E "-" T ["-"]
// E = (*) T [")"]
@ -464,12 +430,12 @@ mod __parse__S {
// T = (*) "Num" [")"]
// T = (*) "Num" ["-"]
//
// "(" -> Shift(S10)
// "Num" -> Shift(S9)
// "(" -> Shift(S9)
// "Num" -> Shift(S8)
//
// E -> S14
// T -> S7
pub fn __state10<
// T -> S10
pub fn __state9<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
@ -484,13 +450,13 @@ mod __parse__S {
let mut __lookbehind = None;
let mut __sym1 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state10(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state9(__lookbehind, __lookahead, __tokens, __sym1));
}
Some(Tok::Num(__tok0)) => {
let mut __lookbehind = None;
let mut __sym1 = &mut Some((__tok0));
let __lookahead = __tokens.next();
__result = try!(__state9(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state8(__lookbehind, __lookahead, __tokens, __sym1));
}
_ => {
return Err(__lookahead);
@ -505,7 +471,7 @@ mod __parse__S {
}
__Nonterminal::T(__nt) => {
let __sym1 = &mut Some(__nt);
__result = try!(__state7(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state10(__lookbehind, __lookahead, __tokens, __sym1));
}
_ => {
return Ok((__lookbehind, __lookahead, __nt));
@ -515,6 +481,40 @@ mod __parse__S {
return Ok(__result);
}
// State 10
// E = T (*) [")"]
// E = T (*) ["-"]
//
// ")" -> Reduce(E = T => Call(ActionFn(3));)
// "-" -> Reduce(E = T => Call(ActionFn(3));)
//
pub fn __state10<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
__lookahead: Option<Tok>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<i32>,
) -> Result<(Option<()>, Option<Tok>, __Nonterminal<>), Option<Tok>>
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(Tok::RParen(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
Some(Tok::Minus(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
_ => {
return Err(__lookahead);
}
}
}
// State 11
// E = E "-" T (*) [EOF]
// E = E "-" T (*) ["-"]
@ -563,8 +563,8 @@ mod __parse__S {
// T = (*) "Num" [")"]
// T = (*) "Num" ["-"]
//
// "(" -> Shift(S10)
// "Num" -> Shift(S9)
// "(" -> Shift(S9)
// "Num" -> Shift(S8)
//
// T -> S15
pub fn __state12<
@ -583,13 +583,13 @@ mod __parse__S {
let mut __lookbehind = None;
let mut __sym2 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state10(__lookbehind, __lookahead, __tokens, __sym2));
__result = try!(__state9(__lookbehind, __lookahead, __tokens, __sym2));
}
Some(Tok::Num(__tok0)) => {
let mut __lookbehind = None;
let mut __sym2 = &mut Some((__tok0));
let __lookahead = __tokens.next();
__result = try!(__state9(__lookbehind, __lookahead, __tokens, __sym2));
__result = try!(__state8(__lookbehind, __lookahead, __tokens, __sym2));
}
_ => {
return Err(__lookahead);
@ -734,8 +734,8 @@ mod __parse__S {
// T = "(" E ")" (*) [")"]
// T = "(" E ")" (*) ["-"]
//
// ")" -> Reduce(T = "(", E, ")" => Call(ActionFn(5));)
// "-" -> Reduce(T = "(", E, ")" => Call(ActionFn(5));)
// ")" -> Reduce(T = "(", E, ")" => Call(ActionFn(5));)
//
pub fn __state16<
__TOKENS: Iterator<Item=Tok>,
@ -750,14 +750,14 @@ mod __parse__S {
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(Tok::RParen(..)) => {
Some(Tok::Minus(..)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __sym2 = __sym2.take().unwrap();
let __nt = super::__actions::__action5(__sym0, __sym1, __sym2);
return Ok((__lookbehind, __lookahead, __Nonterminal::T(__nt)));
}
Some(Tok::Minus(..)) => {
Some(Tok::RParen(..)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __sym2 = __sym2.take().unwrap();

455
lalrpop-test/src/util/lalrpop_tok.rs Normal file
View File

@ -0,0 +1,455 @@
use std::str::CharIndices;
use self::Error::*;
use self::Tok::*;
pub enum Error {
UnrecognizedToken(usize),
UnterminatedEscape(usize),
UnterminatedStringLiteral(usize),
UnterminatedCode(usize),
}
#[derive(Clone, Debug)]
pub enum Tok<'input> {
// Keywords;
Enum,
Extern,
Grammar,
If,
Mut,
Pub,
Token,
Use,
Where,
// Identifiers of various kinds:
Escape(&'input str),
Id(&'input str),
MacroId(&'input str), // identifier followed immediately by `<`
Lifetime(&'input str), // includes the `'`
StringLiteral(&'input str), // excludes the `"`
// Symbols:
Ampersand,
BangEquals,
BangTilde,
Colon,
ColonColon,
Comma,
DotDot,
Equals,
EqualsEquals,
EqualsGreaterThanCode(&'input str),
EqualsGreaterThanLookahead,
EqualsGreaterThanLookbehind,
GreaterThan,
LeftBrace,
LeftBracket,
LeftParen,
LessThan,
Lookahead, // @<
Lookbehind, // @>
Plus,
Question,
RightBrace,
RightBracket,
RightParen,
Semi,
Star,
TildeTilde,
Underscore,
}
struct Tokenizer<'input> {
text: &'input str,
chars: CharIndices<'input>,
lookahead: Option<(usize, char)>,
}
macro_rules! eof {
($x:expr) => {
match $x { Some(v) => v, None => { return None; } }
}
}
type Spanned<T> = (usize, T, usize);
const KEYWORDS: &'static [(&'static str, Tok<'static>)] = &[
("enum", Enum),
("extern", Extern),
("grammar", Grammar),
("if", If),
("mut", Mut),
("pub", Pub),
("token", Token),
("use", Use),
("where", Where),
];
impl<'input> Tokenizer<'input> {
fn bump(&mut self) -> Option<(usize, char)> {
self.lookahead = self.chars.next();
self.lookahead
}
fn right_arrow(&mut self, idx0: usize) -> Result<Spanned<Tok<'input>>, Error> {
// we've seen =>, now we have to choose between:
//
// => code
// =>@<
// =>@>
let idx1 = match self.lookahead {
Some((_, '@')) => {
match self.bump() {
Some((idx2, '<')) => {
self.bump();
return Ok((idx0, EqualsGreaterThanLookahead, idx2+1));
}
Some((idx2, '>')) => {
self.bump();
return Ok((idx0, EqualsGreaterThanLookbehind, idx2+1));
}
_ => {
return Err(UnrecognizedToken(idx0));
}
}
}
None => {
return Err(UnterminatedCode(idx0));
}
Some((idx1, _)) => { idx1 }
};
// This is the interesting case. To find the end of the code,
// we have to scan ahead, matching (), [], and {}, and looking
// for a suitable terminator: `,`, `;`, `]`, `}`, or `)`.
let mut balance = 0; // number of unclosed `(` etc
loop {
match self.lookahead {
Some((_, '(')) |
Some((_, '[')) |
Some((_, '{')) => {
balance += 1;
}
Some((_, ')')) |
Some((_, ']')) |
Some((_, '}')) if balance > 0 => {
balance -= 1;
}
None if balance == 0 => {
// Note: we do not consume the
// terminator. The code is everything *up
// to but not including* the terminating
// `,`, `;`, etc.
let code = &self.text[idx1..];
return Ok((idx0, EqualsGreaterThanCode(code), self.text.len()));
}
Some((idx2, ';')) |
Some((idx2, ',')) |
Some((idx2, ')')) |
Some((idx2, ']')) |
Some((idx2, '}')) if balance == 0 => {
// Note: we do not consume the
// terminator. The code is everything *up
// to but not including* the terminating
// `,`, `;`, etc.
let code = &self.text[idx1..idx2];
return Ok((idx0, EqualsGreaterThanCode(code), idx2));
}
None if balance > 0 => {
// the input should not end with an
// unbalanced number of `{` etc!
return Err(UnterminatedCode(idx0));
}
_ => { }
}
self.bump();
}
}
fn escape(&mut self, idx0: usize) -> Result<Spanned<Tok<'input>>, Error> {
match self.take_until(|c| c == '`') {
Some(idx1) => {
self.bump(); // consume the '`'
let text: &'input str = &self.text[idx0+1..idx1]; // do not include the `` in the str
Ok((idx0, Escape(text), idx1))
}
None => {
Err(UnterminatedEscape(idx0))
}
}
}
fn string_literal(&mut self, idx0: usize) -> Result<Spanned<Tok<'input>>, Error> {
let mut escape = false;
let terminate = |c: char| {
if escape {
escape = false;
false
} else if c == '\\' {
escape = true;
false
} else if c == '"' {
true
} else {
false
}
};
match self.take_until(terminate) {
Some(idx1) => {
self.bump(); // consume the '"'
let text = &self.text[idx0+1..idx1]; // do not include the `` in the str
Ok((idx0, StringLiteral(text), idx1))
}
None => {
Err(UnterminatedStringLiteral(idx0))
}
}
}
fn lifetime(&mut self, idx0: usize) -> Spanned<Tok<'input>> {
let (start, word, end) = self.word(idx0);
(start, Lifetime(word), end)
}
fn identifierish(&mut self, idx0: usize) -> Spanned<Tok<'input>> {
let (start, word, end) = self.word(idx0);
let tok =
// search for a keyword first; if none are found, this is
// either a MacroId or an Id, depending on whether there
// is a `<` immediately afterwards
KEYWORDS.iter()
.filter(|&&(w, _)| w == word)
.map(|&(_, ref t)| t.clone())
.next()
.unwrap_or_else(|| {
match self.lookahead {
Some((_, '<')) => MacroId(word),
_ => Id(word),
}
});
(start, tok, end)
}
fn word(&mut self, idx0: usize) -> Spanned<&'input str> {
match self.take_while(is_identifier_continue) {
Some(end) => (idx0, &self.text[idx0..end], end),
None => (idx0, &self.text[idx0..], self.text.len()),
}
}
fn take_while<F>(&mut self, mut keep_going: F) -> Option<usize>
where F: FnMut(char) -> bool
{
self.take_until(|c| !keep_going(c))
}
fn take_until<F>(&mut self, mut terminate: F) -> Option<usize>
where F: FnMut(char) -> bool
{
loop {
match self.lookahead {
None => {
return None;
}
Some((idx1, c)) => {
if terminate(c) {
return Some(idx1);
} else {
self.bump();
}
}
}
}
}
}
impl<'input> Iterator for Tokenizer<'input> {
type Item = Result<Spanned<Tok<'input>>, Error>;
fn next(&mut self) -> Option<Result<Spanned<Tok<'input>>, Error>> {
match self.lookahead {
Some((idx0, '&')) => {
self.bump();
Some(Ok((idx0, Ampersand, idx0+1)))
}
Some((idx0, '!')) => {
match self.bump() {
Some((idx1, '=')) => {
self.bump();
Some(Ok((idx0, BangEquals, idx1+1)))
}
Some((idx1, '~')) => {
self.bump();
Some(Ok((idx0, BangTilde, idx1+1)))
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
}
}
}
Some((idx0, ':')) => {
match self.bump() {
Some((idx1, ':')) => {
self.bump();
Some(Ok((idx0, ColonColon, idx1+1)))
}
_ => {
Some(Ok((idx0, Colon, idx0+1)))
}
}
}
Some((idx0, ',')) => {
self.bump();
Some(Ok((idx0, Comma, idx0+1)))
}
Some((idx0, '.')) => {
match self.bump() {
Some((idx1, '.')) => {
self.bump();
Some(Ok((idx0, DotDot, idx1+1)))
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
}
}
}
Some((idx0, '=')) => {
match self.bump() {
Some((idx1, '=')) => {
self.bump();
Some(Ok((idx0, EqualsEquals, idx1+1)))
}
Some((_, '>')) => {
self.bump();
Some(self.right_arrow(idx0))
}
_ => {
Some(Ok((idx0, Equals, idx0+1)))
}
}
}
Some((idx0, '>')) => {
self.bump();
Some(Ok((idx0, GreaterThan, idx0+1)))
}
Some((idx0, '{')) => {
self.bump();
Some(Ok((idx0, LeftBrace, idx0+1)))
}
Some((idx0, '[')) => {
self.bump();
Some(Ok((idx0, LeftBracket, idx0+1)))
}
Some((idx0, '(')) => {
self.bump();
Some(Ok((idx0, LeftParen, idx0+1)))
}
Some((idx0, '<')) => {
self.bump();
Some(Ok((idx0, LessThan, idx0+1)))
}
Some((idx0, '@')) => {
match self.bump() {
Some((idx1, '<')) => {
self.bump();
Some(Ok((idx0, Lookahead, idx1+1)))
}
Some((idx1, '>')) => {
self.bump();
Some(Ok((idx0, Lookbehind, idx1+1)))
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
}
}
}
Some((idx0, '+')) => {
self.bump();
Some(Ok((idx0, Plus, idx0+1)))
}
Some((idx0, '?')) => {
self.bump();
Some(Ok((idx0, Question, idx0+1)))
}
Some((idx0, '}')) => {
self.bump();
Some(Ok((idx0, RightBrace, idx0+1)))
}
Some((idx0, ']')) => {
self.bump();
Some(Ok((idx0, RightBracket, idx0+1)))
}
Some((idx0, ')')) => {
self.bump();
Some(Ok((idx0, RightParen, idx0+1)))
}
Some((idx0, ';')) => {
self.bump();
Some(Ok((idx0, Semi, idx0+1)))
}
Some((idx0, '*')) => {
self.bump();
Some(Ok((idx0, Star, idx0+1)))
}
Some((idx0, '~')) => {
match self.bump() {
Some((idx1, '~')) => {
self.bump();
Some(Ok((idx0, TildeTilde, idx1+1)))
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
}
}
}
Some((idx0, '_')) => {
self.bump();
Some(Ok((idx0, Underscore, idx0+1)))
}
Some((idx0, '`')) => {
self.bump();
Some(self.escape(idx0))
}
Some((idx0, '\'')) => {
self.bump();
Some(Ok(self.lifetime(idx0)))
}
Some((idx0, '"')) => {
self.bump();
Some(self.string_literal(idx0))
}
Some((idx0, c)) if is_identifier_start(c) => {
Some(Ok(self.identifierish(idx0)))
}
Some((idx, _)) => {
Some(Err(UnrecognizedToken(idx)))
}
None => {
None
}
}
}
}
fn is_identifier_start(c: char) -> bool {
// for some reason c.is_xid_start() is not stable :(
c.is_alphabetic() || c == '_'
}
fn is_identifier_continue(c: char) -> bool {
// for some reason c.is_xid_continue() is not stable :(
c.is_alphabetic() || c == '_' || c.is_digit(10)
}

1
lalrpop-test/src/util/mod.rs
View File

@ -5,6 +5,7 @@ use util::tok::Tok;
// a simple tokenizer
pub mod tok;
pub mod lalrpop_tok;
pub fn test<R:Debug+Eq,F>(parse_fn: F,
input: &str,