fluencelabs/lalrpop
1
0
Fork 0
mirror of https://github.com/fluencelabs/lalrpop synced 2025-03-16 17:00:53 +00:00
Code Issues Projects Releases Wiki Activity

Move the LALRPOP tokenizer into lalrpop itself; rename the existing

Browse Source 
token to lexer; add a few more tests and fix a bug in `test`
This commit is contained in:
Niko Matsakis 2015-07-19 14:03:00 -04:00
parent a40f774ecc
commit 4c74cc6873
17 changed files with 3832 additions and 3735 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1844
lalrpop-test/src/expr.rs
View File

File diff suppressed because it is too large Load Diff

4926
lalrpop-test/src/expr_arena.rs
View File

File diff suppressed because it is too large Load Diff

100
lalrpop-test/src/loc.rs
View File

@ -23,10 +23,10 @@ mod __parse__Items {
pub enum __Nonterminal<> {
____Items(Vec<(Option<usize>, Option<usize>)>),
_40_3c(::std::option::Option<usize>),
_40_3e(::std::option::Option<usize>),
Items(Vec<(Option<usize>, Option<usize>)>),
_40_3e(::std::option::Option<usize>),
Spanned_3c_22_2b_22_3e((Option<usize>, Option<usize>)),
_40_3c(::std::option::Option<usize>),
}
// State 0
@ -103,8 +103,8 @@ mod __parse__Items {
// "-" -> Shift(S2)
// "+" -> Reduce(@< = => Lookahead;)
//
// @< -> S4
// Spanned<"+"> -> S3
// @< -> S3
// Spanned<"+"> -> S4
pub fn __state1<
__TOKENS: Iterator<Item=(usize,Tok,usize)>,
>(
@ -140,11 +140,11 @@ mod __parse__Items {
match __nt {
__Nonterminal::_40_3c(__nt) => {
let __sym1 = &mut Some(__nt);
__result = try!(__state4(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state3(__lookbehind, __lookahead, __tokens, __sym1));
}
__Nonterminal::Spanned_3c_22_2b_22_3e(__nt) => {
let __sym1 = &mut Some(__nt);
__result = try!(__state3(__lookbehind, __lookahead, __tokens, __sym0, __sym1));
__result = try!(__state4(__lookbehind, __lookahead, __tokens, __sym0, __sym1));
}
_ => {
return Ok((__lookbehind, __lookahead, __nt));
@ -159,8 +159,8 @@ mod __parse__Items {
// Items = Items "-" (*) ["+"]
// Items = Items "-" (*) ["-"]
//
// "+" -> Reduce(Items = Items, "-" => Call(ActionFn(3));)
// "-" -> Reduce(Items = Items, "-" => Call(ActionFn(3));)
// "+" -> Reduce(Items = Items, "-" => Call(ActionFn(3));)
// EOF -> Reduce(Items = Items, "-" => Call(ActionFn(3));)
//
pub fn __state2<
@ -175,13 +175,13 @@ mod __parse__Items {
{
let mut __result: (Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>);
match __lookahead {
Some((_, Tok::Plus(..), _)) => {
Some((_, Tok::Minus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action3(__sym0, __sym1);
return Ok((__lookbehind, __lookahead, __Nonterminal::Items(__nt)));
}
Some((_, Tok::Minus(..), _)) => {
Some((_, Tok::Plus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __nt = super::__actions::__action3(__sym0, __sym1);
@ -200,6 +200,37 @@ mod __parse__Items {
}
// State 3
// Spanned<"+"> = @< (*) "+" @> [EOF]
// Spanned<"+"> = @< (*) "+" @> ["+"]
// Spanned<"+"> = @< (*) "+" @> ["-"]
//
// "+" -> Shift(S5)
//
pub fn __state3<
__TOKENS: Iterator<Item=(usize,Tok,usize)>,
>(
__lookbehind: Option<usize>,
__lookahead: Option<(usize,Tok,usize)>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<::std::option::Option<usize>>,
) -> Result<(Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>), Option<(usize,Tok,usize)>>
{
let mut __result: (Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>);
match __lookahead {
Some((_, __tok @ Tok::Plus(..), __loc)) => {
let mut __lookbehind = Some(__loc);
let mut __sym1 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state5(__lookbehind, __lookahead, __tokens, __sym0, __sym1));
}
_ => {
return Err(__lookahead);
}
}
return Ok(__result);
}
// State 4
// Items = Items Spanned<"+"> (*) [EOF]
// Items = Items Spanned<"+"> (*) ["+"]
// Items = Items Spanned<"+"> (*) ["-"]
@ -208,7 +239,7 @@ mod __parse__Items {
// "+" -> Reduce(Items = Items, Spanned<"+"> => Call(ActionFn(2));)
// "-" -> Reduce(Items = Items, Spanned<"+"> => Call(ActionFn(2));)
//
pub fn __state3<
pub fn __state4<
__TOKENS: Iterator<Item=(usize,Tok,usize)>,
>(
__lookbehind: Option<usize>,
@ -244,37 +275,6 @@ mod __parse__Items {
}
}
// State 4
// Spanned<"+"> = @< (*) "+" @> [EOF]
// Spanned<"+"> = @< (*) "+" @> ["+"]
// Spanned<"+"> = @< (*) "+" @> ["-"]
//
// "+" -> Shift(S5)
//
pub fn __state4<
__TOKENS: Iterator<Item=(usize,Tok,usize)>,
>(
__lookbehind: Option<usize>,
__lookahead: Option<(usize,Tok,usize)>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<::std::option::Option<usize>>,
) -> Result<(Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>), Option<(usize,Tok,usize)>>
{
let mut __result: (Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>);
match __lookahead {
Some((_, __tok @ Tok::Plus(..), __loc)) => {
let mut __lookbehind = Some(__loc);
let mut __sym1 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state5(__lookbehind, __lookahead, __tokens, __sym0, __sym1));
}
_ => {
return Err(__lookahead);
}
}
return Ok(__result);
}
// State 5
// @> = (*) [EOF]
// @> = (*) ["+"]
@ -283,9 +283,9 @@ mod __parse__Items {
// Spanned<"+"> = @< "+" (*) @> ["+"]
// Spanned<"+"> = @< "+" (*) @> ["-"]
//
// EOF -> Reduce(@> = => Lookbehind;)
// "-" -> Reduce(@> = => Lookbehind;)
// "+" -> Reduce(@> = => Lookbehind;)
// EOF -> Reduce(@> = => Lookbehind;)
//
// @> -> S6
pub fn __state5<
@ -300,10 +300,6 @@ mod __parse__Items {
{
let mut __result: (Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>);
match __lookahead {
None => {
let __nt = ::std::clone::Clone::clone(&__lookbehind);
__result = (__lookbehind, __lookahead, __Nonterminal::_40_3e(__nt));
}
Some((_, Tok::Minus(..), _)) => {
let __nt = ::std::clone::Clone::clone(&__lookbehind);
__result = (__lookbehind, __lookahead, __Nonterminal::_40_3e(__nt));
@ -312,6 +308,10 @@ mod __parse__Items {
let __nt = ::std::clone::Clone::clone(&__lookbehind);
__result = (__lookbehind, __lookahead, __Nonterminal::_40_3e(__nt));
}
None => {
let __nt = ::std::clone::Clone::clone(&__lookbehind);
__result = (__lookbehind, __lookahead, __Nonterminal::_40_3e(__nt));
}
_ => {
return Err(__lookahead);
}
@ -336,9 +336,9 @@ mod __parse__Items {
// Spanned<"+"> = @< "+" @> (*) ["+"]
// Spanned<"+"> = @< "+" @> (*) ["-"]
//
// "-" -> Reduce(Spanned<"+"> = @<, "+", @> => Call(ActionFn(4));)
// EOF -> Reduce(Spanned<"+"> = @<, "+", @> => Call(ActionFn(4));)
// "+" -> Reduce(Spanned<"+"> = @<, "+", @> => Call(ActionFn(4));)
// EOF -> Reduce(Spanned<"+"> = @<, "+", @> => Call(ActionFn(4));)
// "-" -> Reduce(Spanned<"+"> = @<, "+", @> => Call(ActionFn(4));)
//
pub fn __state6<
__TOKENS: Iterator<Item=(usize,Tok,usize)>,
@ -353,7 +353,7 @@ mod __parse__Items {
{
let mut __result: (Option<usize>, Option<(usize,Tok,usize)>, __Nonterminal<>);
match __lookahead {
Some((_, Tok::Minus(..), _)) => {
Some((_, Tok::Plus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __sym2 = __sym2.take().unwrap();
@ -367,7 +367,7 @@ mod __parse__Items {
let __nt = super::__actions::__action4(__sym0, __sym1, __sym2);
return Ok((__lookbehind, __lookahead, __Nonterminal::Spanned_3c_22_2b_22_3e(__nt)));
}
Some((_, Tok::Plus(..), _)) => {
Some((_, Tok::Minus(..), _)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __sym2 = __sym2.take().unwrap();

266
lalrpop-test/src/sub.rs
View File

@ -22,10 +22,10 @@ mod __parse__S {
use util::tok::Tok;
pub enum __Nonterminal<> {
____S(i32),
E(i32),
S(i32),
T(i32),
____S(i32),
}
// State 0
@ -40,12 +40,12 @@ mod __parse__S {
// T = (*) "Num" ["-"]
// __S = (*) S [EOF]
//
// "(" -> Shift(S5)
// "Num" -> Shift(S2)
// "(" -> Shift(S4)
//
// T -> S1
// S -> S3
// E -> S4
// E -> S1
// T -> S5
pub fn __state0<
__TOKENS: Iterator<Item=Tok>,
>(
@ -56,18 +56,18 @@ mod __parse__S {
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(__tok @ Tok::LParen(..)) => {
let mut __lookbehind = None;
let mut __sym0 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state5(__lookbehind, __lookahead, __tokens, __sym0));
}
Some(Tok::Num(__tok0)) => {
let mut __lookbehind = None;
let mut __sym0 = &mut Some((__tok0));
let __lookahead = __tokens.next();
__result = try!(__state2(__lookbehind, __lookahead, __tokens, __sym0));
}
Some(__tok @ Tok::LParen(..)) => {
let mut __lookbehind = None;
let mut __sym0 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state4(__lookbehind, __lookahead, __tokens, __sym0));
}
_ => {
return Err(__lookahead);
}
@ -75,17 +75,17 @@ mod __parse__S {
loop {
let (__lookbehind, __lookahead, __nt) = __result;
match __nt {
__Nonterminal::T(__nt) => {
let __sym0 = &mut Some(__nt);
__result = try!(__state1(__lookbehind, __lookahead, __tokens, __sym0));
}
__Nonterminal::S(__nt) => {
let __sym0 = &mut Some(__nt);
__result = try!(__state3(__lookbehind, __lookahead, __tokens, __sym0));
}
__Nonterminal::E(__nt) => {
let __sym0 = &mut Some(__nt);
__result = try!(__state4(__lookbehind, __lookahead, __tokens, __sym0));
__result = try!(__state1(__lookbehind, __lookahead, __tokens, __sym0));
}
__Nonterminal::T(__nt) => {
let __sym0 = &mut Some(__nt);
__result = try!(__state5(__lookbehind, __lookahead, __tokens, __sym0));
}
_ => {
return Ok((__lookbehind, __lookahead, __nt));
@ -95,11 +95,12 @@ mod __parse__S {
}
// State 1
// E = T (*) [EOF]
// E = T (*) ["-"]
// E = E (*) "-" T [EOF]
// E = E (*) "-" T ["-"]
// S = E (*) [EOF]
//
// EOF -> Reduce(E = T => Call(ActionFn(3));)
// "-" -> Reduce(E = T => Call(ActionFn(3));)
// "-" -> Shift(S6)
// EOF -> Reduce(S = E => Call(ActionFn(1));)
//
pub fn __state1<
__TOKENS: Iterator<Item=Tok>,
@ -112,28 +113,30 @@ mod __parse__S {
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(__tok @ Tok::Minus(..)) => {
let mut __lookbehind = None;
let mut __sym1 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state6(__lookbehind, __lookahead, __tokens, __sym0, __sym1));
}
None => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
Some(Tok::Minus(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
let __nt = super::__actions::__action1(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::S(__nt)));
}
_ => {
return Err(__lookahead);
}
}
return Ok(__result);
}
// State 2
// T = "Num" (*) [EOF]
// T = "Num" (*) ["-"]
//
// EOF -> Reduce(T = "Num" => Call(ActionFn(4));)
// "-" -> Reduce(T = "Num" => Call(ActionFn(4));)
// EOF -> Reduce(T = "Num" => Call(ActionFn(4));)
//
pub fn __state2<
__TOKENS: Iterator<Item=Tok>,
@ -146,12 +149,12 @@ mod __parse__S {
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
None => {
Some(Tok::Minus(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action4(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::T(__nt)));
}
Some(Tok::Minus(..)) => {
None => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action4(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::T(__nt)));
@ -190,43 +193,6 @@ mod __parse__S {
}
// State 4
// E = E (*) "-" T [EOF]
// E = E (*) "-" T ["-"]
// S = E (*) [EOF]
//
// EOF -> Reduce(S = E => Call(ActionFn(1));)
// "-" -> Shift(S6)
//
pub fn __state4<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
__lookahead: Option<Tok>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<i32>,
) -> Result<(Option<()>, Option<Tok>, __Nonterminal<>), Option<Tok>>
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(__tok @ Tok::Minus(..)) => {
let mut __lookbehind = None;
let mut __sym1 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state6(__lookbehind, __lookahead, __tokens, __sym0, __sym1));
}
None => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action1(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::S(__nt)));
}
_ => {
return Err(__lookahead);
}
}
return Ok(__result);
}
// State 5
// E = (*) E "-" T [")"]
// E = (*) E "-" T ["-"]
// E = (*) T [")"]
@ -238,12 +204,12 @@ mod __parse__S {
// T = (*) "Num" [")"]
// T = (*) "Num" ["-"]
//
// "Num" -> Shift(S8)
// "(" -> Shift(S9)
// "Num" -> Shift(S9)
// "(" -> Shift(S10)
//
// E -> S7
// T -> S10
pub fn __state5<
// T -> S8
pub fn __state4<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
@ -258,13 +224,13 @@ mod __parse__S {
let mut __lookbehind = None;
let mut __sym1 = &mut Some((__tok0));
let __lookahead = __tokens.next();
__result = try!(__state8(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state9(__lookbehind, __lookahead, __tokens, __sym1));
}
Some(__tok @ Tok::LParen(..)) => {
let mut __lookbehind = None;
let mut __sym1 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state9(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state10(__lookbehind, __lookahead, __tokens, __sym1));
}
_ => {
return Err(__lookahead);
@ -279,7 +245,7 @@ mod __parse__S {
}
__Nonterminal::T(__nt) => {
let __sym1 = &mut Some(__nt);
__result = try!(__state10(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state8(__lookbehind, __lookahead, __tokens, __sym1));
}
_ => {
return Ok((__lookbehind, __lookahead, __nt));
@ -289,6 +255,40 @@ mod __parse__S {
return Ok(__result);
}
// State 5
// E = T (*) [EOF]
// E = T (*) ["-"]
//
// "-" -> Reduce(E = T => Call(ActionFn(3));)
// EOF -> Reduce(E = T => Call(ActionFn(3));)
//
pub fn __state5<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
__lookahead: Option<Tok>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<i32>,
) -> Result<(Option<()>, Option<Tok>, __Nonterminal<>), Option<Tok>>
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(Tok::Minus(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
None => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
_ => {
return Err(__lookahead);
}
}
}
// State 6
// E = E "-" (*) T [EOF]
// E = E "-" (*) T ["-"]
@ -297,7 +297,7 @@ mod __parse__S {
// T = (*) "Num" [EOF]
// T = (*) "Num" ["-"]
//
// "(" -> Shift(S5)
// "(" -> Shift(S4)
// "Num" -> Shift(S2)
//
// T -> S11
@ -317,7 +317,7 @@ mod __parse__S {
let mut __lookbehind = None;
let mut __sym2 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state5(__lookbehind, __lookahead, __tokens, __sym2));
__result = try!(__state4(__lookbehind, __lookahead, __tokens, __sym2));
}
Some(Tok::Num(__tok0)) => {
let mut __lookbehind = None;
@ -385,13 +385,47 @@ mod __parse__S {
}
// State 8
// E = T (*) [")"]
// E = T (*) ["-"]
//
// "-" -> Reduce(E = T => Call(ActionFn(3));)
// ")" -> Reduce(E = T => Call(ActionFn(3));)
//
pub fn __state8<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
__lookahead: Option<Tok>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<i32>,
) -> Result<(Option<()>, Option<Tok>, __Nonterminal<>), Option<Tok>>
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(Tok::Minus(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
Some(Tok::RParen(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
_ => {
return Err(__lookahead);
}
}
}
// State 9
// T = "Num" (*) [")"]
// T = "Num" (*) ["-"]
//
// ")" -> Reduce(T = "Num" => Call(ActionFn(4));)
// "-" -> Reduce(T = "Num" => Call(ActionFn(4));)
//
pub fn __state8<
pub fn __state9<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
@ -418,7 +452,7 @@ mod __parse__S {
}
}
// State 9
// State 10
// E = (*) E "-" T [")"]
// E = (*) E "-" T ["-"]
// E = (*) T [")"]
@ -430,12 +464,12 @@ mod __parse__S {
// T = (*) "Num" [")"]
// T = (*) "Num" ["-"]
//
// "(" -> Shift(S9)
// "Num" -> Shift(S8)
// "(" -> Shift(S10)
// "Num" -> Shift(S9)
//
// T -> S8
// E -> S14
// T -> S10
pub fn __state9<
pub fn __state10<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
@ -450,13 +484,13 @@ mod __parse__S {
let mut __lookbehind = None;
let mut __sym1 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state9(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state10(__lookbehind, __lookahead, __tokens, __sym1));
}
Some(Tok::Num(__tok0)) => {
let mut __lookbehind = None;
let mut __sym1 = &mut Some((__tok0));
let __lookahead = __tokens.next();
__result = try!(__state8(__lookbehind, __lookahead, __tokens, __sym1));
__result = try!(__state9(__lookbehind, __lookahead, __tokens, __sym1));
}
_ => {
return Err(__lookahead);
@ -465,14 +499,14 @@ mod __parse__S {
while __sym0.is_some() {
let (__lookbehind, __lookahead, __nt) = __result;
match __nt {
__Nonterminal::T(__nt) => {
let __sym1 = &mut Some(__nt);
__result = try!(__state8(__lookbehind, __lookahead, __tokens, __sym1));
}
__Nonterminal::E(__nt) => {
let __sym1 = &mut Some(__nt);
__result = try!(__state14(__lookbehind, __lookahead, __tokens, __sym0, __sym1));
}
__Nonterminal::T(__nt) => {
let __sym1 = &mut Some(__nt);
__result = try!(__state10(__lookbehind, __lookahead, __tokens, __sym1));
}
_ => {
return Ok((__lookbehind, __lookahead, __nt));
}
@ -481,40 +515,6 @@ mod __parse__S {
return Ok(__result);
}
// State 10
// E = T (*) [")"]
// E = T (*) ["-"]
//
// ")" -> Reduce(E = T => Call(ActionFn(3));)
// "-" -> Reduce(E = T => Call(ActionFn(3));)
//
pub fn __state10<
__TOKENS: Iterator<Item=Tok>,
>(
__lookbehind: Option<()>,
__lookahead: Option<Tok>,
__tokens: &mut __TOKENS,
__sym0: &mut Option<i32>,
) -> Result<(Option<()>, Option<Tok>, __Nonterminal<>), Option<Tok>>
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(Tok::RParen(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
Some(Tok::Minus(..)) => {
let __sym0 = __sym0.take().unwrap();
let __nt = super::__actions::__action3(__sym0);
return Ok((__lookbehind, __lookahead, __Nonterminal::E(__nt)));
}
_ => {
return Err(__lookahead);
}
}
}
// State 11
// E = E "-" T (*) [EOF]
// E = E "-" T (*) ["-"]
@ -563,8 +563,8 @@ mod __parse__S {
// T = (*) "Num" [")"]
// T = (*) "Num" ["-"]
//
// "(" -> Shift(S9)
// "Num" -> Shift(S8)
// "Num" -> Shift(S9)
// "(" -> Shift(S10)
//
// T -> S15
pub fn __state12<
@ -579,17 +579,17 @@ mod __parse__S {
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
Some(__tok @ Tok::LParen(..)) => {
let mut __lookbehind = None;
let mut __sym2 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state9(__lookbehind, __lookahead, __tokens, __sym2));
}
Some(Tok::Num(__tok0)) => {
let mut __lookbehind = None;
let mut __sym2 = &mut Some((__tok0));
let __lookahead = __tokens.next();
__result = try!(__state8(__lookbehind, __lookahead, __tokens, __sym2));
__result = try!(__state9(__lookbehind, __lookahead, __tokens, __sym2));
}
Some(__tok @ Tok::LParen(..)) => {
let mut __lookbehind = None;
let mut __sym2 = &mut Some((__tok));
let __lookahead = __tokens.next();
__result = try!(__state10(__lookbehind, __lookahead, __tokens, __sym2));
}
_ => {
return Err(__lookahead);
@ -614,8 +614,8 @@ mod __parse__S {
// T = "(" E ")" (*) [EOF]
// T = "(" E ")" (*) ["-"]
//
// EOF -> Reduce(T = "(", E, ")" => Call(ActionFn(5));)
// "-" -> Reduce(T = "(", E, ")" => Call(ActionFn(5));)
// EOF -> Reduce(T = "(", E, ")" => Call(ActionFn(5));)
//
pub fn __state13<
__TOKENS: Iterator<Item=Tok>,
@ -630,14 +630,14 @@ mod __parse__S {
{
let mut __result: (Option<()>, Option<Tok>, __Nonterminal<>);
match __lookahead {
None => {
Some(Tok::Minus(..)) => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __sym2 = __sym2.take().unwrap();
let __nt = super::__actions::__action5(__sym0, __sym1, __sym2);
return Ok((__lookbehind, __lookahead, __Nonterminal::T(__nt)));
}
Some(Tok::Minus(..)) => {
None => {
let __sym0 = __sym0.take().unwrap();
let __sym1 = __sym1.take().unwrap();
let __sym2 = __sym2.take().unwrap();

1
lalrpop-test/src/util/mod.rs
View File

@ -5,7 +5,6 @@ use util::tok::Tok;
// a simple tokenizer
pub mod tok;
pub mod lalrpop_tok;
pub fn test<R:Debug+Eq,F>(parse_fn: F,
input: &str,

4
lalrpop/src/token/dfa/interpret.rs → lalrpop/src/lexer/dfa/interpret.rs
View File

@ -1,5 +1,5 @@
use token::dfa::{DFA, Kind, NFAIndex, START};
use token::re::Test;
use lexer::dfa::{DFA, Kind, NFAIndex, START};
use lexer::re::Test;
pub fn interpret<'text>(dfa: &DFA, input: &'text str) -> Option<(NFAIndex, &'text str)> {
let mut longest: Option<(NFAIndex, usize)> = None;

4
lalrpop/src/token/dfa/mod.rs → lalrpop/src/lexer/dfa/mod.rs
View File

@ -4,8 +4,8 @@
use kernel_set::{Kernel, KernelSet};
use std::fmt::{Debug, Display, Formatter, Error};
use std::rc::Rc;
use token::re;
use token::nfa::{self, NFA, NFAStateIndex};
use lexer::re;
use lexer::nfa::{self, NFA, NFAStateIndex};
use util::Set;
#[cfg(test)]

6
lalrpop/src/token/dfa/test.rs → lalrpop/src/lexer/dfa/test.rs
View File

@ -1,6 +1,6 @@
use token::dfa::{self, Ambiguity, DFA, NFAIndex, Precedence};
use token::dfa::interpret::interpret;
use token::re;
use lexer::dfa::{self, Ambiguity, DFA, NFAIndex, Precedence};
use lexer::dfa::interpret::interpret;
use lexer::re;
fn dfa(inputs: &[(&str, Precedence)]) -> Result<DFA,Ambiguity> {
let regexs: Result<Vec<_>, _> = inputs.iter().map(|&(s, _)| re::parse_regex(s)).collect();

3
lalrpop/src/token/mod.rs → lalrpop/src/lexer/mod.rs
View File

@ -1,7 +1,6 @@
#![allow(dead_code)] // tokenizer, which is not yet deployed.
//! Code related to the builtin tokenizer, which is based on regular
//! expressions.
//! Code related to generating tokenizers.
mod dfa;
mod re;

4
lalrpop/src/token/nfa/interpret.rs → lalrpop/src/lexer/nfa/interpret.rs
View File

@ -1,7 +1,7 @@
//! A depth-first interpreter for NFAs.
use token::nfa::{NFA, NFAStateIndex, START, StateKind, Noop, Other};
use token::re::Test;
use lexer::nfa::{NFA, NFAStateIndex, START, StateKind, Noop, Other};
use lexer::re::Test;
use std::cmp::max;
/// Interpret `nfa` applied to `test`, returning the longest matching

2
lalrpop/src/token/nfa/mod.rs → lalrpop/src/lexer/nfa/mod.rs
View File

@ -4,7 +4,7 @@
use std::fmt::{Debug, Formatter, Error};
use std::usize;
use token::re::{Regex, Alternative, Elem, RepeatOp, Test};
use lexer::re::{Regex, Alternative, Elem, RepeatOp, Test};
#[cfg(test)]
mod interpret;

6
lalrpop/src/token/nfa/test.rs → lalrpop/src/lexer/nfa/test.rs
View File

@ -1,6 +1,6 @@
use token::nfa::{NFA, Noop, Other, StateKind};
use token::nfa::interpret::interpret;
use token::re::{self, Test};
use lexer::nfa::{NFA, Noop, Other, StateKind};
use lexer::nfa::interpret::interpret;
use lexer::re::{self, Test};
#[test]
fn edge_iter() {

0
lalrpop/src/token/re/mod.rs → lalrpop/src/lexer/re/mod.rs
View File

0
lalrpop/src/token/re/test.rs → lalrpop/src/lexer/re/test.rs
View File

3
lalrpop/src/lib.rs
View File

@ -23,11 +23,12 @@ mod rust;
mod build;
mod grammar;
mod intern;
mod lexer;
mod lr1;
mod normalize;
mod parser;
mod kernel_set;
mod token;
mod tok;
mod util;
#[cfg(test)] mod generate;

333
lalrpop-test/src/util/lalrpop_tok.rs → lalrpop/src/tok/mod.rs
View File

@ -1,8 +1,14 @@
//! A tokenizer for use in LALRPOP itself.
use std::str::CharIndices;
use self::Error::*;
use self::Tok::*;
#[cfg(test)]
mod test;
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Error {
UnrecognizedToken(usize),
UnterminatedEscape(usize),
@ -10,7 +16,7 @@ pub enum Error {
UnterminatedCode(usize),
}
#[derive(Clone, Debug)]
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Tok<'input> {
// Keywords;
Enum,
@ -85,9 +91,19 @@ const KEYWORDS: &'static [(&'static str, Tok<'static>)] = &[
("token", Token),
("use", Use),
("where", Where),
];
];
impl<'input> Tokenizer<'input> {
fn new(text: &'input str) -> Tokenizer<'input> {
let mut t = Tokenizer {
text: text,
chars: text.char_indices(),
lookahead: None
};
t.bump();
t
}
fn bump(&mut self) -> Option<(usize, char)> {
self.lookahead = self.chars.next();
self.lookahead
@ -280,166 +296,183 @@ impl<'input> Iterator for Tokenizer<'input> {
type Item = Result<Spanned<Tok<'input>>, Error>;
fn next(&mut self) -> Option<Result<Spanned<Tok<'input>>, Error>> {
match self.lookahead {
Some((idx0, '&')) => {
self.bump();
Some(Ok((idx0, Ampersand, idx0+1)))
}
Some((idx0, '!')) => {
match self.bump() {
Some((idx1, '=')) => {
self.bump();
Some(Ok((idx0, BangEquals, idx1+1)))
}
Some((idx1, '~')) => {
self.bump();
Some(Ok((idx0, BangTilde, idx1+1)))
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
loop {
return match self.lookahead {
Some((idx0, '&')) => {
self.bump();
Some(Ok((idx0, Ampersand, idx0+1)))
}
Some((idx0, '!')) => {
match self.bump() {
Some((idx1, '=')) => {
self.bump();
Some(Ok((idx0, BangEquals, idx1+1)))
}
Some((idx1, '~')) => {
self.bump();
Some(Ok((idx0, BangTilde, idx1+1)))
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
}
}
}
}
Some((idx0, ':')) => {
match self.bump() {
Some((idx1, ':')) => {
self.bump();
Some(Ok((idx0, ColonColon, idx1+1)))
}
_ => {
Some(Ok((idx0, Colon, idx0+1)))
Some((idx0, ':')) => {
match self.bump() {
Some((idx1, ':')) => {
self.bump();
Some(Ok((idx0, ColonColon, idx1+1)))
}
_ => {
Some(Ok((idx0, Colon, idx0+1)))
}
}
}
}
Some((idx0, ',')) => {
self.bump();
Some(Ok((idx0, Comma, idx0+1)))
}
Some((idx0, '.')) => {
match self.bump() {
Some((idx1, '.')) => {
self.bump();
Some(Ok((idx0, DotDot, idx1+1)))
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
Some((idx0, ',')) => {
self.bump();
Some(Ok((idx0, Comma, idx0+1)))
}
Some((idx0, '.')) => {
match self.bump() {
Some((idx1, '.')) => {
self.bump();
Some(Ok((idx0, DotDot, idx1+1)))
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
}
}
}
}
Some((idx0, '=')) => {
match self.bump() {
Some((idx1, '=')) => {
self.bump();
Some(Ok((idx0, EqualsEquals, idx1+1)))
}
Some((_, '>')) => {
self.bump();
Some(self.right_arrow(idx0))
}
_ => {
Some(Ok((idx0, Equals, idx0+1)))
Some((idx0, '=')) => {
match self.bump() {
Some((idx1, '=')) => {
self.bump();
Some(Ok((idx0, EqualsEquals, idx1+1)))
}
Some((_, '>')) => {
self.bump();
Some(self.right_arrow(idx0))
}
_ => {
Some(Ok((idx0, Equals, idx0+1)))
}
}
}
}
Some((idx0, '>')) => {
self.bump();
Some(Ok((idx0, GreaterThan, idx0+1)))
}
Some((idx0, '{')) => {
self.bump();
Some(Ok((idx0, LeftBrace, idx0+1)))
}
Some((idx0, '[')) => {
self.bump();
Some(Ok((idx0, LeftBracket, idx0+1)))
}
Some((idx0, '(')) => {
self.bump();
Some(Ok((idx0, LeftParen, idx0+1)))
}
Some((idx0, '<')) => {
self.bump();
Some(Ok((idx0, LessThan, idx0+1)))
}
Some((idx0, '@')) => {
match self.bump() {
Some((idx1, '<')) => {
self.bump();
Some(Ok((idx0, Lookahead, idx1+1)))
}
Some((idx1, '>')) => {
self.bump();
Some(Ok((idx0, Lookbehind, idx1+1)))
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
Some((idx0, '>')) => {
self.bump();
Some(Ok((idx0, GreaterThan, idx0+1)))
}
Some((idx0, '{')) => {
self.bump();
Some(Ok((idx0, LeftBrace, idx0+1)))
}
Some((idx0, '[')) => {
self.bump();
Some(Ok((idx0, LeftBracket, idx0+1)))
}
Some((idx0, '(')) => {
self.bump();
Some(Ok((idx0, LeftParen, idx0+1)))
}
Some((idx0, '<')) => {
self.bump();
Some(Ok((idx0, LessThan, idx0+1)))
}
Some((idx0, '@')) => {
match self.bump() {
Some((idx1, '<')) => {
self.bump();
Some(Ok((idx0, Lookahead, idx1+1)))
}
Some((idx1, '>')) => {
self.bump();
Some(Ok((idx0, Lookbehind, idx1+1)))
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
}
}
}
}
Some((idx0, '+')) => {
self.bump();
Some(Ok((idx0, Plus, idx0+1)))
}
Some((idx0, '?')) => {
self.bump();
Some(Ok((idx0, Question, idx0+1)))
}
Some((idx0, '}')) => {
self.bump();
Some(Ok((idx0, RightBrace, idx0+1)))
}
Some((idx0, ']')) => {
self.bump();
Some(Ok((idx0, RightBracket, idx0+1)))
}
Some((idx0, ')')) => {
self.bump();
Some(Ok((idx0, RightParen, idx0+1)))
}
Some((idx0, ';')) => {
self.bump();
Some(Ok((idx0, Semi, idx0+1)))
}
Some((idx0, '*')) => {
self.bump();
Some(Ok((idx0, Star, idx0+1)))
}
Some((idx0, '~')) => {
match self.bump() {
Some((idx1, '~')) => {
self.bump();
Some(Ok((idx0, TildeTilde, idx1+1)))
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
Some((idx0, '+')) => {
self.bump();
Some(Ok((idx0, Plus, idx0+1)))
}
Some((idx0, '?')) => {
self.bump();
Some(Ok((idx0, Question, idx0+1)))
}
Some((idx0, '}')) => {
self.bump();
Some(Ok((idx0, RightBrace, idx0+1)))
}
Some((idx0, ']')) => {
self.bump();
Some(Ok((idx0, RightBracket, idx0+1)))
}
Some((idx0, ')')) => {
self.bump();
Some(Ok((idx0, RightParen, idx0+1)))
}
Some((idx0, ';')) => {
self.bump();
Some(Ok((idx0, Semi, idx0+1)))
}
Some((idx0, '*')) => {
self.bump();
Some(Ok((idx0, Star, idx0+1)))
}
Some((idx0, '~')) => {
match self.bump() {
Some((idx1, '~')) => {
self.bump();
Some(Ok((idx0, TildeTilde, idx1+1)))
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
}
}
}
}
Some((idx0, '_')) => {
self.bump();
Some(Ok((idx0, Underscore, idx0+1)))
}
Some((idx0, '`')) => {
self.bump();
Some(self.escape(idx0))
}
Some((idx0, '\'')) => {
self.bump();
Some(Ok(self.lifetime(idx0)))
}
Some((idx0, '"')) => {
self.bump();
Some(self.string_literal(idx0))
}
Some((idx0, c)) if is_identifier_start(c) => {
Some(Ok(self.identifierish(idx0)))
}
Some((idx, _)) => {
Some(Err(UnrecognizedToken(idx)))
}
None => {
None
}
Some((idx0, '_')) => {
self.bump();
Some(Ok((idx0, Underscore, idx0+1)))
}
Some((idx0, '`')) => {
self.bump();
Some(self.escape(idx0))
}
Some((idx0, '\'')) => {
self.bump();
Some(Ok(self.lifetime(idx0)))
}
Some((idx0, '"')) => {
self.bump();
Some(self.string_literal(idx0))
}
Some((idx0, '/')) => {
match self.bump() {
Some((_, '/')) => {
self.take_until(|c| c == '\n');
continue;
}
_ => {
Some(Err(UnrecognizedToken(idx0)))
}
}
}
Some((idx0, c)) if is_identifier_start(c) => {
Some(Ok(self.identifierish(idx0)))
}
Some((_, c)) if c.is_whitespace() => {
self.bump();
continue;
}
Some((idx, _)) => {
Some(Err(UnrecognizedToken(idx)))
}
None => {
None
}
};
}
}
}

65
lalrpop/src/tok/test.rs Normal file
View File

@ -0,0 +1,65 @@
use super::{is_identifier_start, is_identifier_continue, Tok, Tokenizer};
use super::Tok::*;
fn test(input: &str,
expected: Vec<(&str, Tok)>)
{
// use $ to signal EOL because it can be replaced with a single space
// for spans, and because it applies also to r#XXX# style strings:
let input = input.replace("$", "\n");
let tokenizer = Tokenizer::new(&input);
let len = expected.len();
for (token, (expected_span, expected_tok)) in tokenizer.zip(expected.into_iter()) {
println!("token: {:?}", token);
let expected_start = expected_span.find("~").unwrap();
let expected_end = expected_span.rfind("~").unwrap() + 1;
assert_eq!(Ok((expected_start, expected_tok, expected_end)), token);
}
let tokenizer = Tokenizer::new(&input);
assert_eq!(None, tokenizer.skip(len).next());
}
#[test]
fn identifier_start1() {
assert!(is_identifier_start('f'));
}
#[test]
fn identifier_continue1() {
assert!(is_identifier_continue('o'));
}
#[test]
fn basic() {
test("extern foo", vec![
("~~~~~~ ", Extern),
(" ~~~", Id("foo")),
]);
}
#[test]
fn eol_comment() {
test("extern // This is a comment$ foo", vec![
("~~~~~~ ", Extern),
(" ~~~", Id("foo")),
]);
}
#[test]
fn code1() {
test("=> a(b, c),", vec![
("~~~~~~~~~~ ", EqualsGreaterThanCode(" a(b, c)")),
(" ~", Comma),
]);
}
#[test]
#[should_panic]
fn code_forgot_comma() {
test("=> a(b, c),", vec![
("~~~~~~~~~~ ", EqualsGreaterThanCode(" a(b, c)")),
// intentionally forget the comma token; this is more of a test of `test`
]);
}