Add example for tokens with lifetimes

doc/calculator/src/ast.rs

@@ -6,6 +6,12 @@ pub enum Expr {
     Error,
 }
 
+pub enum ExprSymbol<'input>{
+    NumSymbol(&'input str),
+    Op(Box<ExprSymbol<'input>>, Opcode, Box<ExprSymbol<'input>>),
+    Error,
+}
+
 #[derive(Copy, Clone)]
 pub enum Opcode {
     Mul,
@@ -25,6 +31,17 @@ impl Debug for Expr {
     }
 }
 
+impl<'input> Debug for ExprSymbol<'input> {
+    fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
+        use self::ExprSymbol::*;
+        match *self {
+            NumSymbol(n) => write!(fmt, "{:?}", n),
+            Op(ref l, op, ref r) => write!(fmt, "({:?} {:?} {:?})", l, op, r),
+            Error => write!(fmt, "error"),
+        }
+    }
+}
+
 impl Debug for Opcode {
     fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
         use self::Opcode::*;

doc/calculator/src/calculator8.lalrpop

@@ -0,0 +1,37 @@
+use ast::{ExprSymbol, Opcode};
+use tok8::Tok;
+
+grammar<'input>(text: &'input str);
+
+pub Expr: Box<ExprSymbol<'input>> = { // (1)
+    Expr "ExprOp" Factor => Box::new(ExprSymbol::Op(<>)), // (2)
+    Factor,
+};
+
+Factor: Box<ExprSymbol<'input>> = {
+    Factor "FactorOp" Term => Box::new(ExprSymbol::Op(<>)),
+    Term,
+};
+
+
+Term: Box<ExprSymbol<'input>> = {
+    "num" => Box::new(ExprSymbol::NumSymbol(<>)),
+    "(" <Expr> ")"
+};
+
+extern {
+    type Location = usize;
+    type Error = ();
+    
+    enum Tok<'input> {
+        "num" => Tok::NumSymbol(<&'input str>),
+        "FactorOp" => Tok::FactorOp(<Opcode>),
+        "ExprOp" => Tok::ExprOp(<Opcode>),
+        "(" => Tok::ParenOpen,
+        ")" => Tok::ParenClose,
+    }
+
+}
+
+
+

doc/calculator/src/main.rs

@@ -136,6 +136,20 @@ fn calculator7() {
     assert_eq!(&format!("{:?}", expr), "((22 * 44) + 66)");
 }
 
+lalrpop_mod!(pub calculator8);
+mod tok8;
+use tok8::Lexer;
+
+#[test]
+fn calculator8() {
+    let input = "22 * pi + 66";
+    let lexer = Lexer::new(input);
+    let expr = calculator8::ExprParser::new()
+        .parse(input,lexer)
+        .unwrap();
+    assert_eq!(&format!("{:?}", expr), "((\"22\" * \"pi\") + \"66\")");
+}
+
 #[cfg(not(test))]
 fn main() {
     println!("Hello, world!");

doc/calculator/src/tok8.rs

@@ -0,0 +1,59 @@
+use ast::Opcode;
+
+pub type Spanned<Tok, Loc, Error> = Result<(Loc, Tok, Loc), Error>;
+
+#[derive(Copy, Clone, Debug)]
+pub enum Tok<'input> {
+    Num(i32),
+    NumSymbol(&'input str),
+    FactorOp(Opcode),
+    ExprOp(Opcode),
+    ParenOpen,
+    ParenClose,
+}
+
+use std::str::CharIndices;
+
+pub struct Lexer<'input> {
+    chars: std::iter::Peekable<CharIndices<'input>>,
+    input: &'input str,
+}
+
+impl<'input> Lexer<'input> {
+    pub fn new(input: &'input str) -> Self {
+        Lexer {
+            chars: input.char_indices().peekable(),
+            input,
+        }
+    }
+}
+
+impl<'input> Iterator for Lexer<'input> {
+    type Item = Spanned<Tok<'input>, usize, ()>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            match self.chars.next() {
+                Some((_, ' '))  | Some((_, '\n')) | Some((_, '\t')) => continue,
+                Some((i, ')')) => return Some(Ok((i, Tok::ParenClose, i + 1))),
+                Some((i, '(')) => return Some(Ok((i, Tok::ParenOpen, i + 1))),
+                Some((i, '+')) => return Some(Ok((i, Tok::ExprOp(Opcode::Add), i + 1))),
+                Some((i, '-')) => return Some(Ok((i, Tok::ExprOp(Opcode::Sub), i + 1))),
+                Some((i, '*')) => return Some(Ok((i, Tok::FactorOp(Opcode::Mul), i + 1))),
+                Some((i, '/')) => return Some(Ok((i, Tok::FactorOp(Opcode::Div), i + 1))),
+
+                None => return None, // End of file
+                Some((i,_)) => {
+                    loop {
+                        match self.chars.peek() {
+                            Some((j, ')'))|Some((j, '('))|Some((j, '+'))|Some((j, '-'))|Some((j, '*'))|Some((j, '/'))|Some((j,' '))
+                            => return Some(Ok((i, Tok::NumSymbol(&self.input[i..*j]), *j))),
+                            None => return Some(Ok((i, Tok::NumSymbol(&self.input[i..]),self.input.len()))),
+                            _ => {self.chars.next();},
+                        }
+                    }
+                }
+            }
+        }
+    }
+}