Add example for tokens with lifetimes

This commit is contained in:
Sosthène Guédon 2019-12-26 22:58:42 +01:00
parent f1a8e2bd67
commit 16363051e6
4 changed files with 127 additions and 0 deletions

View File

@ -6,6 +6,12 @@ pub enum Expr {
Error,
}
pub enum ExprSymbol<'input>{
NumSymbol(&'input str),
Op(Box<ExprSymbol<'input>>, Opcode, Box<ExprSymbol<'input>>),
Error,
}
#[derive(Copy, Clone)]
pub enum Opcode {
Mul,
@ -25,6 +31,17 @@ impl Debug for Expr {
}
}
impl<'input> Debug for ExprSymbol<'input> {
fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
use self::ExprSymbol::*;
match *self {
NumSymbol(n) => write!(fmt, "{:?}", n),
Op(ref l, op, ref r) => write!(fmt, "({:?} {:?} {:?})", l, op, r),
Error => write!(fmt, "error"),
}
}
}
impl Debug for Opcode {
fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
use self::Opcode::*;

View File

@ -0,0 +1,37 @@
use ast::{ExprSymbol, Opcode};
use tok8::Tok;
grammar<'input>(text: &'input str);
pub Expr: Box<ExprSymbol<'input>> = { // (1)
Expr "ExprOp" Factor => Box::new(ExprSymbol::Op(<>)), // (2)
Factor,
};
Factor: Box<ExprSymbol<'input>> = {
Factor "FactorOp" Term => Box::new(ExprSymbol::Op(<>)),
Term,
};
Term: Box<ExprSymbol<'input>> = {
"num" => Box::new(ExprSymbol::NumSymbol(<>)),
"(" <Expr> ")"
};
extern {
type Location = usize;
type Error = ();
enum Tok<'input> {
"num" => Tok::NumSymbol(<&'input str>),
"FactorOp" => Tok::FactorOp(<Opcode>),
"ExprOp" => Tok::ExprOp(<Opcode>),
"(" => Tok::ParenOpen,
")" => Tok::ParenClose,
}
}

View File

@ -136,6 +136,20 @@ fn calculator7() {
assert_eq!(&format!("{:?}", expr), "((22 * 44) + 66)");
}
lalrpop_mod!(pub calculator8);
mod tok8;
use tok8::Lexer;
#[test]
fn calculator8() {
let input = "22 * pi + 66";
let lexer = Lexer::new(input);
let expr = calculator8::ExprParser::new()
.parse(input,lexer)
.unwrap();
assert_eq!(&format!("{:?}", expr), "((\"22\" * \"pi\") + \"66\")");
}
#[cfg(not(test))]
fn main() {
println!("Hello, world!");

View File

@ -0,0 +1,59 @@
use ast::Opcode;
pub type Spanned<Tok, Loc, Error> = Result<(Loc, Tok, Loc), Error>;
#[derive(Copy, Clone, Debug)]
pub enum Tok<'input> {
Num(i32),
NumSymbol(&'input str),
FactorOp(Opcode),
ExprOp(Opcode),
ParenOpen,
ParenClose,
}
use std::str::CharIndices;
pub struct Lexer<'input> {
chars: std::iter::Peekable<CharIndices<'input>>,
input: &'input str,
}
impl<'input> Lexer<'input> {
pub fn new(input: &'input str) -> Self {
Lexer {
chars: input.char_indices().peekable(),
input,
}
}
}
impl<'input> Iterator for Lexer<'input> {
type Item = Spanned<Tok<'input>, usize, ()>;
fn next(&mut self) -> Option<Self::Item> {
loop {
match self.chars.next() {
Some((_, ' ')) | Some((_, '\n')) | Some((_, '\t')) => continue,
Some((i, ')')) => return Some(Ok((i, Tok::ParenClose, i + 1))),
Some((i, '(')) => return Some(Ok((i, Tok::ParenOpen, i + 1))),
Some((i, '+')) => return Some(Ok((i, Tok::ExprOp(Opcode::Add), i + 1))),
Some((i, '-')) => return Some(Ok((i, Tok::ExprOp(Opcode::Sub), i + 1))),
Some((i, '*')) => return Some(Ok((i, Tok::FactorOp(Opcode::Mul), i + 1))),
Some((i, '/')) => return Some(Ok((i, Tok::FactorOp(Opcode::Div), i + 1))),
None => return None, // End of file
Some((i,_)) => {
loop {
match self.chars.peek() {
Some((j, ')'))|Some((j, '('))|Some((j, '+'))|Some((j, '-'))|Some((j, '*'))|Some((j, '/'))|Some((j,' '))
=> return Some(Ok((i, Tok::NumSymbol(&self.input[i..*j]), *j))),
None => return Some(Ok((i, Tok::NumSymbol(&self.input[i..]),self.input.len()))),
_ => {self.chars.next();},
}
}
}
}
}
}
}