diff --git a/doc/calculator/src/ast.rs b/doc/calculator/src/ast.rs index f0540c1..888feb2 100644 --- a/doc/calculator/src/ast.rs +++ b/doc/calculator/src/ast.rs @@ -6,6 +6,12 @@ pub enum Expr { Error, } +pub enum ExprSymbol<'input>{ + NumSymbol(&'input str), + Op(Box>, Opcode, Box>), + Error, +} + #[derive(Copy, Clone)] pub enum Opcode { Mul, @@ -25,6 +31,17 @@ impl Debug for Expr { } } +impl<'input> Debug for ExprSymbol<'input> { + fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> { + use self::ExprSymbol::*; + match *self { + NumSymbol(n) => write!(fmt, "{:?}", n), + Op(ref l, op, ref r) => write!(fmt, "({:?} {:?} {:?})", l, op, r), + Error => write!(fmt, "error"), + } + } +} + impl Debug for Opcode { fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> { use self::Opcode::*; diff --git a/doc/calculator/src/calculator7.lalrpop b/doc/calculator/src/calculator7.lalrpop new file mode 100644 index 0000000..8fbe919 --- /dev/null +++ b/doc/calculator/src/calculator7.lalrpop @@ -0,0 +1,34 @@ +use std::str::FromStr; +use ast::{Expr, Opcode}; + +grammar(scale: i32); + +pub Expr: Box = { // (1) + Expr ExprOp Factor => Box::new(Expr::Op(<>)), // (2) + Factor, +}; + +ExprOp: Opcode = { // (3) + "+" => Opcode::Add, + "-" => Opcode::Sub, +}; + +Factor: Box = { + Factor FactorOp Term => Box::new(Expr::Op(<>)), + Term, +}; + +FactorOp: Opcode = { + "*" => Opcode::Mul, + "/" => Opcode::Div, +}; + +Term: Box = { + Num => Box::new(Expr::Number(<>)), + "(" ")" +}; + +Num: i32 = { + r"[0-9]+" => i32::from_str(<>).unwrap()*scale, +}; + diff --git a/doc/calculator/src/calculator8.lalrpop b/doc/calculator/src/calculator8.lalrpop new file mode 100644 index 0000000..79f84a6 --- /dev/null +++ b/doc/calculator/src/calculator8.lalrpop @@ -0,0 +1,36 @@ +use ast::{ExprSymbol, Opcode}; +use tok8::Tok; + +grammar<'input>(input: &'input str); + +pub Expr: Box> = { // (1) + Expr "ExprOp" Factor => Box::new(ExprSymbol::Op(<>)), // (2) + Factor, +}; + +Factor: Box> = { + Factor "FactorOp" Term => Box::new(ExprSymbol::Op(<>)), + Term, +}; + + +Term: Box> = { + "num" => Box::new(ExprSymbol::NumSymbol(<>)), + "(" ")" +}; + +extern { + type Location = usize; + type Error = (); + + enum Tok<'input> { + "num" => Tok::NumSymbol(<&'input str>), + "FactorOp" => Tok::FactorOp(), + "ExprOp" => Tok::ExprOp(), + "(" => Tok::ParenOpen, + ")" => Tok::ParenClose, + } +} + + + diff --git a/doc/calculator/src/main.rs b/doc/calculator/src/main.rs index ee8f743..a6656e0 100644 --- a/doc/calculator/src/main.rs +++ b/doc/calculator/src/main.rs @@ -125,6 +125,31 @@ fn calculator6() { assert_eq!(errors.len(), 4); } +lalrpop_mod!(pub calculator7); + +#[test] +fn calculator7() { + let scale = 2; + let expr = calculator7::ExprParser::new() + .parse(scale,"11 * 22 + 33") + .unwrap(); + assert_eq!(&format!("{:?}", expr), "((22 * 44) + 66)"); +} + +lalrpop_mod!(pub calculator8); +mod tok8; +use tok8::Lexer; + +#[test] +fn calculator8() { + let input = "22 * pi + 66"; + let lexer = Lexer::new(input); + let expr = calculator8::ExprParser::new() + .parse(input,lexer) + .unwrap(); + assert_eq!(&format!("{:?}", expr), "((\"22\" * \"pi\") + \"66\")"); +} + #[cfg(not(test))] fn main() { println!("Hello, world!"); diff --git a/doc/calculator/src/tok8.rs b/doc/calculator/src/tok8.rs new file mode 100644 index 0000000..611077c --- /dev/null +++ b/doc/calculator/src/tok8.rs @@ -0,0 +1,59 @@ +use ast::Opcode; + +pub type Spanned = Result<(Loc, Tok, Loc), Error>; + +#[derive(Copy, Clone, Debug)] +pub enum Tok<'input> { + NumSymbol(&'input str), + FactorOp(Opcode), + ExprOp(Opcode), + ParenOpen, + ParenClose, +} + +use std::str::CharIndices; + +pub struct Lexer<'input> { + chars: std::iter::Peekable>, + input: &'input str, +} + +impl<'input> Lexer<'input> { + pub fn new(input: &'input str) -> Self { + Lexer { + chars: input.char_indices().peekable(), + input, + } + } +} + +impl<'input> Iterator for Lexer<'input> { + type Item = Spanned, usize, ()>; + + fn next(&mut self) -> Option { + loop { + match self.chars.next() { + Some((_, ' ')) | Some((_, '\n')) | Some((_, '\t')) => continue, + Some((i, ')')) => return Some(Ok((i, Tok::ParenClose, i + 1))), + Some((i, '(')) => return Some(Ok((i, Tok::ParenOpen, i + 1))), + Some((i, '+')) => return Some(Ok((i, Tok::ExprOp(Opcode::Add), i + 1))), + Some((i, '-')) => return Some(Ok((i, Tok::ExprOp(Opcode::Sub), i + 1))), + Some((i, '*')) => return Some(Ok((i, Tok::FactorOp(Opcode::Mul), i + 1))), + Some((i, '/')) => return Some(Ok((i, Tok::FactorOp(Opcode::Div), i + 1))), + + None => return None, // End of file + Some((i,_)) => { + loop { + match self.chars.peek() { + Some((j, ')'))|Some((j, '('))|Some((j, '+'))|Some((j, '-'))|Some((j, '*'))|Some((j, '/'))|Some((j,' ')) + => return Some(Ok((i, Tok::NumSymbol(&self.input[i..*j]), *j))), + None => return Some(Ok((i, Tok::NumSymbol(&self.input[i..]),self.input.len()))), + _ => {}, + } + self.chars.next(); + } + } + } + } + } +} diff --git a/doc/src/SUMMARY.md b/doc/src/SUMMARY.md index 7cf2ac8..0279ae5 100644 --- a/doc/src/SUMMARY.md +++ b/doc/src/SUMMARY.md @@ -7,12 +7,15 @@ - [Adding LALRPOP to your project](tutorial/001_adding_lalrpop.md) - [Parsing parenthesized numbers](tutorial/002_paren_numbers.md) - [Type inference](tutorial/003_type_inference.md) - - [Controlling the lexer](tutorial/004_controlling_lexer.md) - - [Handling full expressions](tutorial/005_full_expressions.md) - - [Building ASTs](tutorial/006_building_asts.md) - - [Macros](tutorial/007_macros.md) - - [Error recovery](tutorial/008_error_recovery.md) -- [Writing a custom lexer](lexer_tutorial/index.md) + - [Handling full expressions](tutorial/004_full_expressions.md) + - [Building ASTs](tutorial/005_building_asts.md) + - [Macros](tutorial/006_macros.md) + - [Error recovery](tutorial/007_error_recovery.md) + - [Passing state parameter](tutorial/008_state_parameter.md) +- [Controlling the lexer](lexer_tutorial/index.md) + - [LALRPOP's lexer generator](lexer_tutorial/001_lexer_gen.md) + - [Writing a custom lexer](lexer_tutorial/002_writing_custom_lexer.md) + - [Using tokens with references](lexer_tutorial/003_token_references.md) - [Advanced setup](advanced_setup.md) - [Generate in source tree](generate_in_source.md) ----------- diff --git a/doc/src/tutorial/004_controlling_lexer.md b/doc/src/lexer_tutorial/001_lexer_gen.md similarity index 99% rename from doc/src/tutorial/004_controlling_lexer.md rename to doc/src/lexer_tutorial/001_lexer_gen.md index a34a901..89195da 100644 --- a/doc/src/tutorial/004_controlling_lexer.md +++ b/doc/src/lexer_tutorial/001_lexer_gen.md @@ -1,4 +1,4 @@ -# Controlling the lexer +# LALRPOP's lexer generator This example dives a bit deeper into how LALRPOP works. In particular, it dives into the meaning of those strings and regular expression that diff --git a/doc/src/lexer_tutorial/002_writing_custom_lexer.md b/doc/src/lexer_tutorial/002_writing_custom_lexer.md new file mode 100644 index 0000000..7d48c37 --- /dev/null +++ b/doc/src/lexer_tutorial/002_writing_custom_lexer.md @@ -0,0 +1,201 @@ +# Writing a custom lexer + +Let's say we want to parse the Whitespace language, so we've put together a grammar like the following: + +```lalrpop +pub Program = ; + +Statement: ast::Stmt = { + " " , + "\t" " " , + "\t" "\t" , + "\n" , + "\t" "\n" , +}; + +StackOp: ast::Stmt = { + " " => ast::Stmt::Push(<>), + "\n" " " => ast::Stmt::Dup, + "\n" "\t" => ast::Stmt::Swap, + "\n" "\n" => ast::Stmt::Discard, +}; + +MathOp: ast::Stmt = { + " " " " => ast::Stmt::Add, + " " "\t" => ast::Stmt::Sub, + " " "\n" => ast::Stmt::Mul, + "\t" " " => ast::Stmt::Div, + "\t" "\t" => ast::Stmt::Mod, +}; + +// Remainder omitted +``` + +Naturally, it doesn't work. By default, LALRPOP generates a tokenizer that skips all whitespace -- including newlines. What we *want* is to capture whitespace characters and ignore the rest as comments, and LALRPOP does the opposite of that. + +At the moment, LALRPOP doesn't allow you to configure the default tokenizer. In the future it will become quite flexible, but for now we have to write our own. + +Let's start by defining the stream format. The parser will accept an iterator where each item in the stream has the following structure: + +```lalrpop +pub type Spanned = Result<(Loc, Tok, Loc), Error>; +``` + +`Loc` is typically just a `usize`, representing a byte offset into the input string. Each token is accompanied by two of them, marking the start and end positions where it was found. `Error` can be pretty much anything you choose. And of course `Tok` is the meat of the stream, defining what possible values the tokens themselves can have. Following the conventions of Rust iterators, we'll signal a valid token with `Some(Ok(...))`, an error with `Some(Err(...))`, and EOF with `None`. + +(Note that the term "tokenizer" normally refers to a piece of code that simply splits up the stream, whereas a "lexer" also tags each token with its lexical category. What we're writing is the latter.) + +Whitespace is a simple language from a lexical standpoint, with only three valid tokens: + +```lalrpop +pub enum Tok { + Space, + Tab, + Linefeed, +} +``` + +Everything else is a comment. There are no invalid lexes, so we'll define our own error type, a void enum: + +```lalrpop +pub enum LexicalError { + // Not possible +} +``` + +Now for the lexer itself. We'll take a string slice as its input. For each token we process, we'll want to know the character value, and the byte offset in the string where it begins. We can do that by wrapping the `CharIndices` iterator, which yields tuples of `(usize, char)` representing exactly that information. + +```lalrpop +use std::str::CharIndices; + +pub struct Lexer<'input> { + chars: CharIndices<'input>, +} + +impl<'input> Lexer<'input> { + pub fn new(input: &'input str) -> Self { + Lexer { chars: input.char_indices() } + } +} +``` + +(The lifetime parameter `'input` indicates that the Lexer cannot outlive the string it's trying to parse.) + +Let's review our rules: + +- For a space character, we output `Tok::Space`. +- For a tab character, we output `Tok::Tab`. +- For a linefeed (newline) character, we output `Tok::Linefeed`. +- We skip all other characters. +- If we've reached the end of the string, we'll return `None` to signal EOF. + +Writing a lexer for a language with multi-character tokens can get very complicated, but this is so straightforward, we can translate it directly into code without thinking very hard. Here's our `Iterator` implementation: + +```lalrpop +impl<'input> Iterator for Lexer<'input> { + type Item = Spanned; + + fn next(&mut self) -> Option { + loop { + match self.chars.next() { + Some((i, ' ')) => return Some(Ok((i, Tok::Space, i+1))), + Some((i, '\t')) => return Some(Ok((i, Tok::Tab, i+1))), + Some((i, '\n')) => return Some(Ok((i, Tok::Linefeed, i+1))), + + None => return None, // End of file + _ => continue, // Comment; skip this character + } + } + } +} +``` + +That's it. That's all we need. + +## Updating the parser + +To use this with LALRPOP, we need to expose its API to the parser. It's pretty easy to do, but also somewhat magical, so pay close attention. Pick a convenient place in the grammar file (I chose the bottom) and insert an `extern` block: + +```lalrpop +extern { + // ... +} +``` + +Now we tell LALRPOP about the `Location` and `Error` types, as if we're writing a trait: + +```lalrpop +extern { + type Location = usize; + type Error = lexer::LexicalError; + + // ... +} +``` + +We expose the `Tok` type by kinda sorta redeclaring it: + +```lalrpop +extern { + type Location = usize; + type Error = lexer::LexicalError; + + enum lexer::Tok { + // ... + } +} +``` + +Now we have to declare each of our terminals. For each variant of `Tok`, we pick what name the parser will see, and write a pattern of the form `name => lexer::Tok::Variant`, similar to how action code works in grammar rules. The name can be an identifier, or a string literal. We'll use the latter. + +Here's the whole thing: + +```lalrpop +extern { + type Location = usize; + type Error = lexer::LexicalError; + + enum lexer::Tok { + " " => lexer::Tok::Space, + "\t" => lexer::Tok::Tab, + "\n" => lexer::Tok::Linefeed, + } +} +``` + +From now on, the parser will take a `Lexer` as its input instead of a string slice, like so: + +```rust + let lexer = lexer::Lexer::new("\n\n\n"); + match parser::parse_Program(lexer) { + ... + } +``` + +And any time we write a string literal in the grammar, it'll substitute a variant of our `Tok` enum. This means **we don't have to change any of the rules we already wrote!** This will work as-is: + +```lalrpop +FlowCtrl: ast::Stmt = { + " " " "