fluencelabs/lalrpop
1
0
Fork 0
mirror of https://github.com/fluencelabs/lalrpop synced 2025-03-16 17:00:53 +00:00
Code Issues Projects Releases Wiki Activity

Move internal token to __intern_token.

Browse Source 
Re-export it at the toplevel because client visible types
use it.
This commit is contained in:
Joeri van Ruth 2017-10-05 12:26:02 +02:00
parent 9d38aa4722
commit 5c7f81b821
10 changed files with 39 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/calculator/src/calculator6.lalrpop
View File

@ -1,8 +1,8 @@
use std::str::FromStr;
use ast::{Expr, Opcode};
use lalrpop_util::{ErrorRecovery,InternalToken};
use lalrpop_util::ErrorRecovery;
grammar<'err>(errors: &'err mut Vec<ErrorRecovery<usize, InternalToken<'input>, &'static str>>);
grammar<'err>(errors: &'err mut Vec<ErrorRecovery<usize, Token<'input>, &'static str>>);
pub Exprs = Comma<Expr>;

4
lalrpop-test/src/main.rs
View File

@ -158,11 +158,11 @@ fn expr_intern_tok_test_err() {
fn parse_error_map_token_and_location() {
let expr = "(1+\n(2++3))";
let result = expr_intern_tok::parse_Expr(1, expr);
let err : lalrpop_util::ParseError<usize, lalrpop_util::InternalToken,&'static str>
let err : lalrpop_util::ParseError<usize, expr_intern_tok::Token,&'static str>
= result.unwrap_err();
let message = err
.map_token(|lalrpop_util::InternalToken(_,t)| format!("TOKEN {}", t))
.map_token(|expr_intern_tok::Token(_,t)| format!("TOKEN {}", t))
.map_location(|offset| format!("line {}", expr[0..offset].lines().count()))
.to_string();
assert!(message.contains("Unrecognized token `TOKEN +`"));

10
lalrpop-util/src/lib.rs
View File

@ -120,16 +120,6 @@ pub struct ErrorRecovery<L, T, E> {
pub dropped_tokens: Vec<(L, T, L)>,
}
// Only used by the internal tokenizer
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct InternalToken<'a>(pub usize, pub &'a str);
impl<'a> fmt::Display for InternalToken<'a> {
fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> {
self.1.fmt(formatter)
}
}
#[cfg(test)]
mod tests {
use super::*;

1
lalrpop/src/build/mod.rs
View File

@ -404,6 +404,7 @@ fn emit_recursive_ascent(session: &Session, grammar: &r::Grammar, report_file :
if let Some(ref intern_token) = grammar.intern_token {
try!(intern_token::compile(&grammar, intern_token, &mut rust));
rust!(rust, "pub use self::{}intern_token::Token;", grammar.prefix);
}
try!(action::emit_action_code(grammar, &mut rust));

36
lalrpop/src/lexer/intern_token/mod.rs
View File

@ -6,6 +6,15 @@ Generates an iterator type `__Matcher` that looks roughly like
mod __intern_token {
extern crate regex as __regex;
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct Token<'input>(pub usize, pub &'input str);
// ~~~~~~ ~~~~~~~~~~~
// token token
// index text
// (type)
impl<'a> fmt::Display for Token<'a> { ... }
pub struct __Matcher<'input> {
text: &'input str,
consumed: usize,
@ -18,10 +27,9 @@ mod __intern_token {
}
impl<'input> Iterator for __Matcher<'input> {
type Item = Result<(usize, (usize, &'input str), usize), ParseError>;
// ~~~~~ ~~~~~ ~~~~~~~~~~~ ~~~~~
// start token token end
// index text
type Item = Result<(usize, Token<'input>, usize), ParseError>;
// ~~~~~ ~~~~~~~~~~~~~ ~~~~~
// start token end
}
}
```
@ -47,6 +55,16 @@ pub fn compile<W: Write>(
rust!(out, "#![allow(unused_imports)]");
try!(out.write_uses("", &grammar));
rust!(out, "extern crate regex as {}regex;", prefix);
rust!(out, "use std::fmt as {}fmt;", prefix);
rust!(out, "");
rust!(out, "#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]");
rust!(out, "pub struct Token<'input>(pub usize, pub &'input str);");
rust!(out, "impl<'a> {}fmt::Display for Token<'a> {{", prefix);
rust!(out, "fn fmt(&self, formatter: &mut {}fmt::Formatter) -> Result<(), {}fmt::Error> {{", prefix, prefix);
rust!(out, "{}fmt::Display::fmt(self.1, formatter)", prefix);
rust!(out, "}}");
rust!(out, "}}");
rust!(out, "");
rust!(out, "pub struct {}Matcher<'input> {{", prefix);
rust!(out, "text: &'input str,"); // remaining input
rust!(out, "consumed: usize,"); // number of chars consumed thus far
@ -102,9 +120,9 @@ pub fn compile<W: Write>(
rust!(out, "}}"); // impl Matcher<'input>
rust!(out, "");
rust!(out, "impl<'input> Iterator for {}Matcher<'input> {{", prefix);
rust!(out, "type Item = Result<(usize, {}lalrpop_util::InternalToken<'input>, usize), \
{}lalrpop_util::ParseError<usize,{}lalrpop_util::InternalToken<'input>,{}>>;",
prefix, prefix, prefix,
rust!(out, "type Item = Result<(usize, Token<'input>, usize), \
{}lalrpop_util::ParseError<usize,Token<'input>,{}>>;",
prefix,
grammar.types.error_type());
rust!(out, "");
rust!(out, "fn next(&mut self) -> Option<Self::Item> {{");
@ -162,8 +180,8 @@ pub fn compile<W: Write>(
rust!(out, "let {}end_offset = {}start_offset + {}longest_match;", prefix, prefix, prefix);
rust!(out, "self.text = {}remaining;", prefix);
rust!(out, "self.consumed = {}end_offset;", prefix);
rust!(out, "Some(Ok(({}start_offset, {}lalrpop_util::InternalToken({}index, {}result), {}end_offset)))",
prefix, prefix, prefix, prefix, prefix);
rust!(out, "Some(Ok(({}start_offset, Token({}index, {}result), {}end_offset)))",
prefix, prefix, prefix, prefix);
rust!(out, "}}"); // else
rust!(out, "}}"); // else

1
lalrpop/src/lr1/codegen/ascent.rs
View File

@ -877,7 +877,6 @@ impl<'ascent, 'grammar, W: Write> CodeGenerator<'ascent,
pattern_names.last().cloned().unwrap()
});
/* banana */
let mut pattern = format!("{}", pattern);
if pattern_names.is_empty() {
pattern_names.push(format!("{}tok", self.prefix));

4
lalrpop/src/lr1/codegen/base.rs
View File

@ -89,7 +89,9 @@ impl<'codegen, 'grammar, W: Write, C> CodeGenerator<'codegen, 'grammar, W, C> {
pub fn write_uses(&mut self) -> io::Result<()> {
try!(self.out.write_uses(&format!("{}::", self.action_module), &self.grammar));
if self.grammar.intern_token.is_none() {
if self.grammar.intern_token.is_some() {
rust!(self.out, "use {}::{}intern_token::Token;", self.action_module, self.prefix);
} else {
rust!(self.out, "use {}::{}ToTriple;", self.action_module, self.prefix);
}

3
lalrpop/src/normalize/lower/mod.rs
View File

@ -49,8 +49,7 @@ impl<'s> LowerState<'s> {
let mut token_span = None;
let internal_token_path = Path {
absolute: false,
ids: vec![intern(&format!("{}lalrpop_util", grammar.prefix.clone())),
intern("InternalToken")],
ids: vec![intern("Token")],
};
for item in grammar.items {

5
lalrpop/src/normalize/tyinfer/mod.rs
View File

@ -85,12 +85,11 @@ impl<'grammar> TypeInferencer<'grammar> {
mutable: false,
referent: Box::new(TypeRepr::str())
};
let enum_type = // lalrpop_util::InternalToken<'input>
let enum_type = // Token<'input>
TypeRepr::Nominal(NominalTypeRepr {
path: Path {
absolute: false,
ids: vec![intern(&format!("{}lalrpop_util", grammar.prefix)),
intern("InternalToken")],
ids: vec![intern("Token")],
},
types: vec![TypeRepr::Lifetime(intern(INPUT_LIFETIME))]
});

2
lalrpop/src/normalize/tyinfer/test.rs
View File

@ -220,6 +220,6 @@ fn error() {
grammar;
Z = !;
"#, vec![
("Z", "__lalrpop_util::ParseError<usize, __lalrpop_util::InternalToken<'input>, &'static str>")
("Z", "__lalrpop_util::ParseError<usize, Token<'input>, &'static str>")
])
}