mirror of
https://github.com/fluencelabs/lalrpop
synced 2025-03-16 17:00:53 +00:00
Move internal token to __intern_token.
Re-export it at the toplevel because client visible types use it.
This commit is contained in:
parent
9d38aa4722
commit
5c7f81b821
@ -1,8 +1,8 @@
|
||||
use std::str::FromStr;
|
||||
use ast::{Expr, Opcode};
|
||||
use lalrpop_util::{ErrorRecovery,InternalToken};
|
||||
use lalrpop_util::ErrorRecovery;
|
||||
|
||||
grammar<'err>(errors: &'err mut Vec<ErrorRecovery<usize, InternalToken<'input>, &'static str>>);
|
||||
grammar<'err>(errors: &'err mut Vec<ErrorRecovery<usize, Token<'input>, &'static str>>);
|
||||
|
||||
pub Exprs = Comma<Expr>;
|
||||
|
||||
|
@ -158,11 +158,11 @@ fn expr_intern_tok_test_err() {
|
||||
fn parse_error_map_token_and_location() {
|
||||
let expr = "(1+\n(2++3))";
|
||||
let result = expr_intern_tok::parse_Expr(1, expr);
|
||||
let err : lalrpop_util::ParseError<usize, lalrpop_util::InternalToken,&'static str>
|
||||
let err : lalrpop_util::ParseError<usize, expr_intern_tok::Token,&'static str>
|
||||
= result.unwrap_err();
|
||||
|
||||
let message = err
|
||||
.map_token(|lalrpop_util::InternalToken(_,t)| format!("TOKEN {}", t))
|
||||
.map_token(|expr_intern_tok::Token(_,t)| format!("TOKEN {}", t))
|
||||
.map_location(|offset| format!("line {}", expr[0..offset].lines().count()))
|
||||
.to_string();
|
||||
assert!(message.contains("Unrecognized token `TOKEN +`"));
|
||||
|
@ -120,16 +120,6 @@ pub struct ErrorRecovery<L, T, E> {
|
||||
pub dropped_tokens: Vec<(L, T, L)>,
|
||||
}
|
||||
|
||||
// Only used by the internal tokenizer
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct InternalToken<'a>(pub usize, pub &'a str);
|
||||
|
||||
impl<'a> fmt::Display for InternalToken<'a> {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> {
|
||||
self.1.fmt(formatter)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
@ -404,6 +404,7 @@ fn emit_recursive_ascent(session: &Session, grammar: &r::Grammar, report_file :
|
||||
|
||||
if let Some(ref intern_token) = grammar.intern_token {
|
||||
try!(intern_token::compile(&grammar, intern_token, &mut rust));
|
||||
rust!(rust, "pub use self::{}intern_token::Token;", grammar.prefix);
|
||||
}
|
||||
|
||||
try!(action::emit_action_code(grammar, &mut rust));
|
||||
|
@ -6,6 +6,15 @@ Generates an iterator type `__Matcher` that looks roughly like
|
||||
mod __intern_token {
|
||||
extern crate regex as __regex;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Token<'input>(pub usize, pub &'input str);
|
||||
// ~~~~~~ ~~~~~~~~~~~
|
||||
// token token
|
||||
// index text
|
||||
// (type)
|
||||
|
||||
impl<'a> fmt::Display for Token<'a> { ... }
|
||||
|
||||
pub struct __Matcher<'input> {
|
||||
text: &'input str,
|
||||
consumed: usize,
|
||||
@ -18,10 +27,9 @@ mod __intern_token {
|
||||
}
|
||||
|
||||
impl<'input> Iterator for __Matcher<'input> {
|
||||
type Item = Result<(usize, (usize, &'input str), usize), ParseError>;
|
||||
// ~~~~~ ~~~~~ ~~~~~~~~~~~ ~~~~~
|
||||
// start token token end
|
||||
// index text
|
||||
type Item = Result<(usize, Token<'input>, usize), ParseError>;
|
||||
// ~~~~~ ~~~~~~~~~~~~~ ~~~~~
|
||||
// start token end
|
||||
}
|
||||
}
|
||||
```
|
||||
@ -47,6 +55,16 @@ pub fn compile<W: Write>(
|
||||
rust!(out, "#![allow(unused_imports)]");
|
||||
try!(out.write_uses("", &grammar));
|
||||
rust!(out, "extern crate regex as {}regex;", prefix);
|
||||
rust!(out, "use std::fmt as {}fmt;", prefix);
|
||||
rust!(out, "");
|
||||
rust!(out, "#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]");
|
||||
rust!(out, "pub struct Token<'input>(pub usize, pub &'input str);");
|
||||
rust!(out, "impl<'a> {}fmt::Display for Token<'a> {{", prefix);
|
||||
rust!(out, "fn fmt(&self, formatter: &mut {}fmt::Formatter) -> Result<(), {}fmt::Error> {{", prefix, prefix);
|
||||
rust!(out, "{}fmt::Display::fmt(self.1, formatter)", prefix);
|
||||
rust!(out, "}}");
|
||||
rust!(out, "}}");
|
||||
rust!(out, "");
|
||||
rust!(out, "pub struct {}Matcher<'input> {{", prefix);
|
||||
rust!(out, "text: &'input str,"); // remaining input
|
||||
rust!(out, "consumed: usize,"); // number of chars consumed thus far
|
||||
@ -102,9 +120,9 @@ pub fn compile<W: Write>(
|
||||
rust!(out, "}}"); // impl Matcher<'input>
|
||||
rust!(out, "");
|
||||
rust!(out, "impl<'input> Iterator for {}Matcher<'input> {{", prefix);
|
||||
rust!(out, "type Item = Result<(usize, {}lalrpop_util::InternalToken<'input>, usize), \
|
||||
{}lalrpop_util::ParseError<usize,{}lalrpop_util::InternalToken<'input>,{}>>;",
|
||||
prefix, prefix, prefix,
|
||||
rust!(out, "type Item = Result<(usize, Token<'input>, usize), \
|
||||
{}lalrpop_util::ParseError<usize,Token<'input>,{}>>;",
|
||||
prefix,
|
||||
grammar.types.error_type());
|
||||
rust!(out, "");
|
||||
rust!(out, "fn next(&mut self) -> Option<Self::Item> {{");
|
||||
@ -162,8 +180,8 @@ pub fn compile<W: Write>(
|
||||
rust!(out, "let {}end_offset = {}start_offset + {}longest_match;", prefix, prefix, prefix);
|
||||
rust!(out, "self.text = {}remaining;", prefix);
|
||||
rust!(out, "self.consumed = {}end_offset;", prefix);
|
||||
rust!(out, "Some(Ok(({}start_offset, {}lalrpop_util::InternalToken({}index, {}result), {}end_offset)))",
|
||||
prefix, prefix, prefix, prefix, prefix);
|
||||
rust!(out, "Some(Ok(({}start_offset, Token({}index, {}result), {}end_offset)))",
|
||||
prefix, prefix, prefix, prefix);
|
||||
|
||||
rust!(out, "}}"); // else
|
||||
rust!(out, "}}"); // else
|
||||
|
@ -877,7 +877,6 @@ impl<'ascent, 'grammar, W: Write> CodeGenerator<'ascent,
|
||||
pattern_names.last().cloned().unwrap()
|
||||
});
|
||||
|
||||
/* banana */
|
||||
let mut pattern = format!("{}", pattern);
|
||||
if pattern_names.is_empty() {
|
||||
pattern_names.push(format!("{}tok", self.prefix));
|
||||
|
@ -89,7 +89,9 @@ impl<'codegen, 'grammar, W: Write, C> CodeGenerator<'codegen, 'grammar, W, C> {
|
||||
pub fn write_uses(&mut self) -> io::Result<()> {
|
||||
try!(self.out.write_uses(&format!("{}::", self.action_module), &self.grammar));
|
||||
|
||||
if self.grammar.intern_token.is_none() {
|
||||
if self.grammar.intern_token.is_some() {
|
||||
rust!(self.out, "use {}::{}intern_token::Token;", self.action_module, self.prefix);
|
||||
} else {
|
||||
rust!(self.out, "use {}::{}ToTriple;", self.action_module, self.prefix);
|
||||
}
|
||||
|
||||
|
@ -49,8 +49,7 @@ impl<'s> LowerState<'s> {
|
||||
let mut token_span = None;
|
||||
let internal_token_path = Path {
|
||||
absolute: false,
|
||||
ids: vec![intern(&format!("{}lalrpop_util", grammar.prefix.clone())),
|
||||
intern("InternalToken")],
|
||||
ids: vec![intern("Token")],
|
||||
};
|
||||
|
||||
for item in grammar.items {
|
||||
|
@ -85,12 +85,11 @@ impl<'grammar> TypeInferencer<'grammar> {
|
||||
mutable: false,
|
||||
referent: Box::new(TypeRepr::str())
|
||||
};
|
||||
let enum_type = // lalrpop_util::InternalToken<'input>
|
||||
let enum_type = // Token<'input>
|
||||
TypeRepr::Nominal(NominalTypeRepr {
|
||||
path: Path {
|
||||
absolute: false,
|
||||
ids: vec![intern(&format!("{}lalrpop_util", grammar.prefix)),
|
||||
intern("InternalToken")],
|
||||
ids: vec![intern("Token")],
|
||||
},
|
||||
types: vec![TypeRepr::Lifetime(intern(INPUT_LIFETIME))]
|
||||
});
|
||||
|
@ -220,6 +220,6 @@ fn error() {
|
||||
grammar;
|
||||
Z = !;
|
||||
"#, vec![
|
||||
("Z", "__lalrpop_util::ParseError<usize, __lalrpop_util::InternalToken<'input>, &'static str>")
|
||||
("Z", "__lalrpop_util::ParseError<usize, Token<'input>, &'static str>")
|
||||
])
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user