fluencelabs/lalrpop
1
0
Fork 0
mirror of https://github.com/fluencelabs/lalrpop synced 2025-03-16 17:00:53 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #264 from fitzgen/where-clauses

Browse Source 
Create an explicit structure for `where` clauses
This commit is contained in:
Niko Matsakis 2017-09-18 05:01:34 -04:00 committed by GitHub
commit ff250d40a0
11 changed files with 1452 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lalrpop-test/src/main.rs
View File

@ -41,6 +41,10 @@ mod expr_generic;
mod generics_issue_104;
mod generics_issue_104_lib;
/// Grammar parameterized by `F` with where clause `where F: for<'a> FnMut(&'a
/// str)`.
mod where_clause_with_forall;
/// test of inlining
mod inline;
@ -428,6 +432,14 @@ fn generics_issue_104_test1() {
assert!(generics_issue_104::parse_Schema::<()>("grammar { foo }").is_ok());
}
#[test]
fn where_clause_with_forall_test1() {
assert!(where_clause_with_forall::parse_Term(
&mut |s: &str| println!("log: {}", s),
"(((((42)))))"
).is_ok());
}
#[test]
fn test_match_section() {
assert!(match_section::parse_Query("SELECT foo").is_ok());

21
lalrpop-test/src/where_clause_with_forall.lalrpop Normal file
View File

@ -0,0 +1,21 @@
use std::str::FromStr;
grammar<F>(logger: &mut F) where F: for<'a> FnMut(&'a str);
pub Term: i32 = {
<n:Num> => {
let msg = format!("just parsed {}", n);
logger(&msg);
n
},
"(" <t:Term> ")" => {
logger("propagating...");
{
let msg = format!("... {}", t);
logger(&msg);
}
t
},
};
Num: i32 = <s:r"[0-9]+"> => i32::from_str(s).unwrap();

1003
lalrpop-test/src/where_clause_with_forall.rs Normal file
View File

File diff suppressed because it is too large Load Diff

222
lalrpop/src/grammar/parse_tree.rs
View File

@ -23,7 +23,7 @@ pub struct Grammar {
pub span: Span,
pub type_parameters: Vec<TypeParameter>,
pub parameters: Vec<Parameter>,
pub where_clauses: Vec<String>,
pub where_clauses: Vec<WhereClause<TypeRef>>,
pub items: Vec<GrammarItem>,
pub annotations: Vec<Annotation>,
pub module_attributes: Vec<String>,
@ -222,6 +222,105 @@ pub enum TypeRef {
OfSymbol(SymbolKind),
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum WhereClause<T> {
// 'a: 'b + 'c
Lifetime {
lifetime: InternedString,
bounds: Vec<InternedString>,
},
// where for<'a> &'a T: Debug + Into<usize>
Type {
forall: Option<Vec<InternedString>>,
ty: T,
bounds: Vec<TypeBound<T>>,
}
}
impl<T> WhereClause<T> {
pub fn map<F, U>(&self, mut f: F) -> WhereClause<U>
where F: FnMut(&T) -> U
{
match *self {
WhereClause::Lifetime { lifetime, ref bounds } => WhereClause::Lifetime {
lifetime: lifetime,
bounds: bounds.clone(),
},
WhereClause::Type { ref forall, ref ty, ref bounds } => WhereClause::Type {
forall: forall.clone(),
ty: f(ty),
bounds: bounds.iter().map(|b| b.map(&mut f)).collect(),
},
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum TypeBound<T> {
// The `'a` in `T: 'a`.
Lifetime(InternedString),
// `for<'a> FnMut(&'a usize)`
Fn {
forall: Option<Vec<InternedString>>,
path: Path,
parameters: Vec<T>,
ret: Option<T>,
},
// `some::Trait` or `some::Trait<Param, ...>` or `some::Trait<Item = Assoc>`
// or `for<'a> Trait<'a, T>`
Trait {
forall: Option<Vec<InternedString>>,
path: Path,
parameters: Vec<TypeBoundParameter<T>>,
}
}
impl<T> TypeBound<T> {
pub fn map<F, U>(&self, mut f: F) -> TypeBound<U>
where F: FnMut(&T) -> U
{
match *self {
TypeBound::Lifetime(l) => TypeBound::Lifetime(l),
TypeBound::Fn { ref forall, ref path, ref parameters, ref ret } => TypeBound::Fn {
forall: forall.clone(),
path: path.clone(),
parameters: parameters.iter().map(&mut f).collect(),
ret: ret.as_ref().map(f),
},
TypeBound::Trait { ref forall, ref path, ref parameters } => TypeBound::Trait {
forall: forall.clone(),
path: path.clone(),
parameters: parameters.iter().map(|p| p.map(&mut f)).collect(),
}
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum TypeBoundParameter<T> {
// 'a
Lifetime(InternedString),
// `T` or `'a`
TypeParameter(T),
// `Item = T`
Associated(InternedString, T),
}
impl<T> TypeBoundParameter<T> {
pub fn map<F, U>(&self, mut f: F) -> TypeBoundParameter<U>
where F: FnMut(&T) -> U
{
match *self {
TypeBoundParameter::Lifetime(l) =>
TypeBoundParameter::Lifetime(l),
TypeBoundParameter::TypeParameter(ref t) =>
TypeBoundParameter::TypeParameter(f(t)),
TypeBoundParameter::Associated(id, ref t) =>
TypeBoundParameter::Associated(id, f(t)),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum TypeParameter {
Lifetime(InternedString),
@ -342,7 +441,7 @@ pub enum SymbolKind {
// @R
Lookbehind,
Error
}
@ -547,6 +646,123 @@ impl Symbol {
}
}
impl<T: Display> Display for WhereClause<T> {
fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
match *self {
WhereClause::Lifetime { lifetime, ref bounds } => {
write!(fmt, "{}:", lifetime)?;
for (i, b) in bounds.iter().enumerate() {
if i != 0 {
write!(fmt, " +")?;
}
write!(fmt, " {}", b)?;
}
Ok(())
}
WhereClause::Type { ref forall, ref ty, ref bounds } => {
if let Some(ref forall) = *forall {
write!(fmt, "for<")?;
for (i, l) in forall.iter().enumerate() {
if i != 0 {
write!(fmt, ", ")?;
}
write!(fmt, "{}", l)?;
}
write!(fmt, "> ")?;
}
write!(fmt, "{}: ", ty)?;
for (i, b) in bounds.iter().enumerate() {
if i != 0 {
write!(fmt, " +")?;
}
write!(fmt, " {}", b)?;
}
Ok(())
}
}
}
}
impl<T: Display> Display for TypeBound<T> {
fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
match *self {
TypeBound::Lifetime(l) => {
write!(fmt, "{}", l)
}
TypeBound::Fn { ref forall, ref path, ref parameters, ref ret } => {
if let Some(ref forall) = *forall {
write!(fmt, "for<")?;
for (i, l) in forall.iter().enumerate() {
if i != 0 {
write!(fmt, ", ")?;
}
write!(fmt, "{}", l)?;
}
write!(fmt, "> ")?;
}
write!(fmt, "{}(", path)?;
for (i, p) in parameters.iter().enumerate() {
if i != 0 {
write!(fmt, ", ")?;
}
write!(fmt, "{}", p)?;
}
write!(fmt, ")")?;
if let Some(ref ret) = *ret {
write!(fmt, " -> {}", ret)?;
}
Ok(())
}
TypeBound::Trait { ref forall, ref path, ref parameters } => {
if let Some(ref forall) = *forall {
write!(fmt, "for<")?;
for (i, l) in forall.iter().enumerate() {
if i != 0 {
write!(fmt, ", ")?;
}
write!(fmt, "{}", l)?;
}
write!(fmt, "> ")?;
}
write!(fmt, "{}", path)?;
if parameters.is_empty() {
return Ok(());
}
write!(fmt, "<")?;
for (i, p) in parameters.iter().enumerate() {
if i != 0 {
write!(fmt, ", ")?;
}
write!(fmt, "{}", p)?;
}
write!(fmt, ">")
}
}
}
}
impl<T: Display> Display for TypeBoundParameter<T> {
fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
match *self {
TypeBoundParameter::Lifetime(l) => {
write!(fmt, "{}", l)
}
TypeBoundParameter::TypeParameter(ref t) => {
write!(fmt, "{}", t)
}
TypeBoundParameter::Associated(id, ref t) => {
write!(fmt, "{} = {}", id, t)
}
}
}
}
impl Display for TerminalString {
fn fmt(&self, fmt: &mut Formatter) -> Result<(), Error> {
match *self {
@ -554,7 +770,7 @@ impl Display for TerminalString {
write!(fmt, "{}", s),
TerminalString::Bare(s) =>
write!(fmt, "{}", s),
TerminalString::Error =>
TerminalString::Error =>
write!(fmt, "error"),
}
}

4
lalrpop/src/grammar/repr.rs
View File

@ -17,7 +17,7 @@ pub use grammar::parse_tree::{Annotation,
NonterminalString,
Path,
Span,
TerminalLiteral, TerminalString, TypeParameter};
TerminalLiteral, TerminalString, TypeParameter, WhereClause};
#[derive(Clone, Debug)]
pub struct Grammar {
@ -47,7 +47,7 @@ pub struct Grammar {
pub parameters: Vec<Parameter>,
// where clauses declared on the grammar, like `grammar<T> where T: Sized`
pub where_clauses: Vec<String>,
pub where_clauses: Vec<WhereClause<TypeRepr>>,
// optional tokenizer DFA; this is only needed if the user did not supply
// an extern token declaration

9
lalrpop/src/normalize/lower/mod.rs
View File

@ -151,6 +151,11 @@ impl<'s> LowerState<'s> {
})
.collect();
let where_clauses = grammar.where_clauses
.iter()
.map(|wc| wc.map(pt::TypeRef::type_repr))
.collect();
let mut algorithm = r::Algorithm::default();
// FIXME Error recovery only works for parse tables so temporarily only generate parse tables for
@ -189,7 +194,7 @@ impl<'s> LowerState<'s> {
token_span: token_span.unwrap(),
type_parameters: grammar.type_parameters,
parameters: parameters,
where_clauses: grammar.where_clauses,
where_clauses: where_clauses,
algorithm: algorithm,
intern_token: self.intern_token,
terminals: r::TerminalSet {
@ -324,7 +329,7 @@ impl<'s> LowerState<'s> {
match norm_util::check_between_braces(&action) {
norm_util::Presence::None => {
action
}
}
norm_util::Presence::Normal => {
let name_str : String = intern::read(|interner| {
let name_strs: Vec<_> = names.iter().map(|&(_,name,_)| interner.data(name)).collect();

59
lalrpop/src/parser/lrgrammar.lalrpop
View File

@ -14,21 +14,14 @@ pub Grammar: Grammar =
<lo:@L> "grammar" <hi:@R>
<tps:GrammarTypeParameters?>
<parameters:GrammarParameters?>
<where_clauses:"where"?>
<where_clauses:GrammarWhereClauses?>
";"
<items:GrammarItem*> => {
let where_clauses =
where_clauses.iter()
.flat_map(|wc| wc.iter())
.map(|s| strip(s))
.filter(|s| !s.is_empty())
.map(|s| s.to_string())
.collect();
Grammar { prefix: format!("__"), // adjusted by `parse_grammar`
span: Span(lo, hi),
type_parameters: tps.unwrap_or(vec![]),
parameters: parameters.unwrap_or(vec![]),
where_clauses: where_clauses,
where_clauses: where_clauses.unwrap_or(vec![]),
items: uses.into_iter().chain(items).collect(),
annotations: annotations,
module_attributes: module_attributes}
@ -42,6 +35,48 @@ TypeParameter: TypeParameter = {
<l:Id> => TypeParameter::Id(l)
};
pub GrammarWhereClauses: Vec<WhereClause<TypeRef>> =
"where" <Comma<GrammarWhereClause>>;
GrammarWhereClause: WhereClause<TypeRef> = {
<l:Lifetime> ":" <bounds:Plus<Lifetime>> =>
WhereClause::Lifetime { lifetime: l, bounds: bounds },
<f:ForAll?> <ty:TypeRef> ":" <bounds:TypeBounds> =>
WhereClause::Type { forall: f, ty: ty, bounds: bounds }
};
ForAll: Vec<InternedString> =
"for" "<" <Comma<Lifetime>> ">";
TypeBounds: Vec<TypeBound<TypeRef>> =
<Plus<TypeBound>>;
TypeBound: TypeBound<TypeRef> = {
<l:Lifetime> =>
TypeBound::Lifetime(l),
<f:ForAll?> <p:Path> "(" <params:Comma<TypeRef>> ")" <ret:("->" <TypeRef>)?> =>
TypeBound::Fn { forall: f, path: p, parameters: params, ret: ret },
<f:ForAll?> <p:Path> <params:("<" <Comma<TypeBoundParameter>> ">")?> =>
TypeBound::Trait { forall: f, path: p, parameters: params.unwrap_or(vec![]) }
};
TypeBoundParameter: TypeBoundParameter<TypeRef> = {
<l:Lifetime> => TypeBoundParameter::Lifetime(l),
<ty:TypeRef> => TypeBoundParameter::TypeParameter(ty),
<id:Id> "=" <ty:TypeRef> => TypeBoundParameter::Associated(id, ty),
};
Plus<T>: Vec<T> = {
<v:(<T> "+")*> <e:T?> => match e {
None => v,
Some(e) => {
let mut v = v;
v.push(e);
v
}
}
};
GrammarParameters: Vec<Parameter> =
"(" <Comma<GrammarParameter>> ")";
@ -176,7 +211,7 @@ SymbolKind1: SymbolKind = {
"@R" =>
SymbolKind::Lookbehind,
"!" =>
SymbolKind::Error,
};
@ -378,9 +413,10 @@ extern {
"mut" => Tok::Mut,
"pub" => Tok::Pub,
"type" => Tok::Type,
"where" => Tok::Where,
"for" => Tok::For,
"!" => Tok::Bang,
"use" => Tok::Use(<&'input str>),
"where" => Tok::Where(<Vec<&'input str>>),
"Escape" => Tok::Escape(<&'input str>),
"Id" => Tok::Id(<&'input str>),
@ -412,6 +448,7 @@ extern {
"<" => Tok::LessThan,
"@L" => Tok::Lookahead,
"@R" => Tok::Lookbehind,
"->" => Tok::MinusGreaterThan,
"+" => Tok::Plus,
"?" => Tok::Question,
"}" => Tok::RightBrace,

8
lalrpop/src/parser/mod.rs
View File

@ -46,3 +46,11 @@ pub fn parse_type_ref<'input>(input: &'input str)
let tokenizer = tok::Tokenizer::new(input, 0);
lrgrammar::parse_TypeRef(input, tokenizer)
}
#[cfg(test)]
pub fn parse_where_clauses<'input>(input: &'input str)
-> Result<Vec<WhereClause<TypeRef>>, ParseError<'input>>
{
let tokenizer = tok::Tokenizer::new(input, 0);
lrgrammar::parse_GrammarWhereClauses(input, tokenizer)
}

56
lalrpop/src/parser/test.rs
View File

@ -94,3 +94,59 @@ fn match_complex() {
_ => panic!("expected MatchToken, but was: {:?}", first_item)
}
}
#[test]
fn where_clauses() {
let clauses = vec![
"where T: Debug",
"where T: Debug + Display",
"where T: std::ops::Add<usize>",
"where T: IntoIterator<Item = usize>",
"where T: 'a",
"where 'a: 'b",
"where for<'a> &'a T: Debug",
"where T: for<'a> Flobbles<'a>",
"where T: FnMut(usize)",
"where T: FnMut(usize, bool)",
"where T: FnMut() -> bool",
"where T: for<'a> FnMut(&'a usize)",
"where T: Debug, U: Display",
];
for santa in clauses {
assert!(
parser::parse_where_clauses(santa).is_ok(),
"should parse where clauses: {}",
santa
);
}
}
#[test]
fn grammars_with_where_clauses() {
let grammars = vec![
r###"
grammar<T> where T: StaticMethods;
"###,
r###"
grammar<T>(methods: &mut T) where T: MutMethods;
"###,
r###"
grammar<'input, T>(methods: &mut T) where T: 'input + Debug + MutMethods;
"###,
r###"
grammar<F>(methods: &mut F) where F: for<'a> FnMut(&'a usize) -> bool;
"###,
r###"
grammar<F>(logger: &mut F) where F: for<'a> FnMut(&'a str);
"###,
];
for g in grammars {
assert!(parser::parse_grammar(g).is_ok());
}
}

47
lalrpop/src/tok/mod.rs
View File

@ -42,11 +42,12 @@ pub enum Tok<'input> {
Mut,
Pub,
Type,
Where,
For,
// Special keywords: these are accompanied by a series of
// uninterpreted strings representing imports and stuff.
Use(&'input str),
Where(Vec<&'input str>),
// Identifiers of various kinds:
Escape(&'input str),
@ -79,6 +80,7 @@ pub enum Tok<'input> {
LessThan,
Lookahead, // @L
Lookbehind, // @R
MinusGreaterThan,
Plus,
Question,
RightBrace,
@ -111,10 +113,12 @@ const KEYWORDS: &'static [(&'static str, Tok<'static>)] = &[
("mut", Mut),
("pub", Pub),
("type", Type),
("where", Where),
("for", For),
];
/*
* Helper for backtracking.
* Helper for backtracking.
*/
macro_rules! first {
($this:expr, $action:expr, $fallback:expr) => {
@ -266,7 +270,7 @@ impl<'input> Tokenizer<'input> {
}
Some((idx0, '#')) => {
self.bump();
first!(self,
first!(self,
{ self.shebang_attribute(idx0) },
{ Ok((idx0, Hash, idx0+1)) })
}
@ -367,6 +371,17 @@ impl<'input> Tokenizer<'input> {
}
}
}
Some((idx0, '-')) => {
match self.bump() {
Some((idx1, '>')) => {
self.bump();
Some(Ok((idx0, MinusGreaterThan, idx1 + 1)))
}
_ => {
Some(error(UnrecognizedToken, idx0))
}
}
}
Some((idx0, c)) if is_identifier_start(c) => {
if c == 'r' {
// watch out for r"..." or r#"..."# strings
@ -676,31 +691,6 @@ impl<'input> Tokenizer<'input> {
return Ok((start, Tok::Use(code), code_end));
}
if word == "where" {
let mut wcs = vec![];
let mut wc_start = end;
let mut wc_end;
loop {
// Note: do not include `{` as a delimeter here, as
// that is not legal in the trait/where-clause syntax,
// and in fact signals start of the fn body. But do
// include `<`.
wc_end = try!(self.code(wc_start, "([<", ">])"));
let wc = &self.text[wc_start..wc_end];
wcs.push(wc);
// if this ended in a comma, maybe expect another where-clause
if let Some((_, ',')) = self.lookahead {
self.bump();
wc_start = wc_end + 1;
} else {
break;
}
}
return Ok((start, Tok::Where(wcs), wc_end));
}
let tok =
// search for a keyword first; if none are found, this is
// either a MacroId or an Id, depending on whether there
@ -799,4 +789,3 @@ fn is_identifier_start(c: char) -> bool {
fn is_identifier_continue(c: char) -> bool {
UnicodeXID::is_xid_continue(c) || c == '_'
}

62
lalrpop/src/tok/test.rs
View File

@ -241,11 +241,60 @@ fn equalsgreaterthancode_nested_function_with_lifetimes() {
#[test]
fn where_with_lifetimes() {
test(r#"where <'a,bar<'b,'c>>,baz;"#, vec![
(r#"~~~~~~~~~~~~~~~~~~~~~~~~~ "#, Where(vec![" <'a,bar<'b,'c>>", "baz"])),
(r#"~~~~~ "#, Where),
(r#" ~ "#, LessThan),
(r#" ~~ "#, Lifetime("'a")),
(r#" ~ "#, Comma),
(r#" ~~~ "#, MacroId("bar")),
(r#" ~ "#, LessThan),
(r#" ~~ "#, Lifetime("'b")),
(r#" ~ "#, Comma),
(r#" ~~ "#, Lifetime("'c")),
(r#" ~ "#, GreaterThan),
(r#" ~ "#, GreaterThan),
(r#" ~ "#, Comma),
(r#" ~~~ "#, Id("baz")),
(r#" ~"#, Semi),
]);
}
#[test]
fn forall() {
test(r#"for<'a, 'b, 'c> FnMut"#, vec![
(r#"~~~ "#, For),
(r#" ~ "#, LessThan),
(r#" ~~ "#, Lifetime("'a")),
(r#" ~ "#, Comma),
(r#" ~~ "#, Lifetime("'b")),
(r#" ~ "#, Comma),
(r#" ~~ "#, Lifetime("'c")),
(r#" ~ "#, GreaterThan),
(r#" ~~~~~"#, Id("FnMut")),
]);
}
#[test]
fn where_forall_fnmut_with_return_type() {
test(r#"where F: for<'a> FnMut(&'a T) -> U;"#, vec![
(r#"~~~~~ "#, Where),
(r#" ~ "#, Id("F")),
(r#" ~ "#, Colon),
(r#" ~~~ "#, For),
(r#" ~ "#, LessThan),
(r#" ~~ "#, Lifetime("'a")),
(r#" ~ "#, GreaterThan),
(r#" ~~~~~ "#, Id("FnMut")),
(r#" ~ "#, LeftParen),
(r#" ~ "#, Ampersand),
(r#" ~~ "#, Lifetime("'a")),
(r#" ~ "#, Id("T")),
(r#" ~ "#, RightParen),
(r#" ~~ "#, MinusGreaterThan),
(r#" ~ "#, Id("U")),
(r#" ~"#, Semi),
]);
}
#[test]
fn equalsgreaterthancode_error_unbalanced() {
test_err(r#"=> (,"#,
@ -371,7 +420,14 @@ fn use2() {
#[test]
fn where1() {
test(r#"where <foo,bar>,baz;"#, vec![
(r#"~~~~~~~~~~~~~~~~~~~ "#, Where(vec![" <foo,bar>", "baz"])),
(r#"~~~~~ "#, Where),
(r#" ~ "#, LessThan),
(r#" ~~~ "#, Id("foo")),
(r#" ~ "#, Comma),
(r#" ~~~ "#, Id("bar")),
(r#" ~ "#, GreaterThan),
(r#" ~ "#, Comma),
(r#" ~~~ "#, Id("baz")),
(r#" ~"#, Semi),
]);
}
@ -453,5 +509,3 @@ fn char_literals() {
(r#" ~~"#, Lifetime("'c")),
]);
}