use TerminalLiteral instead of TerminalString where appropriate

This avoids a lot of unwraps and panics. I did copy the precedence code
but I plan to refactor that shortly.

lalrpop/src/grammar/parse_tree.rs

@@ -108,7 +108,7 @@ impl MatchItem {
     }
 }
 
-pub type MatchSymbol = TerminalString;
+pub type MatchSymbol = TerminalLiteral;
 pub type MatchMapping = TerminalString;
 
 /// Intern tokens are not typed by the user: they are synthesized in
@@ -119,7 +119,7 @@ pub struct InternToken {
     /// Set of `r"foo"` and `"foo"` literals extracted from the
     /// grammar. Sorted by order of increasing precedence.
     pub literals: Vec<TerminalLiteral>,
-    pub match_to_user_name_map: Option<Map<TerminalString, TerminalString>>,
+    pub match_to_user_name_map: Option<Map<TerminalLiteral, TerminalString>>,
     pub dfa: DFA
 }
 
@@ -340,6 +340,16 @@ impl TerminalLiteral {
             TerminalLiteral::Regex(_, p) => p,
         }
     }
+
+    pub fn with_match_precedence(self, p: usize) -> TerminalLiteral {
+        // Multiply times two since we still want to distinguish
+        // between quoted and regex precedence
+        let base_precedence = p * 2;
+        match self {
+            TerminalLiteral::Quoted(i, _) => TerminalLiteral::Quoted(i, base_precedence+1),
+            TerminalLiteral::Regex(i, _) => TerminalLiteral::Regex(i, base_precedence+0),
+        }
+    }
 }
 
 #[derive(Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)]
@@ -383,17 +393,6 @@ impl TerminalString {
     pub fn regex(i: InternedString) -> TerminalString {
         TerminalString::Literal(TerminalLiteral::Regex(i, 0))
     }
-
-    pub fn with_match_precedence(self, p: usize) -> TerminalString {
-        let base_precedence = p * 2;  // Multiply times two since we still want to distinguish between quoted and regex precedence
-        match self {
-            TerminalString::Literal(TerminalLiteral::Quoted(i, _)) =>
-                TerminalString::Literal(TerminalLiteral::Quoted(i, base_precedence+1)),
-            TerminalString::Literal(TerminalLiteral::Regex(i, _)) =>
-                TerminalString::Literal(TerminalLiteral::Regex(i, base_precedence+0)),
-            _ => panic!("cannot set match precedence for {:?}", self)
-        }
-    }
 }
 
 impl Into<Box<Content>> for TerminalString {

lalrpop/src/normalize/lower/mod.rs

@@ -91,7 +91,7 @@ impl<'s> LowerState<'s> {
 
                                                     // FIXME: This should be cleaner
                                                     if let Some(ref mm) = data.match_to_user_name_map {
-                                                        if let Some(m) = mm.get(&TerminalString::Literal(literal)) {
+                                                        if let Some(m) = mm.get(&literal) {
                                                             return (m.clone(), pattern);
                                                         }
                                                     }

lalrpop/src/normalize/token_check/mod.rs

@@ -36,8 +36,8 @@ pub fn validate(mut grammar: Grammar) -> NormResult<Grammar> {
                     match *item {
                         MatchItem::Unmapped(sym, _) => {
                             let precedence_sym = sym.with_match_precedence(precedence);
-                            match_to_user.insert(precedence_sym, sym);
+                            match_to_user.insert(precedence_sym, TerminalString::Literal(sym));
-                            user_to_match.insert(sym, precedence_sym);
+                            user_to_match.insert(TerminalString::Literal(sym), precedence_sym);
                         },
                         MatchItem::Mapped(sym, mapping, _) => {
                             let precedence_sym = sym.with_match_precedence(precedence);
@@ -94,7 +94,7 @@ struct Validator<'grammar> {
     grammar: &'grammar Grammar,
     all_literals: Map<TerminalLiteral, Span>,
     conversions: Option<Set<TerminalString>>,
-    user_name_to_match_map: Option<Map<TerminalString, TerminalString>>,
+    user_name_to_match_map: Option<Map<TerminalString, TerminalLiteral>>,
     match_catch_all: Option<bool>,
 }
 
@@ -175,9 +175,8 @@ impl<'grammar> Validator<'grammar> {
                 // FIMXE: Should not allow undefined literals if no CatchAll
                 TerminalString::Bare(c) => match self.user_name_to_match_map {
                     Some(ref m) => {
-                        if let Some(v) = m.get(&term) {
+                        if let Some(&vl) = m.get(&term) {
                             // FIXME: I don't think this span here is correct
-                            let vl = v.as_literal().expect("must map to a literal");
                             self.all_literals.entry(vl).or_insert(span);
                         } else {
                             return_err!(span, "terminal `{}` does not have a match mapping defined for it",
@@ -197,9 +196,8 @@ impl<'grammar> Validator<'grammar> {
 
                 TerminalString::Literal(l) => match self.user_name_to_match_map {
                     Some(ref m) => {
-                        if let Some(v) = m.get(&term) {
+                        if let Some(&vl) = m.get(&term) {
                             // FIXME: I don't think this span here is correct
-                            let vl = v.as_literal().expect("must map to a literal");
                             self.all_literals.entry(vl).or_insert(span);
                         } else {
                             // Unwrap should be safe as we shouldn't have match_catch_all without user_name_to_match_map
@@ -229,7 +227,7 @@ impl<'grammar> Validator<'grammar> {
 // Construction phase -- if we are constructing a tokenizer, this
 // phase builds up an internal token DFA.
 
-pub fn construct(grammar: &mut Grammar, literals_map: Map<TerminalLiteral, Span>, match_to_user_name_map: Option<Map<TerminalString, TerminalString>>) -> NormResult<()> {
+pub fn construct(grammar: &mut Grammar, literals_map: Map<TerminalLiteral, Span>, match_to_user_name_map: Option<Map<TerminalLiteral, TerminalString>>) -> NormResult<()> {
     let mut literals: Vec<TerminalLiteral> =
         literals_map.keys()
                     .cloned()

lalrpop/src/normalize/tyinfer/mod.rs

@@ -80,12 +80,11 @@ impl<'grammar> TypeInferencer<'grammar> {
             let mut types = Types::new(&grammar.prefix, Some(loc_type), error_type, enum_type);
 
             for &literal in &intern_token.literals {
-                let terminal_string = TerminalString::Literal(literal);
                 let user_name = intern_token.match_to_user_name_map
                                             .as_ref()
-                                            .and_then(|it| it.get(&terminal_string))
+                                            .and_then(|it| it.get(&literal))
                                             .cloned()
-                                            .unwrap_or(terminal_string);
+                                            .unwrap_or(TerminalString::Literal(literal));
                 types.add_term_type(user_name, input_str.clone());
             }
 

lalrpop/src/parser/lrgrammar.lalrpop

@@ -251,7 +251,7 @@ MatchItem: MatchItem = {
     }
 };
 
-MatchSymbol = QuotedTerminal;
+MatchSymbol = QuotedLiteral;
 
 pub MatchMapping = Terminal;
 
@@ -340,8 +340,12 @@ Terminal: TerminalString = {
 };
 
 QuotedTerminal: TerminalString = {
-    <s:StringLiteral> => TerminalString::quoted(s),
+    QuotedLiteral => TerminalString::Literal(<>),
-    <s:RegexLiteral> => TerminalString::regex(s),
+};
+
+QuotedLiteral: TerminalLiteral = {
+    <s:StringLiteral> => TerminalLiteral::Quoted(s, 1),
+    <s:RegexLiteral> => TerminalLiteral::Regex(s, 0),
 };
 
 StringLiteral: InternedString =

lalrpop/src/parser/test.rs

@@ -49,7 +49,7 @@ fn match_complex() {
             let item00 = contents0.items.get(0).unwrap();
             match *item00 {
                 MatchItem::Mapped(ref sym, ref mapping, _) => {
-                    assert_eq!(format!("{:?}", sym), "r#\"(?i)begin\"#");
+                    assert_eq!(format!("{:?}", sym), "r#\"(?i)begin\"#+0");
                     assert_eq!(format!("{}", mapping), "\"BEGIN\"");
                 },
                 _ => panic!("expected MatchItem::Mapped, but was: {:?}", item00)
@@ -58,7 +58,7 @@ fn match_complex() {
             let item01 = contents0.items.get(1).unwrap();
             match *item01 {
                 MatchItem::Mapped(ref sym, ref mapping, _) => {
-                    assert_eq!(format!("{:?}", sym), "r#\"(?i)end\"#");
+                    assert_eq!(format!("{:?}", sym), "r#\"(?i)end\"#+0");
                     assert_eq!(format!("{}", mapping), "\"END\"");
                 },
                 _ => panic!("expected MatchItem::Mapped, but was: {:?}", item00)
@@ -69,7 +69,7 @@ fn match_complex() {
             let item10 = contents1.items.get(0).unwrap();
             match *item10 {
                 MatchItem::Mapped(ref sym, ref mapping, _) => {
-                    assert_eq!(format!("{:?}", sym), "r#\"[a-zA-Z_][a-zA-Z0-9_]*\"#");
+                    assert_eq!(format!("{:?}", sym), "r#\"[a-zA-Z_][a-zA-Z0-9_]*\"#+0");
                     assert_eq!(format!("{}", mapping), "IDENTIFIER");
                 },
                 _ => panic!("expected MatchItem::Mapped, but was: {:?}", item10)
@@ -80,7 +80,7 @@ fn match_complex() {
             let item20 = contents2.items.get(0).unwrap();
             match *item20 {
                 MatchItem::Unmapped(ref sym, _) => {
-                    assert_eq!(format!("{:?}", sym), "\"other\"");
+                    assert_eq!(format!("{:?}", sym), "\"other\"+1");
                 },
                 _ => panic!("expected MatchItem::Unmapped, but was: {:?}", item20)
             };