doc: Document lexer skipping

doc/calculator/src/calculator3.lalrpop

@@ -2,6 +2,21 @@ use std::str::FromStr;
 
 grammar;
 
+match {
+    "+",
+    "-",
+    "*",
+    "/",
+    "(",
+    ")",
+    r"[0-9]+",
+
+    // Skip whitespace and comments
+    r"\s*" => { },
+    r"//[^\n\r]*[\n\r]*" => { }, // `// comment`
+    r"/\*([^\*]*\*+[^\*/])*([^\*]*\*+|[^\*])*\*/" => { }, // `/* comment */`
+}
+
 pub Expr: i32 = {
     <l:Expr> "+" <r:Factor> => l + r,
     <l:Expr> "-" <r:Factor> => l - r,

doc/src/lexer_tutorial/001_lexer_gen.md

@@ -321,6 +321,20 @@ match {
 And now any reference in your grammar to `"BEGIN"` will actually match
 any capitalization.
 
+#### Customizing skipping between tokens
+
+If we want to support comments we will need to skip more than just whitespace in our lexer.
+To this end `ignore patterns` can be specified.
+
+```
+match {
+    r"\s*" => { }, // The default whitespace skipping is disabled an `ignore pattern` is specified
+    r"//[^\n\r]*[\n\r]*" => { }, // Skip `// comments`
+    r"/\*([^\*]*\*+[^\*/])*([^\*]*\*+|[^\*])*\*/" => { },  // Skip `/* comments */`
+}
+```
+
+
 [lexer tutorial]: index.md
 [calculator2b]: ../../calculator/src/calculator2b.lalrpop
 [calculator3]: ../../calculator/src/calculator3.lalrpop