Reinstate not_followed_by and add to DecimalLiteral also remove _ in dsl keywords.

crates/codegen/grammar/src/dsl.rs

@@ -281,22 +281,22 @@ macro_rules! slang_parser {
         )
     };
 
-    (( $b:tt delimited_by $o:tt and $c:tt )) => {
+    (( $b:tt delimited by $o:tt and $c:tt )) => {
         $crate::ParserDefinitionNode::DelimitedBy(
             Box::new(slang_parser!($o)),
             Box::new(slang_parser!($b)),
             Box::new(slang_parser!($c)),
             slang_location!()
         )
     };
-    (( $b:tt terminated_by $t:tt )) => {
+    (( $b:tt terminated by $t:tt )) => {
         $crate::ParserDefinitionNode::TerminatedBy(
             Box::new(slang_parser!($b)),
             Box::new(slang_parser!($t)),
             slang_location!()
         )
     };
-    (( $b:tt separated_by $s:tt )) => {
+    (( $b:tt separated by $s:tt )) => {
         $crate::ParserDefinitionNode::SeparatedBy(
             Box::new(slang_parser!($b)),
             Box::new(slang_parser!($s)),
@@ -497,6 +497,15 @@ macro_rules! slang_scanner {
         )
     };
 
+    (( $b:tt not followed by $nla:tt )) => {
+        $crate::ScannerDefinitionNode::NotFollowedBy(
+            Box::new(slang_scanner!($b)),
+            Box::new(slang_scanner!($nla)),
+            slang_location!()
+        )
+    };
+
+
     (( $first:tt | $($rest:tt)|+ )) => {
         $crate::ScannerDefinitionNode::Choice(
             vec![slang_scanner!($first), $(slang_scanner!($rest)),+],

crates/codegen/grammar/src/scanner_definition.rs

@@ -30,6 +30,7 @@ pub enum ScannerDefinitionNode {
     Sequence(Vec<Self>, SourceLocation),
     Choice(Vec<Self>, SourceLocation),
     NoneOf(String, SourceLocation),
+    NotFollowedBy(Box<Self>, Box<Self>, SourceLocation),
     CharRange(char, char, SourceLocation),
     Literal(String, SourceLocation),
     ScannerDefinition(ScannerDefinitionRef, SourceLocation),
@@ -56,6 +57,11 @@ impl Visitable for ScannerDefinitionNode {
                 }
             }
 
+            Self::NotFollowedBy(node, lookahead, _) => {
+                node.accept_visitor(visitor);
+                lookahead.accept_visitor(visitor);
+            }
+
             Self::NoneOf(_, _)
             | Self::CharRange(_, _, _)
             | Self::Literal(_, _)

crates/codegen/parser/generator/src/scanner_definition.rs

@@ -74,6 +74,12 @@ impl ScannerDefinitionNodeExtensions for ScannerDefinitionNode {
                 quote! { scan_none_of!(stream, #(#chars),*) }
             }
 
+            ScannerDefinitionNode::NotFollowedBy(node, lookahead, _) => {
+                let scanner = node.to_scanner_code();
+                let negative_lookahead_scanner = lookahead.to_scanner_code();
+                quote! { scan_not_followed_by!(stream, #scanner, #negative_lookahead_scanner) }
+            }
+
             ScannerDefinitionNode::Sequence(nodes, _) => {
                 let scanners = nodes
                     .iter()

crates/codegen/parser/runtime/src/support/scanner_macros.rs

@@ -108,24 +108,6 @@ macro_rules! scan_optional {
     }};
 }
 
-#[allow(unused_macros)]
-macro_rules! scan_difference {
-    ($stream:ident, $minuend:expr, $subtrahend:expr) => {{
-        let start = $stream.position();
-        ($minuend)
-            && ({
-                let end = $stream.position();
-                $stream.set_position(start);
-                if ($subtrahend) && (end == $stream.position()) {
-                    false
-                } else {
-                    $stream.set_position(end);
-                    true
-                }
-            })
-    }};
-}
-
 #[allow(unused_macros)]
 macro_rules! scan_not_followed_by {
     ($stream:ident, $scanner:expr, $not_followed_by:expr) => {

crates/solidity/outputs/cargo/tests/src/scanner/mod.rs

@@ -6,7 +6,7 @@ use slang_solidity::{
 
 #[test]
 fn test_next_token() {
-    let version = Version::parse("0.4.11").unwrap();
+    let version = Version::parse("0.8.0").unwrap();
     let language = Language::new(version).unwrap();
 
     for (s, k) in &[
@@ -18,9 +18,18 @@ fn test_next_token() {
         ("+=", PlusEqual),
         ("1", DecimalLiteral),
         ("\n", EndOfLine),
+        ("unicode'abc'", UnicodeStringLiteral),
+        ("unicode'abc'ZZ", Identifier), // TODO: This needs to be further checked against solc
+        ("hex'abcd'", HexStringLiteral),
+        ("hex'abcd'ZZz", HexKeyword), // TODO: This needs to be further checked against solc
         ("// single line\n", SingleLineComment),
         ("/* multi-line\n   comment */ blah", MultilineComment),
+        ("/* multi-line comment **/ blah", MultilineComment),
     ] {
         assert_eq!(language.scan(LexicalContext::Default, s), Some(*k));
     }
+
+    assert_eq!(language.scan(LexicalContext::Default, "0ZZ"), None);
+    assert_eq!(language.scan(LexicalContext::Default, "0xabZZ"), None);
+    assert_eq!(language.scan(LexicalContext::Default, "'abc'ZZ"), None);
 }