lexer: add namespaces

antimony-lang · Dec 1, 2023 · b53e55a · b53e55a
1 parent 44d45c8
commit b53e55a
Show file tree

Hide file tree

Showing 2 changed files with 60 additions and 1 deletion.
diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs
@@ -72,6 +72,8 @@ pub enum TokenKind {
     Percent,
     /// ":"
     Colon,
+    /// "::"
+    DoubleColon,
     /// ";"
     SemiColon,
     /// "."
@@ -281,7 +283,13 @@ impl Cursor<'_> {
                 }
                 _ => Assign,
             },
-            ':' => Colon,
+            ':' => match self.first() {
+                ':' => {
+                    self.bump();
+                    DoubleColon
+                }
+                _ => Colon,
+            },
             ';' => SemiColon,
             ',' => Comma,
             '<' => match self.first() {

diff --git a/src/lexer/tests.rs b/src/lexer/tests.rs
@@ -299,6 +299,57 @@ fn test_functions() {
     );
 }
 
+#[test]
+fn test_namespaces() {
+    let mut tokens = tokenize("foo::bar").unwrap().into_iter().filter(|t| {
+        t.kind != TokenKind::Whitespace
+            && t.kind != TokenKind::Tab
+            && t.kind != TokenKind::CarriageReturn
+    });
+
+    assert_eq!(
+        tokens.next().unwrap(),
+        Token {
+            len: 3,
+            kind: TokenKind::Identifier("foo".to_string()),
+            raw: "foo".to_owned(),
+            pos: Position {
+                raw: 2,
+                line: 1,
+                offset: 2
+            }
+        }
+    );
+
+    assert_eq!(
+        tokens.next().unwrap(),
+        Token {
+            len: 2,
+            kind: TokenKind::DoubleColon,
+            raw: "::".to_owned(),
+            pos: Position {
+                raw: 4,
+                line: 1,
+                offset: 4
+            }
+        }
+    );
+
+    assert_eq!(
+        tokens.next().unwrap(),
+        Token {
+            len: 3,
+            kind: TokenKind::Identifier("bar".to_string()),
+            raw: "bar".to_owned(),
+            pos: Position {
+                raw: 7,
+                line: 1,
+                offset: 7
+            }
+        }
+    );
+}
+
 #[test]
 fn test_comments() {
     let mut tokens = tokenize(