From b53e55a5b083093843afb3f5bd67dcbdb1eab568 Mon Sep 17 00:00:00 2001 From: Garrit Franke Date: Fri, 1 Dec 2023 15:11:56 +0100 Subject: [PATCH] lexer: add namespaces --- src/lexer/mod.rs | 10 ++++++++- src/lexer/tests.rs | 51 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index 044cf026..9baecad5 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs @@ -72,6 +72,8 @@ pub enum TokenKind { Percent, /// ":" Colon, + /// "::" + DoubleColon, /// ";" SemiColon, /// "." @@ -281,7 +283,13 @@ impl Cursor<'_> { } _ => Assign, }, - ':' => Colon, + ':' => match self.first() { + ':' => { + self.bump(); + DoubleColon + } + _ => Colon, + }, ';' => SemiColon, ',' => Comma, '<' => match self.first() { diff --git a/src/lexer/tests.rs b/src/lexer/tests.rs index 8b9f6ce0..fc9f9423 100644 --- a/src/lexer/tests.rs +++ b/src/lexer/tests.rs @@ -299,6 +299,57 @@ fn test_functions() { ); } +#[test] +fn test_namespaces() { + let mut tokens = tokenize("foo::bar").unwrap().into_iter().filter(|t| { + t.kind != TokenKind::Whitespace + && t.kind != TokenKind::Tab + && t.kind != TokenKind::CarriageReturn + }); + + assert_eq!( + tokens.next().unwrap(), + Token { + len: 3, + kind: TokenKind::Identifier("foo".to_string()), + raw: "foo".to_owned(), + pos: Position { + raw: 2, + line: 1, + offset: 2 + } + } + ); + + assert_eq!( + tokens.next().unwrap(), + Token { + len: 2, + kind: TokenKind::DoubleColon, + raw: "::".to_owned(), + pos: Position { + raw: 4, + line: 1, + offset: 4 + } + } + ); + + assert_eq!( + tokens.next().unwrap(), + Token { + len: 3, + kind: TokenKind::Identifier("bar".to_string()), + raw: "bar".to_owned(), + pos: Position { + raw: 7, + line: 1, + offset: 7 + } + } + ); +} + #[test] fn test_comments() { let mut tokens = tokenize(