diff --git a/pkg/parser/ast/tokenizer.go b/pkg/parser/ast/tokenizer.go index 85788ea..151c902 100644 --- a/pkg/parser/ast/tokenizer.go +++ b/pkg/parser/ast/tokenizer.go @@ -73,7 +73,7 @@ var keywordRegex1 = regexp.MustCompile(`(?i)^(and|or|not|abs|sqrt|sin|cos|tan|as var keywordRegex2 = regexp.MustCompile(`(?i)^(if|then|else|end|goto)`) var keywordRegexes = []*regexp.Regexp{keywordRegex1, keywordRegex2} -var identifierRegex = regexp.MustCompile("^:?[a-zA-Z]+[a-zA-Z0-9_]*") +var identifierRegex = regexp.MustCompile("^:[a-zA-Z0-9_]+|^[a-zA-Z]+[a-zA-Z0-9_]*") var numberRegex = regexp.MustCompile("^[0-9]+(\\.[0-9]+)?") diff --git a/pkg/parser/ast/tokenizer_test.go b/pkg/parser/ast/tokenizer_test.go index 7cde1ce..4e7b8ee 100644 --- a/pkg/parser/ast/tokenizer_test.go +++ b/pkg/parser/ast/tokenizer_test.go @@ -18,6 +18,7 @@ func TestTokenizer(t *testing.T) { // comment 2 :var = "another\"test" :foo = (1+2.75)*3 + :_=1 ` expected := `Line: 1, Coloumn: 1, Type: Newline @@ -116,6 +117,11 @@ func TestTokenizer(t *testing.T) { Line: 10, Coloumn: 18, Type: Number, Value: '3' Line: 10, Coloumn: 19, Type: Newline Line: 11, Coloumn: 1, Type: Whitespace, Value: ' ' + Line: 11, Coloumn: 2, Type: ID, Value: ':_' + Line: 11, Coloumn: 4, Type: Symbol, Value: '=' + Line: 11, Coloumn: 5, Type: Number, Value: '1' + Line: 11, Coloumn: 6, Type: Newline + Line: 12, Coloumn: 1, Type: Whitespace, Value: ' ' ` tk := ast.NewTokenizer()