Trim whitespace for tokenizer, plus tests (#38)

* Trim whitespace for tokenizer, plus tests

* Address review comments

expression_evaluate_test.go

@@ -110,6 +110,16 @@ var testData = map[string]struct {
 		false,
 		"Hello world!",
 	},
+	"with-whitespace": {
+		map[string]any{
+			"message": "Hello world!",
+		},
+		nil,
+		"   $.message    \n",
+		false,
+		false,
+		"Hello world!",
+	},
 	"sub1map": {
 		map[string]any{
 			"message": "Hello world!",

internal/ast/tokenizer.go

@@ -138,7 +138,8 @@ var tokenPatterns = []tokenPattern{
 // initTokenizer initializes the tokenizer struct with the given expression.
 func initTokenizer(expression string, sourceName string) *tokenizer {
 	var t tokenizer
-	t.reader = strings.NewReader(expression)
+	// Need to trim the whitespace first since that can cause unexpected blank tokens.
+	t.reader = strings.NewReader(strings.TrimSpace(expression))
 	t.s.Init(t.reader)
 	t.s.Filename = sourceName
 	return &t

internal/ast/tokenizer_test.go

@@ -10,7 +10,8 @@ import (
 var filename = "example.go"
 
 func TestTokenizer(t *testing.T) {
-	input := `$.steps.read_kubeconfig.output["success"].credentials[f(1,2)]`
+	// Include trailing whitespace to ensure that it has no ill effects.
+	input := `$.steps.read_kubeconfig.output["success"].credentials[f(1,2)]   `
 	tokenizer := initTokenizer(input, filename)
 	expectedValue := []TokenValue{
 		{"$", RootAccessToken, filename, 1, 1},