Optimize IN comparion by using a map instead of slice when possible

TokenKind.go

@@ -27,6 +27,8 @@ const (
 	CLAUSE_CLOSE
 
 	TERNARY
+
+	MAP
 )
 
 /*

benchmarks_test.go

@@ -249,3 +249,20 @@ func BenchmarkNestedAccessors(bench *testing.B) {
 		expression.Evaluate(fooFailureParameters)
 	}
 }
+
+func BenchmarkIn(bench *testing.B) {
+
+	expressionString := "a in (1, 2, 3, 4, 5, 6, 7, 8, 9, 10)"
+	expression, _ := NewEvaluableExpression(expressionString)
+
+	bench.ResetTimer()
+	var val interface{}
+	for i := 0; i < bench.N; i++ {
+		val, _ = expression.Evaluate(map[string]interface{}{
+			"a": 4,
+		})
+	}
+	if val != true {
+		bench.Error("expected true")
+	}
+}

evaluationStage.go

@@ -419,11 +419,16 @@ func separatorStage(left interface{}, right interface{}, parameters Parameters)
 }
 
 func inStage(left interface{}, right interface{}, parameters Parameters) (interface{}, error) {
-
-	for _, value := range right.([]interface{}) {
-		if left == value {
-			return true, nil
+	switch t := right.(type) {
+	case []interface{}:
+		for _, value := range t {
+			if left == value {
+				return true, nil
+			}
 		}
+	case map[interface{}]struct{}:
+		_, hit := t[left]
+		return hit, nil
 	}
 	return false, nil
 }
@@ -496,10 +501,12 @@ func comparatorTypeCheck(left interface{}, right interface{}) bool {
 	return false
 }
 
-func isArray(value interface{}) bool {
+func isArrayOrMap(value interface{}) bool {
 	switch value.(type) {
 	case []interface{}:
 		return true
+	case map[interface{}]struct{}:
+		return true
 	}
 	return false
 }

parsing.go

@@ -350,40 +350,92 @@ func readUntilFalse(stream *lexerStream, includeWhitespace bool, breakWhitespace
 */
 func optimizeTokens(tokens []ExpressionToken) ([]ExpressionToken, error) {
 
-	var token ExpressionToken
 	var symbol OperatorSymbol
 	var err error
-	var index int
 
-	for index, token = range tokens {
+	for index := 0; index < len(tokens); index++ {
+		token := tokens[index]
 
-		// if we find a regex operator, and the right-hand value is a constant, precompile and replace with a pattern.
 		if token.Kind != COMPARATOR {
 			continue
 		}
 
 		symbol = comparatorSymbols[token.Value.(string)]
-		if symbol != REQ && symbol != NREQ {
-			continue
-		}
+		switch symbol {
+		case REQ, NREQ: // if we find a regex operator, and the right-hand value is a constant, precompile and replace with a pattern.
+			nextToken := tokens[index+1]
+			if nextToken.Kind == STRING {
 
-		index++
-		token = tokens[index]
-		if token.Kind == STRING {
+				nextToken.Kind = PATTERN
+				nextToken.Value, err = regexp.Compile(nextToken.Value.(string))
 
-			token.Kind = PATTERN
-			token.Value, err = regexp.Compile(token.Value.(string))
+				if err != nil {
+					return tokens, err
+				}
 
-			if err != nil {
-				return tokens, err
+				tokens[index+1] = nextToken
 			}
+		case IN:
+			nextToken := tokens[index+1]
+
+			if nextToken.Kind != CLAUSE {
+				continue
+			}
+
+			isComp, endIndex := clauseIsComparable(tokens, index+1)
+			if !isComp {
+				break // switch
+			}
+
+			mp := clauseToMap(tokens, index)
+			nextToken.Kind = MAP
+			nextToken.Value = mp
+			tokens[index+1] = nextToken
 
-			tokens[index] = token
+			// remove all tokens that have been condensed into map
+			newTokens := make([]ExpressionToken, 0, len(tokens)-(endIndex-index))
+			newTokens = append(newTokens, tokens[:index+2]...)
+			newTokens = append(newTokens, tokens[endIndex+1:]...)
+			tokens = newTokens
 		}
 	}
+
 	return tokens, nil
 }
 
+func clauseIsComparable(tokens []ExpressionToken, index int) (bool, int) {
+loop:
+	for {
+		index++
+		token := tokens[index]
+		switch token.Kind {
+		case CLAUSE_CLOSE:
+			break loop
+		case SEPARATOR, STRING, NUMERIC:
+			continue
+		default:
+			return false, index
+		}
+	}
+	return true, index
+}
+
+func clauseToMap(tokens []ExpressionToken, index int) map[interface{}]struct{} {
+	mp := make(map[interface{}]struct{})
+loop:
+	for {
+		index++
+		token := tokens[index]
+		switch token.Kind {
+		case CLAUSE_CLOSE:
+			break loop
+		case STRING, NUMERIC:
+			mp[token.Value] = struct{}{}
+		}
+	}
+	return mp
+}
+
 /*
 	Checks the balance of tokens which have multiple parts, such as parenthesis.
 */

stagePlanner.go

@@ -421,6 +421,8 @@ func planValue(stream *tokenStream) (*evaluationStage, error) {
 		fallthrough
 	case PATTERN:
 		fallthrough
+	case MAP:
+		fallthrough
 	case BOOLEAN:
 		symbol = LITERAL
 		operator = makeLiteralStage(token.Value)
@@ -486,7 +488,7 @@ func findTypeChecks(symbol OperatorSymbol) typeChecks {
 		}
 	case IN:
 		return typeChecks{
-			right: isArray,
+			right: isArrayOrMap,
 		}
 	case BITWISE_LSHIFT:
 		fallthrough