From a0e71a7f2ff0c4f1ce6e3258710605f8aa3bb9c1 Mon Sep 17 00:00:00 2001 From: Eric Chlebek Date: Tue, 21 Nov 2017 16:31:41 -0800 Subject: [PATCH] Add package dynamic. Package dynamic can be used to with json.Marshaler and json.Unmarshaler to create types that can support arbitrary custom attributes. The package achieves this by parsing json messages with the json-iterator package in a streaming fashion, and mapping the results onto dynamically created types with package reflect. Package dynamic also implements GetField, which can be used to implement the govaluate.Parameters interface. --- types/dynamic/dynamic.go | 306 ++++++++++++++++++++++++++++++++++ types/dynamic/dynamic_test.go | 249 +++++++++++++++++++++++++++ 2 files changed, 555 insertions(+) create mode 100644 types/dynamic/dynamic.go create mode 100644 types/dynamic/dynamic_test.go diff --git a/types/dynamic/dynamic.go b/types/dynamic/dynamic.go new file mode 100644 index 0000000000..b8f01f2774 --- /dev/null +++ b/types/dynamic/dynamic.go @@ -0,0 +1,306 @@ +package dynamic + +import ( + "fmt" + "reflect" + "sort" + "strings" + + jsoniter "github.com/json-iterator/go" +) + +// CustomAttributer is for use with GetField. It allows GetField to access +// serialized custom attributes. +type CustomAttributer interface { + // CustomAttributes returns json-serialized custom attributes. + CustomAttributes() []byte +} + +// GetField gets a field from v according to its name. +// If GetField doesn't find a struct field with the corresponding name, then +// it will try to dynamically find the corresponding item in the 'Custom' +// field. GetField is case-sensitive, but custom attribute names will be +// converted to CamelCaps. +func GetField(v CustomAttributer, name string) (interface{}, error) { + strukt := reflect.Indirect(reflect.ValueOf(v)) + if kind := strukt.Kind(); kind != reflect.Struct { + return nil, fmt.Errorf("invalid type (want struct): %v", kind) + } + fields := getFields(strukt) + field, ok := fields[name] + if ok { + return field.Value.Interface(), nil + } + // If we get here, we are dealing with custom attributes. + return getCustomAttribute(v.CustomAttributes(), name) +} + +// getCustomAttribute dynamically builds a concrete type. If the concrete +// type is a composite type, then it will either be a struct or a slice. +func getCustomAttribute(msg []byte, name string) (interface{}, error) { + any := jsoniter.Get(msg, name) + if err := any.LastError(); err != nil { + lowerName := fmt.Sprintf("%s%s", strings.ToLower(string(name[0])), name[1:]) + if name != lowerName { + // fall back to lower-case name + return getCustomAttribute(msg, lowerName) + } + return nil, err + } + if any.GetInterface() == nil { + // Fall back to lower-case name + lowerName := fmt.Sprintf("%s%s", strings.ToLower(string(name[0])), name[1:]) + any = jsoniter.Get(msg, lowerName) + } + value, err := anyToValue(any) + return value, err +} + +func anyToValue(any jsoniter.Any) (interface{}, error) { + switch any.ValueType() { + case jsoniter.InvalidValue: + return nil, fmt.Errorf("dynamic: %s", any.LastError()) + case jsoniter.StringValue: + return any.ToString(), nil + case jsoniter.NumberValue: + return any.ToFloat64(), nil + case jsoniter.NilValue: + return nil, nil + case jsoniter.BoolValue: + return any.ToBool(), nil + case jsoniter.ArrayValue: + return buildSliceAny(any) + case jsoniter.ObjectValue: + return buildStructAny(any) + default: + return nil, fmt.Errorf("dynamic: unrecognized value type! %d", any.ValueType()) + } +} + +// buildSliceAny dynamically builds a slice from a jsoniter.Any +func buildSliceAny(any jsoniter.Any) (interface{}, error) { + n := any.Size() + result := make([]interface{}, 0, n) + for i := 0; i < n; i++ { + value, err := anyToValue(any.Get(i)) + if err != nil { + return nil, err + } + result = append(result, value) + } + return result, nil +} + +// buildStructAny dynamically builds a struct from a jsoniter.Any +func buildStructAny(any jsoniter.Any) (interface{}, error) { + keys := any.Keys() + fields := make([]reflect.StructField, 0, len(keys)) + values := make([]interface{}, 0, len(keys)) + for _, key := range keys { + value, err := anyToValue(any.Get(key)) + if err != nil { + return nil, err + } + values = append(values, value) + fields = append(fields, reflect.StructField{ + Name: strings.Title(key), + Type: reflect.TypeOf(value), + }) + } + structType := reflect.StructOf(fields) + structPtr := reflect.New(structType) + structVal := reflect.Indirect(structPtr) + for i, value := range values { + field := structVal.Field(i) + field.Set(reflect.ValueOf(value)) + } + + return structVal.Interface(), nil +} + +// getFields gets a map of struct fields by name from a reflect.Value +func getFields(v reflect.Value) map[string]structField { + typ := v.Type() + numField := v.NumField() + result := make(map[string]structField, numField) + for i := 0; i < numField; i++ { + field := typ.Field(i) + if len(field.PkgPath) != 0 { + // unexported + continue + } + value := v.Field(i) + sf := structField{Field: field, Value: value} + sf.JSONName, sf.OmitEmpty = sf.jsonFieldName() + result[field.Name] = sf + } + return result +} + +// structField is an internal convenience type +type structField struct { + Field reflect.StructField + Value reflect.Value + JSONName string + OmitEmpty bool +} + +func (s structField) IsEmpty() bool { + zeroValue := reflect.Zero(s.Field.Type).Interface() + return reflect.DeepEqual(zeroValue, s.Value.Interface()) +} + +func (s structField) jsonFieldName() (string, bool) { + fieldName := s.Field.Name + tag, ok := s.Field.Tag.Lookup("json") + omitEmpty := false + if ok { + parts := strings.Split(tag, ",") + fieldName = parts[0] + if len(parts) > 1 && parts[1] == "omitempty" { + omitEmpty = true + } + } + return fieldName, omitEmpty +} + +func getJSONFields(v reflect.Value) map[string]structField { + typ := v.Type() + numField := v.NumField() + result := make(map[string]structField, numField) + for i := 0; i < numField; i++ { + field := typ.Field(i) + if len(field.PkgPath) != 0 { + // unexported + continue + } + value := v.Field(i) + sf := structField{Field: field, Value: value} + sf.JSONName, sf.OmitEmpty = sf.jsonFieldName() + if sf.JSONName == "-" { + continue + } + if sf.OmitEmpty && sf.IsEmpty() { + continue + } + // sf is a valid JSON field. + result[sf.JSONName] = sf + } + return result +} + +// ExtractCustomAttributes selects only custom attributes from msg. It will +// ignore any fields in msg that correspond to fields in v. v must be of kind +// reflect.Struct. +func ExtractCustomAttributes(v interface{}, msg []byte) ([]byte, error) { + strukt := reflect.Indirect(reflect.ValueOf(v)) + if kind := strukt.Kind(); kind != reflect.Struct { + return nil, fmt.Errorf("invalid type (want struct): %v", kind) + } + fields := getJSONFields(strukt) + stream := jsoniter.NewStream(jsoniter.ConfigDefault, nil, 4096) + var anys map[string]jsoniter.Any + if err := jsoniter.Unmarshal(msg, &anys); err != nil { + return nil, err + } + objectStarted := false + for field, value := range anys { + _, ok := fields[field] + if ok { + // Not a custom attribute + continue + } + if !objectStarted { + objectStarted = true + stream.WriteObjectStart() + } else { + stream.WriteMore() + } + stream.WriteObjectField(field) + value.WriteTo(stream) + } + if !objectStarted { + stream.WriteObjectStart() + } + stream.WriteObjectEnd() + return stream.Buffer(), nil +} + +// Marshal encodes the struct fields in v that are valid to encode. +// It also encodes any custom attributes that are defined. Marshal +// respects the encoding/json rules regarding exported fields, and tag +// semantics. If v's kind is not reflect.Struct, an error will be returned. +func Marshal(v CustomAttributer) ([]byte, error) { + s := jsoniter.NewStream(jsoniter.ConfigDefault, nil, 4096) + s.WriteObjectStart() + + if err := encodeStructFields(v, s); err != nil { + return nil, err + } + + custom := v.CustomAttributes() + if len(custom) > 0 { + if err := encodeCustomFields(custom, s); err != nil { + return nil, err + } + } + + s.WriteObjectEnd() + + return s.Buffer(), nil +} + +func encodeStructFields(v interface{}, s *jsoniter.Stream) error { + strukt := reflect.Indirect(reflect.ValueOf(v)) + if kind := strukt.Kind(); kind != reflect.Struct { + return fmt.Errorf("invalid type (want struct): %v", kind) + } + m := getJSONFields(strukt) + fields := make([]structField, 0, len(m)) + for _, s := range m { + fields = append(fields, s) + } + sort.Slice(fields, func(i, j int) bool { + return fields[i].JSONName < fields[j].JSONName + }) + objectStarted := false + for _, field := range fields { + if !objectStarted { + objectStarted = true + } else { + s.WriteMore() + } + s.WriteObjectField(field.JSONName) + s.WriteVal(field.Value.Interface()) + } + return nil +} + +type anyT struct { + Name string + jsoniter.Any +} + +func sortAnys(m map[string]jsoniter.Any) []anyT { + anys := make([]anyT, 0, len(m)) + for key, any := range m { + anys = append(anys, anyT{Name: key, Any: any}) + } + sort.Slice(anys, func(i, j int) bool { + return anys[i].Name < anys[j].Name + }) + return anys +} + +func encodeCustomFields(custom []byte, s *jsoniter.Stream) error { + var anys map[string]jsoniter.Any + if err := jsoniter.Unmarshal(custom, &anys); err != nil { + return err + } + for _, any := range sortAnys(anys) { + s.WriteMore() + s.WriteObjectField(any.Name) + any.WriteTo(s) + } + return nil +} diff --git a/types/dynamic/dynamic_test.go b/types/dynamic/dynamic_test.go new file mode 100644 index 0000000000..b4cd8040f2 --- /dev/null +++ b/types/dynamic/dynamic_test.go @@ -0,0 +1,249 @@ +package dynamic + +import ( + "reflect" + "testing" + + "github.com/Knetic/govaluate" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetStructField(t *testing.T) { + assert := assert.New(t) + + test := struct { + Valid int `json:"valid"` + invalidUnexported int `json:"invalid"` + ValidEmpty int `json:"validEmpty"` + InvalidEmpty int `json:"invalidEmpty,omitempty"` + Invalid int `json:"-"` + }{ + Valid: 5, + invalidUnexported: 1, + ValidEmpty: 0, + InvalidEmpty: 0, + Invalid: 10, + } + + fields := getFields(reflect.ValueOf(test)) + require.Equal(t, len(fields), 4) + + field := fields["Valid"] + assert.Equal(field.Value.Interface(), 5) + assert.Equal("valid", field.JSONName) + assert.Equal(false, field.OmitEmpty) + + field = fields["ValidEmpty"] + assert.Equal(field.Value.Interface(), 0) + assert.Equal("validEmpty", field.JSONName) + assert.Equal(false, field.OmitEmpty) + + field = fields["InvalidEmpty"] + assert.Equal(field.Value.Interface(), 0) + + field = fields["Invalid"] + assert.Equal(field.Value.Interface(), 10) +} + +func TestGetJSONStructField(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + test := struct { + Valid int `json:"valid"` + invalidUnexported int `json:"invalid"` + ValidEmpty int `json:"validEmpty"` + InvalidEmpty int `json:"invalidEmpty,omitempty"` + Invalid int `json:"-"` + }{ + Valid: 5, + invalidUnexported: 1, + ValidEmpty: 0, + InvalidEmpty: 0, + Invalid: 10, + } + + fields := getJSONFields(reflect.ValueOf(test)) + require.Equal(2, len(fields)) + + field, ok := fields["valid"] + require.Equal(true, ok) + assert.Equal(field.Value.Interface(), 5) + assert.Equal("valid", field.JSONName) + assert.Equal(false, field.OmitEmpty) + + field = fields["validEmpty"] + assert.Equal(field.Value.Interface(), 0) + assert.Equal("validEmpty", field.JSONName) + assert.Equal(false, field.OmitEmpty) +} + +type MyType struct { + Foo string `json:"foo"` + Bar []MyType `json:"bar"` + + custom []byte +} + +func (m MyType) CustomAttributes() []byte { + return m.custom +} + +func (m MyType) Get(name string) (interface{}, error) { + return GetField(m, name) +} + +func TestExtractEmptyCustomAttributes(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + msg := []byte(`{"foo": "hello, world!","bar":[{"foo":"o hai"}]}`) + var m MyType + + attrs, err := ExtractCustomAttributes(m, msg) + require.Nil(err) + assert.Equal([]byte("{}"), attrs) +} + +func TestExtractCustomAttributes(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + msg := []byte(`{"foo": "hello, world!","bar":[{"foo":"o hai"}], "customattr": "such custom"}`) + var m MyType + + attrs, err := ExtractCustomAttributes(m, msg) + require.Nil(err) + assert.Equal([]byte(`{"customattr":"such custom"}`), attrs) +} + +func TestMarshal(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + customBytes := []byte(`{"a":1,"b":2.0,"c":true,"d":"false","e":[1,2,3],"f":{"foo":"bar"}}`) + expBytes := []byte(`{"bar":null,"foo":"hello world!","a":1,"b":2.0,"c":true,"d":"false","e":[1,2,3],"f":{"foo":"bar"}}`) + + m := MyType{ + Foo: "hello world!", + Bar: nil, + custom: customBytes, + } + + b, err := Marshal(m) + require.Nil(err) + assert.Equal(expBytes, b) +} + +func TestGetField(t *testing.T) { + m := MyType{ + Foo: "hello", + Bar: []MyType{{Foo: "there"}}, + custom: []byte(`{"a":"a","b":1,"c":2.0,"d":true,"e":null,"foo":{"hello":5},"bar":[true,10.5]}`), + } + + tests := []struct { + Field string + Exp interface{} + Kind reflect.Kind + }{ + { + Field: "a", + Exp: "a", + Kind: reflect.String, + }, + { + Field: "b", + Exp: 1.0, + Kind: reflect.Float64, + }, + { + Field: "c", + Exp: 2.0, + Kind: reflect.Float64, + }, + { + Field: "d", + Exp: true, + Kind: reflect.Bool, + }, + { + Field: "e", + Exp: nil, + Kind: reflect.Invalid, + }, + { + Field: "foo", + Exp: struct { + Hello float64 + }{Hello: 5.0}, + Kind: reflect.Struct, + }, + { + Field: "bar", + Exp: []interface{}{true, 10.5}, + Kind: reflect.Slice, + }, + } + + for _, test := range tests { + t.Run(test.Field, func(t *testing.T) { + field, err := GetField(m, test.Field) + require.Nil(t, err) + assert.Equal(t, test.Exp, field) + assert.Equal(t, test.Kind, reflect.ValueOf(field).Kind()) + }) + } +} + +func TestQueryGovaluateSimple(t *testing.T) { + m := MyType{ + custom: []byte(`{"hello":5}`), + } + + expr, err := govaluate.NewEvaluableExpression("hello == 5") + require.Nil(t, err) + require.NotNil(t, expr) + + result, err := expr.Eval(m) + require.Nil(t, err) + require.Equal(t, true, result) + + expr, err = govaluate.NewEvaluableExpression("Hello == 5") + require.Nil(t, err) + require.NotNil(t, expr) + + result, err = expr.Eval(m) + require.Nil(t, err) + require.Equal(t, true, result) +} + +func TestQueryGovaluateComplex(t *testing.T) { + m := MyType{ + custom: []byte(`{"hello":{"foo":5,"bar":6.0}}`), + } + + expr, err := govaluate.NewEvaluableExpression("hello.Foo == 5") + require.Nil(t, err) + require.NotNil(t, expr) + + result, err := expr.Eval(m) + require.Nil(t, err) + require.Equal(t, true, result) + + expr, err = govaluate.NewEvaluableExpression("Hello.Foo == 5") + require.Nil(t, err) + require.NotNil(t, expr) + + result, err = expr.Eval(m) + require.Nil(t, err) + require.Equal(t, true, result) + + expr, err = govaluate.NewEvaluableExpression("Hello.Foo < Hello.Bar") + require.Nil(t, err) + require.NotNil(t, expr) + + result, err = expr.Eval(m) + require.Nil(t, err) + require.Equal(t, true, result) +}