diff --git a/.gitignore b/.gitignore index c5601b6..4168218 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.dylib *.html .vscode/* +.idea/ # Test binary, built with `go test -c` *.test diff --git a/Magefile.go b/Magefile.go new file mode 100644 index 0000000..a1b2e94 --- /dev/null +++ b/Magefile.go @@ -0,0 +1,231 @@ +//+build mage + +package main + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" +) + +var ldflags = "" + +// allow user to override go executable by running as GOEXE=xxx make ... on unix-like systems +var goexe = "go" + +// Build is the default that fmt, vet, runs test and builds +var Default = Build + +func init() { + if exe := os.Getenv("GOEXE"); exe != "" { + goexe = exe + } + + // We want to use Go 1.11 modules even if the source lives inside GOPATH. + // The default is "auto". + os.Setenv("GO111MODULE", "on") +} + +// Fmt run gofmt linter +func Fmt() error { + if !isGoLatest() { + return nil + } + pkgs, err := packages() + if err != nil { + return err + } + failed := false + first := true + for _, pkg := range pkgs { + files, err := filepath.Glob(filepath.Join(pkg, "*.go")) + if err != nil { + return nil + } + for _, f := range files { + // gofmt doesn't exit with non-zero when it finds unformatted code + // so we have to explicitly look for output, and if we find any, we + // should fail this target. + s, err := sh.Output("gofmt", "-l", f) + if err != nil { + fmt.Printf("ERROR: running gofmt on %q: %v\n", f, err) + failed = true + } + if s != "" { + if first { + fmt.Println("The following files are not gofmt'ed:") + first = false + } + failed = true + fmt.Println(s) + } + } + } + if failed { + return errors.New("improperly formatted go files") + } + return nil +} + +// Vet run go vet linter +func Vet() error { + if err := sh.Run(goexe, "vet", "./..."); err != nil { + return fmt.Errorf("error running go vet: %v", err) + } + return nil +} + +// Run tests +func Test() error { + env := map[string]string{"GOFLAGS": testGoFlags()} + return runCmd(env, goexe, "test", "./...", buildFlags(), "-tags", buildTags()) +} + +// TestRace run tests with race detector +func TestRace() error { + env := map[string]string{"GOFLAGS": testGoFlags()} + return runCmd(env, goexe, "test", "-race", "./...", buildFlags(), "-tags", buildTags()) +} + +// TestCoverHTML generates test coverage report +func TestCoverHTML() error { + const ( + coverAll = "coverage-all.out" + cover = "coverage.out" + ) + f, err := os.Create(coverAll) + if err != nil { + return err + } + defer f.Close() + if _, err := f.Write([]byte("mode: count")); err != nil { + return err + } + pkgs, err := packages() + if err != nil { + return err + } + for _, pkg := range pkgs { + if err := sh.Run(goexe, "test", "-coverprofile="+cover, pkg); err != nil { + return err + } + b, err := ioutil.ReadFile(cover) + if err != nil { + if os.IsNotExist(err) { + continue + } + return err + } + idx := bytes.Index(b, []byte{'\n'}) + b = b[idx+1:] + if _, err := f.Write(b); err != nil { + return err + } + } + if err := f.Close(); err != nil { + return err + } + return sh.Run(goexe, "tool", "cover", "-html="+coverAll) +} + +// Build run linters, vet and tests +func Build() error { + if strings.Contains(runtime.Version(), "1.8") { + // Go 1.8 doesn't play along with go test ./... and /vendor. + // We could fix that, but that would take time. + fmt.Printf("Skip Build on %s\n", runtime.Version()) + return nil + } + + // TODO: Add lint after fixing errors + mg.Deps(Fmt, Vet, TestRace) + return nil +} + +var ( + pkgPrefixLen = len("github.com/grafadruid/go-druid") + pkgs []string + pkgsInit sync.Once +) + +// testGoFlags returns test flags that need to be set +func testGoFlags() string { + return "-v" +} + +func packages() ([]string, error) { + var err error + pkgsInit.Do(func() { + var s string + s, err = sh.Output(goexe, "list", "./...") + if err != nil { + return + } + pkgs = strings.Split(s, "\n") + for i := range pkgs { + pkgs[i] = "." + pkgs[i][pkgPrefixLen:] + } + }) + return pkgs, err +} + +func buildFlags() []string { + if runtime.GOOS == "windows" { + return []string{"-buildmode", "exe"} + } + return nil +} + +func buildTags() string { + return "none" +} + +func isGoLatest() bool { + return strings.Contains(runtime.Version(), "1.14") +} + +func runCmd(env map[string]string, cmd string, args ...interface{}) error { + if mg.Verbose() { + return runWith(env, cmd, args...) + } + output, err := sh.OutputWith(env, cmd, argsToStrings(args...)...) + if err != nil { + fmt.Fprint(os.Stderr, output) + } + + return err +} + +func runWith(env map[string]string, cmd string, inArgs ...interface{}) error { + s := argsToStrings(inArgs...) + return sh.RunWith(env, cmd, s...) +} + +func argsToStrings(v ...interface{}) []string { + var args []string + for _, arg := range v { + switch v := arg.(type) { + case string: + if v != "" { + args = append(args, v) + } + case []string: + if v != nil { + args = append(args, v...) + } + default: + panic("invalid type") + } + } + + return args +} diff --git a/README.md b/README.md index a0048bd..af0ad0e 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,28 @@ # go-druid A Golang client for Druid. Now supports Query API and Common API. + +### Development + +#### Testing +`go-druid` uses mage to run tests locally. + Install Mage: +``` + git clone https://github.com/magefile/mage + cd mage + go run bootstrap.go +``` +`mage -l` provides a list of targets that can be run. Default is `Check` + +``` +Targets: + build runs go mod download and then installs the binary. + check* run linters and tests + fmt run gofmt linter + lint run golint linter https://github.com/golang/lint + testCoverHTML generates test coverage report + testRace run tests with race detector + vet run go vet linter + +* default target +``` \ No newline at end of file diff --git a/builder/query/group_by.go b/builder/query/group_by.go index 61d90db..3d94e4e 100644 --- a/builder/query/group_by.go +++ b/builder/query/group_by.go @@ -23,7 +23,7 @@ type GroupBy struct { PostAggregations []builder.PostAggregator `json:"postAggregations,omitempty"` Having builder.HavingSpec `json:"having,omitempty"` LimitSpec builder.LimitSpec `json:"limitSpec,omitempty"` - SubtotalsSpec [][]string `json:"subtotalsSpec",omitempty` + SubtotalsSpec [][]string `json:"subtotalsSpec,omitempty"` } func NewGroupBy() *GroupBy { diff --git a/examples/tdigestExample.go b/examples/tdigestExample.go index e4890a0..a4fe27a 100644 --- a/examples/tdigestExample.go +++ b/examples/tdigestExample.go @@ -33,7 +33,6 @@ func getConnection() *druid.Client { To experiment, you can use the doubles_sketch_data.tsv file attached in this repo. It is a copy of https://github.com/apache/druid/blob/master/extensions-contrib/tdigestsketch/src/test/resources/doubles_sketch_data.tsv */ - // tdigestSketchUsingBuilder example using Builder Pattern func tdigestSketchUsingBuilder() { d := getConnection() @@ -49,18 +48,16 @@ func tdigestSketchUsingBuilder() { atds := aggregation.NewTDigestSketch().SetName("merged_sketch").SetFieldName("valuesTDS") a := []builder.Aggregator{atds} - //TDigest Post Aggregation qf := postaggregation.NewQuantilesFromTDigestSketchField(). SetType("fieldAccess"). SetFieldName("merged_sketch") qa := postaggregation.NewQuantilesFromTDigestSketch(). SetField(qf). - SetFractions([]float64{0.25, 0.5, 0.75, 0.9, 0.95, 0.99}). // add additional quantiles as needed + SetFractions([]float64{0.25, 0.5, 0.75, 0.9, 0.95, 0.99}). // add additional quantiles as needed SetName("quantiles") pa := []builder.PostAggregator{qa} - ts := query.NewTimeseries().SetDataSource(table).SetIntervals(is).SetAggregations(a).SetPostAggregations(pa).SetGranularity(m).SetLimit(10) var results interface{} _, err := d.Query().Execute(ts, &results) diff --git a/go.mod b/go.mod index 0d8faa8..53facbf 100644 --- a/go.mod +++ b/go.mod @@ -6,5 +6,6 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/google/go-querystring v1.0.0 github.com/hashicorp/go-retryablehttp v0.6.7 + github.com/magefile/mage v1.11.0 github.com/stretchr/testify v1.2.2 ) diff --git a/go.sum b/go.sum index f3d8a04..93813bb 100644 --- a/go.sum +++ b/go.sum @@ -8,7 +8,37 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-retryablehttp v0.6.7 h1:8/CAEZt/+F7kR7GevNHulKkUjLht3CPmn7egmhieNKo= github.com/hashicorp/go-retryablehttp v0.6.7/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/magefile/mage v1.11.0 h1:C/55Ywp9BpgVVclD3lRnSYCwXTYxmSppIgLeDYlNuls= +github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 h1:EBZoQjiKKPaLbPrbpssUfuHtwM6KV/vb4U85g/cigFY= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=