diff --git a/carbonserver/list.go b/carbonserver/list.go index 98057065e..3d6d35374 100644 --- a/carbonserver/list.go +++ b/carbonserver/list.go @@ -3,7 +3,6 @@ package carbonserver import ( "encoding/json" "fmt" - "log" "net/http" _ "net/http/pprof" "strconv" @@ -143,6 +142,8 @@ type ListQueryResult struct { PhysicalSize int64 LogicalSize int64 + Limited bool + Metrics []ListMetricInfo } @@ -156,14 +157,21 @@ func (listener *CarbonserverListener) queryMetricsList(query string, limit int, return nil, errMetricsListEmpty } - log.Println(strings.ReplaceAll(query, ".", "/")) - names, isFiles, nodes, err := fidx.trieIdx.query(strings.ReplaceAll(query, ".", "/"), limit, nil) + // TODO: support topk queries? + + names, isFiles, nodes, limited, err := fidx.trieIdx.query(strings.ReplaceAll(query, ".", "/"), limit, nil) if err != nil { return nil, err } + // WHY: it's impossible for /list_query api user to be aware that a + // query is limited in nodes lookup. + // Returning a Limited signal allows api user to decide that if it should + // make a separate a call or show a warning. + result.Limited = limited for i, name := range names { if len(result.Metrics) >= limit { + result.Limited = true break } diff --git a/carbonserver/trie.go b/carbonserver/trie.go index 7d4d1dbcf..cd32c4736 100644 --- a/carbonserver/trie.go +++ b/carbonserver/trie.go @@ -695,7 +695,7 @@ func (ti *trieIndex) newDir() *trieNode { // TODO: add some defensive logics agains bad queries? // depth first search -func (ti *trieIndex) query(expr string, limit int, expand func(globs []string) ([]string, error)) (files []string, isFiles []bool, nodes []*trieNode, err error) { +func (ti *trieIndex) query(expr string, limit int, expand func(globs []string) ([]string, error)) (files []string, isFiles []bool, nodes []*trieNode, limited bool, err error) { expr = strings.TrimSpace(expr) if expr == "" { expr = "*" @@ -708,14 +708,14 @@ func (ti *trieIndex) query(expr string, limit int, expand func(globs []string) ( } gs, err := newGlobState(node, expand) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, false, err } exact = exact && gs.exact matchers = append(matchers, gs) } if len(matchers) == 0 { - return nil, nil, nil, nil + return nil, nil, nil, false, nil } var cur = ti.root @@ -829,7 +829,12 @@ func (ti *trieIndex) query(expr string, limit int, expand func(globs []string) ( nodes = append(nodes, dirNode) } - if len(files) >= limit || exact { + if len(files) >= limit { + limited = true + break + } + + if exact { break } @@ -861,7 +866,7 @@ func (ti *trieIndex) query(expr string, limit int, expand func(globs []string) ( continue } - return files, isFiles, nodes, nil + return files, isFiles, nodes, limited, nil } // note: tn might be a root or file node, which has a nil c @@ -1475,7 +1480,7 @@ func (listener *CarbonserverListener) expandGlobsTrie(query string) ([]string, [ var leafs []bool for _, g := range globs { - f, l, _, err := fidx.trieIdx.query(g, listener.maxMetricsGlobbed-len(files), listener.expandGlobBraces) + f, l, _, _, err := fidx.trieIdx.query(g, listener.maxMetricsGlobbed-len(files), listener.expandGlobBraces) if err != nil { return nil, nil, err } @@ -1655,7 +1660,7 @@ func (ti *trieIndex) applyQuotas(resetFrequency time.Duration, quotas ...*Quota) continue } - paths, _, nodes, err := ti.query(strings.ReplaceAll(quota.Pattern, ".", "/"), 1<<31-1, nil) + paths, _, nodes, _, err := ti.query(strings.ReplaceAll(quota.Pattern, ".", "/"), 1<<31-1, nil) if err != nil { return nil, err } diff --git a/carbonserver/trie_fuzz_query.go b/carbonserver/trie_fuzz_query.go index 59b43947e..602082791 100644 --- a/carbonserver/trie_fuzz_query.go +++ b/carbonserver/trie_fuzz_query.go @@ -41,7 +41,7 @@ var trie = func() *trieIndex { }() func Fuzz(data []byte) int { - _, _, err := trie.query(string(data), 1000, func([]string) ([]string, error) { return nil, nil }) + _, _,_, _, err := trie.query(string(data), 1000, func([]string) ([]string, error) { return nil, nil }) if err != nil { // panic(err) return 0 diff --git a/carbonserver/trie_test.go b/carbonserver/trie_test.go index 2997d6888..03ef2fcb1 100644 --- a/carbonserver/trie_test.go +++ b/carbonserver/trie_test.go @@ -745,7 +745,7 @@ func TestTrieIndex(t *testing.T) { func TestTrieEdgeCases(t *testing.T) { var trie = newTrie(".wsp", nil) - _, _, _, err := trie.query("[\xff\xff-\xff", 1000, func([]string) ([]string, error) { return nil, nil }) + _, _, _, _, err := trie.query("[\xff\xff-\xff", 1000, func([]string) ([]string, error) { return nil, nil }) if err == nil || err.Error() != "glob: range overflow" { t.Errorf("trie should return an range overflow error") } @@ -775,7 +775,7 @@ func TestTrieQueryOpts(t *testing.T) { "sys.app.host-02.cpu.system", } - files, _, nodes, err := trieIndex.query("sys/app/host-0{1,2}", 1000, nil) + files, _, nodes, _, err := trieIndex.query("sys/app/host-0{1,2}", 1000, nil) if err != nil { t.Error(err) } @@ -837,7 +837,7 @@ func TestTrieConcurrentReadWrite(t *testing.T) { // case <-filec: default: // skipcq: GSC-G404 - files, _, _, err := trieIndex.query(fmt.Sprintf("level-0-%d/level-1-%d/level-2-%d*", rand.Intn(factor), rand.Intn(factor), rand.Intn(factor)), int(math.MaxInt64), nil) + files, _, _, _, err := trieIndex.query(fmt.Sprintf("level-0-%d/level-1-%d/level-2-%d*", rand.Intn(factor), rand.Intn(factor), rand.Intn(factor)), int(math.MaxInt64), nil) if err != nil { panic(err) }