From 22235531eda279fde84b753b759ccfc45df8ff87 Mon Sep 17 00:00:00 2001 From: Alvin Lin Date: Fri, 24 Jun 2022 10:48:12 -0700 Subject: [PATCH 1/4] Update 1.13.0 first RC release date (#4761) Signed-off-by: Alvin Lin --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 7fbe53833c..a89b477d6f 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -28,7 +28,7 @@ Our goal is to provide a new minor release every 6 weeks. This is a new process | v1.10.0 | 2021-06-25 | Bryan Boreham (@bboreham) | | v1.11.0 | 2021-08-06 | Marco Pracucci (@pracucci) | | v1.12.0 | 2022-02-17 | _Abandoned_ | -| v1.13.0 | TBD | Alvin Lin (@alvinlin123) | +| v1.13.0 | 2022-06-23 | Alvin Lin (@alvinlin123) | ## Release shepherd responsibilities From 9ae152c573a8a84784cb28d6e97e238b19840c63 Mon Sep 17 00:00:00 2001 From: Alvin Lin Date: Fri, 24 Jun 2022 10:48:28 -0700 Subject: [PATCH 2/4] Update 1.x Guarantees doc to call out compactor shuffle sharding is experimental (#4762) Signed-off-by: Alvin Lin --- docs/configuration/v1-guarantees.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/configuration/v1-guarantees.md b/docs/configuration/v1-guarantees.md index 3af5dd4cd1..e0fefc69bf 100644 --- a/docs/configuration/v1-guarantees.md +++ b/docs/configuration/v1-guarantees.md @@ -95,3 +95,5 @@ Currently experimental features are: - `-alertmanager.sharding-ring.heartbeat-period=0` - `-compactor.ring.heartbeat-period=0` - `-store-gateway.sharding-ring.heartbeat-period=0` +- Compactor shuffle sharding + - Enabled via `-compactor.sharding-enabled=true`, `-compactor.sharding-strategy=shuffle-sharding`, and `-compactor.tenant-shard-size` set to a value larger than 0. \ No newline at end of file From eb84489ae9a5eba82da6b5fc97cb968411bc1426 Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Tue, 28 Jun 2022 19:09:57 -0700 Subject: [PATCH 3/4] Removing fingerprint calculator from the lock on the GetSeries API (#4765) * Removing fingerprint calculator from the lock on the GetSeries API * Add test to prove order does not matter on QueryLimiter,AddSeries --- pkg/distributor/distributor.go | 9 +++++--- pkg/util/limiter/query_limiter_test.go | 32 +++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index a80144c076..2093ed4c7d 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -1028,8 +1028,9 @@ func (d *Distributor) MetricsForLabelMatchers(ctx context.Context, from, through if err := queryLimiter.AddSeries(cortexpb.FromMetricsToLabelAdapters(m)); err != nil { return nil, err } + fingerprint := m.Fingerprint() mutex.Lock() - (*metrics)[m.Fingerprint()] = m + (*metrics)[fingerprint] = m mutex.Unlock() } @@ -1060,11 +1061,13 @@ func (d *Distributor) MetricsForLabelMatchersStream(ctx context.Context, from, t for _, metric := range resp.Metric { m := cortexpb.FromLabelAdaptersToMetricWithCopy(metric.Labels) - if err := queryLimiter.AddSeries(cortexpb.FromMetricsToLabelAdapters(m)); err != nil { + if err := queryLimiter.AddSeries(metric.Labels); err != nil { return nil, err } + + fingerprint := m.Fingerprint() mutex.Lock() - (*metrics)[m.Fingerprint()] = m + (*metrics)[fingerprint] = m mutex.Unlock() } } diff --git a/pkg/util/limiter/query_limiter_test.go b/pkg/util/limiter/query_limiter_test.go index 92f06e7cdb..8ee93e205d 100644 --- a/pkg/util/limiter/query_limiter_test.go +++ b/pkg/util/limiter/query_limiter_test.go @@ -45,9 +45,31 @@ func TestQueryLimiter_AddSeriers_ShouldReturnErrorOnLimitExceeded(t *testing.T) ) var ( - series1 = labels.FromMap(map[string]string{ - labels.MetricName: metricName + "_1", + series1 = []cortexpb.LabelAdapter{ + { + Name: labels.MetricName, + Value: metricName + "_1", + }, + { + Name: "series1", + Value: "1", + }, + } + + series1OtherOrderLabels = []cortexpb.LabelAdapter{ + { + Name: "series1", + Value: "1", + }, + { + Name: labels.MetricName, + Value: metricName + "_1", + }, + } + + series1FromMap = labels.FromMap(map[string]string{ "series1": "1", + labels.MetricName: metricName + "_1", }) series2 = labels.FromMap(map[string]string{ labels.MetricName: metricName + "_2", @@ -55,7 +77,11 @@ func TestQueryLimiter_AddSeriers_ShouldReturnErrorOnLimitExceeded(t *testing.T) }) limiter = NewQueryLimiter(1, 0, 0) ) - err := limiter.AddSeries(cortexpb.FromLabelsToLabelAdapters(series1)) + err := limiter.AddSeries(series1) + require.NoError(t, err) + err = limiter.AddSeries(cortexpb.FromLabelsToLabelAdapters(series1FromMap)) + require.NoError(t, err) + err = limiter.AddSeries(series1OtherOrderLabels) require.NoError(t, err) err = limiter.AddSeries(cortexpb.FromLabelsToLabelAdapters(series2)) require.Error(t, err) From f7a9d2742043052a7f48cc30d2d6b1b4a2eb964e Mon Sep 17 00:00:00 2001 From: Harry John Date: Tue, 5 Jul 2022 22:10:20 -0700 Subject: [PATCH 4/4] Ensure that compaction continues for block with incomplete time-range (#4771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Ensure that compaction continues for block with incomplete time-range Signed-off-by: 🌲 Harry 🌊 John 🏔 * Ensure that compaction continues for block with incomplete time-range Signed-off-by: 🌲 Harry 🌊 John 🏔 * Ensure that compaction continues for block with incomplete time-range Signed-off-by: 🌲 Harry 🌊 John 🏔 * Ensure that compaction continues for block with incomplete time-range Signed-off-by: 🌲 Harry 🌊 John 🏔 --- CHANGELOG.md | 1 + pkg/compactor/shuffle_sharding_grouper.go | 8 +- .../shuffle_sharding_grouper_test.go | 163 +++++++++--------- 3 files changed, 94 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac2b32aa7f..e7805377d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## master / unreleased +[BUGFIX] Compactor: Fixes #4770 - an edge case in compactor where compaction stops when a tenant stops ingesting samples. #4771 ## 1.13.0 in progress * [CHANGE] Changed default for `-ingester.min-ready-duration` from 1 minute to 15 seconds. #4539 diff --git a/pkg/compactor/shuffle_sharding_grouper.go b/pkg/compactor/shuffle_sharding_grouper.go index a631b72814..74d8625c93 100644 --- a/pkg/compactor/shuffle_sharding_grouper.go +++ b/pkg/compactor/shuffle_sharding_grouper.go @@ -361,7 +361,6 @@ func groupBlocksByCompactableRanges(blocks []*metadata.Meta, ranges []int64) []b // the same size still fits in the range. To do it, we consider valid a group only // if it's before the most recent block or if it fully covers the range. highestMinTime := blocks[len(blocks)-1].MinTime - for idx := 0; idx < len(groups); { group := groups[idx] @@ -377,6 +376,13 @@ func groupBlocksByCompactableRanges(blocks []*metadata.Meta, ranges []int64) []b continue } + // If the group's maxTime is after 1 block range, we can compact assuming that + // all the required blocks have already been uploaded. + if int64(ulid.Now()) > group.maxTime()+group.rangeLength() { + idx++ + continue + } + // We hit into a group which would compact recent blocks prematurely, // so we need to filter it out. diff --git a/pkg/compactor/shuffle_sharding_grouper_test.go b/pkg/compactor/shuffle_sharding_grouper_test.go index 7fc69cf66a..60fa25bfc4 100644 --- a/pkg/compactor/shuffle_sharding_grouper_test.go +++ b/pkg/compactor/shuffle_sharding_grouper_test.go @@ -18,82 +18,92 @@ import ( ) func TestShuffleShardingGrouper_Groups(t *testing.T) { - block1ulid := ulid.MustNew(1, nil) - block2ulid := ulid.MustNew(2, nil) - block3ulid := ulid.MustNew(3, nil) - block4ulid := ulid.MustNew(4, nil) - block5ulid := ulid.MustNew(5, nil) - block6ulid := ulid.MustNew(6, nil) - block7ulid := ulid.MustNew(7, nil) - block8ulid := ulid.MustNew(8, nil) - block9ulid := ulid.MustNew(9, nil) - block10ulid := ulid.MustNew(10, nil) - block11ulid := ulid.MustNew(11, nil) - block12ulid := ulid.MustNew(12, nil) - block13ulid := ulid.MustNew(13, nil) - block14ulid := ulid.MustNew(14, nil) - block15ulid := ulid.MustNew(15, nil) + block1hto2hExt1Ulid := ulid.MustNew(1, nil) + block3hto4hExt1Ulid := ulid.MustNew(2, nil) + block0hto1hExt1Ulid := ulid.MustNew(3, nil) + block2hto3hExt1Ulid := ulid.MustNew(4, nil) + block1hto2hExt2Ulid := ulid.MustNew(5, nil) + block0hto1hExt2Ulid := ulid.MustNew(6, nil) + block0to1hExt3Ulid := ulid.MustNew(7, nil) + block4hto6hExt2Ulid := ulid.MustNew(8, nil) + block6hto8hExt2Ulid := ulid.MustNew(9, nil) + block1hto2hExt1UlidCopy := ulid.MustNew(10, nil) + block0hto20hExt1Ulid := ulid.MustNew(11, nil) + block21hto40hExt1Ulid := ulid.MustNew(12, nil) + block21hto40hExt1UlidCopy := ulid.MustNew(13, nil) + block0hto45mExt1Ulid := ulid.MustNew(14, nil) + block0hto1h30mExt1Ulid := ulid.MustNew(15, nil) + blocklast1hExt1Ulid := ulid.MustNew(16, nil) + blocklast1hExt1UlidCopy := ulid.MustNew(17, nil) blocks := map[ulid.ULID]*metadata.Meta{ - block1ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block1ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()}, + block1hto2hExt1Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block1hto2hExt1Ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, }, - block2ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block2ulid, MinTime: 3 * time.Hour.Milliseconds(), MaxTime: 4 * time.Hour.Milliseconds()}, + block1hto2hExt1UlidCopy: { + BlockMeta: tsdb.BlockMeta{ULID: block1hto2hExt1UlidCopy, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, }, - block3ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block3ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()}, + block3hto4hExt1Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block3hto4hExt1Ulid, MinTime: 3 * time.Hour.Milliseconds(), MaxTime: 4 * time.Hour.Milliseconds()}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, }, - block4ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block4ulid, MinTime: 2 * time.Hour.Milliseconds(), MaxTime: 3 * time.Hour.Milliseconds()}, + block0hto1hExt1Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block0hto1hExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, }, - block5ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block5ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()}, - Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}}, + block2hto3hExt1Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block2hto3hExt1Ulid, MinTime: 2 * time.Hour.Milliseconds(), MaxTime: 3 * time.Hour.Milliseconds()}, + Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, }, - block6ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block6ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()}, + block1hto2hExt2Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block1hto2hExt2Ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}}, }, - block7ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block7ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()}, - Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, - }, - block8ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block8ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()}, + block0hto1hExt2Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block0hto1hExt2Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}}, }, - block9ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block9ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()}, + block0to1hExt3Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block0to1hExt3Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1 * time.Hour.Milliseconds()}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "3"}}, }, - block10ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block10ulid, MinTime: 4 * time.Hour.Milliseconds(), MaxTime: 6 * time.Hour.Milliseconds()}, + block4hto6hExt2Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block4hto6hExt2Ulid, MinTime: 4 * time.Hour.Milliseconds(), MaxTime: 6 * time.Hour.Milliseconds()}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}}, }, - block11ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block11ulid, MinTime: 6 * time.Hour.Milliseconds(), MaxTime: 8 * time.Hour.Milliseconds()}, + block6hto8hExt2Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block6hto8hExt2Ulid, MinTime: 6 * time.Hour.Milliseconds(), MaxTime: 8 * time.Hour.Milliseconds()}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "2"}}, }, - block12ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block12ulid, MinTime: 1 * time.Hour.Milliseconds(), MaxTime: 2 * time.Hour.Milliseconds()}, + block0hto20hExt1Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block0hto20hExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 20 * time.Hour.Milliseconds()}, + Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, + }, + block21hto40hExt1Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block21hto40hExt1Ulid, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, }, - block13ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block13ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 20 * time.Hour.Milliseconds()}, + block21hto40hExt1UlidCopy: { + BlockMeta: tsdb.BlockMeta{ULID: block21hto40hExt1UlidCopy, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, }, - block14ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block14ulid, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()}, + block0hto45mExt1Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block0hto45mExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 45 * time.Minute.Milliseconds()}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, }, - block15ulid: { - BlockMeta: tsdb.BlockMeta{ULID: block15ulid, MinTime: 21 * time.Hour.Milliseconds(), MaxTime: 40 * time.Hour.Milliseconds()}, + block0hto1h30mExt1Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: block0hto1h30mExt1Ulid, MinTime: 0 * time.Hour.Milliseconds(), MaxTime: 1*time.Hour.Milliseconds() + 30*time.Minute.Milliseconds()}, + Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, + }, + blocklast1hExt1Ulid: { + BlockMeta: tsdb.BlockMeta{ULID: blocklast1hExt1Ulid, MinTime: int64(ulid.Now()) - 1*time.Hour.Milliseconds(), MaxTime: int64(ulid.Now())}, + Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, + }, + blocklast1hExt1UlidCopy: { + BlockMeta: tsdb.BlockMeta{ULID: blocklast1hExt1UlidCopy, MinTime: int64(ulid.Now()) - 1*time.Hour.Milliseconds(), MaxTime: int64(ulid.Now())}, Thanos: metadata.Thanos{Labels: map[string]string{"external": "1"}}, }, } @@ -105,38 +115,52 @@ func TestShuffleShardingGrouper_Groups(t *testing.T) { }{ "test basic grouping": { ranges: []time.Duration{2 * time.Hour, 4 * time.Hour}, - blocks: map[ulid.ULID]*metadata.Meta{block1ulid: blocks[block1ulid], block2ulid: blocks[block2ulid], block3ulid: blocks[block3ulid], block4ulid: blocks[block4ulid], block5ulid: blocks[block5ulid], block6ulid: blocks[block6ulid]}, + blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block2hto3hExt1Ulid: blocks[block2hto3hExt1Ulid], block1hto2hExt2Ulid: blocks[block1hto2hExt2Ulid], block0hto1hExt2Ulid: blocks[block0hto1hExt2Ulid]}, expected: [][]ulid.ULID{ - {block5ulid, block6ulid}, - {block1ulid, block3ulid}, - {block2ulid, block4ulid}, + {block1hto2hExt2Ulid, block0hto1hExt2Ulid}, + {block1hto2hExt1Ulid, block0hto1hExt1Ulid}, + {block3hto4hExt1Ulid, block2hto3hExt1Ulid}, }, }, "test no compaction": { ranges: []time.Duration{2 * time.Hour, 4 * time.Hour}, - blocks: map[ulid.ULID]*metadata.Meta{block7ulid: blocks[block7ulid], block8ulid: blocks[block8ulid], block9ulid: blocks[block9ulid]}, + blocks: map[ulid.ULID]*metadata.Meta{block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block0hto1hExt2Ulid: blocks[block0hto1hExt2Ulid], block0to1hExt3Ulid: blocks[block0to1hExt3Ulid]}, expected: [][]ulid.ULID{}, }, "test smallest range first": { ranges: []time.Duration{2 * time.Hour, 4 * time.Hour}, - blocks: map[ulid.ULID]*metadata.Meta{block1ulid: blocks[block1ulid], block2ulid: blocks[block2ulid], block3ulid: blocks[block3ulid], block4ulid: blocks[block4ulid], block10ulid: blocks[block10ulid], block11ulid: blocks[block11ulid]}, + blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block2hto3hExt1Ulid: blocks[block2hto3hExt1Ulid], block4hto6hExt2Ulid: blocks[block4hto6hExt2Ulid], block6hto8hExt2Ulid: blocks[block6hto8hExt2Ulid]}, expected: [][]ulid.ULID{ - {block1ulid, block3ulid}, - {block2ulid, block4ulid}, - {block10ulid, block11ulid}, + {block1hto2hExt1Ulid, block0hto1hExt1Ulid}, + {block3hto4hExt1Ulid, block2hto3hExt1Ulid}, + {block4hto6hExt2Ulid, block6hto8hExt2Ulid}, }, }, "test oldest min time first": { ranges: []time.Duration{2 * time.Hour, 4 * time.Hour}, - blocks: map[ulid.ULID]*metadata.Meta{block1ulid: blocks[block1ulid], block2ulid: blocks[block2ulid], block3ulid: blocks[block3ulid], block4ulid: blocks[block4ulid], block12ulid: blocks[block12ulid]}, + blocks: map[ulid.ULID]*metadata.Meta{block1hto2hExt1Ulid: blocks[block1hto2hExt1Ulid], block3hto4hExt1Ulid: blocks[block3hto4hExt1Ulid], block0hto1hExt1Ulid: blocks[block0hto1hExt1Ulid], block2hto3hExt1Ulid: blocks[block2hto3hExt1Ulid], block1hto2hExt1UlidCopy: blocks[block1hto2hExt1UlidCopy]}, expected: [][]ulid.ULID{ - {block1ulid, block3ulid, block12ulid}, - {block2ulid, block4ulid}, + {block1hto2hExt1Ulid, block0hto1hExt1Ulid, block1hto2hExt1UlidCopy}, + {block3hto4hExt1Ulid, block2hto3hExt1Ulid}, }, }, "test overlapping blocks": { - ranges: []time.Duration{20 * time.Hour, 40 * time.Hour}, - blocks: map[ulid.ULID]*metadata.Meta{block13ulid: blocks[block13ulid], block14ulid: blocks[block14ulid], block15ulid: blocks[block15ulid]}, + ranges: []time.Duration{20 * time.Hour, 40 * time.Hour}, + blocks: map[ulid.ULID]*metadata.Meta{block0hto20hExt1Ulid: blocks[block0hto20hExt1Ulid], block21hto40hExt1Ulid: blocks[block21hto40hExt1Ulid], block21hto40hExt1UlidCopy: blocks[block21hto40hExt1UlidCopy]}, + expected: [][]ulid.ULID{ + {block21hto40hExt1Ulid, block21hto40hExt1UlidCopy}, + }, + }, + "test imperfect maxTime blocks": { + ranges: []time.Duration{2 * time.Hour}, + blocks: map[ulid.ULID]*metadata.Meta{block0hto1h30mExt1Ulid: blocks[block0hto1h30mExt1Ulid], block0hto45mExt1Ulid: blocks[block0hto45mExt1Ulid]}, + expected: [][]ulid.ULID{ + {block0hto45mExt1Ulid, block0hto1h30mExt1Ulid}, + }, + }, + "test prematurely created blocks": { + ranges: []time.Duration{2 * time.Hour}, + blocks: map[ulid.ULID]*metadata.Meta{blocklast1hExt1UlidCopy: blocks[blocklast1hExt1UlidCopy], blocklast1hExt1Ulid: blocks[blocklast1hExt1Ulid]}, expected: [][]ulid.ULID{}, }, } @@ -374,21 +398,6 @@ func TestGroupBlocksByCompactableRanges(t *testing.T) { }}, }, }, - "a range containing the most recent block shouldn't be prematurely compacted if doesn't cover the full range": { - ranges: []int64{10, 20, 40}, - blocks: []*metadata.Meta{ - {BlockMeta: tsdb.BlockMeta{MinTime: 5, MaxTime: 8}}, - {BlockMeta: tsdb.BlockMeta{MinTime: 7, MaxTime: 9}}, - {BlockMeta: tsdb.BlockMeta{MinTime: 10, MaxTime: 12}}, - {BlockMeta: tsdb.BlockMeta{MinTime: 13, MaxTime: 15}}, - }, - expected: []blocksGroup{ - {rangeStart: 0, rangeEnd: 10, blocks: []*metadata.Meta{ - {BlockMeta: tsdb.BlockMeta{MinTime: 5, MaxTime: 8}}, - {BlockMeta: tsdb.BlockMeta{MinTime: 7, MaxTime: 9}}, - }}, - }, - }, } for testName, testData := range tests {