From 3748c364a0c78f4860bb1e1f5b785573f16941dc Mon Sep 17 00:00:00 2001 From: devanbenz Date: Mon, 13 Jan 2025 13:41:44 -0600 Subject: [PATCH] feat: Begin 'compacting' tests in to single test --- tsdb/engine/tsm1/compact_test.go | 1379 +++++++++--------------------- 1 file changed, 413 insertions(+), 966 deletions(-) diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go index f0faa7e5c0b..05e8022990a 100644 --- a/tsdb/engine/tsm1/compact_test.go +++ b/tsdb/engine/tsm1/compact_test.go @@ -3,6 +3,7 @@ package tsm1_test import ( "errors" "fmt" + "golang.org/x/exp/slices" "io/fs" "math" "os" @@ -2269,465 +2270,380 @@ func TestDefaultPlanner_PlanOptimize_NoLevel4(t *testing.T) { } } -// This test is added to acount for many TSM files within a group being over 2 GB -// we want to ensure that the shard will be planned. -func TestDefaultPlanner_PlanOptimize_LargeMultiGeneration(t *testing.T) { - data := []tsm1.FileStat{ - { - Path: "01-05.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "01-06.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "01-07.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "01-08.tsm1", - Size: 1048 * 1024 * 1024, - }, - { - Path: "02-05.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "02-06.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "02-07.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "02-08.tsm1", - Size: 1048 * 1024 * 1024, - }, - { - Path: "03-04.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "03-05.tsm1", - Size: 512 * 1024 * 1024, - }, - } - - cp := tsm1.NewDefaultPlanner( - &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - }, tsdb.DefaultCompactFullWriteColdDuration, - ) - - expFiles := make([]tsm1.FileStat, 0) - for _, file := range data { - expFiles = append(expFiles, file) - } - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") - - tsm, pLen, _ := cp.PlanOptimize() - require.Equal(t, 1, len(tsm), "compaction group") - require.Equal(t, int64(len(tsm)), pLen, "compaction group length") - require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files") -} - -func TestDefaultPlanner_PlanOptimize_LargeMultiGenerationSmallFirstFiles(t *testing.T) { - data := []tsm1.FileStat{ - { - Path: "03-04.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "03-05.tsm1", - Size: 512 * 1024 * 1024, - }, - { - Path: "01-05.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "01-06.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "01-07.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "01-08.tsm1", - Size: 1048 * 1024 * 1024, - }, - { - Path: "02-05.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "02-06.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "02-07.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "02-08.tsm1", - Size: 1048 * 1024 * 1024, - }, - } - - cp := tsm1.NewDefaultPlanner( - &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - }, tsdb.DefaultCompactFullWriteColdDuration, - ) - - expFiles := make([]tsm1.FileStat, 0) - for _, file := range data { - expFiles = append(expFiles, file) - } - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") - - tsm, pLen, _ := cp.PlanOptimize() - require.Equal(t, 1, len(tsm), "compaction group") - require.Equal(t, int64(len(tsm)), pLen, "compaction group length") - require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files") -} - -// This test is added to account for a single generation that has a group size -// under 2 GB so it should be further compacted to a single file. -func TestDefaultPlanner_PlanOptimize_SmallSingleGeneration(t *testing.T) { - // ~650 MB total group size - data := []tsm1.FileStat{ - { - Path: "01-05.tsm1", - Size: 300 * 1024 * 1024, - }, - { - Path: "01-06.tsm1", - Size: 200 * 1024 * 1024, - }, - { - Path: "01-07.tsm1", - Size: 100 * 1024 * 1024, - }, - { - Path: "01-08.tsm1", - Size: 50 * 1024 * 1024, - }, - } - - cp := tsm1.NewDefaultPlanner( - &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - }, tsdb.DefaultCompactFullWriteColdDuration, - ) - - expFiles := make([]tsm1.FileStat, 0) - for _, file := range data { - expFiles = append(expFiles, file) - } - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") - - tsm, pLen, gLen := cp.PlanOptimize() - require.Equal(t, 1, len(tsm), "compaction group") - require.Equal(t, int64(len(tsm)), pLen, "compaction group length") - require.Equal(t, int64(1), gLen, "generation of TSM files") - require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files") -} - -func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationReverse(t *testing.T) { - // ~650 MB total group size - data := []tsm1.FileStat{ - { - Path: "01-05.tsm1", - Size: 50 * 1024 * 1024, - }, - { - Path: "01-06.tsm1", - Size: 100 * 1024 * 1024, - }, - { - Path: "01-07.tsm1", - Size: 200 * 1024 * 1024, - }, - { - Path: "01-08.tsm1", - Size: 300 * 1024 * 1024, - }, - } - - cp := tsm1.NewDefaultPlanner( - &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - }, tsdb.DefaultCompactFullWriteColdDuration, - ) - - expFiles := make([]tsm1.FileStat, 0) - for _, file := range data { - expFiles = append(expFiles, file) - } - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") - - tsm, pLen, gLen := cp.PlanOptimize() - require.Equal(t, 1, len(tsm), "compaction group") - require.Equal(t, int64(len(tsm)), pLen, "compaction group length") - require.Equal(t, int64(1), gLen, "generation of TSM files") - require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files") -} - -// This test is added to account for a single generation that has a group size -// under 2 GB and has less then level 4 files it should be further compacted to a single file. -func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing.T) { - // ~650 MB total group size - data := []tsm1.FileStat{ - { - Path: "01-02.tsm1", - Size: 300 * 1024 * 1024, - }, - { - Path: "01-03.tsm1", - Size: 200 * 1024 * 1024, - }, - { - Path: "01-04.tsm1", - Size: 100 * 1024 * 1024, - }, - } - - cp := tsm1.NewDefaultPlanner( - &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - }, tsdb.DefaultCompactFullWriteColdDuration, - ) - - expFiles := make([]tsm1.FileStat, 0) - for _, file := range data { - expFiles = append(expFiles, file) - } - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") - - tsm, pLen, gLen := cp.PlanOptimize() - require.Equal(t, 1, len(tsm), "compaction group") - require.Equal(t, int64(len(tsm)), pLen, "compaction group length") - require.Equal(t, int64(1), gLen, "generation of TSM files") - require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files") -} - -func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4SmallFileFirst(t *testing.T) { - // ~650 MB total group size - data := []tsm1.FileStat{ - { - Path: "01-02.tsm1", - Size: 100 * 1024 * 1024, - }, - { - Path: "01-03.tsm1", - Size: 200 * 1024 * 1024, - }, - { - Path: "01-04.tsm1", - Size: 300 * 1024 * 1024, - }, - } - - cp := tsm1.NewDefaultPlanner( - &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - }, tsdb.DefaultCompactFullWriteColdDuration, - ) - - expFiles := make([]tsm1.FileStat, 0) - for _, file := range data { - expFiles = append(expFiles, file) - } - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") - - tsm, pLen, gLen := cp.PlanOptimize() - require.Equal(t, 1, len(tsm), "compaction group") - require.Equal(t, int64(len(tsm)), pLen, "compaction group length") - require.Equal(t, int64(1), gLen, "generation of TSM files") - require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files") -} - -// This test is added to account for a single generation that has a group size -// under 2 GB and all files at max default points per block of 1000. -// This should be planned for compaction at a more aggressive points per block. -func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) { - // ~650 MB total group size - data := []tsm1.FileStat{ - { - Path: "01-05.tsm1", - Size: 300 * 1024 * 1024, - }, - { - Path: "01-06.tsm1", - Size: 200 * 1024 * 1024, - }, - { - Path: "01-07.tsm1", - Size: 100 * 1024 * 1024, - }, - { - Path: "01-08.tsm1", - Size: 50 * 1024 * 1024, - }, - } - - fs := &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - } - - err := fs.SetBlockCounts([]int{tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock}) - require.NoError(t, err, "SetBlockCounts") - - cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration) - - compacted, reason := cp.FullyCompacted() - require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason") - require.False(t, compacted, "is fully compacted") - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") - - _, cgLen, genLen := cp.PlanOptimize() - require.Equal(t, int64(1), cgLen, "compaction group length") - require.Equal(t, int64(1), genLen, "generation count") -} - -func TestDefaultPlanner_FullyCompacted_SmallSingleGenerationSmallFileFirst(t *testing.T) { - // ~650 MB total group size - data := []tsm1.FileStat{ - { - Path: "01-05.tsm1", - Size: 50 * 1024 * 1024, - }, - { - Path: "01-06.tsm1", - Size: 100 * 1024 * 1024, - }, - { - Path: "01-07.tsm1", - Size: 200 * 1024 * 1024, - }, - { - Path: "01-08.tsm1", - Size: 300 * 1024 * 1024, - }, - } - - fs := &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - } - - err := fs.SetBlockCounts([]int{tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock}) - require.NoError(t, err, "SetBlockCounts") - - cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration) +func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) { + // bc are the block counts for the tsm files + // bc can be an empty list + type PlanOptimizeTests struct { + fs []tsm1.FileStat + bc []int + fullyCompactedReasonExp string + generationCount int64 + } + + fileStats := []PlanOptimizeTests{ + // Large multi generation group with files at and under 2GB + PlanOptimizeTests{ + []tsm1.FileStat{ + { + Path: "01-05.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "01-06.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "01-07.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "01-08.tsm1", + Size: 1048 * 1024 * 1024, + }, + { + Path: "02-05.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "02-06.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "02-07.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "02-08.tsm1", + Size: 1048 * 1024 * 1024, + }, + { + Path: "03-04.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "03-05.tsm1", + Size: 512 * 1024 * 1024, + }, + }, + []int{}, + "not fully compacted and not idle because of more than one generation", + 3, + }, + // ~650mb group size + PlanOptimizeTests{ + []tsm1.FileStat{ + { + Path: "01-05.tsm1", + Size: 300 * 1024 * 1024, + }, + { + Path: "01-06.tsm1", + Size: 200 * 1024 * 1024, + }, + { + Path: "01-07.tsm1", + Size: 100 * 1024 * 1024, + }, + { + Path: "01-08.tsm1", + Size: 50 * 1024 * 1024, + }, + }, + []int{}, + tsdb.SingleGenerationReasonText, + 1, + }, + // ~650 MB total group size with generations under 4 + PlanOptimizeTests{ + []tsm1.FileStat{ + { + Path: "01-02.tsm1", + Size: 300 * 1024 * 1024, + }, + { + Path: "01-03.tsm1", + Size: 200 * 1024 * 1024, + }, + { + Path: "01-04.tsm1", + Size: 100 * 1024 * 1024, + }, + }, + []int{}, + tsdb.SingleGenerationReasonText, + 1, + }, + { + []tsm1.FileStat{ + { + Path: "01-05.tsm1", + Size: 300 * 1024 * 1024, + }, + { + Path: "01-06.tsm1", + Size: 200 * 1024 * 1024, + }, + { + Path: "01-07.tsm1", + Size: 100 * 1024 * 1024, + }, + { + Path: "01-08.tsm1", + Size: 50 * 1024 * 1024, + }, + }, []int{tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock}, + tsdb.SingleGenerationReasonText, + 1, + }, + + // > 2 GB total group size + // 50% of files are at aggressive max block size + { + []tsm1.FileStat{ + { + Path: "01-05.tsm1", + Size: 700 * 1024 * 1024, + }, + { + Path: "01-06.tsm1", + Size: 500 * 1024 * 1024, + }, + { + Path: "01-07.tsm1", + Size: 400 * 1024 * 1024, + }, + { + Path: "01-08.tsm1", + Size: 300 * 1024 * 1024, + }, + { + Path: "01-09.tsm1", + Size: 200 * 1024 * 1024, + }, + { + Path: "01-10.tsm1", + Size: 100 * 1024 * 1024, + }, + { + Path: "01-11.tsm1", + Size: 50 * 1024 * 1024, + }, + { + Path: "01-12.tsm1", + Size: 25 * 1024 * 1024, + }, + }, + []int{ + tsdb.AggressiveMaxPointsPerBlock, + tsdb.AggressiveMaxPointsPerBlock, + tsdb.AggressiveMaxPointsPerBlock, + tsdb.AggressiveMaxPointsPerBlock, + tsdb.DefaultMaxPointsPerBlock, + tsdb.DefaultMaxPointsPerBlock, + tsdb.DefaultMaxPointsPerBlock, + tsdb.DefaultMaxPointsPerBlock, + }, + tsdb.SingleGenerationReasonText, + 1, + }, + { + // > 2 GB total group size + // 100% of files are at aggressive max block size + []tsm1.FileStat{ + { + Path: "01-13.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "01-14.tsm1", + Size: 650 * 1024 * 1024, + }, + { + Path: "01-15.tsm1", + Size: 450 * 1024 * 1024, + }, + }, []int{ + tsdb.AggressiveMaxPointsPerBlock, + tsdb.DefaultMaxPointsPerBlock, + tsdb.DefaultMaxPointsPerBlock, + }, + tsdb.SingleGenerationReasonText, + 1, + }, + { + // Last files are lower than first files generations + // Mix of less than 4 level files and > level 4 files + []tsm1.FileStat{ + { + Path: "01-05.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "01-06.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "01-07.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "01-08.tsm1", + Size: 1048 * 1024 * 1024, + }, + { + Path: "02-05.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "02-06.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "02-07.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "02-08.tsm1", + Size: 1048 * 1024 * 1024, + }, + { + Path: "03-03.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "03-04.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "03-04.tsm1", + Size: 600 * 1024 * 1024, + }, + { + Path: "03-06.tsm1", + Size: 500 * 1024 * 1024, + }, + }, []int{}, "not fully compacted and not idle because of more than one generation", 3, + }, + { + // This test will mock a 'backfill' condition where we have a single + // shard with many generations. The initial generation should be fully + // compacted, but we have some new generations that are not. We need to ensure + // the optimize planner will pick these up and compact everything together. + []tsm1.FileStat{ + { + Path: "01-05.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "01-06.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "01-07.tsm1", + Size: 2048 * 1024 * 1024, + }, + { + Path: "02-04.tsm1", + Size: 700 * 1024 * 1024, + }, + { + Path: "02-05.tsm1", + Size: 500 * 1024 * 1024, + }, + { + Path: "02-06.tsm1", + Size: 400 * 1024 * 1024, + }, + { + Path: "03-02.tsm1", + Size: 700 * 1024 * 1024, + }, + { + Path: "03-03.tsm1", + Size: 500 * 1024 * 1024, + }, + { + Path: "03-04.tsm1", + Size: 400 * 1024 * 1024, + }, + { + Path: "04-01.tsm1", + Size: 700 * 1024 * 1024, + }, + { + Path: "04-02.tsm1", + Size: 500 * 1024 * 1024, + }, + { + Path: "03-03.tsm1", + Size: 400 * 1024 * 1024, + }, + }, []int{ + tsdb.AggressiveMaxPointsPerBlock, + tsdb.AggressiveMaxPointsPerBlock, + tsdb.AggressiveMaxPointsPerBlock, + + tsdb.AggressiveMaxPointsPerBlock, + tsdb.AggressiveMaxPointsPerBlock, + tsdb.DefaultMaxPointsPerBlock, + + tsdb.AggressiveMaxPointsPerBlock, + tsdb.AggressiveMaxPointsPerBlock, + tsdb.DefaultMaxPointsPerBlock, + + tsdb.DefaultMaxPointsPerBlock, + // Use some magic numbers but these are just small values for block counts + 100, + 10, + }, + "not fully compacted and not idle because of more than one generation", + 4, + }, + } + + expected := func(cp *tsm1.DefaultPlanner, reasonExp string, generationCountExp int64) { + compacted, reason := cp.FullyCompacted() + require.Equal(t, reason, reasonExp, "fullyCompacted reason") + require.False(t, compacted, "is fully compacted") + + _, cgLen := cp.PlanLevel(1) + require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") + _, cgLen = cp.PlanLevel(2) + require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") + _, cgLen = cp.PlanLevel(3) + require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") + + tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) + require.Zero(t, len(tsmP), "compaction group; Plan()") + require.Zero(t, pLenP, "compaction group length; Plan()") + + _, cgLen, genLen := cp.PlanOptimize() + require.Equal(t, int64(1), cgLen, "compaction group length") + require.Equal(t, generationCountExp, genLen, "generation count") + + } + + for _, fileStat := range fileStats { + ffs := &fakeFileStore{ + PathsFn: func() []tsm1.FileStat { + return fileStat.fs + }, + } - compacted, reason := cp.FullyCompacted() - require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason") - require.False(t, compacted, "is fully compacted") + if len(fileStat.bc) > 0 { + err := ffs.SetBlockCounts(fileStat.bc) + require.NoError(t, err, "setting block counts") + } - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") + cp := tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration) + expected(cp, fileStat.fullyCompactedReasonExp, fileStat.generationCount) - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") + // Reverse test files and re-run tests + slices.Reverse(fileStat.fs) + cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration) + expected(cp, fileStat.fullyCompactedReasonExp, fileStat.generationCount) + } - _, cgLen, genLen := cp.PlanOptimize() - require.Equal(t, int64(1), cgLen, "compaction group length") - require.Equal(t, int64(1), genLen, "generation count") } // This test is added to account for halting state after @@ -2772,163 +2688,6 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T) require.Zero(t, genLen, "generation count") } -// This test is added to account for a single generation that has a group size -// under 2 GB and a mix of aggressive max blocks and default max blocks -// it should be further compacted. -func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlocks(t *testing.T) { - // > 2 GB total group size - // 50% of files are at aggressive max block size - data := []tsm1.FileStat{ - { - Path: "01-05.tsm1", - Size: 700 * 1024 * 1024, - }, - { - Path: "01-06.tsm1", - Size: 500 * 1024 * 1024, - }, - { - Path: "01-07.tsm1", - Size: 400 * 1024 * 1024, - }, - { - Path: "01-08.tsm1", - Size: 300 * 1024 * 1024, - }, - { - Path: "01-09.tsm1", - Size: 200 * 1024 * 1024, - }, - { - Path: "01-10.tsm1", - Size: 100 * 1024 * 1024, - }, - { - Path: "01-11.tsm1", - Size: 50 * 1024 * 1024, - }, - { - Path: "01-12.tsm1", - Size: 25 * 1024 * 1024, - }, - } - - fs := &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - } - blocks := []int{ - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, - } - err := fs.SetBlockCounts(blocks) - require.NoError(t, err, "SetBlockCounts") - - cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration) - compacted, reason := cp.FullyCompacted() - require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason") - require.False(t, compacted, "is fully compacted") - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") - - _, cgLen, genLen := cp.PlanOptimize() - require.Equal(t, int64(1), cgLen, "compaction group length") - require.Equal(t, int64(1), genLen, "generation count") -} - -func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlocksMixFileSize(t *testing.T) { - // > 2 GB total group size - // 50% of files are at aggressive max block size - data := []tsm1.FileStat{ - { - Path: "01-05.tsm1", - Size: 300 * 1024 * 1024, - }, - { - Path: "01-06.tsm1", - Size: 700 * 1024 * 1024, - }, - { - Path: "01-07.tsm1", - Size: 25 * 1024 * 1024, - }, - { - Path: "01-08.tsm1", - Size: 400 * 1024 * 1024, - }, - { - Path: "01-09.tsm1", - Size: 50 * 1024 * 1024, - }, - { - Path: "01-10.tsm1", - Size: 100 * 1024 * 1024, - }, - { - Path: "01-11.tsm1", - Size: 200 * 1024 * 1024, - }, - { - Path: "01-12.tsm1", - Size: 500 * 1024 * 1024, - }, - } - - fs := &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - } - blocks := []int{ - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, - } - err := fs.SetBlockCounts(blocks) - require.NoError(t, err, "SetBlockCounts") - - cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration) - compacted, reason := cp.FullyCompacted() - require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason") - require.False(t, compacted, "is fully compacted") - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") - - _, cgLen, genLen := cp.PlanOptimize() - require.Equal(t, int64(1), cgLen, "compaction group length") - require.Equal(t, int64(1), genLen, "generation count") -} - // This test is added to account for a single generation that has a group size // over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction. // It should not compact any further. @@ -3045,114 +2804,8 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *t Size: 2048 * 1024 * 1024, }, { - Path: "01-14.tsm1", - Size: 691 * 1024 * 1024, - }, - } - - fs := &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - } - blocks := []int{ - tsdb.AggressiveMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, - } - err := fs.SetBlockCounts(blocks) - require.NoError(t, err, "SetBlockCounts") - - cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration) - - compacted, reason := cp.FullyCompacted() - reasonExp := "" - require.Equal(t, reason, reasonExp, "fullyCompacted reason") - require.True(t, compacted, "is fully compacted") - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") - - cgroup, cgLen, genLen := cp.PlanOptimize() - require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group") - require.Equal(t, int64(0), cgLen, "compaction group length") - require.Equal(t, int64(0), genLen, "generation count") -} - -func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocksSmallFileFirst(t *testing.T) { - // > 2 GB total group size - // 100% of files are at aggressive max block size - data := []tsm1.FileStat{ - { - Path: "01-13.tsm1", - Size: 691 * 1024 * 1024, - }, - { - Path: "01-14.tsm1", - Size: 2048 * 1024 * 1024, - }, - } - - fs := &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - } - blocks := []int{ - tsdb.DefaultMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - } - err := fs.SetBlockCounts(blocks) - require.NoError(t, err, "SetBlockCounts") - - cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration) - - compacted, reason := cp.FullyCompacted() - reasonExp := "" - require.Equal(t, reason, reasonExp, "fullyCompacted reason") - require.True(t, compacted, "is fully compacted") - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") - - cgroup, cgLen, genLen := cp.PlanOptimize() - require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group") - require.Equal(t, int64(0), cgLen, "compaction group length") - require.Equal(t, int64(0), genLen, "generation count") -} - -// This test is added to account for a single generation that has a group size -// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction. -func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *testing.T) { - // > 2 GB total group size - // 100% of files are at aggressive max block size - data := []tsm1.FileStat{ - { - Path: "01-13.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "01-14.tsm1", - Size: 650 * 1024 * 1024, - }, - { - Path: "01-15.tsm1", - Size: 450 * 1024 * 1024, + Path: "01-14.tsm1", + Size: 691 * 1024 * 1024, }, } @@ -3163,8 +2816,7 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t * } blocks := []int{ tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, + tsdb.DefaultMaxPointsPerBlock, } err := fs.SetBlockCounts(blocks) require.NoError(t, err, "SetBlockCounts") @@ -3189,24 +2841,20 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t * cgroup, cgLen, genLen := cp.PlanOptimize() require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group") - require.Zero(t, cgLen, "compaction group length") - require.Zero(t, genLen, "generation count") + require.Equal(t, int64(0), cgLen, "compaction group length") + require.Equal(t, int64(0), genLen, "generation count") } -func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocksSmallFileFirst(t *testing.T) { +func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocksSmallFileFirst(t *testing.T) { // > 2 GB total group size // 100% of files are at aggressive max block size data := []tsm1.FileStat{ { Path: "01-13.tsm1", - Size: 450 * 1024 * 1024, + Size: 691 * 1024 * 1024, }, { Path: "01-14.tsm1", - Size: 650 * 1024 * 1024, - }, - { - Path: "01-15.tsm1", Size: 2048 * 1024 * 1024, }, } @@ -3217,8 +2865,7 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocksSmal }, } blocks := []int{ - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, + tsdb.DefaultMaxPointsPerBlock, tsdb.AggressiveMaxPointsPerBlock, } err := fs.SetBlockCounts(blocks) @@ -3244,14 +2891,13 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocksSmal cgroup, cgLen, genLen := cp.PlanOptimize() require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group") - require.Zero(t, cgLen, "compaction group length") - require.Zero(t, genLen, "generation count") + require.Equal(t, int64(0), cgLen, "compaction group length") + require.Equal(t, int64(0), genLen, "generation count") } // This test is added to account for a single generation that has a group size -// over 2 GB and multiple files under 2 GB with multiple files under aggressive -// max points per block. This should further compact. -func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(t *testing.T) { +// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction. +func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *testing.T) { // > 2 GB total group size // 100% of files are at aggressive max block size data := []tsm1.FileStat{ @@ -3276,8 +2922,8 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks( } blocks := []int{ tsdb.AggressiveMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, + tsdb.AggressiveMaxPointsPerBlock, + tsdb.AggressiveMaxPointsPerBlock, } err := fs.SetBlockCounts(blocks) require.NoError(t, err, "SetBlockCounts") @@ -3285,8 +2931,9 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks( cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration) compacted, reason := cp.FullyCompacted() - require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason") - require.False(t, compacted, "is fully compacted") + reasonExp := "" + require.Equal(t, reason, reasonExp, "fullyCompacted reason") + require.True(t, compacted, "is fully compacted") _, cgLen := cp.PlanLevel(1) require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") @@ -3299,66 +2946,28 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks( require.Zero(t, len(tsmP), "compaction group; Plan()") require.Zero(t, pLenP, "compaction group length; Plan()") - _, cgLen, genLen := cp.PlanOptimize() - require.Equal(t, int64(1), cgLen, "compaction group length") - require.Equal(t, int64(1), genLen, "generation count") + cgroup, cgLen, genLen := cp.PlanOptimize() + require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group") + require.Zero(t, cgLen, "compaction group length") + require.Zero(t, genLen, "generation count") } -// This test is added to account for multiple generations over level 4 -// compaction and over 2 GB group size, with a level 3 start generation -// over 2 GB group size. -func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T) { +func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocksSmallFileFirst(t *testing.T) { // > 2 GB total group size // 100% of files are at aggressive max block size data := []tsm1.FileStat{ { - Path: "01-05.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "01-06.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "01-07.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "01-08.tsm1", - Size: 1048 * 1024 * 1024, - }, - { - Path: "02-05.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "02-06.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "02-07.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "02-08.tsm1", - Size: 1048 * 1024 * 1024, + Path: "01-13.tsm1", + Size: 450 * 1024 * 1024, }, { - Path: "03-03.tsm1", - Size: 2048 * 1024 * 1024, + Path: "01-14.tsm1", + Size: 650 * 1024 * 1024, }, { - Path: "03-04.tsm1", + Path: "01-15.tsm1", Size: 2048 * 1024 * 1024, }, - { - Path: "03-04.tsm1", - Size: 600 * 1024 * 1024, - }, - { - Path: "03-06.tsm1", - Size: 500 * 1024 * 1024, - }, } fs := &fakeFileStore{ @@ -3366,13 +2975,20 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T) return data }, } + blocks := []int{ + tsdb.AggressiveMaxPointsPerBlock, + tsdb.AggressiveMaxPointsPerBlock, + tsdb.AggressiveMaxPointsPerBlock, + } + err := fs.SetBlockCounts(blocks) + require.NoError(t, err, "SetBlockCounts") cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration) - expFiles := make([]tsm1.FileStat, 0) - for _, file := range data { - expFiles = append(expFiles, file) - } + compacted, reason := cp.FullyCompacted() + reasonExp := "" + require.Equal(t, reason, reasonExp, "fullyCompacted reason") + require.True(t, compacted, "is fully compacted") _, cgLen := cp.PlanLevel(1) require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") @@ -3385,10 +3001,10 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T) require.Zero(t, len(tsmP), "compaction group; Plan()") require.Zero(t, pLenP, "compaction group length; Plan()") - tsm, cgLen, genLen := cp.PlanOptimize() - require.Equal(t, int64(1), cgLen, "compaction group length") - require.Equal(t, int64(3), genLen, "generation count") - require.Equal(t, len(expFiles), len(tsm[0]), "tsm files in compaction group") + cgroup, cgLen, genLen := cp.PlanOptimize() + require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group") + require.Zero(t, cgLen, "compaction group length") + require.Zero(t, genLen, "generation count") } func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2MixedSizes(t *testing.T) { @@ -3475,175 +3091,6 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2MixedSizes(t *t require.Equal(t, len(expFiles), len(tsm[0]), "tsm files in compaction group") } -// This test will check to ensure that any TSM generations planned with the default planner -// using Plan() and PlanLevel() over default block size are skipped. -func TestDefaultPlanner_PlanOverAggressiveBlocks(t *testing.T) { - data := []tsm1.FileStat{ - { - Path: "01-02.tsm1", - Size: 251 * 1024 * 1024, - }, - { - Path: "01-03.tsm1", - Size: 1 * 1024 * 1024, - }, - { - Path: "02-02.tsm1", - Size: 251 * 1024 * 1024, - }, - { - Path: "02-03.tsm1", - Size: 1 * 1024 * 1024, - }, - { - Path: "03-02.tsm1", - Size: 251 * 1024 * 1024, - }, - { - Path: "03-03.tsm1", - Size: 1 * 1024 * 1024, - }, - } - - fs := &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - } - blocks := []int{ - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - } - err := fs.SetBlockCounts(blocks) - require.NoError(t, err, "SetBlockCounts") - - cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration) - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") -} - -// This test will mock a 'backfill' condition where we have a single -// shard with many generations. The initial generation should be fully -// compacted, but we have some new generations that are not. We need to ensure -// the optimize planner will pick these up and compact everything together. -func TestDefaultPlanner_BackfillMock(t *testing.T) { - // > 2 GB total group size - // 50% of files are at aggressive max block size - data := []tsm1.FileStat{ - { - Path: "01-05.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "01-06.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "01-07.tsm1", - Size: 2048 * 1024 * 1024, - }, - { - Path: "02-04.tsm1", - Size: 700 * 1024 * 1024, - }, - { - Path: "02-05.tsm1", - Size: 500 * 1024 * 1024, - }, - { - Path: "02-06.tsm1", - Size: 400 * 1024 * 1024, - }, - { - Path: "03-02.tsm1", - Size: 700 * 1024 * 1024, - }, - { - Path: "03-03.tsm1", - Size: 500 * 1024 * 1024, - }, - { - Path: "03-04.tsm1", - Size: 400 * 1024 * 1024, - }, - { - Path: "04-01.tsm1", - Size: 700 * 1024 * 1024, - }, - { - Path: "04-02.tsm1", - Size: 500 * 1024 * 1024, - }, - { - Path: "03-03.tsm1", - Size: 400 * 1024 * 1024, - }, - } - - fs := &fakeFileStore{ - PathsFn: func() []tsm1.FileStat { - return data - }, - } - blocks := []int{ - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, - - tsdb.AggressiveMaxPointsPerBlock, - tsdb.AggressiveMaxPointsPerBlock, - tsdb.DefaultMaxPointsPerBlock, - - tsdb.DefaultMaxPointsPerBlock, - // Use some magic numbers but these are just small values for block counts - 100, - 10, - } - err := fs.SetBlockCounts(blocks) - require.NoError(t, err, "SetBlockCounts") - - cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration) - - _, cgLen := cp.PlanLevel(1) - require.Zero(t, cgLen, "compaction group length; PlanLevel(1)") - _, cgLen = cp.PlanLevel(2) - require.Zero(t, cgLen, "compaction group length; PlanLevel(2)") - _, cgLen = cp.PlanLevel(3) - require.Zero(t, cgLen, "compaction group length; PlanLevel(3)") - - tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second)) - require.Zero(t, len(tsmP), "compaction group; Plan()") - require.Zero(t, pLenP, "compaction group length; Plan()") - - expFiles := make([]tsm1.FileStat, 0) - for _, file := range data { - expFiles = append(expFiles, file) - } - - tsm, cgLen, genLen := cp.PlanOptimize() - require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files") - require.Equal(t, int64(1), cgLen, "compaction group length") - // Should pick up 4 generations for compaction - require.Equal(t, int64(4), genLen, "generation count") -} - func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) { data := []tsm1.FileStat{ {