package compact import ( "math" "path/filepath" "testing" "git.dvdt.dev/david/ingot/internal/block" "git.dvdt.dev/david/ingot/internal/chunkenc" "git.dvdt.dev/david/ingot/labels" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) const ( hour = 3600 * 1000 // 1 hour in ms ) // makeChunk creates a chunk with the given samples and returns its raw bytes. func makeChunk(t *testing.T, timestamps []int64, values []float64) []byte { t.Helper() c := chunkenc.NewXORChunk() a, err := c.Appender() require.NoError(t, err) for i := range timestamps { a.Append(timestamps[i], values[i]) } return append([]byte(nil), c.Bytes()...) } // flushTestBlock creates a block in dataDir and returns an opened Reader. func flushTestBlock(t *testing.T, dataDir string, series []block.SeriesFlush, level int, sources []string) *block.Reader { t.Helper() var ulid string var err error if level == 1 && sources == nil { ulid, err = block.Flush(dataDir, series) } else { ulid, err = block.FlushCompacted(dataDir, series, level, sources) } require.NoError(t, err) r, err := block.Open(filepath.Join(dataDir, ulid)) require.NoError(t, err) return r } // collectBlockSamples reads all samples for a series ref from a block. func collectBlockSamples(t *testing.T, r *block.Reader, ref uint64) []sample { t.Helper() it, err := r.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64) require.NoError(t, err) var out []sample for it.Next() { ts, v := it.At() out = append(out, sample{ts, v}) } require.NoError(t, it.Err()) return out } type sample struct { t int64 v float64 } func TestPlan(t *testing.T) { tests := []struct { name string blocks []blockSpec // blocks to create wantGroup bool // expect a compaction group wantLevel int // expected resulting level wantCount int // expected number of sources in group }{ { name: "no_blocks", blocks: nil, wantGroup: false, }, { name: "single_block", blocks: []blockSpec{ {minT: 0, maxT: 2 * hour, level: 1}, }, wantGroup: false, }, { name: "two_level1_blocks_within_8h", blocks: []blockSpec{ {minT: 0, maxT: 2 * hour, level: 1}, {minT: 2 * hour, maxT: 4 * hour, level: 1}, }, wantGroup: true, wantLevel: 2, wantCount: 2, }, { name: "four_level1_blocks", blocks: []blockSpec{ {minT: 0, maxT: 2 * hour, level: 1}, {minT: 2 * hour, maxT: 4 * hour, level: 1}, {minT: 4 * hour, maxT: 6 * hour, level: 1}, {minT: 6 * hour, maxT: 8 * hour, level: 1}, }, wantGroup: true, wantLevel: 2, wantCount: 4, }, { name: "level1_blocks_exceed_8h_span", blocks: []blockSpec{ {minT: 0, maxT: 2 * hour, level: 1}, {minT: 7 * hour, maxT: 9 * hour, level: 1}, }, wantGroup: false, }, { name: "two_level2_blocks_within_32h", blocks: []blockSpec{ {minT: 0, maxT: 8 * hour, level: 2}, {minT: 8 * hour, maxT: 16 * hour, level: 2}, }, wantGroup: true, wantLevel: 3, wantCount: 2, }, { name: "max_level_blocks_not_compacted", blocks: []blockSpec{ {minT: 0, maxT: 32 * hour, level: 3}, {minT: 32 * hour, maxT: 64 * hour, level: 3}, }, wantGroup: false, }, { name: "mixed_levels_lower_prioritized", blocks: []blockSpec{ {minT: 0, maxT: 2 * hour, level: 1}, {minT: 2 * hour, maxT: 4 * hour, level: 1}, {minT: 10 * hour, maxT: 18 * hour, level: 2}, {minT: 18 * hour, maxT: 26 * hour, level: 2}, }, wantGroup: true, wantLevel: 2, // level-1 group found first wantCount: 2, }, } levels := []int64{2 * hour, 8 * hour, 32 * hour} for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { dataDir := t.TempDir() c := New(dataDir, levels, 0, nil) var blocks []*block.Reader for _, spec := range tc.blocks { r := createBlockWithMeta(t, dataDir, spec) blocks = append(blocks, r) t.Cleanup(func() { r.Close() }) } group := c.Plan(blocks) if !tc.wantGroup { assert.Nil(t, group, "expected no compaction group") return } require.NotNil(t, group, "expected a compaction group") assert.Equal(t, tc.wantLevel, group.Level, "compaction level") assert.Equal(t, tc.wantCount, len(group.Sources), "source count") }) } } func TestCompact(t *testing.T) { tests := []struct { name string sourceBlocks []sourceBlock wantSeriesRefs []uint64 wantSamples map[uint64][]sample wantLevel int }{ { name: "merge_two_blocks_disjoint_series", sourceBlocks: []sourceBlock{ { level: 1, series: []seriesData{ {ref: 1, labels: labels.FromStrings("__name__", "a"), samples: []sample{{1000, 1.0}, {2000, 2.0}}}, }, }, { level: 1, series: []seriesData{ {ref: 2, labels: labels.FromStrings("__name__", "b"), samples: []sample{{1000, 3.0}, {2000, 4.0}}}, }, }, }, wantSeriesRefs: []uint64{1, 2}, wantSamples: map[uint64][]sample{ 1: {{1000, 1.0}, {2000, 2.0}}, 2: {{1000, 3.0}, {2000, 4.0}}, }, wantLevel: 2, }, { name: "merge_two_blocks_same_series", sourceBlocks: []sourceBlock{ { level: 1, series: []seriesData{ {ref: 1, labels: labels.FromStrings("__name__", "temp"), samples: []sample{{1000, 1.0}, {2000, 2.0}}}, }, }, { level: 1, series: []seriesData{ {ref: 1, labels: labels.FromStrings("__name__", "temp"), samples: []sample{{3000, 3.0}, {4000, 4.0}}}, }, }, }, wantSeriesRefs: []uint64{1}, wantSamples: map[uint64][]sample{ 1: {{1000, 1.0}, {2000, 2.0}, {3000, 3.0}, {4000, 4.0}}, }, wantLevel: 2, }, { name: "merge_level2_blocks", sourceBlocks: []sourceBlock{ { level: 2, series: []seriesData{ {ref: 1, labels: labels.FromStrings("__name__", "a"), samples: []sample{{1000, 1.0}}}, }, }, { level: 2, series: []seriesData{ {ref: 1, labels: labels.FromStrings("__name__", "a"), samples: []sample{{5000, 5.0}}}, }, }, }, wantSeriesRefs: []uint64{1}, wantSamples: map[uint64][]sample{ 1: {{1000, 1.0}, {5000, 5.0}}, }, wantLevel: 3, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { dataDir := t.TempDir() c := New(dataDir, []int64{2 * hour, 8 * hour, 32 * hour}, 0, nil) // Create source blocks. var sources []*block.Reader for _, sb := range tc.sourceBlocks { r := createSourceBlock(t, dataDir, sb) sources = append(sources, r) } // Compact. newULID, err := c.Compact(sources) require.NoError(t, err) require.NotEmpty(t, newULID) // Close source blocks. for _, s := range sources { s.Close() } // Open compacted block. compacted, err := block.Open(filepath.Join(dataDir, newULID)) require.NoError(t, err) defer compacted.Close() // Verify compaction level. assert.Equal(t, tc.wantLevel, compacted.Meta.Compaction.Level, "compaction level") assert.Equal(t, len(tc.sourceBlocks), len(compacted.Meta.Compaction.Sources), "source count") // Verify series count. assert.Equal(t, len(tc.wantSeriesRefs), compacted.Meta.Stats.NumSeries, "series count") // Verify each series' samples. for _, ref := range tc.wantSeriesRefs { got := collectBlockSamples(t, compacted, ref) want := tc.wantSamples[ref] require.Equal(t, len(want), len(got), "sample count for ref %d", ref) for i := range want { assert.Equal(t, want[i].t, got[i].t, "ref %d sample %d t", ref, i) assert.Equal(t, want[i].v, got[i].v, "ref %d sample %d v", ref, i) } } }) } } func TestExpired(t *testing.T) { tests := []struct { name string blocks []blockSpec now int64 retention int64 wantCount int }{ { name: "no_retention", blocks: []blockSpec{{minT: 0, maxT: 1000}}, now: 100000, retention: 0, wantCount: 0, }, { name: "all_within_retention", blocks: []blockSpec{ {minT: 80000, maxT: 90000}, }, now: 100000, retention: 50000, wantCount: 0, }, { name: "one_expired", blocks: []blockSpec{ {minT: 0, maxT: 10000}, {minT: 80000, maxT: 90000}, }, now: 100000, retention: 50000, wantCount: 1, }, { name: "all_expired", blocks: []blockSpec{ {minT: 0, maxT: 10000}, {minT: 20000, maxT: 30000}, }, now: 100000, retention: 50000, wantCount: 2, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { dataDir := t.TempDir() clock := func() int64 { return tc.now } c := New(dataDir, nil, tc.retention, clock) var blocks []*block.Reader for _, spec := range tc.blocks { r := createBlockWithMeta(t, dataDir, spec) blocks = append(blocks, r) t.Cleanup(func() { r.Close() }) } expired := c.Expired(blocks) assert.Equal(t, tc.wantCount, len(expired), "expired block count") }) } } // --- helpers --- type blockSpec struct { minT int64 maxT int64 level int } type sourceBlock struct { level int series []seriesData } type seriesData struct { ref uint64 labels []labels.Label samples []sample } // createBlockWithMeta creates a minimal block with the given time range and level. func createBlockWithMeta(t *testing.T, dataDir string, spec blockSpec) *block.Reader { t.Helper() // Need at least one series with samples spanning [minT, maxT]. timestamps := []int64{spec.minT, spec.maxT} values := []float64{1.0, 2.0} data := makeChunk(t, timestamps, values) series := []block.SeriesFlush{{ Ref: 1, Labels: labels.FromStrings("__name__", "test"), Chunks: []block.ChunkData{{MinT: spec.minT, MaxT: spec.maxT, Data: data}}, }} level := spec.level if level == 0 { level = 1 } return flushTestBlock(t, dataDir, series, level, []string{"src"}) } // createSourceBlock creates a block with the given series data. func createSourceBlock(t *testing.T, dataDir string, sb sourceBlock) *block.Reader { t.Helper() var flushData []block.SeriesFlush for _, sd := range sb.series { timestamps := make([]int64, len(sd.samples)) values := make([]float64, len(sd.samples)) for i, s := range sd.samples { timestamps[i] = s.t values[i] = s.v } data := makeChunk(t, timestamps, values) minT, maxT := sd.samples[0].t, sd.samples[len(sd.samples)-1].t flushData = append(flushData, block.SeriesFlush{ Ref: sd.ref, Labels: sd.labels, Chunks: []block.ChunkData{{MinT: minT, MaxT: maxT, Data: data}}, }) } level := sb.level if level == 0 { level = 1 } return flushTestBlock(t, dataDir, flushData, level, []string{"src"}) }