Compaction, retention, and soak testing
Block refcounting (atomic refs + condemned flag) lets live queries survive concurrent compaction and retention. Levelled compactor merges 2h->8h->32h blocks via raw chunk passthrough. Retention drops blocks older than the configured window. Background goroutine drives flush/compact/retain cycles; exported RunCompaction/ApplyRetention allow deterministic test control via injectable clock. Soak test: 10k series x 48h simulated at 15s intervals (115M samples). Validates flat memory (158 MiB peak), bounded disk (13 blocks), and zero query errors during compaction. All existing tests refactored to table-driven with uniform assertions.
This commit is contained in:
@@ -0,0 +1,430 @@
|
||||
package compact
|
||||
|
||||
import (
|
||||
"math"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.dvdt.dev/david/ingot/internal/block"
|
||||
"git.dvdt.dev/david/ingot/internal/chunkenc"
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
hour = 3600 * 1000 // 1 hour in ms
|
||||
)
|
||||
|
||||
// makeChunk creates a chunk with the given samples and returns its raw bytes.
|
||||
func makeChunk(t *testing.T, timestamps []int64, values []float64) []byte {
|
||||
t.Helper()
|
||||
c := chunkenc.NewXORChunk()
|
||||
a, err := c.Appender()
|
||||
require.NoError(t, err)
|
||||
for i := range timestamps {
|
||||
a.Append(timestamps[i], values[i])
|
||||
}
|
||||
return append([]byte(nil), c.Bytes()...)
|
||||
}
|
||||
|
||||
// flushTestBlock creates a block in dataDir and returns an opened Reader.
|
||||
func flushTestBlock(t *testing.T, dataDir string, series []block.SeriesFlush, level int, sources []string) *block.Reader {
|
||||
t.Helper()
|
||||
var ulid string
|
||||
var err error
|
||||
if level == 1 && sources == nil {
|
||||
ulid, err = block.Flush(dataDir, series)
|
||||
} else {
|
||||
ulid, err = block.FlushCompacted(dataDir, series, level, sources)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
r, err := block.Open(filepath.Join(dataDir, ulid))
|
||||
require.NoError(t, err)
|
||||
return r
|
||||
}
|
||||
|
||||
// collectBlockSamples reads all samples for a series ref from a block.
|
||||
func collectBlockSamples(t *testing.T, r *block.Reader, ref uint64) []sample {
|
||||
t.Helper()
|
||||
it, err := r.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
var out []sample
|
||||
for it.Next() {
|
||||
ts, v := it.At()
|
||||
out = append(out, sample{ts, v})
|
||||
}
|
||||
require.NoError(t, it.Err())
|
||||
return out
|
||||
}
|
||||
|
||||
type sample struct {
|
||||
t int64
|
||||
v float64
|
||||
}
|
||||
|
||||
func TestPlan(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
blocks []blockSpec // blocks to create
|
||||
wantGroup bool // expect a compaction group
|
||||
wantLevel int // expected resulting level
|
||||
wantCount int // expected number of sources in group
|
||||
}{
|
||||
{
|
||||
name: "no_blocks",
|
||||
blocks: nil,
|
||||
wantGroup: false,
|
||||
},
|
||||
{
|
||||
name: "single_block",
|
||||
blocks: []blockSpec{
|
||||
{minT: 0, maxT: 2 * hour, level: 1},
|
||||
},
|
||||
wantGroup: false,
|
||||
},
|
||||
{
|
||||
name: "two_level1_blocks_within_8h",
|
||||
blocks: []blockSpec{
|
||||
{minT: 0, maxT: 2 * hour, level: 1},
|
||||
{minT: 2 * hour, maxT: 4 * hour, level: 1},
|
||||
},
|
||||
wantGroup: true,
|
||||
wantLevel: 2,
|
||||
wantCount: 2,
|
||||
},
|
||||
{
|
||||
name: "four_level1_blocks",
|
||||
blocks: []blockSpec{
|
||||
{minT: 0, maxT: 2 * hour, level: 1},
|
||||
{minT: 2 * hour, maxT: 4 * hour, level: 1},
|
||||
{minT: 4 * hour, maxT: 6 * hour, level: 1},
|
||||
{minT: 6 * hour, maxT: 8 * hour, level: 1},
|
||||
},
|
||||
wantGroup: true,
|
||||
wantLevel: 2,
|
||||
wantCount: 4,
|
||||
},
|
||||
{
|
||||
name: "level1_blocks_exceed_8h_span",
|
||||
blocks: []blockSpec{
|
||||
{minT: 0, maxT: 2 * hour, level: 1},
|
||||
{minT: 7 * hour, maxT: 9 * hour, level: 1},
|
||||
},
|
||||
wantGroup: false,
|
||||
},
|
||||
{
|
||||
name: "two_level2_blocks_within_32h",
|
||||
blocks: []blockSpec{
|
||||
{minT: 0, maxT: 8 * hour, level: 2},
|
||||
{minT: 8 * hour, maxT: 16 * hour, level: 2},
|
||||
},
|
||||
wantGroup: true,
|
||||
wantLevel: 3,
|
||||
wantCount: 2,
|
||||
},
|
||||
{
|
||||
name: "max_level_blocks_not_compacted",
|
||||
blocks: []blockSpec{
|
||||
{minT: 0, maxT: 32 * hour, level: 3},
|
||||
{minT: 32 * hour, maxT: 64 * hour, level: 3},
|
||||
},
|
||||
wantGroup: false,
|
||||
},
|
||||
{
|
||||
name: "mixed_levels_lower_prioritized",
|
||||
blocks: []blockSpec{
|
||||
{minT: 0, maxT: 2 * hour, level: 1},
|
||||
{minT: 2 * hour, maxT: 4 * hour, level: 1},
|
||||
{minT: 10 * hour, maxT: 18 * hour, level: 2},
|
||||
{minT: 18 * hour, maxT: 26 * hour, level: 2},
|
||||
},
|
||||
wantGroup: true,
|
||||
wantLevel: 2, // level-1 group found first
|
||||
wantCount: 2,
|
||||
},
|
||||
}
|
||||
|
||||
levels := []int64{2 * hour, 8 * hour, 32 * hour}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
c := New(dataDir, levels, 0, nil)
|
||||
|
||||
var blocks []*block.Reader
|
||||
for _, spec := range tc.blocks {
|
||||
r := createBlockWithMeta(t, dataDir, spec)
|
||||
blocks = append(blocks, r)
|
||||
t.Cleanup(func() { r.Close() })
|
||||
}
|
||||
|
||||
group := c.Plan(blocks)
|
||||
|
||||
if !tc.wantGroup {
|
||||
assert.Nil(t, group, "expected no compaction group")
|
||||
return
|
||||
}
|
||||
require.NotNil(t, group, "expected a compaction group")
|
||||
assert.Equal(t, tc.wantLevel, group.Level, "compaction level")
|
||||
assert.Equal(t, tc.wantCount, len(group.Sources), "source count")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompact(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
sourceBlocks []sourceBlock
|
||||
wantSeriesRefs []uint64
|
||||
wantSamples map[uint64][]sample
|
||||
wantLevel int
|
||||
}{
|
||||
{
|
||||
name: "merge_two_blocks_disjoint_series",
|
||||
sourceBlocks: []sourceBlock{
|
||||
{
|
||||
level: 1,
|
||||
series: []seriesData{
|
||||
{ref: 1, labels: labels.FromStrings("__name__", "a"), samples: []sample{{1000, 1.0}, {2000, 2.0}}},
|
||||
},
|
||||
},
|
||||
{
|
||||
level: 1,
|
||||
series: []seriesData{
|
||||
{ref: 2, labels: labels.FromStrings("__name__", "b"), samples: []sample{{1000, 3.0}, {2000, 4.0}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantSeriesRefs: []uint64{1, 2},
|
||||
wantSamples: map[uint64][]sample{
|
||||
1: {{1000, 1.0}, {2000, 2.0}},
|
||||
2: {{1000, 3.0}, {2000, 4.0}},
|
||||
},
|
||||
wantLevel: 2,
|
||||
},
|
||||
{
|
||||
name: "merge_two_blocks_same_series",
|
||||
sourceBlocks: []sourceBlock{
|
||||
{
|
||||
level: 1,
|
||||
series: []seriesData{
|
||||
{ref: 1, labels: labels.FromStrings("__name__", "temp"), samples: []sample{{1000, 1.0}, {2000, 2.0}}},
|
||||
},
|
||||
},
|
||||
{
|
||||
level: 1,
|
||||
series: []seriesData{
|
||||
{ref: 1, labels: labels.FromStrings("__name__", "temp"), samples: []sample{{3000, 3.0}, {4000, 4.0}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantSeriesRefs: []uint64{1},
|
||||
wantSamples: map[uint64][]sample{
|
||||
1: {{1000, 1.0}, {2000, 2.0}, {3000, 3.0}, {4000, 4.0}},
|
||||
},
|
||||
wantLevel: 2,
|
||||
},
|
||||
{
|
||||
name: "merge_level2_blocks",
|
||||
sourceBlocks: []sourceBlock{
|
||||
{
|
||||
level: 2,
|
||||
series: []seriesData{
|
||||
{ref: 1, labels: labels.FromStrings("__name__", "a"), samples: []sample{{1000, 1.0}}},
|
||||
},
|
||||
},
|
||||
{
|
||||
level: 2,
|
||||
series: []seriesData{
|
||||
{ref: 1, labels: labels.FromStrings("__name__", "a"), samples: []sample{{5000, 5.0}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantSeriesRefs: []uint64{1},
|
||||
wantSamples: map[uint64][]sample{
|
||||
1: {{1000, 1.0}, {5000, 5.0}},
|
||||
},
|
||||
wantLevel: 3,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
c := New(dataDir, []int64{2 * hour, 8 * hour, 32 * hour}, 0, nil)
|
||||
|
||||
// Create source blocks.
|
||||
var sources []*block.Reader
|
||||
for _, sb := range tc.sourceBlocks {
|
||||
r := createSourceBlock(t, dataDir, sb)
|
||||
sources = append(sources, r)
|
||||
}
|
||||
|
||||
// Compact.
|
||||
newULID, err := c.Compact(sources)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, newULID)
|
||||
|
||||
// Close source blocks.
|
||||
for _, s := range sources {
|
||||
s.Close()
|
||||
}
|
||||
|
||||
// Open compacted block.
|
||||
compacted, err := block.Open(filepath.Join(dataDir, newULID))
|
||||
require.NoError(t, err)
|
||||
defer compacted.Close()
|
||||
|
||||
// Verify compaction level.
|
||||
assert.Equal(t, tc.wantLevel, compacted.Meta.Compaction.Level, "compaction level")
|
||||
assert.Equal(t, len(tc.sourceBlocks), len(compacted.Meta.Compaction.Sources), "source count")
|
||||
|
||||
// Verify series count.
|
||||
assert.Equal(t, len(tc.wantSeriesRefs), compacted.Meta.Stats.NumSeries, "series count")
|
||||
|
||||
// Verify each series' samples.
|
||||
for _, ref := range tc.wantSeriesRefs {
|
||||
got := collectBlockSamples(t, compacted, ref)
|
||||
want := tc.wantSamples[ref]
|
||||
require.Equal(t, len(want), len(got), "sample count for ref %d", ref)
|
||||
for i := range want {
|
||||
assert.Equal(t, want[i].t, got[i].t, "ref %d sample %d t", ref, i)
|
||||
assert.Equal(t, want[i].v, got[i].v, "ref %d sample %d v", ref, i)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpired(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
blocks []blockSpec
|
||||
now int64
|
||||
retention int64
|
||||
wantCount int
|
||||
}{
|
||||
{
|
||||
name: "no_retention",
|
||||
blocks: []blockSpec{{minT: 0, maxT: 1000}},
|
||||
now: 100000,
|
||||
retention: 0,
|
||||
wantCount: 0,
|
||||
},
|
||||
{
|
||||
name: "all_within_retention",
|
||||
blocks: []blockSpec{
|
||||
{minT: 80000, maxT: 90000},
|
||||
},
|
||||
now: 100000,
|
||||
retention: 50000,
|
||||
wantCount: 0,
|
||||
},
|
||||
{
|
||||
name: "one_expired",
|
||||
blocks: []blockSpec{
|
||||
{minT: 0, maxT: 10000},
|
||||
{minT: 80000, maxT: 90000},
|
||||
},
|
||||
now: 100000,
|
||||
retention: 50000,
|
||||
wantCount: 1,
|
||||
},
|
||||
{
|
||||
name: "all_expired",
|
||||
blocks: []blockSpec{
|
||||
{minT: 0, maxT: 10000},
|
||||
{minT: 20000, maxT: 30000},
|
||||
},
|
||||
now: 100000,
|
||||
retention: 50000,
|
||||
wantCount: 2,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
clock := func() int64 { return tc.now }
|
||||
c := New(dataDir, nil, tc.retention, clock)
|
||||
|
||||
var blocks []*block.Reader
|
||||
for _, spec := range tc.blocks {
|
||||
r := createBlockWithMeta(t, dataDir, spec)
|
||||
blocks = append(blocks, r)
|
||||
t.Cleanup(func() { r.Close() })
|
||||
}
|
||||
|
||||
expired := c.Expired(blocks)
|
||||
assert.Equal(t, tc.wantCount, len(expired), "expired block count")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
type blockSpec struct {
|
||||
minT int64
|
||||
maxT int64
|
||||
level int
|
||||
}
|
||||
|
||||
type sourceBlock struct {
|
||||
level int
|
||||
series []seriesData
|
||||
}
|
||||
|
||||
type seriesData struct {
|
||||
ref uint64
|
||||
labels []labels.Label
|
||||
samples []sample
|
||||
}
|
||||
|
||||
// createBlockWithMeta creates a minimal block with the given time range and level.
|
||||
func createBlockWithMeta(t *testing.T, dataDir string, spec blockSpec) *block.Reader {
|
||||
t.Helper()
|
||||
// Need at least one series with samples spanning [minT, maxT].
|
||||
timestamps := []int64{spec.minT, spec.maxT}
|
||||
values := []float64{1.0, 2.0}
|
||||
data := makeChunk(t, timestamps, values)
|
||||
|
||||
series := []block.SeriesFlush{{
|
||||
Ref: 1,
|
||||
Labels: labels.FromStrings("__name__", "test"),
|
||||
Chunks: []block.ChunkData{{MinT: spec.minT, MaxT: spec.maxT, Data: data}},
|
||||
}}
|
||||
|
||||
level := spec.level
|
||||
if level == 0 {
|
||||
level = 1
|
||||
}
|
||||
return flushTestBlock(t, dataDir, series, level, []string{"src"})
|
||||
}
|
||||
|
||||
// createSourceBlock creates a block with the given series data.
|
||||
func createSourceBlock(t *testing.T, dataDir string, sb sourceBlock) *block.Reader {
|
||||
t.Helper()
|
||||
var flushData []block.SeriesFlush
|
||||
for _, sd := range sb.series {
|
||||
timestamps := make([]int64, len(sd.samples))
|
||||
values := make([]float64, len(sd.samples))
|
||||
for i, s := range sd.samples {
|
||||
timestamps[i] = s.t
|
||||
values[i] = s.v
|
||||
}
|
||||
data := makeChunk(t, timestamps, values)
|
||||
minT, maxT := sd.samples[0].t, sd.samples[len(sd.samples)-1].t
|
||||
flushData = append(flushData, block.SeriesFlush{
|
||||
Ref: sd.ref,
|
||||
Labels: sd.labels,
|
||||
Chunks: []block.ChunkData{{MinT: minT, MaxT: maxT, Data: data}},
|
||||
})
|
||||
}
|
||||
|
||||
level := sb.level
|
||||
if level == 0 {
|
||||
level = 1
|
||||
}
|
||||
return flushTestBlock(t, dataDir, flushData, level, []string{"src"})
|
||||
}
|
||||
Reference in New Issue
Block a user