Compaction, retention, and soak testing

Block refcounting (atomic refs + condemned flag) lets live queries
survive concurrent compaction and retention. Levelled compactor merges
2h->8h->32h blocks via raw chunk passthrough. Retention drops blocks
older than the configured window. Background goroutine drives
flush/compact/retain cycles; exported RunCompaction/ApplyRetention allow
deterministic test control via injectable clock.

Soak test: 10k series x 48h simulated at 15s intervals (115M samples).
Validates flat memory (158 MiB peak), bounded disk (13 blocks), and zero
query errors during compaction.

All existing tests refactored to table-driven with uniform assertions.
This commit is contained in:
2026-07-04 17:02:57 -04:00
parent 323a6f2951
commit 0356f2e082
10 changed files with 1748 additions and 341 deletions
+206
View File
@@ -0,0 +1,206 @@
// Package compact implements levelled compaction and retention for ingot blocks.
package compact
import (
"fmt"
"sort"
"git.dvdt.dev/david/ingot/internal/block"
"git.dvdt.dev/david/ingot/labels"
)
// Clock returns the current time in milliseconds since epoch.
type Clock func() int64
// Compactor manages levelled compaction and retention.
type Compactor struct {
dataDir string
levels []int64 // level durations in ms, e.g. [2h, 8h, 32h]
retention int64 // retention window in ms (0 = disabled)
clock Clock
}
// New creates a Compactor. levels are the compaction level durations in
// ascending order (e.g. 2h, 8h, 32h in milliseconds). retention is the
// maximum age of data in milliseconds (0 disables retention).
func New(dataDir string, levels []int64, retention int64, clock Clock) *Compactor {
return &Compactor{
dataDir: dataDir,
levels: levels,
retention: retention,
clock: clock,
}
}
// CompactionGroup describes a set of source blocks to compact.
type CompactionGroup struct {
Sources []*block.Reader
Level int // resulting compaction level
}
// Plan returns the first eligible compaction group, or nil if no compaction
// is needed. Lower levels are prioritized. A group requires at least 2
// blocks at the same compaction level whose combined time span fits within
// the next level's duration.
func (c *Compactor) Plan(blocks []*block.Reader) *CompactionGroup {
if len(blocks) < 2 {
return nil
}
// Group blocks by compaction level.
byLevel := make(map[int][]*block.Reader)
for _, b := range blocks {
lvl := b.Meta.Compaction.Level
byLevel[lvl] = append(byLevel[lvl], b)
}
// Sort levels ascending.
var levels []int
for lvl := range byLevel {
levels = append(levels, lvl)
}
sort.Ints(levels)
for _, lvl := range levels {
group := c.planLevel(byLevel[lvl], lvl)
if group != nil {
return group
}
}
return nil
}
// planLevel finds a compactable group within blocks at the same level.
func (c *Compactor) planLevel(blocks []*block.Reader, level int) *CompactionGroup {
if len(blocks) < 2 {
return nil
}
// Sort by MinTime.
sorted := make([]*block.Reader, len(blocks))
copy(sorted, blocks)
sort.Slice(sorted, func(i, j int) bool {
return sorted[i].Meta.MinTime < sorted[j].Meta.MinTime
})
// Determine the max span for the next level.
var maxSpan int64
if level-1 < len(c.levels) && level >= 1 {
// Find the next level's duration. Level 1 blocks should compact
// into level 2 when they span up to levels[1] (8h), etc.
if level < len(c.levels) {
maxSpan = c.levels[level]
}
}
if maxSpan == 0 {
// No higher level defined, or level 0 — use the second level duration.
if level < len(c.levels) {
maxSpan = c.levels[level]
} else {
return nil // already at max level
}
}
// Find the first group of consecutive blocks whose span fits maxSpan.
for i := 0; i < len(sorted)-1; i++ {
group := []*block.Reader{sorted[i]}
for j := i + 1; j < len(sorted); j++ {
span := sorted[j].Meta.MaxTime - sorted[i].Meta.MinTime
if span > maxSpan {
break
}
group = append(group, sorted[j])
}
if len(group) >= 2 {
return &CompactionGroup{
Sources: group,
Level: level + 1,
}
}
}
return nil
}
// Compact merges source blocks into a single new block. Returns the new
// block's ULID. The caller is responsible for swapping the block set and
// releasing source blocks.
func (c *Compactor) Compact(sources []*block.Reader) (string, error) {
if len(sources) == 0 {
return "", fmt.Errorf("compact: no source blocks")
}
merged := make(map[uint64]*mergedEntry)
for _, src := range sources {
for _, entry := range src.Series() {
me, ok := merged[entry.Ref]
if !ok {
me = &mergedEntry{
ref: entry.Ref,
labels: entry.Labels,
}
merged[entry.Ref] = me
}
for _, cm := range entry.Chunks {
raw, err := src.RawChunkData(cm.Ref)
if err != nil {
return "", fmt.Errorf("compact: read chunk ref %v from %s: %w",
cm.Ref, src.Meta.ULID, err)
}
// Copy the raw bytes since the source may be munmapped later.
data := make([]byte, len(raw))
copy(data, raw)
me.chunks = append(me.chunks, block.ChunkData{
MinT: cm.MinT,
MaxT: cm.MaxT,
Data: data,
})
}
}
}
// Build flush data sorted by ref for deterministic output.
flushData := make([]block.SeriesFlush, 0, len(merged))
for _, me := range merged {
flushData = append(flushData, block.SeriesFlush{
Ref: me.ref,
Labels: me.labels,
Chunks: me.chunks,
})
}
sort.Slice(flushData, func(i, j int) bool { return flushData[i].Ref < flushData[j].Ref })
// Determine new compaction level and collect source ULIDs.
maxLevel := 0
sourceULIDs := make([]string, 0, len(sources))
for _, src := range sources {
if src.Meta.Compaction.Level > maxLevel {
maxLevel = src.Meta.Compaction.Level
}
sourceULIDs = append(sourceULIDs, src.Meta.ULID)
}
return block.FlushCompacted(c.dataDir, flushData, maxLevel+1, sourceULIDs)
}
// Expired returns blocks whose MaxTime is older than the retention window.
// Returns nil if retention is disabled (zero).
func (c *Compactor) Expired(blocks []*block.Reader) []*block.Reader {
if c.retention == 0 {
return nil
}
cutoff := c.clock() - c.retention
var expired []*block.Reader
for _, b := range blocks {
if b.Meta.MaxTime < cutoff {
expired = append(expired, b)
}
}
return expired
}
type mergedEntry struct {
ref uint64
labels []labels.Label
chunks []block.ChunkData
}
+430
View File
@@ -0,0 +1,430 @@
package compact
import (
"math"
"path/filepath"
"testing"
"git.dvdt.dev/david/ingot/internal/block"
"git.dvdt.dev/david/ingot/internal/chunkenc"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
hour = 3600 * 1000 // 1 hour in ms
)
// makeChunk creates a chunk with the given samples and returns its raw bytes.
func makeChunk(t *testing.T, timestamps []int64, values []float64) []byte {
t.Helper()
c := chunkenc.NewXORChunk()
a, err := c.Appender()
require.NoError(t, err)
for i := range timestamps {
a.Append(timestamps[i], values[i])
}
return append([]byte(nil), c.Bytes()...)
}
// flushTestBlock creates a block in dataDir and returns an opened Reader.
func flushTestBlock(t *testing.T, dataDir string, series []block.SeriesFlush, level int, sources []string) *block.Reader {
t.Helper()
var ulid string
var err error
if level == 1 && sources == nil {
ulid, err = block.Flush(dataDir, series)
} else {
ulid, err = block.FlushCompacted(dataDir, series, level, sources)
}
require.NoError(t, err)
r, err := block.Open(filepath.Join(dataDir, ulid))
require.NoError(t, err)
return r
}
// collectBlockSamples reads all samples for a series ref from a block.
func collectBlockSamples(t *testing.T, r *block.Reader, ref uint64) []sample {
t.Helper()
it, err := r.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64)
require.NoError(t, err)
var out []sample
for it.Next() {
ts, v := it.At()
out = append(out, sample{ts, v})
}
require.NoError(t, it.Err())
return out
}
type sample struct {
t int64
v float64
}
func TestPlan(t *testing.T) {
tests := []struct {
name string
blocks []blockSpec // blocks to create
wantGroup bool // expect a compaction group
wantLevel int // expected resulting level
wantCount int // expected number of sources in group
}{
{
name: "no_blocks",
blocks: nil,
wantGroup: false,
},
{
name: "single_block",
blocks: []blockSpec{
{minT: 0, maxT: 2 * hour, level: 1},
},
wantGroup: false,
},
{
name: "two_level1_blocks_within_8h",
blocks: []blockSpec{
{minT: 0, maxT: 2 * hour, level: 1},
{minT: 2 * hour, maxT: 4 * hour, level: 1},
},
wantGroup: true,
wantLevel: 2,
wantCount: 2,
},
{
name: "four_level1_blocks",
blocks: []blockSpec{
{minT: 0, maxT: 2 * hour, level: 1},
{minT: 2 * hour, maxT: 4 * hour, level: 1},
{minT: 4 * hour, maxT: 6 * hour, level: 1},
{minT: 6 * hour, maxT: 8 * hour, level: 1},
},
wantGroup: true,
wantLevel: 2,
wantCount: 4,
},
{
name: "level1_blocks_exceed_8h_span",
blocks: []blockSpec{
{minT: 0, maxT: 2 * hour, level: 1},
{minT: 7 * hour, maxT: 9 * hour, level: 1},
},
wantGroup: false,
},
{
name: "two_level2_blocks_within_32h",
blocks: []blockSpec{
{minT: 0, maxT: 8 * hour, level: 2},
{minT: 8 * hour, maxT: 16 * hour, level: 2},
},
wantGroup: true,
wantLevel: 3,
wantCount: 2,
},
{
name: "max_level_blocks_not_compacted",
blocks: []blockSpec{
{minT: 0, maxT: 32 * hour, level: 3},
{minT: 32 * hour, maxT: 64 * hour, level: 3},
},
wantGroup: false,
},
{
name: "mixed_levels_lower_prioritized",
blocks: []blockSpec{
{minT: 0, maxT: 2 * hour, level: 1},
{minT: 2 * hour, maxT: 4 * hour, level: 1},
{minT: 10 * hour, maxT: 18 * hour, level: 2},
{minT: 18 * hour, maxT: 26 * hour, level: 2},
},
wantGroup: true,
wantLevel: 2, // level-1 group found first
wantCount: 2,
},
}
levels := []int64{2 * hour, 8 * hour, 32 * hour}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
dataDir := t.TempDir()
c := New(dataDir, levels, 0, nil)
var blocks []*block.Reader
for _, spec := range tc.blocks {
r := createBlockWithMeta(t, dataDir, spec)
blocks = append(blocks, r)
t.Cleanup(func() { r.Close() })
}
group := c.Plan(blocks)
if !tc.wantGroup {
assert.Nil(t, group, "expected no compaction group")
return
}
require.NotNil(t, group, "expected a compaction group")
assert.Equal(t, tc.wantLevel, group.Level, "compaction level")
assert.Equal(t, tc.wantCount, len(group.Sources), "source count")
})
}
}
func TestCompact(t *testing.T) {
tests := []struct {
name string
sourceBlocks []sourceBlock
wantSeriesRefs []uint64
wantSamples map[uint64][]sample
wantLevel int
}{
{
name: "merge_two_blocks_disjoint_series",
sourceBlocks: []sourceBlock{
{
level: 1,
series: []seriesData{
{ref: 1, labels: labels.FromStrings("__name__", "a"), samples: []sample{{1000, 1.0}, {2000, 2.0}}},
},
},
{
level: 1,
series: []seriesData{
{ref: 2, labels: labels.FromStrings("__name__", "b"), samples: []sample{{1000, 3.0}, {2000, 4.0}}},
},
},
},
wantSeriesRefs: []uint64{1, 2},
wantSamples: map[uint64][]sample{
1: {{1000, 1.0}, {2000, 2.0}},
2: {{1000, 3.0}, {2000, 4.0}},
},
wantLevel: 2,
},
{
name: "merge_two_blocks_same_series",
sourceBlocks: []sourceBlock{
{
level: 1,
series: []seriesData{
{ref: 1, labels: labels.FromStrings("__name__", "temp"), samples: []sample{{1000, 1.0}, {2000, 2.0}}},
},
},
{
level: 1,
series: []seriesData{
{ref: 1, labels: labels.FromStrings("__name__", "temp"), samples: []sample{{3000, 3.0}, {4000, 4.0}}},
},
},
},
wantSeriesRefs: []uint64{1},
wantSamples: map[uint64][]sample{
1: {{1000, 1.0}, {2000, 2.0}, {3000, 3.0}, {4000, 4.0}},
},
wantLevel: 2,
},
{
name: "merge_level2_blocks",
sourceBlocks: []sourceBlock{
{
level: 2,
series: []seriesData{
{ref: 1, labels: labels.FromStrings("__name__", "a"), samples: []sample{{1000, 1.0}}},
},
},
{
level: 2,
series: []seriesData{
{ref: 1, labels: labels.FromStrings("__name__", "a"), samples: []sample{{5000, 5.0}}},
},
},
},
wantSeriesRefs: []uint64{1},
wantSamples: map[uint64][]sample{
1: {{1000, 1.0}, {5000, 5.0}},
},
wantLevel: 3,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
dataDir := t.TempDir()
c := New(dataDir, []int64{2 * hour, 8 * hour, 32 * hour}, 0, nil)
// Create source blocks.
var sources []*block.Reader
for _, sb := range tc.sourceBlocks {
r := createSourceBlock(t, dataDir, sb)
sources = append(sources, r)
}
// Compact.
newULID, err := c.Compact(sources)
require.NoError(t, err)
require.NotEmpty(t, newULID)
// Close source blocks.
for _, s := range sources {
s.Close()
}
// Open compacted block.
compacted, err := block.Open(filepath.Join(dataDir, newULID))
require.NoError(t, err)
defer compacted.Close()
// Verify compaction level.
assert.Equal(t, tc.wantLevel, compacted.Meta.Compaction.Level, "compaction level")
assert.Equal(t, len(tc.sourceBlocks), len(compacted.Meta.Compaction.Sources), "source count")
// Verify series count.
assert.Equal(t, len(tc.wantSeriesRefs), compacted.Meta.Stats.NumSeries, "series count")
// Verify each series' samples.
for _, ref := range tc.wantSeriesRefs {
got := collectBlockSamples(t, compacted, ref)
want := tc.wantSamples[ref]
require.Equal(t, len(want), len(got), "sample count for ref %d", ref)
for i := range want {
assert.Equal(t, want[i].t, got[i].t, "ref %d sample %d t", ref, i)
assert.Equal(t, want[i].v, got[i].v, "ref %d sample %d v", ref, i)
}
}
})
}
}
func TestExpired(t *testing.T) {
tests := []struct {
name string
blocks []blockSpec
now int64
retention int64
wantCount int
}{
{
name: "no_retention",
blocks: []blockSpec{{minT: 0, maxT: 1000}},
now: 100000,
retention: 0,
wantCount: 0,
},
{
name: "all_within_retention",
blocks: []blockSpec{
{minT: 80000, maxT: 90000},
},
now: 100000,
retention: 50000,
wantCount: 0,
},
{
name: "one_expired",
blocks: []blockSpec{
{minT: 0, maxT: 10000},
{minT: 80000, maxT: 90000},
},
now: 100000,
retention: 50000,
wantCount: 1,
},
{
name: "all_expired",
blocks: []blockSpec{
{minT: 0, maxT: 10000},
{minT: 20000, maxT: 30000},
},
now: 100000,
retention: 50000,
wantCount: 2,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
dataDir := t.TempDir()
clock := func() int64 { return tc.now }
c := New(dataDir, nil, tc.retention, clock)
var blocks []*block.Reader
for _, spec := range tc.blocks {
r := createBlockWithMeta(t, dataDir, spec)
blocks = append(blocks, r)
t.Cleanup(func() { r.Close() })
}
expired := c.Expired(blocks)
assert.Equal(t, tc.wantCount, len(expired), "expired block count")
})
}
}
// --- helpers ---
type blockSpec struct {
minT int64
maxT int64
level int
}
type sourceBlock struct {
level int
series []seriesData
}
type seriesData struct {
ref uint64
labels []labels.Label
samples []sample
}
// createBlockWithMeta creates a minimal block with the given time range and level.
func createBlockWithMeta(t *testing.T, dataDir string, spec blockSpec) *block.Reader {
t.Helper()
// Need at least one series with samples spanning [minT, maxT].
timestamps := []int64{spec.minT, spec.maxT}
values := []float64{1.0, 2.0}
data := makeChunk(t, timestamps, values)
series := []block.SeriesFlush{{
Ref: 1,
Labels: labels.FromStrings("__name__", "test"),
Chunks: []block.ChunkData{{MinT: spec.minT, MaxT: spec.maxT, Data: data}},
}}
level := spec.level
if level == 0 {
level = 1
}
return flushTestBlock(t, dataDir, series, level, []string{"src"})
}
// createSourceBlock creates a block with the given series data.
func createSourceBlock(t *testing.T, dataDir string, sb sourceBlock) *block.Reader {
t.Helper()
var flushData []block.SeriesFlush
for _, sd := range sb.series {
timestamps := make([]int64, len(sd.samples))
values := make([]float64, len(sd.samples))
for i, s := range sd.samples {
timestamps[i] = s.t
values[i] = s.v
}
data := makeChunk(t, timestamps, values)
minT, maxT := sd.samples[0].t, sd.samples[len(sd.samples)-1].t
flushData = append(flushData, block.SeriesFlush{
Ref: sd.ref,
Labels: sd.labels,
Chunks: []block.ChunkData{{MinT: minT, MaxT: maxT, Data: data}},
})
}
level := sb.level
if level == 0 {
level = 1
}
return flushTestBlock(t, dataDir, flushData, level, []string{"src"})
}