Compaction, retention, and soak testing

Block refcounting (atomic refs + condemned flag) lets live queries
survive concurrent compaction and retention. Levelled compactor merges
2h->8h->32h blocks via raw chunk passthrough. Retention drops blocks
older than the configured window. Background goroutine drives
flush/compact/retain cycles; exported RunCompaction/ApplyRetention allow
deterministic test control via injectable clock.

Soak test: 10k series x 48h simulated at 15s intervals (115M samples).
Validates flat memory (158 MiB peak), bounded disk (13 blocks), and zero
query errors during compaction.

All existing tests refactored to table-driven with uniform assertions.
This commit is contained in:
2026-07-04 17:02:57 -04:00
parent 323a6f2951
commit 0356f2e082
10 changed files with 1748 additions and 341 deletions
+16 -1
View File
@@ -30,6 +30,16 @@ type SeriesFlush struct {
//
// Returns the block ULID and any error.
func Flush(dataDir string, series []SeriesFlush) (string, error) {
return flushBlock(dataDir, series, 1, nil)
}
// FlushCompacted writes a new immutable block from compacted series data,
// recording the compaction level and source block ULIDs.
func FlushCompacted(dataDir string, series []SeriesFlush, level int, sources []string) (string, error) {
return flushBlock(dataDir, series, level, sources)
}
func flushBlock(dataDir string, series []SeriesFlush, level int, sources []string) (string, error) {
ulid := newULID()
blockDir := filepath.Join(dataDir, ulid)
@@ -49,7 +59,12 @@ func Flush(dataDir string, series []SeriesFlush) (string, error) {
)
meta.ULID = ulid
meta.Version = 1
meta.Compaction = CompactionInfo{Level: 1, Sources: []string{ulid}}
meta.Compaction = CompactionInfo{Level: level}
if sources != nil {
meta.Compaction.Sources = sources
} else {
meta.Compaction.Sources = []string{ulid}
}
meta.MinTime = int64(^uint64(0) >> 1) // max int64
meta.MaxTime = int64(0)