Compaction, retention, and soak testing
Block refcounting (atomic refs + condemned flag) lets live queries survive concurrent compaction and retention. Levelled compactor merges 2h->8h->32h blocks via raw chunk passthrough. Retention drops blocks older than the configured window. Background goroutine drives flush/compact/retain cycles; exported RunCompaction/ApplyRetention allow deterministic test control via injectable clock. Soak test: 10k series x 48h simulated at 15s intervals (115M samples). Validates flat memory (158 MiB peak), bounded disk (13 blocks), and zero query errors during compaction. All existing tests refactored to table-driven with uniform assertions.
This commit is contained in:
@@ -30,6 +30,16 @@ type SeriesFlush struct {
|
||||
//
|
||||
// Returns the block ULID and any error.
|
||||
func Flush(dataDir string, series []SeriesFlush) (string, error) {
|
||||
return flushBlock(dataDir, series, 1, nil)
|
||||
}
|
||||
|
||||
// FlushCompacted writes a new immutable block from compacted series data,
|
||||
// recording the compaction level and source block ULIDs.
|
||||
func FlushCompacted(dataDir string, series []SeriesFlush, level int, sources []string) (string, error) {
|
||||
return flushBlock(dataDir, series, level, sources)
|
||||
}
|
||||
|
||||
func flushBlock(dataDir string, series []SeriesFlush, level int, sources []string) (string, error) {
|
||||
ulid := newULID()
|
||||
blockDir := filepath.Join(dataDir, ulid)
|
||||
|
||||
@@ -49,7 +59,12 @@ func Flush(dataDir string, series []SeriesFlush) (string, error) {
|
||||
)
|
||||
meta.ULID = ulid
|
||||
meta.Version = 1
|
||||
meta.Compaction = CompactionInfo{Level: 1, Sources: []string{ulid}}
|
||||
meta.Compaction = CompactionInfo{Level: level}
|
||||
if sources != nil {
|
||||
meta.Compaction.Sources = sources
|
||||
} else {
|
||||
meta.Compaction.Sources = []string{ulid}
|
||||
}
|
||||
meta.MinTime = int64(^uint64(0) >> 1) // max int64
|
||||
meta.MaxTime = int64(0)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user