Compaction, retention, and soak testing

Block refcounting (atomic refs + condemned flag) lets live queries
survive concurrent compaction and retention. Levelled compactor merges
2h->8h->32h blocks via raw chunk passthrough. Retention drops blocks
older than the configured window. Background goroutine drives
flush/compact/retain cycles; exported RunCompaction/ApplyRetention allow
deterministic test control via injectable clock.

Soak test: 10k series x 48h simulated at 15s intervals (115M samples).
Validates flat memory (158 MiB peak), bounded disk (13 blocks), and zero
query errors during compaction.

All existing tests refactored to table-driven with uniform assertions.
This commit is contained in:
2026-07-04 17:02:57 -04:00
parent 323a6f2951
commit 0356f2e082
10 changed files with 1748 additions and 341 deletions
+35 -6
View File
@@ -3,6 +3,7 @@ package block
import (
"os"
"path/filepath"
"sync/atomic"
"git.dvdt.dev/david/ingot/internal/chunkenc"
"git.dvdt.dev/david/ingot/internal/index"
@@ -11,10 +12,12 @@ import (
// Reader provides read access to an immutable on-disk block.
type Reader struct {
dir string
Meta BlockMeta
idx *index.Reader
chunks *chunkReader
dir string
Meta BlockMeta
idx *index.Reader
chunks *chunkReader
refs atomic.Int32
condemned atomic.Bool
}
// Open opens a block directory for reading. Chunk files are mmap'd.
@@ -39,12 +42,14 @@ func Open(dir string) (*Reader, error) {
return nil, err
}
return &Reader{
r := &Reader{
dir: dir,
Meta: meta,
idx: idx,
chunks: cr,
}, nil
}
r.refs.Store(1) // DB's ownership ref
return r, nil
}
// Series returns all series entries from the index.
@@ -115,6 +120,30 @@ func (r *Reader) AllPostings() []uint64 {
return r.idx.AllPostings()
}
// Dir returns the block directory path.
func (r *Reader) Dir() string { return r.dir }
// Ref increments the refcount. Called by Querier on snapshot.
func (r *Reader) Ref() { r.refs.Add(1) }
// Release decrements the refcount. Returns true if the refcount hit zero
// and the block is condemned (caller should delete the directory).
func (r *Reader) Release() bool {
if r.refs.Add(-1) == 0 {
r.Close()
return r.condemned.Load()
}
return false
}
// Condemn marks the block for directory deletion when refcount reaches zero.
func (r *Reader) Condemn() { r.condemned.Store(true) }
// RawChunkData returns the raw chunk bytes at the given ref (for compaction).
func (r *Reader) RawChunkData(ref index.ChunkRef) ([]byte, error) {
return r.chunks.chunkData(ref)
}
// Close releases all resources (munmaps chunk files).
func (r *Reader) Close() error {
return r.chunks.close()