Compaction, retention, and soak testing
Block refcounting (atomic refs + condemned flag) lets live queries survive concurrent compaction and retention. Levelled compactor merges 2h->8h->32h blocks via raw chunk passthrough. Retention drops blocks older than the configured window. Background goroutine drives flush/compact/retain cycles; exported RunCompaction/ApplyRetention allow deterministic test control via injectable clock. Soak test: 10k series x 48h simulated at 15s intervals (115M samples). Validates flat memory (158 MiB peak), bounded disk (13 blocks), and zero query errors during compaction. All existing tests refactored to table-driven with uniform assertions.
This commit is contained in:
+162
-94
@@ -8,6 +8,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"git.dvdt.dev/david/ingot/internal/chunkenc"
|
||||
"git.dvdt.dev/david/ingot/internal/index"
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -181,13 +182,49 @@ func TestBlockRoundTrip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockMultipleChunksPerSeries(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
|
||||
func TestBlockSeriesChunkIterator(t *testing.T) {
|
||||
chunk1Samples := []sample{s(1000, 1.0), s(1015, 2.0), s(1030, 3.0)}
|
||||
chunk2Samples := []sample{s(2000, 4.0), s(2015, 5.0), s(2030, 6.0)}
|
||||
allSamples := append(chunk1Samples, chunk2Samples...)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
ref uint64
|
||||
mint int64
|
||||
maxt int64
|
||||
wantSamples []sample
|
||||
}{
|
||||
{
|
||||
name: "full_range",
|
||||
ref: 1,
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
wantSamples: append(chunk1Samples, chunk2Samples...),
|
||||
},
|
||||
{
|
||||
name: "second_chunk_only",
|
||||
ref: 1,
|
||||
mint: 2000,
|
||||
maxt: 3000,
|
||||
wantSamples: chunk2Samples,
|
||||
},
|
||||
{
|
||||
name: "no_overlap",
|
||||
ref: 1,
|
||||
mint: 5000,
|
||||
maxt: 6000,
|
||||
wantSamples: nil,
|
||||
},
|
||||
{
|
||||
name: "unknown_ref",
|
||||
ref: 999,
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
wantSamples: nil,
|
||||
},
|
||||
}
|
||||
|
||||
// Setup: create a block with two chunks for series ref=1.
|
||||
dataDir := t.TempDir()
|
||||
flushData := []SeriesFlush{
|
||||
{
|
||||
Ref: 1,
|
||||
@@ -198,71 +235,76 @@ func TestBlockMultipleChunksPerSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ulid, err := Flush(dataDir, flushData)
|
||||
require.NoError(t, err)
|
||||
|
||||
r, err := Open(filepath.Join(dataDir, ulid))
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
|
||||
// Full range.
|
||||
it, err := r.SeriesChunkIterator(1, math.MinInt64, math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
got := collectIterator(t, it)
|
||||
require.Equal(t, len(allSamples), len(got))
|
||||
for i, want := range allSamples {
|
||||
assert.Equal(t, want.t, got[i].t, "sample %d t", i)
|
||||
assert.Equal(t, want.vBits, got[i].vBits, "sample %d v", i)
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
it, err := r.SeriesChunkIterator(tc.ref, tc.mint, tc.maxt)
|
||||
require.NoError(t, err)
|
||||
got := collectIterator(t, it)
|
||||
assert.Equal(t, len(tc.wantSamples), len(got), "sample count")
|
||||
for i, want := range tc.wantSamples {
|
||||
assert.Equal(t, want.t, got[i].t, "sample %d t", i)
|
||||
assert.Equal(t, want.vBits, got[i].vBits, "sample %d v", i)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Query only second chunk's range.
|
||||
it, err = r.SeriesChunkIterator(1, 2000, 3000)
|
||||
require.NoError(t, err)
|
||||
got = collectIterator(t, it)
|
||||
require.Equal(t, len(chunk2Samples), len(got))
|
||||
for i, want := range chunk2Samples {
|
||||
assert.Equal(t, want.t, got[i].t, "sample %d t", i)
|
||||
assert.Equal(t, want.vBits, got[i].vBits, "sample %d v", i)
|
||||
}
|
||||
|
||||
// Query with no overlap.
|
||||
it, err = r.SeriesChunkIterator(1, 5000, 6000)
|
||||
require.NoError(t, err)
|
||||
got = collectIterator(t, it)
|
||||
assert.Empty(t, got)
|
||||
|
||||
// Unknown ref.
|
||||
it, err = r.SeriesChunkIterator(999, math.MinInt64, math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, it.Next())
|
||||
}
|
||||
|
||||
func TestBlockMetaTimeBounds(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
|
||||
flushData := []SeriesFlush{
|
||||
tests := []struct {
|
||||
name string
|
||||
series []SeriesFlush
|
||||
wantMinT int64
|
||||
wantMaxT int64
|
||||
}{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "a"}},
|
||||
Chunks: []ChunkData{{MinT: 500, MaxT: 1000, Data: makeChunk(t, []sample{s(500, 1.0), s(1000, 2.0)})}},
|
||||
name: "two_series_different_ranges",
|
||||
series: []SeriesFlush{
|
||||
{Ref: 1, Labels: []labels.Label{{Name: "__name__", Value: "a"}}, Chunks: []ChunkData{{MinT: 500, MaxT: 1000, Data: makeChunkFromPairs([]int64{500, 1000}, []float64{1.0, 2.0})}}},
|
||||
{Ref: 2, Labels: []labels.Label{{Name: "__name__", Value: "b"}}, Chunks: []ChunkData{{MinT: 200, MaxT: 800, Data: makeChunkFromPairs([]int64{200, 800}, []float64{3.0, 4.0})}}},
|
||||
},
|
||||
wantMinT: 200,
|
||||
wantMaxT: 1000,
|
||||
},
|
||||
{
|
||||
Ref: 2,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "b"}},
|
||||
Chunks: []ChunkData{{MinT: 200, MaxT: 800, Data: makeChunk(t, []sample{s(200, 3.0), s(800, 4.0)})}},
|
||||
name: "single_series",
|
||||
series: []SeriesFlush{
|
||||
{Ref: 1, Labels: []labels.Label{{Name: "__name__", Value: "a"}}, Chunks: []ChunkData{{MinT: 100, MaxT: 500, Data: makeChunkFromPairs([]int64{100, 500}, []float64{1.0, 2.0})}}},
|
||||
},
|
||||
wantMinT: 100,
|
||||
wantMaxT: 500,
|
||||
},
|
||||
{
|
||||
name: "multiple_chunks",
|
||||
series: []SeriesFlush{
|
||||
{Ref: 1, Labels: []labels.Label{{Name: "__name__", Value: "a"}}, Chunks: []ChunkData{
|
||||
{MinT: 100, MaxT: 200, Data: makeChunkFromPairs([]int64{100, 200}, []float64{1.0, 2.0})},
|
||||
{MinT: 300, MaxT: 900, Data: makeChunkFromPairs([]int64{300, 900}, []float64{3.0, 4.0})},
|
||||
}},
|
||||
},
|
||||
wantMinT: 100,
|
||||
wantMaxT: 900,
|
||||
},
|
||||
}
|
||||
|
||||
ulid, err := Flush(dataDir, flushData)
|
||||
require.NoError(t, err)
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
ulid, err := Flush(dataDir, tc.series)
|
||||
require.NoError(t, err)
|
||||
r, err := Open(filepath.Join(dataDir, ulid))
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
|
||||
r, err := Open(filepath.Join(dataDir, ulid))
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
|
||||
assert.Equal(t, int64(200), r.Meta.MinTime)
|
||||
assert.Equal(t, int64(1000), r.Meta.MaxTime)
|
||||
assert.Equal(t, tc.wantMinT, r.Meta.MinTime, "MinTime")
|
||||
assert.Equal(t, tc.wantMaxT, r.Meta.MaxTime, "MaxTime")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestULIDRoundTrip(t *testing.T) {
|
||||
@@ -276,57 +318,83 @@ func TestULIDRoundTrip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorruptChunkCRC(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
|
||||
samples := []sample{s(1000, 71.3), s(1015, 71.4)}
|
||||
flushData := []SeriesFlush{
|
||||
func TestBlockCorruption(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
corruptFunc func(t *testing.T, blockDir string, chunkRef index.ChunkRef)
|
||||
wantErr error
|
||||
}{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "temp"}},
|
||||
Chunks: []ChunkData{{MinT: 1000, MaxT: 1015, Data: makeChunk(t, samples)}},
|
||||
name: "corrupt_chunk_data_byte",
|
||||
corruptFunc: func(t *testing.T, blockDir string, chunkRef index.ChunkRef) {
|
||||
chunkPath := filepath.Join(blockDir, chunksDirName, segmentName(int(chunkRef.Segment())))
|
||||
data, err := os.ReadFile(chunkPath)
|
||||
require.NoError(t, err)
|
||||
off := int(chunkRef.Offset()) + chunkEntryHeaderLen + 1
|
||||
data[off] ^= 0xFF
|
||||
require.NoError(t, os.WriteFile(chunkPath, data, 0644))
|
||||
},
|
||||
wantErr: ErrCorruptChunk,
|
||||
},
|
||||
{
|
||||
name: "corrupt_chunk_crc",
|
||||
corruptFunc: func(t *testing.T, blockDir string, chunkRef index.ChunkRef) {
|
||||
chunkPath := filepath.Join(blockDir, chunksDirName, segmentName(int(chunkRef.Segment())))
|
||||
data, err := os.ReadFile(chunkPath)
|
||||
require.NoError(t, err)
|
||||
// Corrupt the last byte of the CRC.
|
||||
off := int(chunkRef.Offset()) + chunkEntryHeaderLen
|
||||
// Read dataLen to find CRC position.
|
||||
dataLen := int(data[chunkRef.Offset()])*16777216 + int(data[chunkRef.Offset()+1])*65536 +
|
||||
int(data[chunkRef.Offset()+2])*256 + int(data[chunkRef.Offset()+3])
|
||||
crcOff := off + dataLen + 3 // last byte of CRC
|
||||
data[crcOff] ^= 0xFF
|
||||
require.NoError(t, os.WriteFile(chunkPath, data, 0644))
|
||||
},
|
||||
wantErr: ErrCorruptChunk,
|
||||
},
|
||||
}
|
||||
|
||||
ulid, err := Flush(dataDir, flushData)
|
||||
require.NoError(t, err)
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
samples := []sample{s(1000, 71.3), s(1015, 71.4)}
|
||||
flushData := []SeriesFlush{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "temp"}},
|
||||
Chunks: []ChunkData{{MinT: 1000, MaxT: 1015, Data: makeChunk(t, samples)}},
|
||||
},
|
||||
}
|
||||
ulid, err := Flush(dataDir, flushData)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Open block, find the chunk ref, then corrupt the chunk file.
|
||||
blockDir := filepath.Join(dataDir, ulid)
|
||||
r, err := Open(blockDir)
|
||||
require.NoError(t, err)
|
||||
blockDir := filepath.Join(dataDir, ulid)
|
||||
r, err := Open(blockDir)
|
||||
require.NoError(t, err)
|
||||
series := r.Series()
|
||||
require.Equal(t, 1, len(series))
|
||||
chunkRef := series[0].Chunks[0].Ref
|
||||
r.Close()
|
||||
|
||||
series := r.Series()
|
||||
require.Equal(t, 1, len(series))
|
||||
chunkRef := series[0].Chunks[0].Ref
|
||||
r.Close()
|
||||
tc.corruptFunc(t, blockDir, chunkRef)
|
||||
|
||||
// Corrupt chunk data on disk.
|
||||
chunkSeg := chunkRef.Segment()
|
||||
chunkPath := filepath.Join(blockDir, chunksDirName, segmentName(int(chunkSeg)))
|
||||
r, err = Open(blockDir)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
|
||||
// Read, corrupt a data byte, write back.
|
||||
chunkFile, err := readFileBytes(chunkPath)
|
||||
require.NoError(t, err)
|
||||
off := int(chunkRef.Offset()) + chunkEntryHeaderLen + 1 // corrupt a data byte
|
||||
if off < len(chunkFile) {
|
||||
chunkFile[off] ^= 0xFF
|
||||
_, err = r.ChunkIterator(chunkRef)
|
||||
assert.Equal(t, tc.wantErr, err)
|
||||
})
|
||||
}
|
||||
require.NoError(t, writeFileBytes(chunkPath, chunkFile))
|
||||
|
||||
// Re-open and try to read the corrupt chunk.
|
||||
r, err = Open(blockDir)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
|
||||
_, err = r.ChunkIterator(chunkRef)
|
||||
assert.Equal(t, ErrCorruptChunk, err)
|
||||
}
|
||||
|
||||
func readFileBytes(path string) ([]byte, error) {
|
||||
return os.ReadFile(path)
|
||||
}
|
||||
|
||||
func writeFileBytes(path string, data []byte) error {
|
||||
return os.WriteFile(path, data, 0644)
|
||||
// makeChunkFromPairs creates a chunk from timestamp/value slices.
|
||||
func makeChunkFromPairs(ts []int64, vs []float64) []byte {
|
||||
c := chunkenc.NewXORChunk()
|
||||
a, _ := c.Appender()
|
||||
for i := range ts {
|
||||
a.Append(ts[i], vs[i])
|
||||
}
|
||||
return append([]byte(nil), c.Bytes()...)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package block
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
|
||||
"git.dvdt.dev/david/ingot/internal/chunkenc"
|
||||
"git.dvdt.dev/david/ingot/internal/index"
|
||||
@@ -11,10 +12,12 @@ import (
|
||||
|
||||
// Reader provides read access to an immutable on-disk block.
|
||||
type Reader struct {
|
||||
dir string
|
||||
Meta BlockMeta
|
||||
idx *index.Reader
|
||||
chunks *chunkReader
|
||||
dir string
|
||||
Meta BlockMeta
|
||||
idx *index.Reader
|
||||
chunks *chunkReader
|
||||
refs atomic.Int32
|
||||
condemned atomic.Bool
|
||||
}
|
||||
|
||||
// Open opens a block directory for reading. Chunk files are mmap'd.
|
||||
@@ -39,12 +42,14 @@ func Open(dir string) (*Reader, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Reader{
|
||||
r := &Reader{
|
||||
dir: dir,
|
||||
Meta: meta,
|
||||
idx: idx,
|
||||
chunks: cr,
|
||||
}, nil
|
||||
}
|
||||
r.refs.Store(1) // DB's ownership ref
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Series returns all series entries from the index.
|
||||
@@ -115,6 +120,30 @@ func (r *Reader) AllPostings() []uint64 {
|
||||
return r.idx.AllPostings()
|
||||
}
|
||||
|
||||
// Dir returns the block directory path.
|
||||
func (r *Reader) Dir() string { return r.dir }
|
||||
|
||||
// Ref increments the refcount. Called by Querier on snapshot.
|
||||
func (r *Reader) Ref() { r.refs.Add(1) }
|
||||
|
||||
// Release decrements the refcount. Returns true if the refcount hit zero
|
||||
// and the block is condemned (caller should delete the directory).
|
||||
func (r *Reader) Release() bool {
|
||||
if r.refs.Add(-1) == 0 {
|
||||
r.Close()
|
||||
return r.condemned.Load()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Condemn marks the block for directory deletion when refcount reaches zero.
|
||||
func (r *Reader) Condemn() { r.condemned.Store(true) }
|
||||
|
||||
// RawChunkData returns the raw chunk bytes at the given ref (for compaction).
|
||||
func (r *Reader) RawChunkData(ref index.ChunkRef) ([]byte, error) {
|
||||
return r.chunks.chunkData(ref)
|
||||
}
|
||||
|
||||
// Close releases all resources (munmaps chunk files).
|
||||
func (r *Reader) Close() error {
|
||||
return r.chunks.close()
|
||||
|
||||
@@ -30,6 +30,16 @@ type SeriesFlush struct {
|
||||
//
|
||||
// Returns the block ULID and any error.
|
||||
func Flush(dataDir string, series []SeriesFlush) (string, error) {
|
||||
return flushBlock(dataDir, series, 1, nil)
|
||||
}
|
||||
|
||||
// FlushCompacted writes a new immutable block from compacted series data,
|
||||
// recording the compaction level and source block ULIDs.
|
||||
func FlushCompacted(dataDir string, series []SeriesFlush, level int, sources []string) (string, error) {
|
||||
return flushBlock(dataDir, series, level, sources)
|
||||
}
|
||||
|
||||
func flushBlock(dataDir string, series []SeriesFlush, level int, sources []string) (string, error) {
|
||||
ulid := newULID()
|
||||
blockDir := filepath.Join(dataDir, ulid)
|
||||
|
||||
@@ -49,7 +59,12 @@ func Flush(dataDir string, series []SeriesFlush) (string, error) {
|
||||
)
|
||||
meta.ULID = ulid
|
||||
meta.Version = 1
|
||||
meta.Compaction = CompactionInfo{Level: 1, Sources: []string{ulid}}
|
||||
meta.Compaction = CompactionInfo{Level: level}
|
||||
if sources != nil {
|
||||
meta.Compaction.Sources = sources
|
||||
} else {
|
||||
meta.Compaction.Sources = []string{ulid}
|
||||
}
|
||||
meta.MinTime = int64(^uint64(0) >> 1) // max int64
|
||||
meta.MaxTime = int64(0)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user