david
42b03db2fa
Flush sealed head chunks to ULID-named block directories on disk. Each block contains CRC'd chunk segment files (mmap'd for reads), a binary index (symbol table, series, postings with TOC), and a meta.json written last as the immutability gate. WAL is truncated after block fsync, preserving the crash-safety ording invariant.
333 lines
8.3 KiB
Go
333 lines
8.3 KiB
Go
package block
|
|
|
|
import (
|
|
"math"
|
|
"math/rand"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"git.dvdt.dev/david/ingot/internal/chunkenc"
|
|
"git.dvdt.dev/david/ingot/labels"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
type sample struct {
|
|
t int64
|
|
vBits uint64
|
|
}
|
|
|
|
func s(t int64, v float64) sample { return sample{t, math.Float64bits(v)} }
|
|
|
|
// makeChunk creates a chunk with the given samples and returns its raw bytes.
|
|
func makeChunk(t *testing.T, samples []sample) []byte {
|
|
t.Helper()
|
|
c := chunkenc.NewXORChunk()
|
|
a, err := c.Appender()
|
|
require.NoError(t, err)
|
|
for _, s := range samples {
|
|
a.Append(s.t, math.Float64frombits(s.vBits))
|
|
}
|
|
return append([]byte(nil), c.Bytes()...)
|
|
}
|
|
|
|
// collectIterator reads all samples from a ChunkIterator.
|
|
func collectIterator(t *testing.T, it chunkenc.ChunkIterator) []sample {
|
|
t.Helper()
|
|
var out []sample
|
|
for it.Next() {
|
|
ts, v := it.At()
|
|
out = append(out, sample{ts, math.Float64bits(v)})
|
|
}
|
|
require.NoError(t, it.Err())
|
|
return out
|
|
}
|
|
|
|
func TestBlockRoundTrip(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
series []struct {
|
|
ref uint64
|
|
labels []labels.Label
|
|
samples []sample
|
|
}
|
|
}{
|
|
{
|
|
name: "single_series_single_chunk",
|
|
series: []struct {
|
|
ref uint64
|
|
labels []labels.Label
|
|
samples []sample
|
|
}{
|
|
{
|
|
ref: 1,
|
|
labels: []labels.Label{{Name: "__name__", Value: "temp"}},
|
|
samples: []sample{
|
|
s(1000, 71.3), s(1015, 71.4), s(1030, 71.5),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "multiple_series",
|
|
series: []struct {
|
|
ref uint64
|
|
labels []labels.Label
|
|
samples []sample
|
|
}{
|
|
{
|
|
ref: 1,
|
|
labels: []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}},
|
|
samples: []sample{
|
|
s(1000, 71.3), s(1015, 71.4), s(1030, 71.5),
|
|
},
|
|
},
|
|
{
|
|
ref: 2,
|
|
labels: []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "office"}},
|
|
samples: []sample{
|
|
s(1000, 55.0), s(1015, 54.5), s(1030, 54.0),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "large_chunk",
|
|
series: []struct {
|
|
ref uint64
|
|
labels []labels.Label
|
|
samples []sample
|
|
}{
|
|
{
|
|
ref: 1,
|
|
labels: []labels.Label{{Name: "__name__", Value: "counter"}},
|
|
samples: func() []sample {
|
|
rnd := rand.New(rand.NewSource(42))
|
|
out := make([]sample, 120)
|
|
ts, v := int64(0), 0.0
|
|
for i := range out {
|
|
out[i] = s(ts, v)
|
|
ts += 15000 + int64(rnd.Intn(100)) - 50
|
|
v += rnd.Float64()
|
|
}
|
|
return out
|
|
}(),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
dataDir := t.TempDir()
|
|
|
|
// Build SeriesFlush data.
|
|
var flushData []SeriesFlush
|
|
for _, s := range tc.series {
|
|
data := makeChunk(t, s.samples)
|
|
minT, maxT := s.samples[0].t, s.samples[len(s.samples)-1].t
|
|
flushData = append(flushData, SeriesFlush{
|
|
Ref: s.ref,
|
|
Labels: s.labels,
|
|
Chunks: []ChunkData{{MinT: minT, MaxT: maxT, Data: data}},
|
|
})
|
|
}
|
|
|
|
// Write block.
|
|
ulid, err := Flush(dataDir, flushData)
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, ulid)
|
|
|
|
// Open block for reading.
|
|
blockDir := filepath.Join(dataDir, ulid)
|
|
r, err := Open(blockDir)
|
|
require.NoError(t, err)
|
|
defer r.Close()
|
|
|
|
// Verify meta.
|
|
assert.Equal(t, ulid, r.Meta.ULID)
|
|
assert.Equal(t, 1, r.Meta.Version)
|
|
assert.Equal(t, len(tc.series), r.Meta.Stats.NumSeries)
|
|
assert.Equal(t, len(tc.series), r.Meta.Stats.NumChunks)
|
|
|
|
// Verify each series' data via iteration.
|
|
for _, s := range tc.series {
|
|
it, err := r.SeriesChunkIterator(s.ref, math.MinInt64, math.MaxInt64)
|
|
require.NoError(t, err)
|
|
got := collectIterator(t, it)
|
|
require.Equal(t, len(s.samples), len(got), "ref %d sample count", s.ref)
|
|
for i, want := range s.samples {
|
|
assert.Equal(t, want.t, got[i].t, "ref %d sample %d t", s.ref, i)
|
|
assert.Equal(t, want.vBits, got[i].vBits, "ref %d sample %d v", s.ref, i)
|
|
}
|
|
}
|
|
|
|
// Verify postings.
|
|
for _, s := range tc.series {
|
|
for _, l := range s.labels {
|
|
refs := r.Postings(l.Name, l.Value)
|
|
assert.Contains(t, refs, s.ref, "postings for %s=%s", l.Name, l.Value)
|
|
}
|
|
}
|
|
|
|
// Verify labels lookup.
|
|
for _, s := range tc.series {
|
|
ls, ok := r.Labels(s.ref)
|
|
assert.True(t, ok)
|
|
assert.Equal(t, s.labels, ls)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBlockMultipleChunksPerSeries(t *testing.T) {
|
|
dataDir := t.TempDir()
|
|
|
|
chunk1Samples := []sample{s(1000, 1.0), s(1015, 2.0), s(1030, 3.0)}
|
|
chunk2Samples := []sample{s(2000, 4.0), s(2015, 5.0), s(2030, 6.0)}
|
|
allSamples := append(chunk1Samples, chunk2Samples...)
|
|
|
|
flushData := []SeriesFlush{
|
|
{
|
|
Ref: 1,
|
|
Labels: []labels.Label{{Name: "__name__", Value: "temp"}},
|
|
Chunks: []ChunkData{
|
|
{MinT: 1000, MaxT: 1030, Data: makeChunk(t, chunk1Samples)},
|
|
{MinT: 2000, MaxT: 2030, Data: makeChunk(t, chunk2Samples)},
|
|
},
|
|
},
|
|
}
|
|
|
|
ulid, err := Flush(dataDir, flushData)
|
|
require.NoError(t, err)
|
|
|
|
r, err := Open(filepath.Join(dataDir, ulid))
|
|
require.NoError(t, err)
|
|
defer r.Close()
|
|
|
|
// Full range.
|
|
it, err := r.SeriesChunkIterator(1, math.MinInt64, math.MaxInt64)
|
|
require.NoError(t, err)
|
|
got := collectIterator(t, it)
|
|
require.Equal(t, len(allSamples), len(got))
|
|
for i, want := range allSamples {
|
|
assert.Equal(t, want.t, got[i].t, "sample %d t", i)
|
|
assert.Equal(t, want.vBits, got[i].vBits, "sample %d v", i)
|
|
}
|
|
|
|
// Query only second chunk's range.
|
|
it, err = r.SeriesChunkIterator(1, 2000, 3000)
|
|
require.NoError(t, err)
|
|
got = collectIterator(t, it)
|
|
require.Equal(t, len(chunk2Samples), len(got))
|
|
for i, want := range chunk2Samples {
|
|
assert.Equal(t, want.t, got[i].t, "sample %d t", i)
|
|
assert.Equal(t, want.vBits, got[i].vBits, "sample %d v", i)
|
|
}
|
|
|
|
// Query with no overlap.
|
|
it, err = r.SeriesChunkIterator(1, 5000, 6000)
|
|
require.NoError(t, err)
|
|
got = collectIterator(t, it)
|
|
assert.Empty(t, got)
|
|
|
|
// Unknown ref.
|
|
it, err = r.SeriesChunkIterator(999, math.MinInt64, math.MaxInt64)
|
|
require.NoError(t, err)
|
|
assert.False(t, it.Next())
|
|
}
|
|
|
|
func TestBlockMetaTimeBounds(t *testing.T) {
|
|
dataDir := t.TempDir()
|
|
|
|
flushData := []SeriesFlush{
|
|
{
|
|
Ref: 1,
|
|
Labels: []labels.Label{{Name: "__name__", Value: "a"}},
|
|
Chunks: []ChunkData{{MinT: 500, MaxT: 1000, Data: makeChunk(t, []sample{s(500, 1.0), s(1000, 2.0)})}},
|
|
},
|
|
{
|
|
Ref: 2,
|
|
Labels: []labels.Label{{Name: "__name__", Value: "b"}},
|
|
Chunks: []ChunkData{{MinT: 200, MaxT: 800, Data: makeChunk(t, []sample{s(200, 3.0), s(800, 4.0)})}},
|
|
},
|
|
}
|
|
|
|
ulid, err := Flush(dataDir, flushData)
|
|
require.NoError(t, err)
|
|
|
|
r, err := Open(filepath.Join(dataDir, ulid))
|
|
require.NoError(t, err)
|
|
defer r.Close()
|
|
|
|
assert.Equal(t, int64(200), r.Meta.MinTime)
|
|
assert.Equal(t, int64(1000), r.Meta.MaxTime)
|
|
}
|
|
|
|
func TestULIDRoundTrip(t *testing.T) {
|
|
for i := 0; i < 100; i++ {
|
|
u := newULID()
|
|
assert.Equal(t, 26, len(u), "ULID length")
|
|
|
|
// Should parse without error.
|
|
_, err := parseULID(u)
|
|
assert.NoError(t, err, "parse ULID %q", u)
|
|
}
|
|
}
|
|
|
|
func TestCorruptChunkCRC(t *testing.T) {
|
|
dataDir := t.TempDir()
|
|
|
|
samples := []sample{s(1000, 71.3), s(1015, 71.4)}
|
|
flushData := []SeriesFlush{
|
|
{
|
|
Ref: 1,
|
|
Labels: []labels.Label{{Name: "__name__", Value: "temp"}},
|
|
Chunks: []ChunkData{{MinT: 1000, MaxT: 1015, Data: makeChunk(t, samples)}},
|
|
},
|
|
}
|
|
|
|
ulid, err := Flush(dataDir, flushData)
|
|
require.NoError(t, err)
|
|
|
|
// Open block, find the chunk ref, then corrupt the chunk file.
|
|
blockDir := filepath.Join(dataDir, ulid)
|
|
r, err := Open(blockDir)
|
|
require.NoError(t, err)
|
|
|
|
series := r.Series()
|
|
require.Equal(t, 1, len(series))
|
|
chunkRef := series[0].Chunks[0].Ref
|
|
r.Close()
|
|
|
|
// Corrupt chunk data on disk.
|
|
chunkSeg := chunkRef.Segment()
|
|
chunkPath := filepath.Join(blockDir, chunksDirName, segmentName(int(chunkSeg)))
|
|
|
|
// Read, corrupt a data byte, write back.
|
|
chunkFile, err := readFileBytes(chunkPath)
|
|
require.NoError(t, err)
|
|
off := int(chunkRef.Offset()) + chunkEntryHeaderLen + 1 // corrupt a data byte
|
|
if off < len(chunkFile) {
|
|
chunkFile[off] ^= 0xFF
|
|
}
|
|
require.NoError(t, writeFileBytes(chunkPath, chunkFile))
|
|
|
|
// Re-open and try to read the corrupt chunk.
|
|
r, err = Open(blockDir)
|
|
require.NoError(t, err)
|
|
defer r.Close()
|
|
|
|
_, err = r.ChunkIterator(chunkRef)
|
|
assert.Equal(t, ErrCorruptChunk, err)
|
|
}
|
|
|
|
func readFileBytes(path string) ([]byte, error) {
|
|
return os.ReadFile(path)
|
|
}
|
|
|
|
func writeFileBytes(path string, data []byte) error {
|
|
return os.WriteFile(path, data, 0644)
|
|
}
|