Immutable blocks with mmap reads
Flush sealed head chunks to ULID-named block directories on disk. Each block contains CRC'd chunk segment files (mmap'd for reads), a binary index (symbol table, series, postings with TOC), and a meta.json written last as the immutability gate. WAL is truncated after block fsync, preserving the crash-safety ording invariant.
This commit is contained in:
@@ -0,0 +1,332 @@
|
||||
package block
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.dvdt.dev/david/ingot/internal/chunkenc"
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type sample struct {
|
||||
t int64
|
||||
vBits uint64
|
||||
}
|
||||
|
||||
func s(t int64, v float64) sample { return sample{t, math.Float64bits(v)} }
|
||||
|
||||
// makeChunk creates a chunk with the given samples and returns its raw bytes.
|
||||
func makeChunk(t *testing.T, samples []sample) []byte {
|
||||
t.Helper()
|
||||
c := chunkenc.NewXORChunk()
|
||||
a, err := c.Appender()
|
||||
require.NoError(t, err)
|
||||
for _, s := range samples {
|
||||
a.Append(s.t, math.Float64frombits(s.vBits))
|
||||
}
|
||||
return append([]byte(nil), c.Bytes()...)
|
||||
}
|
||||
|
||||
// collectIterator reads all samples from a ChunkIterator.
|
||||
func collectIterator(t *testing.T, it chunkenc.ChunkIterator) []sample {
|
||||
t.Helper()
|
||||
var out []sample
|
||||
for it.Next() {
|
||||
ts, v := it.At()
|
||||
out = append(out, sample{ts, math.Float64bits(v)})
|
||||
}
|
||||
require.NoError(t, it.Err())
|
||||
return out
|
||||
}
|
||||
|
||||
func TestBlockRoundTrip(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
series []struct {
|
||||
ref uint64
|
||||
labels []labels.Label
|
||||
samples []sample
|
||||
}
|
||||
}{
|
||||
{
|
||||
name: "single_series_single_chunk",
|
||||
series: []struct {
|
||||
ref uint64
|
||||
labels []labels.Label
|
||||
samples []sample
|
||||
}{
|
||||
{
|
||||
ref: 1,
|
||||
labels: []labels.Label{{Name: "__name__", Value: "temp"}},
|
||||
samples: []sample{
|
||||
s(1000, 71.3), s(1015, 71.4), s(1030, 71.5),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple_series",
|
||||
series: []struct {
|
||||
ref uint64
|
||||
labels []labels.Label
|
||||
samples []sample
|
||||
}{
|
||||
{
|
||||
ref: 1,
|
||||
labels: []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}},
|
||||
samples: []sample{
|
||||
s(1000, 71.3), s(1015, 71.4), s(1030, 71.5),
|
||||
},
|
||||
},
|
||||
{
|
||||
ref: 2,
|
||||
labels: []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "office"}},
|
||||
samples: []sample{
|
||||
s(1000, 55.0), s(1015, 54.5), s(1030, 54.0),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "large_chunk",
|
||||
series: []struct {
|
||||
ref uint64
|
||||
labels []labels.Label
|
||||
samples []sample
|
||||
}{
|
||||
{
|
||||
ref: 1,
|
||||
labels: []labels.Label{{Name: "__name__", Value: "counter"}},
|
||||
samples: func() []sample {
|
||||
rnd := rand.New(rand.NewSource(42))
|
||||
out := make([]sample, 120)
|
||||
ts, v := int64(0), 0.0
|
||||
for i := range out {
|
||||
out[i] = s(ts, v)
|
||||
ts += 15000 + int64(rnd.Intn(100)) - 50
|
||||
v += rnd.Float64()
|
||||
}
|
||||
return out
|
||||
}(),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
|
||||
// Build SeriesFlush data.
|
||||
var flushData []SeriesFlush
|
||||
for _, s := range tc.series {
|
||||
data := makeChunk(t, s.samples)
|
||||
minT, maxT := s.samples[0].t, s.samples[len(s.samples)-1].t
|
||||
flushData = append(flushData, SeriesFlush{
|
||||
Ref: s.ref,
|
||||
Labels: s.labels,
|
||||
Chunks: []ChunkData{{MinT: minT, MaxT: maxT, Data: data}},
|
||||
})
|
||||
}
|
||||
|
||||
// Write block.
|
||||
ulid, err := Flush(dataDir, flushData)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, ulid)
|
||||
|
||||
// Open block for reading.
|
||||
blockDir := filepath.Join(dataDir, ulid)
|
||||
r, err := Open(blockDir)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
|
||||
// Verify meta.
|
||||
assert.Equal(t, ulid, r.Meta.ULID)
|
||||
assert.Equal(t, 1, r.Meta.Version)
|
||||
assert.Equal(t, len(tc.series), r.Meta.Stats.NumSeries)
|
||||
assert.Equal(t, len(tc.series), r.Meta.Stats.NumChunks)
|
||||
|
||||
// Verify each series' data via iteration.
|
||||
for _, s := range tc.series {
|
||||
it, err := r.SeriesChunkIterator(s.ref, math.MinInt64, math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
got := collectIterator(t, it)
|
||||
require.Equal(t, len(s.samples), len(got), "ref %d sample count", s.ref)
|
||||
for i, want := range s.samples {
|
||||
assert.Equal(t, want.t, got[i].t, "ref %d sample %d t", s.ref, i)
|
||||
assert.Equal(t, want.vBits, got[i].vBits, "ref %d sample %d v", s.ref, i)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify postings.
|
||||
for _, s := range tc.series {
|
||||
for _, l := range s.labels {
|
||||
refs := r.Postings(l.Name, l.Value)
|
||||
assert.Contains(t, refs, s.ref, "postings for %s=%s", l.Name, l.Value)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify labels lookup.
|
||||
for _, s := range tc.series {
|
||||
ls, ok := r.Labels(s.ref)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, s.labels, ls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlockMultipleChunksPerSeries(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
|
||||
chunk1Samples := []sample{s(1000, 1.0), s(1015, 2.0), s(1030, 3.0)}
|
||||
chunk2Samples := []sample{s(2000, 4.0), s(2015, 5.0), s(2030, 6.0)}
|
||||
allSamples := append(chunk1Samples, chunk2Samples...)
|
||||
|
||||
flushData := []SeriesFlush{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "temp"}},
|
||||
Chunks: []ChunkData{
|
||||
{MinT: 1000, MaxT: 1030, Data: makeChunk(t, chunk1Samples)},
|
||||
{MinT: 2000, MaxT: 2030, Data: makeChunk(t, chunk2Samples)},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ulid, err := Flush(dataDir, flushData)
|
||||
require.NoError(t, err)
|
||||
|
||||
r, err := Open(filepath.Join(dataDir, ulid))
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
|
||||
// Full range.
|
||||
it, err := r.SeriesChunkIterator(1, math.MinInt64, math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
got := collectIterator(t, it)
|
||||
require.Equal(t, len(allSamples), len(got))
|
||||
for i, want := range allSamples {
|
||||
assert.Equal(t, want.t, got[i].t, "sample %d t", i)
|
||||
assert.Equal(t, want.vBits, got[i].vBits, "sample %d v", i)
|
||||
}
|
||||
|
||||
// Query only second chunk's range.
|
||||
it, err = r.SeriesChunkIterator(1, 2000, 3000)
|
||||
require.NoError(t, err)
|
||||
got = collectIterator(t, it)
|
||||
require.Equal(t, len(chunk2Samples), len(got))
|
||||
for i, want := range chunk2Samples {
|
||||
assert.Equal(t, want.t, got[i].t, "sample %d t", i)
|
||||
assert.Equal(t, want.vBits, got[i].vBits, "sample %d v", i)
|
||||
}
|
||||
|
||||
// Query with no overlap.
|
||||
it, err = r.SeriesChunkIterator(1, 5000, 6000)
|
||||
require.NoError(t, err)
|
||||
got = collectIterator(t, it)
|
||||
assert.Empty(t, got)
|
||||
|
||||
// Unknown ref.
|
||||
it, err = r.SeriesChunkIterator(999, math.MinInt64, math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, it.Next())
|
||||
}
|
||||
|
||||
func TestBlockMetaTimeBounds(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
|
||||
flushData := []SeriesFlush{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "a"}},
|
||||
Chunks: []ChunkData{{MinT: 500, MaxT: 1000, Data: makeChunk(t, []sample{s(500, 1.0), s(1000, 2.0)})}},
|
||||
},
|
||||
{
|
||||
Ref: 2,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "b"}},
|
||||
Chunks: []ChunkData{{MinT: 200, MaxT: 800, Data: makeChunk(t, []sample{s(200, 3.0), s(800, 4.0)})}},
|
||||
},
|
||||
}
|
||||
|
||||
ulid, err := Flush(dataDir, flushData)
|
||||
require.NoError(t, err)
|
||||
|
||||
r, err := Open(filepath.Join(dataDir, ulid))
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
|
||||
assert.Equal(t, int64(200), r.Meta.MinTime)
|
||||
assert.Equal(t, int64(1000), r.Meta.MaxTime)
|
||||
}
|
||||
|
||||
func TestULIDRoundTrip(t *testing.T) {
|
||||
for i := 0; i < 100; i++ {
|
||||
u := newULID()
|
||||
assert.Equal(t, 26, len(u), "ULID length")
|
||||
|
||||
// Should parse without error.
|
||||
_, err := parseULID(u)
|
||||
assert.NoError(t, err, "parse ULID %q", u)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorruptChunkCRC(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
|
||||
samples := []sample{s(1000, 71.3), s(1015, 71.4)}
|
||||
flushData := []SeriesFlush{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "temp"}},
|
||||
Chunks: []ChunkData{{MinT: 1000, MaxT: 1015, Data: makeChunk(t, samples)}},
|
||||
},
|
||||
}
|
||||
|
||||
ulid, err := Flush(dataDir, flushData)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Open block, find the chunk ref, then corrupt the chunk file.
|
||||
blockDir := filepath.Join(dataDir, ulid)
|
||||
r, err := Open(blockDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
series := r.Series()
|
||||
require.Equal(t, 1, len(series))
|
||||
chunkRef := series[0].Chunks[0].Ref
|
||||
r.Close()
|
||||
|
||||
// Corrupt chunk data on disk.
|
||||
chunkSeg := chunkRef.Segment()
|
||||
chunkPath := filepath.Join(blockDir, chunksDirName, segmentName(int(chunkSeg)))
|
||||
|
||||
// Read, corrupt a data byte, write back.
|
||||
chunkFile, err := readFileBytes(chunkPath)
|
||||
require.NoError(t, err)
|
||||
off := int(chunkRef.Offset()) + chunkEntryHeaderLen + 1 // corrupt a data byte
|
||||
if off < len(chunkFile) {
|
||||
chunkFile[off] ^= 0xFF
|
||||
}
|
||||
require.NoError(t, writeFileBytes(chunkPath, chunkFile))
|
||||
|
||||
// Re-open and try to read the corrupt chunk.
|
||||
r, err = Open(blockDir)
|
||||
require.NoError(t, err)
|
||||
defer r.Close()
|
||||
|
||||
_, err = r.ChunkIterator(chunkRef)
|
||||
assert.Equal(t, ErrCorruptChunk, err)
|
||||
}
|
||||
|
||||
func readFileBytes(path string) ([]byte, error) {
|
||||
return os.ReadFile(path)
|
||||
}
|
||||
|
||||
func writeFileBytes(path string, data []byte) error {
|
||||
return os.WriteFile(path, data, 0644)
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package block
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"syscall"
|
||||
|
||||
"git.dvdt.dev/david/ingot/internal/chunkenc"
|
||||
"git.dvdt.dev/david/ingot/internal/index"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidChunkMagic = errors.New("block: invalid chunk file magic")
|
||||
ErrInvalidChunkVersion = errors.New("block: unsupported chunk file version")
|
||||
ErrCorruptChunk = errors.New("block: corrupt chunk (CRC mismatch)")
|
||||
ErrChunkNotFound = errors.New("block: chunk ref out of bounds")
|
||||
)
|
||||
|
||||
// chunkReader reads chunk data from mmap'd segment files.
|
||||
type chunkReader struct {
|
||||
segments map[int][]byte // segment index -> mmap'd data
|
||||
}
|
||||
|
||||
func newChunkReader(blockDir string) (*chunkReader, error) {
|
||||
chunksDir := filepath.Join(blockDir, chunksDirName)
|
||||
entries, err := os.ReadDir(chunksDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cr := &chunkReader{segments: make(map[int][]byte)}
|
||||
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
idx := parseSegmentName(e.Name())
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
data, err := mmapFile(filepath.Join(chunksDir, e.Name()))
|
||||
if err != nil {
|
||||
cr.close()
|
||||
return nil, fmt.Errorf("block: mmap segment %s: %w", e.Name(), err)
|
||||
}
|
||||
|
||||
// Validate header.
|
||||
if len(data) < chunkHeaderLen {
|
||||
cr.close()
|
||||
return nil, ErrInvalidChunkMagic
|
||||
}
|
||||
magic := binary.BigEndian.Uint32(data[:4])
|
||||
if magic != chunkMagic {
|
||||
cr.close()
|
||||
return nil, ErrInvalidChunkMagic
|
||||
}
|
||||
if data[4] != chunkVersion {
|
||||
cr.close()
|
||||
return nil, ErrInvalidChunkVersion
|
||||
}
|
||||
|
||||
cr.segments[idx] = data
|
||||
}
|
||||
|
||||
return cr, nil
|
||||
}
|
||||
|
||||
// chunkData reads the raw chunk bytes at the given ref, validates CRC.
|
||||
func (cr *chunkReader) chunkData(ref index.ChunkRef) ([]byte, error) {
|
||||
seg := int(ref.Segment())
|
||||
off := int(ref.Offset())
|
||||
|
||||
data, ok := cr.segments[seg]
|
||||
if !ok {
|
||||
return nil, ErrChunkNotFound
|
||||
}
|
||||
|
||||
if off+chunkEntryHeaderLen > len(data) {
|
||||
return nil, ErrChunkNotFound
|
||||
}
|
||||
|
||||
dataLen := int(binary.BigEndian.Uint32(data[off : off+4]))
|
||||
encoding := data[off+4]
|
||||
off += chunkEntryHeaderLen
|
||||
|
||||
end := off + dataLen + chunkEntryCRCLen
|
||||
if end > len(data) {
|
||||
return nil, ErrChunkNotFound
|
||||
}
|
||||
|
||||
chunkBytes := data[off : off+dataLen]
|
||||
off += dataLen
|
||||
|
||||
// Validate CRC.
|
||||
wantCRC := binary.BigEndian.Uint32(data[off : off+4])
|
||||
crc := crc32.New(castagnoliTable)
|
||||
crc.Write([]byte{encoding})
|
||||
crc.Write(chunkBytes)
|
||||
if crc.Sum32() != wantCRC {
|
||||
return nil, ErrCorruptChunk
|
||||
}
|
||||
|
||||
return chunkBytes, nil
|
||||
}
|
||||
|
||||
// chunkIterator returns a ChunkIterator for the chunk at the given ref.
|
||||
func (cr *chunkReader) chunkIterator(ref index.ChunkRef) (chunkenc.ChunkIterator, error) {
|
||||
data, err := cr.chunkData(ref)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return chunkenc.XORChunkFromBytes(data).Iterator(), nil
|
||||
}
|
||||
|
||||
func (cr *chunkReader) close() error {
|
||||
for _, data := range cr.segments {
|
||||
syscall.Munmap(data)
|
||||
}
|
||||
cr.segments = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// segmentIndices returns sorted segment indices.
|
||||
func (cr *chunkReader) segmentIndices() []int {
|
||||
idxs := make([]int, 0, len(cr.segments))
|
||||
for idx := range cr.segments {
|
||||
idxs = append(idxs, idx)
|
||||
}
|
||||
sort.Ints(idxs)
|
||||
return idxs
|
||||
}
|
||||
|
||||
func mmapFile(path string) ([]byte, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
return nil, fmt.Errorf("block: empty file %s", path)
|
||||
}
|
||||
|
||||
data, err := syscall.Mmap(int(f.Fd()), 0, int(info.Size()),
|
||||
syscall.PROT_READ, syscall.MAP_PRIVATE)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func parseSegmentName(name string) int {
|
||||
if len(name) != 6 {
|
||||
return -1
|
||||
}
|
||||
n := 0
|
||||
for _, c := range name {
|
||||
if c < '0' || c > '9' {
|
||||
return -1
|
||||
}
|
||||
n = n*10 + int(c-'0')
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package block
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.dvdt.dev/david/ingot/internal/index"
|
||||
)
|
||||
|
||||
const (
|
||||
chunkMagic uint32 = 0x0FACE5C4
|
||||
chunkVersion byte = 1
|
||||
chunkHeaderLen = 5 // 4-byte magic + 1-byte version
|
||||
chunkEntryHeaderLen = 5 // 4-byte length + 1-byte encoding
|
||||
chunkEntryCRCLen = 4
|
||||
encodingXOR byte = 1
|
||||
chunkSegmentMaxSize = 512 * 1024 * 1024 // 512 MiB
|
||||
chunksDirName = "chunks"
|
||||
)
|
||||
|
||||
var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
|
||||
|
||||
// chunkWriter writes chunk data to segmented chunk files inside a block directory.
|
||||
type chunkWriter struct {
|
||||
dir string
|
||||
segmentIdx int
|
||||
segmentOff int
|
||||
f *os.File
|
||||
}
|
||||
|
||||
func newChunkWriter(blockDir string) (*chunkWriter, error) {
|
||||
chunksDir := filepath.Join(blockDir, chunksDirName)
|
||||
if err := os.MkdirAll(chunksDir, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cw := &chunkWriter{dir: chunksDir, segmentIdx: 1}
|
||||
if err := cw.newSegment(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cw, nil
|
||||
}
|
||||
|
||||
func (cw *chunkWriter) newSegment() error {
|
||||
if cw.f != nil {
|
||||
if err := cw.f.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cw.f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
name := segmentName(cw.segmentIdx)
|
||||
f, err := os.Create(filepath.Join(cw.dir, name))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cw.f = f
|
||||
cw.segmentOff = 0
|
||||
|
||||
// Write chunk file header.
|
||||
var hdr [chunkHeaderLen]byte
|
||||
binary.BigEndian.PutUint32(hdr[:4], chunkMagic)
|
||||
hdr[4] = chunkVersion
|
||||
n, err := f.Write(hdr[:])
|
||||
cw.segmentOff += n
|
||||
return err
|
||||
}
|
||||
|
||||
// writeChunk writes a chunk entry and returns its ChunkRef.
|
||||
// Format: dataLen(4) | encoding(1) | data(dataLen) | CRC32C(4)
|
||||
func (cw *chunkWriter) writeChunk(data []byte) (index.ChunkRef, error) {
|
||||
entrySize := chunkEntryHeaderLen + len(data) + chunkEntryCRCLen
|
||||
|
||||
// Rotate if this entry would exceed segment max size.
|
||||
if cw.segmentOff+entrySize > chunkSegmentMaxSize {
|
||||
cw.segmentIdx++
|
||||
if err := cw.newSegment(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
ref := index.NewChunkRef(uint32(cw.segmentIdx), uint32(cw.segmentOff))
|
||||
|
||||
// Header: length + encoding.
|
||||
var hdr [chunkEntryHeaderLen]byte
|
||||
binary.BigEndian.PutUint32(hdr[:4], uint32(len(data)))
|
||||
hdr[4] = encodingXOR
|
||||
if _, err := cw.f.Write(hdr[:]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Chunk data.
|
||||
if _, err := cw.f.Write(data); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// CRC over encoding + data.
|
||||
crc := crc32.New(castagnoliTable)
|
||||
crc.Write(hdr[4:5]) // encoding byte
|
||||
crc.Write(data)
|
||||
var crcBuf [4]byte
|
||||
binary.BigEndian.PutUint32(crcBuf[:], crc.Sum32())
|
||||
if _, err := cw.f.Write(crcBuf[:]); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
cw.segmentOff += entrySize
|
||||
return ref, nil
|
||||
}
|
||||
|
||||
func (cw *chunkWriter) close() error {
|
||||
if cw.f == nil {
|
||||
return nil
|
||||
}
|
||||
if err := cw.f.Sync(); err != nil {
|
||||
cw.f.Close()
|
||||
return err
|
||||
}
|
||||
return cw.f.Close()
|
||||
}
|
||||
|
||||
func segmentName(idx int) string {
|
||||
var buf [6]byte
|
||||
s := idx
|
||||
for i := 5; i >= 0; i-- {
|
||||
buf[i] = '0' + byte(s%10)
|
||||
s /= 10
|
||||
}
|
||||
return string(buf[:])
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package block
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// BlockMeta describes a block on disk.
|
||||
type BlockMeta struct {
|
||||
ULID string `json:"ulid"`
|
||||
MinTime int64 `json:"minTime"`
|
||||
MaxTime int64 `json:"maxTime"`
|
||||
Stats BlockStats `json:"stats"`
|
||||
Compaction CompactionInfo `json:"compaction"`
|
||||
Version int `json:"version"`
|
||||
}
|
||||
|
||||
// BlockStats holds summary statistics for a block.
|
||||
type BlockStats struct {
|
||||
NumSamples int `json:"numSamples"`
|
||||
NumSeries int `json:"numSeries"`
|
||||
NumChunks int `json:"numChunks"`
|
||||
}
|
||||
|
||||
// CompactionInfo records the block's compaction lineage.
|
||||
type CompactionInfo struct {
|
||||
Level int `json:"level"`
|
||||
Sources []string `json:"sources"`
|
||||
}
|
||||
|
||||
const metaFilename = "meta.json"
|
||||
|
||||
// readMeta reads a block's meta.json from the given block directory.
|
||||
func readMeta(dir string) (BlockMeta, error) {
|
||||
data, err := os.ReadFile(filepath.Join(dir, metaFilename))
|
||||
if err != nil {
|
||||
return BlockMeta{}, err
|
||||
}
|
||||
var m BlockMeta
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
return BlockMeta{}, err
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// writeMeta writes a block's meta.json to the given block directory.
|
||||
// This is the last file written when creating a block — the immutability gate.
|
||||
func writeMeta(dir string, m BlockMeta) error {
|
||||
data, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(filepath.Join(dir, metaFilename), data, 0644)
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package block
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.dvdt.dev/david/ingot/internal/chunkenc"
|
||||
"git.dvdt.dev/david/ingot/internal/index"
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
)
|
||||
|
||||
// Reader provides read access to an immutable on-disk block.
|
||||
type Reader struct {
|
||||
dir string
|
||||
Meta BlockMeta
|
||||
idx *index.Reader
|
||||
chunks *chunkReader
|
||||
}
|
||||
|
||||
// Open opens a block directory for reading. Chunk files are mmap'd.
|
||||
func Open(dir string) (*Reader, error) {
|
||||
meta, err := readMeta(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Read the index file into memory.
|
||||
indexData, err := os.ReadFile(filepath.Join(dir, "index"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
idx, err := index.NewReader(indexData)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cr, err := newChunkReader(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Reader{
|
||||
dir: dir,
|
||||
Meta: meta,
|
||||
idx: idx,
|
||||
chunks: cr,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Series returns all series entries from the index.
|
||||
func (r *Reader) Series() []index.SeriesEntry {
|
||||
return r.idx.Series()
|
||||
}
|
||||
|
||||
// SeriesByRef looks up a series by ref.
|
||||
func (r *Reader) SeriesByRef(ref uint64) (index.SeriesEntry, bool) {
|
||||
return r.idx.SeriesByRef(ref)
|
||||
}
|
||||
|
||||
// Postings returns sorted series refs matching label name=value.
|
||||
func (r *Reader) Postings(name, value string) []uint64 {
|
||||
return r.idx.Postings(name, value)
|
||||
}
|
||||
|
||||
// ChunkIterator returns an iterator for a chunk at the given ref.
|
||||
func (r *Reader) ChunkIterator(ref index.ChunkRef) (chunkenc.ChunkIterator, error) {
|
||||
return r.chunks.chunkIterator(ref)
|
||||
}
|
||||
|
||||
// SeriesChunkIterator returns an iterator over all chunks for a series in a
|
||||
// time range. Chunks are iterated in order.
|
||||
func (r *Reader) SeriesChunkIterator(ref uint64, mint, maxt int64) (chunkenc.ChunkIterator, error) {
|
||||
entry, ok := r.idx.SeriesByRef(ref)
|
||||
if !ok {
|
||||
return &emptyIterator{}, nil
|
||||
}
|
||||
|
||||
var iters []chunkenc.ChunkIterator
|
||||
for _, cm := range entry.Chunks {
|
||||
if cm.MaxT < mint || cm.MinT > maxt {
|
||||
continue
|
||||
}
|
||||
it, err := r.chunks.chunkIterator(cm.Ref)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
iters = append(iters, it)
|
||||
}
|
||||
|
||||
if len(iters) == 0 {
|
||||
return &emptyIterator{}, nil
|
||||
}
|
||||
if len(iters) == 1 {
|
||||
return iters[0], nil
|
||||
}
|
||||
return &multiIterator{iters: iters}, nil
|
||||
}
|
||||
|
||||
// Labels returns the labels for a series by ref.
|
||||
func (r *Reader) Labels(ref uint64) ([]labels.Label, bool) {
|
||||
entry, ok := r.idx.SeriesByRef(ref)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return entry.Labels, true
|
||||
}
|
||||
|
||||
// Close releases all resources (munmaps chunk files).
|
||||
func (r *Reader) Close() error {
|
||||
return r.chunks.close()
|
||||
}
|
||||
|
||||
// multiIterator chains multiple ChunkIterators in order.
|
||||
type multiIterator struct {
|
||||
iters []chunkenc.ChunkIterator
|
||||
cur int
|
||||
}
|
||||
|
||||
func (m *multiIterator) Next() bool {
|
||||
for m.cur < len(m.iters) {
|
||||
if m.iters[m.cur].Next() {
|
||||
return true
|
||||
}
|
||||
if m.iters[m.cur].Err() != nil {
|
||||
return false
|
||||
}
|
||||
m.cur++
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *multiIterator) At() (int64, float64) {
|
||||
return m.iters[m.cur].At()
|
||||
}
|
||||
|
||||
func (m *multiIterator) Err() error {
|
||||
if m.cur < len(m.iters) {
|
||||
return m.iters[m.cur].Err()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type emptyIterator struct{}
|
||||
|
||||
func (e *emptyIterator) Next() bool { return false }
|
||||
func (e *emptyIterator) At() (int64, float64) { return 0, 0 }
|
||||
func (e *emptyIterator) Err() error { return nil }
|
||||
@@ -0,0 +1,119 @@
|
||||
package block
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ULID encoding alphabet (Crockford's Base32).
|
||||
const ulidEncoding = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
|
||||
|
||||
// newULID generates a ULID with the current time and random entropy.
|
||||
// Zero external dependencies — the encoding is simple enough to inline.
|
||||
func newULID() string {
|
||||
var b [16]byte
|
||||
|
||||
// Timestamp: upper 48 bits = milliseconds since epoch.
|
||||
ms := uint64(time.Now().UnixMilli())
|
||||
b[0] = byte(ms >> 40)
|
||||
b[1] = byte(ms >> 32)
|
||||
b[2] = byte(ms >> 24)
|
||||
b[3] = byte(ms >> 16)
|
||||
b[4] = byte(ms >> 8)
|
||||
b[5] = byte(ms)
|
||||
|
||||
// Randomness: lower 80 bits.
|
||||
rand.Read(b[6:])
|
||||
|
||||
return encodeULID(b)
|
||||
}
|
||||
|
||||
// parseULID decodes a ULID string back to its 16-byte binary form.
|
||||
func parseULID(s string) ([16]byte, error) {
|
||||
var b [16]byte
|
||||
if len(s) != 26 {
|
||||
return b, errInvalidULID
|
||||
}
|
||||
|
||||
// Decode 26 base32 chars -> 130 bits -> 16 bytes + 2 spare bits.
|
||||
var val [26]byte
|
||||
for i := 0; i < 26; i++ {
|
||||
idx := strings.IndexByte(ulidEncoding, s[i])
|
||||
if idx < 0 {
|
||||
// Try lowercase.
|
||||
idx = strings.IndexByte(ulidEncoding, s[i]&^0x20)
|
||||
}
|
||||
if idx < 0 {
|
||||
return b, errInvalidULID
|
||||
}
|
||||
val[i] = byte(idx)
|
||||
}
|
||||
|
||||
// Pack 5-bit groups into bytes.
|
||||
b[0] = val[0]<<5 | val[1]
|
||||
b[1] = val[2]<<3 | val[3]>>2
|
||||
b[2] = val[3]<<6 | val[4]<<1 | val[5]>>4
|
||||
b[3] = val[5]<<4 | val[6]>>1
|
||||
b[4] = val[6]<<7 | val[7]<<2 | val[8]>>3
|
||||
b[5] = val[8]<<5 | val[9]
|
||||
b[6] = val[10]<<3 | val[11]>>2
|
||||
b[7] = val[11]<<6 | val[12]<<1 | val[13]>>4
|
||||
b[8] = val[13]<<4 | val[14]>>1
|
||||
b[9] = val[14]<<7 | val[15]<<2 | val[16]>>3
|
||||
b[10] = val[16]<<5 | val[17]
|
||||
b[11] = val[18]<<3 | val[19]>>2
|
||||
b[12] = val[19]<<6 | val[20]<<1 | val[21]>>4
|
||||
b[13] = val[21]<<4 | val[22]>>1
|
||||
b[14] = val[22]<<7 | val[23]<<2 | val[24]>>3
|
||||
b[15] = val[24]<<5 | val[25]
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// ulidTime extracts the millisecond timestamp from a ULID string.
|
||||
func ulidTime(s string) (int64, error) {
|
||||
b, err := parseULID(s)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Timestamp is in the upper 6 bytes.
|
||||
ms := int64(binary.BigEndian.Uint64(append([]byte{0, 0}, b[:6]...)))
|
||||
return ms, nil
|
||||
}
|
||||
|
||||
func encodeULID(b [16]byte) string {
|
||||
// 16 bytes = 128 bits, encoded in 26 base32 chars (130 bits, 2 spare).
|
||||
var dst [26]byte
|
||||
dst[0] = ulidEncoding[(b[0]&0xE0)>>5]
|
||||
dst[1] = ulidEncoding[b[0]&0x1F]
|
||||
dst[2] = ulidEncoding[(b[1]&0xF8)>>3]
|
||||
dst[3] = ulidEncoding[(b[1]&0x07)<<2|(b[2]&0xC0)>>6]
|
||||
dst[4] = ulidEncoding[(b[2]&0x3E)>>1]
|
||||
dst[5] = ulidEncoding[(b[2]&0x01)<<4|(b[3]&0xF0)>>4]
|
||||
dst[6] = ulidEncoding[(b[3]&0x0F)<<1|(b[4]&0x80)>>7]
|
||||
dst[7] = ulidEncoding[(b[4]&0x7C)>>2]
|
||||
dst[8] = ulidEncoding[(b[4]&0x03)<<3|(b[5]&0xE0)>>5]
|
||||
dst[9] = ulidEncoding[b[5]&0x1F]
|
||||
dst[10] = ulidEncoding[(b[6]&0xF8)>>3]
|
||||
dst[11] = ulidEncoding[(b[6]&0x07)<<2|(b[7]&0xC0)>>6]
|
||||
dst[12] = ulidEncoding[(b[7]&0x3E)>>1]
|
||||
dst[13] = ulidEncoding[(b[7]&0x01)<<4|(b[8]&0xF0)>>4]
|
||||
dst[14] = ulidEncoding[(b[8]&0x0F)<<1|(b[9]&0x80)>>7]
|
||||
dst[15] = ulidEncoding[(b[9]&0x7C)>>2]
|
||||
dst[16] = ulidEncoding[(b[9]&0x03)<<3|(b[10]&0xE0)>>5]
|
||||
dst[17] = ulidEncoding[b[10]&0x1F]
|
||||
dst[18] = ulidEncoding[(b[11]&0xF8)>>3]
|
||||
dst[19] = ulidEncoding[(b[11]&0x07)<<2|(b[12]&0xC0)>>6]
|
||||
dst[20] = ulidEncoding[(b[12]&0x3E)>>1]
|
||||
dst[21] = ulidEncoding[(b[12]&0x01)<<4|(b[13]&0xF0)>>4]
|
||||
dst[22] = ulidEncoding[(b[13]&0x0F)<<1|(b[14]&0x80)>>7]
|
||||
dst[23] = ulidEncoding[(b[14]&0x7C)>>2]
|
||||
dst[24] = ulidEncoding[(b[14]&0x03)<<3|(b[15]&0xE0)>>5]
|
||||
dst[25] = ulidEncoding[b[15]&0x1F]
|
||||
return string(dst[:])
|
||||
}
|
||||
|
||||
var errInvalidULID = errors.New("block: invalid ULID")
|
||||
@@ -0,0 +1,141 @@
|
||||
package block
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.dvdt.dev/david/ingot/internal/index"
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
)
|
||||
|
||||
// ChunkData describes a single chunk to be flushed to a block.
|
||||
type ChunkData struct {
|
||||
MinT int64
|
||||
MaxT int64
|
||||
Data []byte // raw XOR chunk bytes (including 2-byte sample count header)
|
||||
}
|
||||
|
||||
// SeriesFlush describes a series with its sealed chunks for block writing.
|
||||
type SeriesFlush struct {
|
||||
Ref uint64
|
||||
Labels []labels.Label
|
||||
Chunks []ChunkData
|
||||
}
|
||||
|
||||
// Flush writes a new immutable block from the given series data.
|
||||
// It creates a ULID-named directory under dataDir containing:
|
||||
// - chunks/ with segment files
|
||||
// - index file
|
||||
// - meta.json (written last as the immutability gate)
|
||||
//
|
||||
// Returns the block ULID and any error.
|
||||
func Flush(dataDir string, series []SeriesFlush) (string, error) {
|
||||
ulid := newULID()
|
||||
blockDir := filepath.Join(dataDir, ulid)
|
||||
|
||||
if err := os.MkdirAll(blockDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Write chunk files and collect index entries.
|
||||
cw, err := newChunkWriter(blockDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var (
|
||||
indexEntries []index.SeriesEntry
|
||||
meta BlockMeta
|
||||
)
|
||||
meta.ULID = ulid
|
||||
meta.Version = 1
|
||||
meta.Compaction = CompactionInfo{Level: 1, Sources: []string{ulid}}
|
||||
meta.MinTime = int64(^uint64(0) >> 1) // max int64
|
||||
meta.MaxTime = int64(0)
|
||||
|
||||
for _, sf := range series {
|
||||
var chunks []index.ChunkMeta
|
||||
for _, cd := range sf.Chunks {
|
||||
ref, err := cw.writeChunk(cd.Data)
|
||||
if err != nil {
|
||||
cw.close()
|
||||
return "", err
|
||||
}
|
||||
chunks = append(chunks, index.ChunkMeta{
|
||||
MinT: cd.MinT,
|
||||
MaxT: cd.MaxT,
|
||||
Ref: ref,
|
||||
})
|
||||
if cd.MinT < meta.MinTime {
|
||||
meta.MinTime = cd.MinT
|
||||
}
|
||||
if cd.MaxT > meta.MaxTime {
|
||||
meta.MaxTime = cd.MaxT
|
||||
}
|
||||
meta.Stats.NumChunks++
|
||||
// Count samples from the chunk's 2-byte header.
|
||||
if len(cd.Data) >= 2 {
|
||||
meta.Stats.NumSamples += int(uint16(cd.Data[0])<<8 | uint16(cd.Data[1]))
|
||||
}
|
||||
}
|
||||
indexEntries = append(indexEntries, index.SeriesEntry{
|
||||
Ref: sf.Ref,
|
||||
Labels: sf.Labels,
|
||||
Chunks: chunks,
|
||||
})
|
||||
meta.Stats.NumSeries++
|
||||
}
|
||||
|
||||
if err := cw.close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Write index file.
|
||||
indexPath := filepath.Join(blockDir, "index")
|
||||
indexFile, err := os.Create(indexPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
iw := index.NewWriter(indexFile)
|
||||
for _, e := range indexEntries {
|
||||
iw.AddSeries(e)
|
||||
}
|
||||
if _, err := iw.WriteTo(); err != nil {
|
||||
indexFile.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := indexFile.Sync(); err != nil {
|
||||
indexFile.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := indexFile.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Fsync the block directory to ensure all files are durable.
|
||||
if err := syncDir(blockDir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Write meta.json last — the immutability gate.
|
||||
if err := writeMeta(blockDir, meta); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Fsync the data directory so the block directory entry is visible.
|
||||
if err := syncDir(dataDir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return ulid, nil
|
||||
}
|
||||
|
||||
func syncDir(dir string) error {
|
||||
d, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer d.Close()
|
||||
return d.Sync()
|
||||
}
|
||||
Reference in New Issue
Block a user