Immutable blocks with mmap reads

Flush sealed head chunks to ULID-named block directories on disk. Each
block contains CRC'd chunk segment files (mmap'd for reads), a binary
index (symbol table, series, postings with TOC), and a meta.json written
last as the immutability gate. WAL is truncated after block fsync,
preserving the crash-safety ording invariant.
This commit is contained in:
2026-07-04 14:59:39 -04:00
parent 376d3faf25
commit 42b03db2fa
16 changed files with 2176 additions and 2 deletions
+141
View File
@@ -0,0 +1,141 @@
package block
import (
"os"
"path/filepath"
"git.dvdt.dev/david/ingot/internal/index"
"git.dvdt.dev/david/ingot/labels"
)
// ChunkData describes a single chunk to be flushed to a block.
type ChunkData struct {
MinT int64
MaxT int64
Data []byte // raw XOR chunk bytes (including 2-byte sample count header)
}
// SeriesFlush describes a series with its sealed chunks for block writing.
type SeriesFlush struct {
Ref uint64
Labels []labels.Label
Chunks []ChunkData
}
// Flush writes a new immutable block from the given series data.
// It creates a ULID-named directory under dataDir containing:
// - chunks/ with segment files
// - index file
// - meta.json (written last as the immutability gate)
//
// Returns the block ULID and any error.
func Flush(dataDir string, series []SeriesFlush) (string, error) {
ulid := newULID()
blockDir := filepath.Join(dataDir, ulid)
if err := os.MkdirAll(blockDir, 0755); err != nil {
return "", err
}
// Write chunk files and collect index entries.
cw, err := newChunkWriter(blockDir)
if err != nil {
return "", err
}
var (
indexEntries []index.SeriesEntry
meta BlockMeta
)
meta.ULID = ulid
meta.Version = 1
meta.Compaction = CompactionInfo{Level: 1, Sources: []string{ulid}}
meta.MinTime = int64(^uint64(0) >> 1) // max int64
meta.MaxTime = int64(0)
for _, sf := range series {
var chunks []index.ChunkMeta
for _, cd := range sf.Chunks {
ref, err := cw.writeChunk(cd.Data)
if err != nil {
cw.close()
return "", err
}
chunks = append(chunks, index.ChunkMeta{
MinT: cd.MinT,
MaxT: cd.MaxT,
Ref: ref,
})
if cd.MinT < meta.MinTime {
meta.MinTime = cd.MinT
}
if cd.MaxT > meta.MaxTime {
meta.MaxTime = cd.MaxT
}
meta.Stats.NumChunks++
// Count samples from the chunk's 2-byte header.
if len(cd.Data) >= 2 {
meta.Stats.NumSamples += int(uint16(cd.Data[0])<<8 | uint16(cd.Data[1]))
}
}
indexEntries = append(indexEntries, index.SeriesEntry{
Ref: sf.Ref,
Labels: sf.Labels,
Chunks: chunks,
})
meta.Stats.NumSeries++
}
if err := cw.close(); err != nil {
return "", err
}
// Write index file.
indexPath := filepath.Join(blockDir, "index")
indexFile, err := os.Create(indexPath)
if err != nil {
return "", err
}
iw := index.NewWriter(indexFile)
for _, e := range indexEntries {
iw.AddSeries(e)
}
if _, err := iw.WriteTo(); err != nil {
indexFile.Close()
return "", err
}
if err := indexFile.Sync(); err != nil {
indexFile.Close()
return "", err
}
if err := indexFile.Close(); err != nil {
return "", err
}
// Fsync the block directory to ensure all files are durable.
if err := syncDir(blockDir); err != nil {
return "", err
}
// Write meta.json last — the immutability gate.
if err := writeMeta(blockDir, meta); err != nil {
return "", err
}
// Fsync the data directory so the block directory entry is visible.
if err := syncDir(dataDir); err != nil {
return "", err
}
return ulid, nil
}
func syncDir(dir string) error {
d, err := os.Open(dir)
if err != nil {
return err
}
defer d.Close()
return d.Sync()
}