Immutable blocks with mmap reads
Flush sealed head chunks to ULID-named block directories on disk. Each block contains CRC'd chunk segment files (mmap'd for reads), a binary index (symbol table, series, postings with TOC), and a meta.json written last as the immutability gate. WAL is truncated after block fsync, preserving the crash-safety ording invariant.
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
package block
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.dvdt.dev/david/ingot/internal/index"
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
)
|
||||
|
||||
// ChunkData describes a single chunk to be flushed to a block.
|
||||
type ChunkData struct {
|
||||
MinT int64
|
||||
MaxT int64
|
||||
Data []byte // raw XOR chunk bytes (including 2-byte sample count header)
|
||||
}
|
||||
|
||||
// SeriesFlush describes a series with its sealed chunks for block writing.
|
||||
type SeriesFlush struct {
|
||||
Ref uint64
|
||||
Labels []labels.Label
|
||||
Chunks []ChunkData
|
||||
}
|
||||
|
||||
// Flush writes a new immutable block from the given series data.
|
||||
// It creates a ULID-named directory under dataDir containing:
|
||||
// - chunks/ with segment files
|
||||
// - index file
|
||||
// - meta.json (written last as the immutability gate)
|
||||
//
|
||||
// Returns the block ULID and any error.
|
||||
func Flush(dataDir string, series []SeriesFlush) (string, error) {
|
||||
ulid := newULID()
|
||||
blockDir := filepath.Join(dataDir, ulid)
|
||||
|
||||
if err := os.MkdirAll(blockDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Write chunk files and collect index entries.
|
||||
cw, err := newChunkWriter(blockDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var (
|
||||
indexEntries []index.SeriesEntry
|
||||
meta BlockMeta
|
||||
)
|
||||
meta.ULID = ulid
|
||||
meta.Version = 1
|
||||
meta.Compaction = CompactionInfo{Level: 1, Sources: []string{ulid}}
|
||||
meta.MinTime = int64(^uint64(0) >> 1) // max int64
|
||||
meta.MaxTime = int64(0)
|
||||
|
||||
for _, sf := range series {
|
||||
var chunks []index.ChunkMeta
|
||||
for _, cd := range sf.Chunks {
|
||||
ref, err := cw.writeChunk(cd.Data)
|
||||
if err != nil {
|
||||
cw.close()
|
||||
return "", err
|
||||
}
|
||||
chunks = append(chunks, index.ChunkMeta{
|
||||
MinT: cd.MinT,
|
||||
MaxT: cd.MaxT,
|
||||
Ref: ref,
|
||||
})
|
||||
if cd.MinT < meta.MinTime {
|
||||
meta.MinTime = cd.MinT
|
||||
}
|
||||
if cd.MaxT > meta.MaxTime {
|
||||
meta.MaxTime = cd.MaxT
|
||||
}
|
||||
meta.Stats.NumChunks++
|
||||
// Count samples from the chunk's 2-byte header.
|
||||
if len(cd.Data) >= 2 {
|
||||
meta.Stats.NumSamples += int(uint16(cd.Data[0])<<8 | uint16(cd.Data[1]))
|
||||
}
|
||||
}
|
||||
indexEntries = append(indexEntries, index.SeriesEntry{
|
||||
Ref: sf.Ref,
|
||||
Labels: sf.Labels,
|
||||
Chunks: chunks,
|
||||
})
|
||||
meta.Stats.NumSeries++
|
||||
}
|
||||
|
||||
if err := cw.close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Write index file.
|
||||
indexPath := filepath.Join(blockDir, "index")
|
||||
indexFile, err := os.Create(indexPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
iw := index.NewWriter(indexFile)
|
||||
for _, e := range indexEntries {
|
||||
iw.AddSeries(e)
|
||||
}
|
||||
if _, err := iw.WriteTo(); err != nil {
|
||||
indexFile.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := indexFile.Sync(); err != nil {
|
||||
indexFile.Close()
|
||||
return "", err
|
||||
}
|
||||
if err := indexFile.Close(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Fsync the block directory to ensure all files are durable.
|
||||
if err := syncDir(blockDir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Write meta.json last — the immutability gate.
|
||||
if err := writeMeta(blockDir, meta); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Fsync the data directory so the block directory entry is visible.
|
||||
if err := syncDir(dataDir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return ulid, nil
|
||||
}
|
||||
|
||||
func syncDir(dir string) error {
|
||||
d, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer d.Close()
|
||||
return d.Sync()
|
||||
}
|
||||
Reference in New Issue
Block a user