david
42b03db2fa
Flush sealed head chunks to ULID-named block directories on disk. Each block contains CRC'd chunk segment files (mmap'd for reads), a binary index (symbol table, series, postings with TOC), and a meta.json written last as the immutability gate. WAL is truncated after block fsync, preserving the crash-safety ording invariant.
134 lines
3.0 KiB
Go
134 lines
3.0 KiB
Go
package block
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"hash/crc32"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"git.dvdt.dev/david/ingot/internal/index"
|
|
)
|
|
|
|
const (
|
|
chunkMagic uint32 = 0x0FACE5C4
|
|
chunkVersion byte = 1
|
|
chunkHeaderLen = 5 // 4-byte magic + 1-byte version
|
|
chunkEntryHeaderLen = 5 // 4-byte length + 1-byte encoding
|
|
chunkEntryCRCLen = 4
|
|
encodingXOR byte = 1
|
|
chunkSegmentMaxSize = 512 * 1024 * 1024 // 512 MiB
|
|
chunksDirName = "chunks"
|
|
)
|
|
|
|
var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
|
|
|
|
// chunkWriter writes chunk data to segmented chunk files inside a block directory.
|
|
type chunkWriter struct {
|
|
dir string
|
|
segmentIdx int
|
|
segmentOff int
|
|
f *os.File
|
|
}
|
|
|
|
func newChunkWriter(blockDir string) (*chunkWriter, error) {
|
|
chunksDir := filepath.Join(blockDir, chunksDirName)
|
|
if err := os.MkdirAll(chunksDir, 0755); err != nil {
|
|
return nil, err
|
|
}
|
|
cw := &chunkWriter{dir: chunksDir, segmentIdx: 1}
|
|
if err := cw.newSegment(); err != nil {
|
|
return nil, err
|
|
}
|
|
return cw, nil
|
|
}
|
|
|
|
func (cw *chunkWriter) newSegment() error {
|
|
if cw.f != nil {
|
|
if err := cw.f.Sync(); err != nil {
|
|
return err
|
|
}
|
|
if err := cw.f.Close(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
name := segmentName(cw.segmentIdx)
|
|
f, err := os.Create(filepath.Join(cw.dir, name))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cw.f = f
|
|
cw.segmentOff = 0
|
|
|
|
// Write chunk file header.
|
|
var hdr [chunkHeaderLen]byte
|
|
binary.BigEndian.PutUint32(hdr[:4], chunkMagic)
|
|
hdr[4] = chunkVersion
|
|
n, err := f.Write(hdr[:])
|
|
cw.segmentOff += n
|
|
return err
|
|
}
|
|
|
|
// writeChunk writes a chunk entry and returns its ChunkRef.
|
|
// Format: dataLen(4) | encoding(1) | data(dataLen) | CRC32C(4)
|
|
func (cw *chunkWriter) writeChunk(data []byte) (index.ChunkRef, error) {
|
|
entrySize := chunkEntryHeaderLen + len(data) + chunkEntryCRCLen
|
|
|
|
// Rotate if this entry would exceed segment max size.
|
|
if cw.segmentOff+entrySize > chunkSegmentMaxSize {
|
|
cw.segmentIdx++
|
|
if err := cw.newSegment(); err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
|
|
ref := index.NewChunkRef(uint32(cw.segmentIdx), uint32(cw.segmentOff))
|
|
|
|
// Header: length + encoding.
|
|
var hdr [chunkEntryHeaderLen]byte
|
|
binary.BigEndian.PutUint32(hdr[:4], uint32(len(data)))
|
|
hdr[4] = encodingXOR
|
|
if _, err := cw.f.Write(hdr[:]); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
// Chunk data.
|
|
if _, err := cw.f.Write(data); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
// CRC over encoding + data.
|
|
crc := crc32.New(castagnoliTable)
|
|
crc.Write(hdr[4:5]) // encoding byte
|
|
crc.Write(data)
|
|
var crcBuf [4]byte
|
|
binary.BigEndian.PutUint32(crcBuf[:], crc.Sum32())
|
|
if _, err := cw.f.Write(crcBuf[:]); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
cw.segmentOff += entrySize
|
|
return ref, nil
|
|
}
|
|
|
|
func (cw *chunkWriter) close() error {
|
|
if cw.f == nil {
|
|
return nil
|
|
}
|
|
if err := cw.f.Sync(); err != nil {
|
|
cw.f.Close()
|
|
return err
|
|
}
|
|
return cw.f.Close()
|
|
}
|
|
|
|
func segmentName(idx int) string {
|
|
var buf [6]byte
|
|
s := idx
|
|
for i := 5; i >= 0; i-- {
|
|
buf[i] = '0' + byte(s%10)
|
|
s /= 10
|
|
}
|
|
return string(buf[:])
|
|
}
|