Immutable blocks with mmap reads

Flush sealed head chunks to ULID-named block directories on disk. Each block contains CRC'd chunk segment files (mmap'd for reads), a binary index (symbol table, series, postings with TOC), and a meta.json written last as the immutability gate. WAL is truncated after block fsync, preserving the crash-safety ording invariant.
2026-07-04 14:59:39 -04:00
parent 376d3faf25
commit 42b03db2fa
16 changed files with 2176 additions and 2 deletions
@@ -0,0 +1,133 @@
+package block
+
+import (
+	"encoding/binary"
+	"hash/crc32"
+	"os"
+	"path/filepath"
+
+	"git.dvdt.dev/david/ingot/internal/index"
+)
+
+const (
+	chunkMagic          uint32 = 0x0FACE5C4
+	chunkVersion        byte   = 1
+	chunkHeaderLen             = 5 // 4-byte magic + 1-byte version
+	chunkEntryHeaderLen        = 5 // 4-byte length + 1-byte encoding
+	chunkEntryCRCLen           = 4
+	encodingXOR         byte   = 1
+	chunkSegmentMaxSize        = 512 * 1024 * 1024 // 512 MiB
+	chunksDirName              = "chunks"
+)
+
+var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
+
+// chunkWriter writes chunk data to segmented chunk files inside a block directory.
+type chunkWriter struct {
+	dir        string
+	segmentIdx int
+	segmentOff int
+	f          *os.File
+}
+
+func newChunkWriter(blockDir string) (*chunkWriter, error) {
+	chunksDir := filepath.Join(blockDir, chunksDirName)
+	if err := os.MkdirAll(chunksDir, 0755); err != nil {
+		return nil, err
+	}
+	cw := &chunkWriter{dir: chunksDir, segmentIdx: 1}
+	if err := cw.newSegment(); err != nil {
+		return nil, err
+	}
+	return cw, nil
+}
+
+func (cw *chunkWriter) newSegment() error {
+	if cw.f != nil {
+		if err := cw.f.Sync(); err != nil {
+			return err
+		}
+		if err := cw.f.Close(); err != nil {
+			return err
+		}
+	}
+
+	name := segmentName(cw.segmentIdx)
+	f, err := os.Create(filepath.Join(cw.dir, name))
+	if err != nil {
+		return err
+	}
+	cw.f = f
+	cw.segmentOff = 0
+
+	// Write chunk file header.
+	var hdr [chunkHeaderLen]byte
+	binary.BigEndian.PutUint32(hdr[:4], chunkMagic)
+	hdr[4] = chunkVersion
+	n, err := f.Write(hdr[:])
+	cw.segmentOff += n
+	return err
+}
+
+// writeChunk writes a chunk entry and returns its ChunkRef.
+// Format: dataLen(4) | encoding(1) | data(dataLen) | CRC32C(4)
+func (cw *chunkWriter) writeChunk(data []byte) (index.ChunkRef, error) {
+	entrySize := chunkEntryHeaderLen + len(data) + chunkEntryCRCLen
+
+	// Rotate if this entry would exceed segment max size.
+	if cw.segmentOff+entrySize > chunkSegmentMaxSize {
+		cw.segmentIdx++
+		if err := cw.newSegment(); err != nil {
+			return 0, err
+		}
+	}
+
+	ref := index.NewChunkRef(uint32(cw.segmentIdx), uint32(cw.segmentOff))
+
+	// Header: length + encoding.
+	var hdr [chunkEntryHeaderLen]byte
+	binary.BigEndian.PutUint32(hdr[:4], uint32(len(data)))
+	hdr[4] = encodingXOR
+	if _, err := cw.f.Write(hdr[:]); err != nil {
+		return 0, err
+	}
+
+	// Chunk data.
+	if _, err := cw.f.Write(data); err != nil {
+		return 0, err
+	}
+
+	// CRC over encoding + data.
+	crc := crc32.New(castagnoliTable)
+	crc.Write(hdr[4:5]) // encoding byte
+	crc.Write(data)
+	var crcBuf [4]byte
+	binary.BigEndian.PutUint32(crcBuf[:], crc.Sum32())
+	if _, err := cw.f.Write(crcBuf[:]); err != nil {
+		return 0, err
+	}
+
+	cw.segmentOff += entrySize
+	return ref, nil
+}
+
+func (cw *chunkWriter) close() error {
+	if cw.f == nil {
+		return nil
+	}
+	if err := cw.f.Sync(); err != nil {
+		cw.f.Close()
+		return err
+	}
+	return cw.f.Close()
+}
+
+func segmentName(idx int) string {
+	var buf [6]byte
+	s := idx
+	for i := 5; i >= 0; i-- {
+		buf[i] = '0' + byte(s%10)
+		s /= 10
+	}
+	return string(buf[:])
+}