Immutable blocks with mmap reads

Flush sealed head chunks to ULID-named block directories on disk. Each block contains CRC'd chunk segment files (mmap'd for reads), a binary index (symbol table, series, postings with TOC), and a meta.json written last as the immutability gate. WAL is truncated after block fsync, preserving the crash-safety ording invariant.
2026-07-04 14:59:39 -04:00
parent 376d3faf25
commit 42b03db2fa
16 changed files with 2176 additions and 2 deletions
@@ -4,8 +4,10 @@ package head

 import (
 	"fmt"
+	"path/filepath"
 	"sync/atomic"

+	"git.dvdt.dev/david/ingot/internal/block"
 	"git.dvdt.dev/david/ingot/internal/chunkenc"
 	"git.dvdt.dev/david/ingot/internal/wal"
 	"git.dvdt.dev/david/ingot/labels"
@@ -13,6 +15,7 @@ import (

 // Head is the in-memory store for active series and their chunks.
 type Head struct {
+	dataDir string // parent directory containing WAL and block dirs
 	series  *seriesMap
 	wal     *wal.WAL
 	nextRef atomic.Uint64
@@ -23,6 +26,7 @@ type Head struct {
 }

 // Open creates or recovers a Head backed by a WAL in walDir.
+// The dataDir (parent of walDir) is used for writing blocks.
 func Open(walDir string, walOpts wal.Options) (*Head, error) {
 	w, err := wal.Open(walDir, walOpts)
 	if err != nil {
@@ -30,8 +34,9 @@ func Open(walDir string, walOpts wal.Options) (*Head, error) {
 	}

 	h := &Head{
-		series: newSeriesMap(),
-		wal:    w,
+		dataDir: filepath.Dir(walDir),
+		series:  newSeriesMap(),
+		wal:     w,
 	}

 	if err := h.replay(); err != nil {
@@ -145,3 +150,71 @@ func (h *Head) MaxTime() int64 { return h.maxTime.Load() }
 func (h *Head) Close() error {
 	return h.wal.Close()
 }
+
+// FlushOlderThan collects all sealed chunks with maxT <= threshold from all
+// series, writes them to an immutable block, and truncates the WAL.
+//
+// The ordering invariant is enforced: block fsync -> meta.json write -> WAL truncate.
+// Returns the block ULID (empty string if nothing to flush) and any error.
+func (h *Head) FlushOlderThan(maxT int64) (string, error) {
+	var flushData []block.SeriesFlush
+
+	h.series.forEach(func(s *memSeries) {
+		s.mu.Lock()
+		defer s.mu.Unlock()
+
+		var toFlush []chunkMeta
+		var remaining []chunkMeta
+		for _, cm := range s.sealed {
+			if cm.maxT <= maxT {
+				toFlush = append(toFlush, cm)
+			} else {
+				remaining = append(remaining, cm)
+			}
+		}
+		if len(toFlush) == 0 {
+			return
+		}
+
+		sf := block.SeriesFlush{
+			Ref:    s.ref,
+			Labels: s.labels,
+		}
+		for _, cm := range toFlush {
+			sf.Chunks = append(sf.Chunks, block.ChunkData{
+				MinT: cm.minT,
+				MaxT: cm.maxT,
+				Data: append([]byte(nil), cm.chunk.Bytes()...),
+			})
+		}
+		flushData = append(flushData, sf)
+
+		// Clear flushed chunks from the series.
+		s.sealed = remaining
+	})
+
+	if len(flushData) == 0 {
+		return "", nil
+	}
+
+	// Write block. Flush handles: chunk files + index + fsync + meta.json.
+	ulid, err := block.Flush(h.dataDir, flushData)
+	if err != nil {
+		return "", fmt.Errorf("head: flush block: %w", err)
+	}
+
+	// WAL truncation: safe because the block is fully fsynced.
+	// Truncate all segments below the current one — the flushed data is now
+	// in the block and doesn't need WAL replay.
+	lastSeg := h.wal.LastSegment()
+	if err := h.wal.Truncate(lastSeg); err != nil {
+		return ulid, fmt.Errorf("head: truncate WAL: %w", err)
+	}
+
+	return ulid, nil
+}
+
+// DataDir returns the data directory (parent of WAL dir).
+func (h *Head) DataDir() string {
+	return h.dataDir
+}
@@ -2,10 +2,12 @@ package head

 import (
 	"math"
+	"os"
 	"path/filepath"
 	"sync"
 	"testing"

+	"git.dvdt.dev/david/ingot/internal/block"
 	"git.dvdt.dev/david/ingot/internal/wal"
 	"git.dvdt.dev/david/ingot/labels"
 	"github.com/stretchr/testify/assert"
@@ -432,6 +434,181 @@ func TestWALReplay(t *testing.T) {
 	}
 }

+func TestFlushOlderThan(t *testing.T) {
+	tests := []struct {
+		name            string
+		numSamples      int // samples per series (each at 15s intervals)
+		flushMaxT       int64
+		wantBlockSeries int // number of series in the block
+		wantBlockExists bool
+	}{
+		{
+			name:            "flush_sealed_chunks",
+			numSamples:      250, // 2 sealed chunks (120 each) + 10 active
+			flushMaxT:       math.MaxInt64,
+			wantBlockSeries: 1,
+			wantBlockExists: true,
+		},
+		{
+			name:            "nothing_to_flush",
+			numSamples:      50, // only active chunk, no sealed
+			flushMaxT:       math.MaxInt64,
+			wantBlockSeries: 0,
+			wantBlockExists: false,
+		},
+		{
+			name:            "partial_flush_by_time",
+			numSamples:      250,
+			flushMaxT:       120 * 15000, // only flush first sealed chunk
+			wantBlockSeries: 1,
+			wantBlockExists: true,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			h := openHead(t)
+
+			// Append samples.
+			app := h.Appender()
+			ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
+			require.NoError(t, err)
+			for i := 1; i < tc.numSamples; i++ {
+				_, err = app.Append(ref, nil, int64(i*15000), float64(i))
+				require.NoError(t, err)
+			}
+			require.NoError(t, app.Commit())
+
+			// Flush.
+			ulid, err := h.FlushOlderThan(tc.flushMaxT)
+			require.NoError(t, err)
+
+			if !tc.wantBlockExists {
+				assert.Empty(t, ulid)
+				return
+			}
+
+			assert.NotEmpty(t, ulid)
+
+			// Verify block exists and is readable.
+			blockDir := filepath.Join(h.DataDir(), ulid)
+			br, err := block.Open(blockDir)
+			require.NoError(t, err)
+			defer br.Close()
+
+			assert.Equal(t, tc.wantBlockSeries, br.Meta.Stats.NumSeries)
+
+			// Verify block data is correct by iterating.
+			if tc.wantBlockSeries > 0 {
+				it, err := br.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64)
+				require.NoError(t, err)
+				count := 0
+				for it.Next() {
+					count++
+				}
+				require.NoError(t, it.Err())
+				assert.Greater(t, count, 0, "block should contain samples")
+			}
+
+			// Head should still have its active chunk data.
+			allSamples := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64)
+			assert.Greater(t, len(allSamples), 0, "head should still have active chunk")
+		})
+	}
+}
+
+func TestFlushThenContinueAppending(t *testing.T) {
+	h := openHead(t)
+
+	// Append enough to seal two chunks (240 samples), plus a few more.
+	app := h.Appender()
+	ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
+	require.NoError(t, err)
+	for i := 1; i < 250; i++ {
+		_, err = app.Append(ref, nil, int64(i*15000), float64(i))
+		require.NoError(t, err)
+	}
+	require.NoError(t, app.Commit())
+
+	// Flush sealed chunks.
+	ulid, err := h.FlushOlderThan(math.MaxInt64)
+	require.NoError(t, err)
+	require.NotEmpty(t, ulid)
+
+	// Continue appending after flush.
+	app = h.Appender()
+	for i := 250; i < 260; i++ {
+		_, err = app.Append(ref, nil, int64(i*15000), float64(i))
+		require.NoError(t, err)
+	}
+	require.NoError(t, app.Commit())
+
+	// Head should have the active chunk data (unflushed).
+	allSamples := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64)
+	assert.Greater(t, len(allSamples), 0)
+
+	// Block should have the flushed data.
+	blockDir := filepath.Join(h.DataDir(), ulid)
+	br, err := block.Open(blockDir)
+	require.NoError(t, err)
+	defer br.Close()
+
+	it, err := br.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64)
+	require.NoError(t, err)
+	blockCount := 0
+	for it.Next() {
+		blockCount++
+	}
+	require.NoError(t, it.Err())
+	assert.Equal(t, 240, blockCount, "block should contain 2 sealed chunks of 120 samples each")
+}
+
+func TestFlushWALTruncation(t *testing.T) {
+	dir := t.TempDir()
+	walDir := filepath.Join(dir, "wal")
+
+	h, err := Open(walDir, wal.Options{})
+	require.NoError(t, err)
+
+	// Append enough to seal chunks.
+	app := h.Appender()
+	ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
+	require.NoError(t, err)
+	for i := 1; i < 250; i++ {
+		_, err = app.Append(ref, nil, int64(i*15000), float64(i))
+		require.NoError(t, err)
+	}
+	require.NoError(t, app.Commit())
+
+	// Count WAL segments before flush.
+	walEntries, err := os.ReadDir(walDir)
+	require.NoError(t, err)
+	segsBefore := len(walEntries)
+
+	// Flush.
+	_, err = h.FlushOlderThan(math.MaxInt64)
+	require.NoError(t, err)
+
+	// WAL segments should have been truncated (or at least not grown).
+	walEntries, err = os.ReadDir(walDir)
+	require.NoError(t, err)
+	segsAfter := len(walEntries)
+	assert.LessOrEqual(t, segsAfter, segsBefore, "WAL should be truncated after flush")
+
+	require.NoError(t, h.Close())
+
+	// Re-open: head should recover from WAL (only unflushed data).
+	h2, err := Open(walDir, wal.Options{})
+	require.NoError(t, err)
+	defer h2.Close()
+
+	// The re-opened head should be functional.
+	app = h2.Appender()
+	_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "new_series"}}, 5000000, 42.0)
+	require.NoError(t, err)
+	require.NoError(t, app.Commit())
+}
+
 func TestConcurrentAppend(t *testing.T) {
 	tests := []struct {
 		name                string
@@ -70,6 +70,18 @@ func (sm *seriesMap) set(hash uint64, s *memSeries) {
 	rs.mu.Unlock()
 }

+// forEach calls fn for every series in the map. The series lock is NOT held.
+func (sm *seriesMap) forEach(fn func(s *memSeries)) {
+	for i := range sm.refStripes {
+		rs := &sm.refStripes[i]
+		rs.mu.RLock()
+		for _, s := range rs.m {
+			fn(s)
+		}
+		rs.mu.RUnlock()
+	}
+}
+
 // remove deletes a series from both maps.
 func (sm *seriesMap) remove(hash uint64, s *memSeries) {
 	hs := &sm.hashStripes[hash%numStripes]