Segmented write-ahead log

Append-only WAL with CRC32-validated records, automatic segment rotation at 128 MiB, background periodic fsync, and crash recovery by truncation at the first corrupt or incomplete record. Record format: type(1) | len(4) | payload | crc32c(4). Two payload types: series (ref + labels) and samples (batch of ref/t/v). Fixed-width encoding througout. Torn-write harness truncates at every byte offset and asserts recovery produces a valid prefix of the original sequnce, both single-segment and multi-segment. DESIGN.md, NOTICE.md, and README.md.
2026-07-04 14:09:09 -04:00
parent 06a95597e9
commit 9a322274a5
12 changed files with 1804 additions and 0 deletions
@@ -0,0 +1,251 @@
+package wal
+
+import (
+	"os"
+	"sync"
+	"time"
+)
+
+// Options configures WAL behavior.
+type Options struct {
+	// SegmentMaxSize is the maximum size of a single segment file in bytes.
+	// A new segment is created when the current one would exceed this.
+	// Default: 128 MiB.
+	SegmentMaxSize int
+
+	// SyncInterval controls background fsync frequency.
+	// Default (zero): 1s. Negative: sync on every Log call.
+	SyncInterval time.Duration
+}
+
+func (o *Options) segmentMaxSize() int {
+	if o.SegmentMaxSize > 0 {
+		return o.SegmentMaxSize
+	}
+	return defaultSegmentMaxSize
+}
+
+func (o *Options) syncInterval() time.Duration {
+	if o.SyncInterval < 0 {
+		return -1 // sync-per-write sentinel
+	}
+	if o.SyncInterval == 0 {
+		return time.Second
+	}
+	return o.SyncInterval
+}
+
+// WAL is a segmented write-ahead log.
+type WAL struct {
+	dir  string
+	opts Options
+
+	mu         sync.Mutex
+	segment    *os.File
+	segmentIdx int
+	segmentOff int64
+	buf        []byte
+
+	done chan struct{}
+	wg   sync.WaitGroup
+}
+
+// Open opens or creates a WAL in dir. If segments already exist, it runs
+// recovery (truncating at the first corrupt record) before returning.
+func Open(dir string, opts Options) (*WAL, error) {
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return nil, err
+	}
+
+	// Recover existing segments.
+	if err := recover(dir); err != nil {
+		return nil, err
+	}
+
+	w := &WAL{
+		dir:  dir,
+		opts: opts,
+		done: make(chan struct{}),
+	}
+
+	// Open or create the active segment.
+	segs, err := listSegments(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(segs) == 0 {
+		// Fresh WAL.
+		w.segmentIdx = 1
+		f, err := createSegment(dir, 1)
+		if err != nil {
+			return nil, err
+		}
+		w.segment = f
+	} else {
+		// Append to the last segment.
+		idx := segs[len(segs)-1]
+		w.segmentIdx = idx
+		f, err := os.OpenFile(segmentPath(dir, idx), os.O_WRONLY|os.O_APPEND, 0644)
+		if err != nil {
+			return nil, err
+		}
+		info, err := f.Stat()
+		if err != nil {
+			f.Close()
+			return nil, err
+		}
+		w.segment = f
+		w.segmentOff = info.Size()
+	}
+
+	// Start background syncer.
+	if interval := opts.syncInterval(); interval > 0 {
+		w.wg.Add(1)
+		go w.syncLoop(interval)
+	}
+
+	return w, nil
+}
+
+// Log writes a framed record to the WAL. The payload is wrapped with
+// the record envelope (type + length + CRC).
+func (w *WAL) Log(typ RecordType, payload []byte) error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	w.buf = EncodeRecord(w.buf[:0], typ, payload)
+
+	// Rotate if this write would exceed the segment size limit.
+	if w.segmentOff+int64(len(w.buf)) > int64(w.opts.segmentMaxSize()) {
+		if err := w.rotate(); err != nil {
+			return err
+		}
+	}
+
+	n, err := w.segment.Write(w.buf)
+	w.segmentOff += int64(n)
+	if err != nil {
+		return err
+	}
+
+	// Sync-per-write mode.
+	if w.opts.syncInterval() < 0 {
+		return w.segment.Sync()
+	}
+
+	return nil
+}
+
+// LogSeries encodes and writes a series record.
+func (w *WAL) LogSeries(recs []SeriesRecord) error {
+	for _, rec := range recs {
+		payload := EncodeSeriesRecord(nil, rec)
+		if err := w.Log(RecordSeries, payload); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// LogSamples encodes and writes a samples record.
+func (w *WAL) LogSamples(samples []RefSample) error {
+	payload := EncodeSamplesRecord(nil, samples)
+	return w.Log(RecordSamples, payload)
+}
+
+// Replay returns a Reader over all WAL segments. The caller must Close
+// the reader when done.
+func (w *WAL) Replay() (*Reader, error) {
+	return NewReader(w.dir)
+}
+
+// Sync forces an fsync of the current segment.
+func (w *WAL) Sync() error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.segment.Sync()
+}
+
+// Truncate deletes all segments with index less than below.
+func (w *WAL) Truncate(below int) error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	segs, err := listSegments(w.dir)
+	if err != nil {
+		return err
+	}
+
+	for _, idx := range segs {
+		if idx >= below {
+			break
+		}
+		if err := os.Remove(segmentPath(w.dir, idx)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// LastSegment returns the index of the current active segment.
+func (w *WAL) LastSegment() int {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.segmentIdx
+}
+
+// Close stops the background syncer, fsyncs, and closes the active segment.
+func (w *WAL) Close() error {
+	close(w.done)
+	w.wg.Wait()
+
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	if w.segment == nil {
+		return nil
+	}
+	if err := w.segment.Sync(); err != nil {
+		w.segment.Close()
+		return err
+	}
+	return w.segment.Close()
+}
+
+// rotate fsyncs the current segment, closes it, and creates a new one.
+// Caller must hold w.mu.
+func (w *WAL) rotate() error {
+	if err := w.segment.Sync(); err != nil {
+		return err
+	}
+	if err := w.segment.Close(); err != nil {
+		return err
+	}
+
+	w.segmentIdx++
+	f, err := createSegment(w.dir, w.segmentIdx)
+	if err != nil {
+		return err
+	}
+	w.segment = f
+	w.segmentOff = 0
+	return nil
+}
+
+func (w *WAL) syncLoop(interval time.Duration) {
+	defer w.wg.Done()
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-w.done:
+			return
+		case <-ticker.C:
+			w.mu.Lock()
+			w.segment.Sync()
+			w.mu.Unlock()
+		}
+	}
+}