Segmented write-ahead log

Append-only WAL with CRC32-validated records, automatic segment rotation
at 128 MiB, background periodic fsync, and crash recovery by truncation
at the first corrupt or incomplete record.

Record format: type(1) | len(4) | payload | crc32c(4). Two payload
types: series (ref + labels) and samples (batch of ref/t/v). Fixed-width
encoding througout.

Torn-write harness truncates at every byte offset and asserts recovery
produces a valid prefix of the original sequnce, both single-segment and
multi-segment.

DESIGN.md, NOTICE.md, and README.md.
This commit is contained in:
2026-07-04 14:09:09 -04:00
parent 06a95597e9
commit 9a322274a5
12 changed files with 1804 additions and 0 deletions
+63
View File
@@ -0,0 +1,63 @@
package wal
import (
"fmt"
"os"
"path/filepath"
"sort"
"strconv"
)
const (
defaultSegmentMaxSize = 128 * 1024 * 1024 // 128 MiB
segmentNameLen = 8 // "00000001"
)
// segmentFileName returns the zero-padded filename for a segment index.
func segmentFileName(index int) string {
return fmt.Sprintf("%0*d", segmentNameLen, index)
}
// parseSegmentIndex parses a segment filename back to its index.
// Returns -1 if the name is not a valid segment file.
func parseSegmentIndex(name string) int {
if len(name) != segmentNameLen {
return -1
}
n, err := strconv.Atoi(name)
if err != nil {
return -1
}
return n
}
// listSegments returns the sorted indices of all segment files in dir.
func listSegments(dir string) ([]int, error) {
entries, err := os.ReadDir(dir)
if err != nil {
return nil, err
}
var indices []int
for _, e := range entries {
if e.IsDir() {
continue
}
idx := parseSegmentIndex(e.Name())
if idx >= 0 {
indices = append(indices, idx)
}
}
sort.Ints(indices)
return indices, nil
}
// segmentPath returns the full path for a segment index within dir.
func segmentPath(dir string, index int) string {
return filepath.Join(dir, segmentFileName(index))
}
// createSegment creates a new segment file and returns it open for writing.
func createSegment(dir string, index int) (*os.File, error) {
return os.OpenFile(segmentPath(dir, index), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644)
}