Segmented write-ahead log
Append-only WAL with CRC32-validated records, automatic segment rotation at 128 MiB, background periodic fsync, and crash recovery by truncation at the first corrupt or incomplete record. Record format: type(1) | len(4) | payload | crc32c(4). Two payload types: series (ref + labels) and samples (batch of ref/t/v). Fixed-width encoding througout. Torn-write harness truncates at every byte offset and asserts recovery produces a valid prefix of the original sequnce, both single-segment and multi-segment. DESIGN.md, NOTICE.md, and README.md.
This commit is contained in:
@@ -0,0 +1,270 @@
|
||||
package wal
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSeriesRecord(t *testing.T) {
|
||||
type result struct {
|
||||
rec SeriesRecord
|
||||
err error
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
want result
|
||||
}{
|
||||
{
|
||||
name: "single_label",
|
||||
data: EncodeSeriesRecord(nil, SeriesRecord{
|
||||
Ref: 42,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "temp"}},
|
||||
}),
|
||||
want: result{SeriesRecord{
|
||||
Ref: 42,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "temp"}},
|
||||
}, nil},
|
||||
},
|
||||
{
|
||||
name: "multiple_labels",
|
||||
data: EncodeSeriesRecord(nil, SeriesRecord{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{
|
||||
{Name: "__name__", Value: "cpu_usage"},
|
||||
{Name: "host", Value: "web-01"},
|
||||
{Name: "region", Value: "us-east"},
|
||||
},
|
||||
}),
|
||||
want: result{SeriesRecord{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{
|
||||
{Name: "__name__", Value: "cpu_usage"},
|
||||
{Name: "host", Value: "web-01"},
|
||||
{Name: "region", Value: "us-east"},
|
||||
},
|
||||
}, nil},
|
||||
},
|
||||
{
|
||||
name: "zero_labels",
|
||||
data: EncodeSeriesRecord(nil, SeriesRecord{Ref: 99, Labels: nil}),
|
||||
want: result{SeriesRecord{Ref: 99, Labels: []labels.Label{}}, nil},
|
||||
},
|
||||
{
|
||||
name: "unicode_labels",
|
||||
data: EncodeSeriesRecord(nil, SeriesRecord{
|
||||
Ref: 7,
|
||||
Labels: []labels.Label{{Name: "名前", Value: "温度"}},
|
||||
}),
|
||||
want: result{SeriesRecord{
|
||||
Ref: 7,
|
||||
Labels: []labels.Label{{Name: "名前", Value: "温度"}},
|
||||
}, nil},
|
||||
},
|
||||
{
|
||||
name: "empty_label_strings",
|
||||
data: EncodeSeriesRecord(nil, SeriesRecord{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{{Name: "", Value: ""}},
|
||||
}),
|
||||
want: result{SeriesRecord{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{{Name: "", Value: ""}},
|
||||
}, nil},
|
||||
},
|
||||
|
||||
// Error cases.
|
||||
{
|
||||
name: "nil",
|
||||
data: nil,
|
||||
want: result{SeriesRecord{}, ErrShortPayload},
|
||||
},
|
||||
{
|
||||
name: "truncated_ref",
|
||||
data: make([]byte, 6),
|
||||
want: result{SeriesRecord{}, ErrShortPayload},
|
||||
},
|
||||
{
|
||||
name: "truncated_nlabels",
|
||||
data: make([]byte, 10),
|
||||
want: result{SeriesRecord{}, ErrShortPayload},
|
||||
},
|
||||
{
|
||||
name: "truncated_name_len",
|
||||
data: func() []byte {
|
||||
d := make([]byte, 13) // ref(8) + nlabels=1(4) + 1 byte (short)
|
||||
binary.BigEndian.PutUint32(d[8:], 1)
|
||||
return d
|
||||
}(),
|
||||
want: result{SeriesRecord{}, ErrShortPayload},
|
||||
},
|
||||
{
|
||||
name: "truncated_name_data",
|
||||
data: func() []byte {
|
||||
d := make([]byte, 16) // ref(8) + nlabels=1(4) + namelen=10(2) + 2 bytes
|
||||
binary.BigEndian.PutUint32(d[8:], 1)
|
||||
binary.BigEndian.PutUint16(d[12:], 10) // claims 10 bytes, only 2 available
|
||||
return d
|
||||
}(),
|
||||
want: result{SeriesRecord{}, ErrShortPayload},
|
||||
},
|
||||
{
|
||||
name: "truncated_value_len",
|
||||
data: func() []byte {
|
||||
d := make([]byte, 15) // ref(8) + nlabels=1(4) + namelen=0(2) + 1 byte
|
||||
binary.BigEndian.PutUint32(d[8:], 1)
|
||||
binary.BigEndian.PutUint16(d[12:], 0) // 0-length name
|
||||
return d
|
||||
}(),
|
||||
want: result{SeriesRecord{}, ErrShortPayload},
|
||||
},
|
||||
{
|
||||
name: "truncated_value_data",
|
||||
data: func() []byte {
|
||||
d := make([]byte, 18) // ref(8) + nlabels=1(4) + namelen=0(2) + vallen=5(2) + 2 bytes
|
||||
binary.BigEndian.PutUint32(d[8:], 1)
|
||||
binary.BigEndian.PutUint16(d[12:], 0)
|
||||
binary.BigEndian.PutUint16(d[14:], 5) // claims 5 bytes, only 2 available
|
||||
return d
|
||||
}(),
|
||||
want: result{SeriesRecord{}, ErrShortPayload},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rec, err := DecodeSeriesRecord(tc.data)
|
||||
assert.Equal(t, tc.want.rec, rec, "record")
|
||||
assert.Equal(t, tc.want.err, err, "error")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// refSampleBits holds a RefSample with the value stored as raw bits
|
||||
// so NaN and negative zero compare correctly.
|
||||
type refSampleBits struct {
|
||||
Ref uint64
|
||||
T int64
|
||||
VBits uint64
|
||||
}
|
||||
|
||||
func toBits(s RefSample) refSampleBits {
|
||||
return refSampleBits{s.Ref, s.T, math.Float64bits(s.V)}
|
||||
}
|
||||
|
||||
func samplesToBits(ss []RefSample) []refSampleBits {
|
||||
out := make([]refSampleBits, len(ss))
|
||||
for i, s := range ss {
|
||||
out[i] = toBits(s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestSamplesRecord(t *testing.T) {
|
||||
type result struct {
|
||||
samples []refSampleBits
|
||||
err error
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
want result
|
||||
}{
|
||||
{
|
||||
name: "single_sample",
|
||||
data: EncodeSamplesRecord(nil, []RefSample{
|
||||
{Ref: 1, T: 1000, V: 71.3},
|
||||
}),
|
||||
want: result{samplesToBits([]RefSample{{Ref: 1, T: 1000, V: 71.3}}), nil},
|
||||
},
|
||||
{
|
||||
name: "multiple_samples",
|
||||
data: EncodeSamplesRecord(nil, []RefSample{
|
||||
{Ref: 1, T: 1000, V: 71.3},
|
||||
{Ref: 1, T: 1015, V: 71.4},
|
||||
{Ref: 2, T: 1000, V: 0},
|
||||
}),
|
||||
want: result{samplesToBits([]RefSample{
|
||||
{Ref: 1, T: 1000, V: 71.3},
|
||||
{Ref: 1, T: 1015, V: 71.4},
|
||||
{Ref: 2, T: 1000, V: 0},
|
||||
}), nil},
|
||||
},
|
||||
{
|
||||
name: "zero_samples",
|
||||
data: EncodeSamplesRecord(nil, nil),
|
||||
want: result{samplesToBits([]RefSample{}), nil},
|
||||
},
|
||||
{
|
||||
name: "special_float_values",
|
||||
data: EncodeSamplesRecord(nil, []RefSample{
|
||||
{Ref: 1, T: 0, V: math.NaN()},
|
||||
{Ref: 2, T: 0, V: math.Inf(1)},
|
||||
{Ref: 3, T: 0, V: math.Inf(-1)},
|
||||
{Ref: 4, T: 0, V: math.Copysign(0, -1)},
|
||||
}),
|
||||
want: result{samplesToBits([]RefSample{
|
||||
{Ref: 1, T: 0, V: math.NaN()},
|
||||
{Ref: 2, T: 0, V: math.Inf(1)},
|
||||
{Ref: 3, T: 0, V: math.Inf(-1)},
|
||||
{Ref: 4, T: 0, V: math.Copysign(0, -1)},
|
||||
}), nil},
|
||||
},
|
||||
{
|
||||
name: "negative_timestamp",
|
||||
data: EncodeSamplesRecord(nil, []RefSample{
|
||||
{Ref: 1, T: -5000, V: 1.5},
|
||||
}),
|
||||
want: result{samplesToBits([]RefSample{{Ref: 1, T: -5000, V: 1.5}}), nil},
|
||||
},
|
||||
|
||||
// Error cases.
|
||||
{
|
||||
name: "nil",
|
||||
data: nil,
|
||||
want: result{nil, ErrShortPayload},
|
||||
},
|
||||
{
|
||||
name: "truncated_count",
|
||||
data: []byte{0, 0},
|
||||
want: result{nil, ErrShortPayload},
|
||||
},
|
||||
{
|
||||
name: "truncated_mid_sample",
|
||||
data: func() []byte {
|
||||
d := make([]byte, 20) // nsamples=1(4) + 16 bytes (need 24)
|
||||
binary.BigEndian.PutUint32(d, 1)
|
||||
return d
|
||||
}(),
|
||||
want: result{nil, ErrShortPayload},
|
||||
},
|
||||
{
|
||||
name: "count_exceeds_data",
|
||||
data: func() []byte {
|
||||
d := make([]byte, 28) // nsamples=2(4) + 24 bytes (only 1 sample)
|
||||
binary.BigEndian.PutUint32(d, 2)
|
||||
return d
|
||||
}(),
|
||||
want: result{nil, ErrShortPayload},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
samples, err := DecodeSamplesRecord(tc.data)
|
||||
got := samplesToBits(samples)
|
||||
assert.Equal(t, tc.want.err, err, "error")
|
||||
require.Equal(t, len(tc.want.samples), len(got), "sample count")
|
||||
for i := range tc.want.samples {
|
||||
assert.Equal(t, tc.want.samples[i], got[i], "sample %d", i)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user