Remove testify dependency

This commit is contained in:
2026-07-04 18:51:08 -04:00
parent 199e555e3d
commit 4a8b1768a1
20 changed files with 1335 additions and 495 deletions
+13 -5
View File
@@ -31,7 +31,7 @@ db.Close()
## Status
**Pre-alpha.** The API is frozen (M4) and the system survives a 48h soak test under sustained load (M5), but this hasn't seen production use yet.
**Alpha.** The API is frozen (M4) and the system survives a 48h soak test under sustained load (M5), but this hasn't seen production use yet.
| Milestone | State |
|---|---|
@@ -44,14 +44,24 @@ db.Close()
See [DESIGN.md](DESIGN.md) for architecture, on-disk format, and the non-goals table. See [ROADMAP.md](ROADMAP.md) for what's next.
## Why
I needed a library to store time-series data locally and the options out there didn't quite work for my case. Prometheus was too heavy for what I needed but I wanted that level of compression. tstorage was close but it didn't have the compression or label indexing.
## Install
```sh
go get git.dvdt.dev/david/ingot
```
## Features
- **Gorilla XOR compression** — ~1 byte/sample on regular metric data (see benchmarks below)
- **Crash-safe** — WAL with CRC32C records; kill -9 at any point loses at most uncommitted samples
- **Crash-safe** — WAL with CRC32C records. Committed data is persisted
- **Query by label matchers** — equality, negation, regex, negative regex; merged across head and blocks
- **Levelled compaction** — 2h → 8h → 32h blocks, background merging, retention-based expiry
- **Self-instrumentation** — the DB records its own metrics (series/chunk counts, compactions, WAL fsync duration) through the normal write path, queryable like any other series
- **Zero external dependencies** (except testify for tests)
- **Zero external dependencies**
## Tools
@@ -138,8 +148,6 @@ The chunk encoding comes from the Gorilla paper (Pelkonen et al., VLDB 2015) via
[tstorage](https://github.com/nakabonne/tstorage) is the closest existing embedded TSDB for Go. It doesn't do Gorilla compression or label-based indexing, which is most of why ingot exists.
I needed a library to store time-series data locally and the options out there didn't quite fit. Prometheus was too heavy for what I needed but I wanted that level of compression. tstorage was close but it didn't have the compression or label indexing.
## Non-goals
Replication, query languages, non-float64 values, deletes, multi-process access, out-of-order ingestion, Windows (sorry not my thing). Check DESIGN.md for reasoning.
+42 -16
View File
@@ -9,15 +9,15 @@ import (
"git.dvdt.dev/david/ingot/internal/block"
"git.dvdt.dev/david/ingot/internal/chunkenc"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func makeChunk(t *testing.T, samples []struct{ t int64; v float64 }) []byte {
t.Helper()
c := chunkenc.NewXORChunk()
a, err := c.Appender()
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for _, s := range samples {
a.Append(s.t, s.v)
}
@@ -51,7 +51,9 @@ func setupTestData(t *testing.T) string {
}
_, err := block.Flush(dir, series)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return dir
}
@@ -59,7 +61,9 @@ func setupTestData(t *testing.T) string {
func blockDir(t *testing.T, dataDir string) string {
t.Helper()
entries, err := os.ReadDir(dataDir)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for _, e := range entries {
if e.IsDir() && e.Name() != "wal" {
return filepath.Join(dataDir, e.Name())
@@ -124,10 +128,16 @@ func TestCmdBlocks(t *testing.T) {
return cmdErr
}()
assert.Equal(t, tc.wantErr != "", err != nil, "error presence")
assert.Contains(t, errString(err), tc.wantErr)
if got := (err != nil); got != (tc.wantErr != "") {
t.Errorf("error presence: got %v, want %v", got, tc.wantErr != "")
}
if !strings.Contains(errString(err), tc.wantErr) {
t.Errorf("got %q, want substring %q", errString(err), tc.wantErr)
}
for _, want := range tc.wantOutputs {
assert.Contains(t, output, want)
if !strings.Contains(output, want) {
t.Errorf("got %q, want substring %q", output, want)
}
}
})
}
@@ -166,10 +176,16 @@ func TestCmdInspect(t *testing.T) {
return cmdErr
}()
assert.Equal(t, tc.wantErr != "", err != nil, "error presence")
assert.Contains(t, errString(err), tc.wantErr)
if got := (err != nil); got != (tc.wantErr != "") {
t.Errorf("error presence: got %v, want %v", got, tc.wantErr != "")
}
if !strings.Contains(errString(err), tc.wantErr) {
t.Errorf("got %q, want substring %q", errString(err), tc.wantErr)
}
for _, want := range tc.wantOutputs {
assert.Contains(t, output, want)
if !strings.Contains(output, want) {
t.Errorf("got %q, want substring %q", output, want)
}
}
})
}
@@ -220,10 +236,16 @@ func TestCmdChunks(t *testing.T) {
return cmdErr
}()
assert.Equal(t, tc.wantErr != "", err != nil, "error presence")
assert.Contains(t, errString(err), tc.wantErr)
if got := (err != nil); got != (tc.wantErr != "") {
t.Errorf("error presence: got %v, want %v", got, tc.wantErr != "")
}
if !strings.Contains(errString(err), tc.wantErr) {
t.Errorf("got %q, want substring %q", errString(err), tc.wantErr)
}
for _, want := range tc.wantOutputs {
assert.Contains(t, output, want)
if !strings.Contains(output, want) {
t.Errorf("got %q, want substring %q", output, want)
}
}
})
}
@@ -263,8 +285,12 @@ func TestCmdFsck(t *testing.T) {
return cmdErr
}()
assert.Equal(t, tc.wantErr != "", err != nil, "error presence")
assert.Contains(t, errString(err), tc.wantErr)
if got := (err != nil); got != (tc.wantErr != "") {
t.Errorf("error presence: got %v, want %v", got, tc.wantErr != "")
}
if !strings.Contains(errString(err), tc.wantErr) {
t.Errorf("got %q, want substring %q", errString(err), tc.wantErr)
}
})
}
}
+39 -15
View File
@@ -10,14 +10,14 @@ import (
"git.dvdt.dev/david/ingot"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func openTestDB(t *testing.T) *ingot.DB {
t.Helper()
db, err := ingot.Open(t.TempDir(), ingot.Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
t.Cleanup(func() { db.Close() })
return db
}
@@ -26,12 +26,20 @@ func seedDB(t *testing.T, db *ingot.DB) {
t.Helper()
app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 1000, 71.3)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(ref, nil, 2000, 71.4)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(0, labels.FromStrings("__name__", "humidity", "room", "office"), 1000, 55.0)
require.NoError(t, err)
require.NoError(t, app.Commit())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestQueryRange(t *testing.T) {
@@ -117,12 +125,16 @@ func TestQueryRange(t *testing.T) {
rec := httptest.NewRecorder()
h.queryRange(rec, req)
assert.Equal(t, tc.wantStatus, rec.Code)
if rec.Code != tc.wantStatus {
t.Errorf("got %v, want %v", rec.Code, tc.wantStatus)
}
// Always decode — error responses produce zero-valued struct.
var resp queryRangeResponse
json.Unmarshal(rec.Body.Bytes(), &resp)
assert.Equal(t, tc.wantCount, len(resp.Data.Result))
if len(resp.Data.Result) != tc.wantCount {
t.Errorf("got %v, want %v", len(resp.Data.Result), tc.wantCount)
}
})
}
}
@@ -200,13 +212,17 @@ func TestRead(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
body, err := json.Marshal(tc.request)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
req := httptest.NewRequest(tc.method, "/api/v1/read", bytes.NewReader(body))
rec := httptest.NewRecorder()
h.read(rec, req)
assert.Equal(t, tc.wantStatus, rec.Code)
if rec.Code != tc.wantStatus {
t.Errorf("got %v, want %v", rec.Code, tc.wantStatus)
}
// Always decode — error responses produce zero-valued struct.
var resp ReadResponse
@@ -215,7 +231,9 @@ func TestRead(t *testing.T) {
if len(resp.Results) > 0 {
firstResultLen = len(resp.Results[0].Timeseries)
}
assert.Equal(t, tc.wantSeries, firstResultLen)
if firstResultLen != tc.wantSeries {
t.Errorf("got %v, want %v", firstResultLen, tc.wantSeries)
}
})
}
}
@@ -267,13 +285,19 @@ func TestStatus(t *testing.T) {
rec := httptest.NewRecorder()
h.status(rec, req)
assert.Equal(t, tc.wantStatus, rec.Code)
if rec.Code != tc.wantStatus {
t.Errorf("got %v, want %v", rec.Code, tc.wantStatus)
}
// Always decode — error responses produce zero-valued struct.
var resp statusResponse
json.Unmarshal(rec.Body.Bytes(), &resp)
assert.Equal(t, tc.wantHeadSeries, resp.HeadSeries)
assert.Equal(t, tc.wantBlocks, resp.Blocks)
if resp.HeadSeries != tc.wantHeadSeries {
t.Errorf("got %v, want %v", resp.HeadSeries, tc.wantHeadSeries)
}
if resp.Blocks != tc.wantBlocks {
t.Errorf("got %v, want %v", resp.Blocks, tc.wantBlocks)
}
})
}
}
-8
View File
@@ -1,11 +1,3 @@
module git.dvdt.dev/david/ingot
go 1.26.1
require github.com/stretchr/testify v1.11.1
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
-10
View File
@@ -1,10 +0,0 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+247 -84
View File
@@ -3,12 +3,11 @@ package ingot
import (
"math"
"os"
"reflect"
"testing"
"time"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// sample is a test convenience type.
@@ -92,13 +91,17 @@ func collectSeriesSet(t *testing.T, ss SeriesSet) map[uint64][]sample {
st, sv := it.At()
samples = append(samples, sample{st, sv})
}
require.NoError(t, it.Err())
if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(samples) > 0 {
h := labels.Hash(ls)
result[h] = samples
}
}
require.NoError(t, ss.Err())
if err := ss.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return result
}
@@ -117,7 +120,9 @@ func oracleByLabelHash(o *oracle, mint, maxt int64, matchers ...*labels.Matcher)
func openTestDB(t *testing.T) *DB {
t.Helper()
db, err := Open(t.TempDir(), Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
t.Cleanup(func() { db.Close() })
return db
}
@@ -133,20 +138,28 @@ func TestQueryOracle(t *testing.T) {
setup: func(t *testing.T, db *DB, o *oracle) {
app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 1000, 71.3)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref, labels.FromStrings("__name__", "temp", "room", "office"))
o.addSample(ref, 1000, 71.3)
_, err = app.Append(ref, nil, 2000, 71.4)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, 2000, 71.4)
ref2, err := app.Append(0, labels.FromStrings("__name__", "humidity", "room", "office"), 1000, 55.0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref2, labels.FromStrings("__name__", "humidity", "room", "office"))
o.addSample(ref2, 1000, 55.0)
require.NoError(t, app.Commit())
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
queries: []queryCase{
{
@@ -199,19 +212,27 @@ func TestQueryOracle(t *testing.T) {
app := db.Appender()
// Write enough samples to seal chunks (need >120 for a sealed chunk).
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref, labels.FromStrings("__name__", "temp"))
o.addSample(ref, 0, 0)
for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, int64(i*15000), float64(i))
}
require.NoError(t, app.Commit())
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Flush to block.
_, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
queries: []queryCase{
{
@@ -233,30 +254,42 @@ func TestQueryOracle(t *testing.T) {
setup: func(t *testing.T, db *DB, o *oracle) {
app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 0, 0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref, labels.FromStrings("__name__", "temp", "room", "office"))
o.addSample(ref, 0, 0)
// Write 250 samples (2 sealed chunks + 10 active).
for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, int64(i*15000), float64(i))
}
require.NoError(t, app.Commit())
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Flush sealed chunks to block.
_, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Append more samples to head (after flush).
app = db.Appender()
for i := 250; i < 260; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, int64(i*15000), float64(i))
}
require.NoError(t, app.Commit())
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
queries: []queryCase{
{
@@ -300,15 +333,21 @@ func TestQueryOracle(t *testing.T) {
app := db.Appender()
for _, s := range series {
ref, err := app.Append(0, s.ls, 1000, 1.0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref, s.ls)
o.addSample(ref, 1000, 1.0)
_, err = app.Append(ref, nil, 2000, 2.0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, 2000, 2.0)
}
require.NoError(t, app.Commit())
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
queries: []queryCase{
{
@@ -357,39 +396,57 @@ func TestQueryOracle(t *testing.T) {
setup: func(t *testing.T, db *DB, o *oracle) {
app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref, labels.FromStrings("__name__", "temp"))
o.addSample(ref, 0, 0)
for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, int64(i*15000), float64(i))
}
require.NoError(t, app.Commit())
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// First flush.
_, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// More data -> second flush.
app = db.Appender()
for i := 250; i < 500; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, int64(i*15000), float64(i))
}
require.NoError(t, app.Commit())
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Some data in head.
app = db.Appender()
for i := 500; i < 510; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, int64(i*15000), float64(i))
}
require.NoError(t, app.Commit())
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
queries: []queryCase{
{
@@ -411,10 +468,14 @@ func TestQueryOracle(t *testing.T) {
setup: func(t *testing.T, db *DB, o *oracle) {
app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 1000, 1.0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref, labels.FromStrings("__name__", "temp"))
o.addSample(ref, 1000, 1.0)
require.NoError(t, app.Commit())
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
queries: []queryCase{
{
@@ -443,18 +504,26 @@ func TestQueryOracle(t *testing.T) {
for _, qc := range tc.queries {
t.Run(qc.name, func(t *testing.T) {
q, err := db.Querier(qc.mint, qc.maxt)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer q.Close()
ss := q.Select(qc.matchers...)
got := collectSeriesSet(t, ss)
want := oracleByLabelHash(o, qc.mint, qc.maxt, qc.matchers...)
assert.Equal(t, len(want), len(got), "series count mismatch")
if len(got) != len(want) {
t.Errorf("series count mismatch: got %v, want %v", len(got), len(want))
}
for h, wantSamples := range want {
gotSamples, ok := got[h]
assert.True(t, ok, "missing series with hash %d", h)
assert.Equal(t, wantSamples, gotSamples, "sample mismatch for hash %d", h)
if !ok {
t.Errorf("missing series with hash %d", h)
}
if !reflect.DeepEqual(gotSamples, wantSamples) {
t.Errorf("sample mismatch for hash %d: got %v, want %v", h, gotSamples, wantSamples)
}
}
})
}
@@ -483,13 +552,21 @@ func TestDBLifecycle(t *testing.T) {
name: "append_and_query_back",
setup: func(t *testing.T, dir string) *DB {
db, err := Open(dir, Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 1000, 71.3)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(ref, nil, 2000, 71.4)
require.NoError(t, err)
require.NoError(t, app.Commit())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return db
},
wantSampleCount: 2,
@@ -502,21 +579,35 @@ func TestDBLifecycle(t *testing.T) {
name: "reopen_with_blocks",
setup: func(t *testing.T, dir string) *DB {
db, err := Open(dir, Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, app.Commit())
_, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err)
require.NoError(t, db.Close())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := db.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
db2, err := Open(dir, Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return db2
},
wantSampleCount: 250,
@@ -529,7 +620,9 @@ func TestDBLifecycle(t *testing.T) {
name: "compacted_blocks_queryable",
setup: func(t *testing.T, dir string) *DB {
db, err := Open(dir, Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
const samplesPerBlock = 130
ref := uint64(0)
for b := 0; b < 4; b++ {
@@ -537,14 +630,22 @@ func TestDBLifecycle(t *testing.T) {
for i := 0; i < samplesPerBlock; i++ {
ts := int64((b*samplesPerBlock + i) * 15000)
r, err := app.Append(ref, labels.FromStrings("__name__", "temp"), ts, float64(ts))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
ref = r
}
require.NoError(t, app.Commit())
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err := db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := db.RunCompaction(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, db.RunCompaction())
return db
},
wantSampleCount: 520,
@@ -561,29 +662,47 @@ func TestDBLifecycle(t *testing.T) {
Retention: 24 * time.Hour,
Clock: func() int64 { return now },
})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Old data (50 hours ago).
app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "old"), 50*3600*1000, 1.0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(50*3600*1000+i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, app.Commit())
_, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Recent data (1 hour ago).
app = db.Appender()
ref2, err := app.Append(0, labels.FromStrings("__name__", "recent"), 99*3600*1000, 1.0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 1; i < 250; i++ {
_, err = app.Append(ref2, nil, int64(99*3600*1000+i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, app.Commit())
_, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
db.ApplyRetention()
return db
},
@@ -602,7 +721,9 @@ func TestDBLifecycle(t *testing.T) {
defer db.Close()
q, err := db.Querier(tc.mint, tc.maxt)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer q.Close()
ss := q.Select(tc.matchers...)
@@ -614,11 +735,19 @@ func TestDBLifecycle(t *testing.T) {
for it.Next() {
sampleCount++
}
require.NoError(t, it.Err())
if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := ss.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if seriesCount != tc.wantSeriesCount {
t.Errorf("series count: got %v, want %v", seriesCount, tc.wantSeriesCount)
}
if sampleCount != tc.wantSampleCount {
t.Errorf("sample count: got %v, want %v", sampleCount, tc.wantSampleCount)
}
require.NoError(t, ss.Err())
assert.Equal(t, tc.wantSeriesCount, seriesCount, "series count")
assert.Equal(t, tc.wantSampleCount, sampleCount, "sample count")
})
}
}
@@ -645,7 +774,9 @@ func TestQueryDuringCompaction(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
db, err := Open(dir, Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer db.Close()
ref := uint64(0)
@@ -654,12 +785,18 @@ func TestQueryDuringCompaction(t *testing.T) {
for i := 0; i < tc.samplesPerBlock; i++ {
ts := int64((b*tc.samplesPerBlock + i) * 15000)
r, err := app.Append(ref, labels.FromStrings("__name__", "temp"), ts, float64(ts))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
ref = r
}
require.NoError(t, app.Commit())
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err := db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
// Snapshot source block dirs.
@@ -672,21 +809,31 @@ func TestQueryDuringCompaction(t *testing.T) {
// Start query holding refs on all blocks.
q, err := db.Querier(math.MinInt64, math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
ss := q.Select(labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp"))
require.True(t, ss.Next())
if !ss.Next() {
t.Fatalf("expected true")
}
it := ss.At().Iterator()
require.True(t, it.Next())
if !it.Next() {
t.Fatalf("expected true")
}
// Compact while query is open.
err = db.RunCompaction()
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Source dirs still exist (query holds refs).
for _, d := range sourceDirs {
_, statErr := os.Stat(d)
assert.NoError(t, statErr, "source dir should exist while query holds ref")
if statErr != nil {
t.Errorf("source dir should exist while query holds ref: %v", statErr)
}
}
// Finish iterating — all data still readable.
@@ -694,19 +841,29 @@ func TestQueryDuringCompaction(t *testing.T) {
for it.Next() {
count++
}
require.NoError(t, it.Err())
assert.Equal(t, tc.samplesPerBlock*tc.numBlocks, count, "all samples readable during compaction")
if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if count != tc.samplesPerBlock*tc.numBlocks {
t.Errorf("all samples readable during compaction: got %v, want %v", count, tc.samplesPerBlock*tc.numBlocks)
}
// Close querier — source dirs should be deleted.
require.NoError(t, q.Close())
if err := q.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
for _, d := range sourceDirs {
_, statErr := os.Stat(d)
assert.True(t, os.IsNotExist(statErr), "source dir should be deleted after query close")
if !os.IsNotExist(statErr) {
t.Errorf("source dir should be deleted after query close")
}
}
// Compacted block still queryable.
q2, err := db.Querier(math.MinInt64, math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer q2.Close()
ss2 := q2.Select(labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp"))
count = 0
@@ -715,10 +872,16 @@ func TestQueryDuringCompaction(t *testing.T) {
for it2.Next() {
count++
}
require.NoError(t, it2.Err())
if err := it2.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := ss2.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if count != tc.samplesPerBlock*tc.numBlocks {
t.Errorf("compacted block has all samples: got %v, want %v", count, tc.samplesPerBlock*tc.numBlocks)
}
require.NoError(t, ss2.Err())
assert.Equal(t, tc.samplesPerBlock*tc.numBlocks, count, "compacted block has all samples")
})
}
}
+119 -39
View File
@@ -5,13 +5,12 @@ import (
"math/rand"
"os"
"path/filepath"
"reflect"
"testing"
"git.dvdt.dev/david/ingot/internal/chunkenc"
"git.dvdt.dev/david/ingot/internal/index"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type sample struct {
@@ -26,7 +25,9 @@ func makeChunk(t *testing.T, samples []sample) []byte {
t.Helper()
c := chunkenc.NewXORChunk()
a, err := c.Appender()
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for _, s := range samples {
a.Append(s.t, math.Float64frombits(s.vBits))
}
@@ -41,7 +42,9 @@ func collectIterator(t *testing.T, it chunkenc.ChunkIterator) []sample {
ts, v := it.At()
out = append(out, sample{ts, math.Float64bits(v)})
}
require.NoError(t, it.Err())
if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return out
}
@@ -137,30 +140,52 @@ func TestBlockRoundTrip(t *testing.T) {
// Write block.
ulid, err := Flush(dataDir, flushData)
require.NoError(t, err)
require.NotEmpty(t, ulid)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if ulid == "" {
t.Fatalf("got empty ULID")
}
// Open block for reading.
blockDir := filepath.Join(dataDir, ulid)
r, err := Open(blockDir)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer r.Close()
// Verify meta.
assert.Equal(t, ulid, r.Meta.ULID)
assert.Equal(t, 1, r.Meta.Version)
assert.Equal(t, len(tc.series), r.Meta.Stats.NumSeries)
assert.Equal(t, len(tc.series), r.Meta.Stats.NumChunks)
if got, want := r.Meta.ULID, ulid; got != want {
t.Errorf("got %v, want %v", got, want)
}
if got, want := r.Meta.Version, 1; got != want {
t.Errorf("got %v, want %v", got, want)
}
if got, want := r.Meta.Stats.NumSeries, len(tc.series); got != want {
t.Errorf("got %v, want %v", got, want)
}
if got, want := r.Meta.Stats.NumChunks, len(tc.series); got != want {
t.Errorf("got %v, want %v", got, want)
}
// Verify each series' data via iteration.
for _, s := range tc.series {
it, err := r.SeriesChunkIterator(s.ref, math.MinInt64, math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
got := collectIterator(t, it)
require.Equal(t, len(s.samples), len(got), "ref %d sample count", s.ref)
if len(got) != len(s.samples) {
t.Fatalf("ref %d sample count: got %v, want %v", s.ref, len(got), len(s.samples))
}
for i, want := range s.samples {
assert.Equal(t, want.t, got[i].t, "ref %d sample %d t", s.ref, i)
assert.Equal(t, want.vBits, got[i].vBits, "ref %d sample %d v", s.ref, i)
if got[i].t != want.t {
t.Errorf("ref %d sample %d t: got %v, want %v", s.ref, i, got[i].t, want.t)
}
if got[i].vBits != want.vBits {
t.Errorf("ref %d sample %d v: got %v, want %v", s.ref, i, got[i].vBits, want.vBits)
}
}
}
@@ -168,15 +193,28 @@ func TestBlockRoundTrip(t *testing.T) {
for _, s := range tc.series {
for _, l := range s.labels {
refs := r.Postings(l.Name, l.Value)
assert.Contains(t, refs, s.ref, "postings for %s=%s", l.Name, l.Value)
found := false
for _, ref := range refs {
if ref == s.ref {
found = true
break
}
}
if !found {
t.Errorf("postings for %s=%s: %v does not contain %d", l.Name, l.Value, refs, s.ref)
}
}
}
// Verify labels lookup.
for _, s := range tc.series {
ls, ok := r.Labels(s.ref)
assert.True(t, ok)
assert.Equal(t, s.labels, ls)
if !ok {
t.Errorf("Labels(%d) returned ok=false", s.ref)
}
if !reflect.DeepEqual(ls, s.labels) {
t.Errorf("got %v, want %v", ls, s.labels)
}
}
})
}
@@ -236,20 +274,32 @@ func TestBlockSeriesChunkIterator(t *testing.T) {
},
}
ulid, err := Flush(dataDir, flushData)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
r, err := Open(filepath.Join(dataDir, ulid))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer r.Close()
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
it, err := r.SeriesChunkIterator(tc.ref, tc.mint, tc.maxt)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
got := collectIterator(t, it)
assert.Equal(t, len(tc.wantSamples), len(got), "sample count")
if len(got) != len(tc.wantSamples) {
t.Errorf("sample count: got %v, want %v", len(got), len(tc.wantSamples))
}
for i, want := range tc.wantSamples {
assert.Equal(t, want.t, got[i].t, "sample %d t", i)
assert.Equal(t, want.vBits, got[i].vBits, "sample %d v", i)
if got[i].t != want.t {
t.Errorf("sample %d t: got %v, want %v", i, got[i].t, want.t)
}
if got[i].vBits != want.vBits {
t.Errorf("sample %d v: got %v, want %v", i, got[i].vBits, want.vBits)
}
}
})
}
@@ -296,13 +346,21 @@ func TestBlockMetaTimeBounds(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
dataDir := t.TempDir()
ulid, err := Flush(dataDir, tc.series)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
r, err := Open(filepath.Join(dataDir, ulid))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer r.Close()
assert.Equal(t, tc.wantMinT, r.Meta.MinTime, "MinTime")
assert.Equal(t, tc.wantMaxT, r.Meta.MaxTime, "MaxTime")
if got, want := r.Meta.MinTime, tc.wantMinT; got != want {
t.Errorf("MinTime: got %v, want %v", got, want)
}
if got, want := r.Meta.MaxTime, tc.wantMaxT; got != want {
t.Errorf("MaxTime: got %v, want %v", got, want)
}
})
}
}
@@ -310,11 +368,15 @@ func TestBlockMetaTimeBounds(t *testing.T) {
func TestULIDRoundTrip(t *testing.T) {
for i := 0; i < 100; i++ {
u := newULID()
assert.Equal(t, 26, len(u), "ULID length")
if len(u) != 26 {
t.Errorf("ULID length: got %v, want %v", len(u), 26)
}
// Should parse without error.
_, err := parseULID(u)
assert.NoError(t, err, "parse ULID %q", u)
if err != nil {
t.Errorf("parse ULID %q: %v", u, err)
}
}
}
@@ -329,10 +391,14 @@ func TestBlockCorruption(t *testing.T) {
corruptFunc: func(t *testing.T, blockDir string, chunkRef index.ChunkRef) {
chunkPath := filepath.Join(blockDir, chunksDirName, segmentName(int(chunkRef.Segment())))
data, err := os.ReadFile(chunkPath)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
off := int(chunkRef.Offset()) + chunkEntryHeaderLen + 1
data[off] ^= 0xFF
require.NoError(t, os.WriteFile(chunkPath, data, 0644))
if err := os.WriteFile(chunkPath, data, 0644); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
wantErr: ErrCorruptChunk,
},
@@ -341,7 +407,9 @@ func TestBlockCorruption(t *testing.T) {
corruptFunc: func(t *testing.T, blockDir string, chunkRef index.ChunkRef) {
chunkPath := filepath.Join(blockDir, chunksDirName, segmentName(int(chunkRef.Segment())))
data, err := os.ReadFile(chunkPath)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Corrupt the last byte of the CRC.
off := int(chunkRef.Offset()) + chunkEntryHeaderLen
// Read dataLen to find CRC position.
@@ -349,7 +417,9 @@ func TestBlockCorruption(t *testing.T) {
int(data[chunkRef.Offset()+2])*256 + int(data[chunkRef.Offset()+3])
crcOff := off + dataLen + 3 // last byte of CRC
data[crcOff] ^= 0xFF
require.NoError(t, os.WriteFile(chunkPath, data, 0644))
if err := os.WriteFile(chunkPath, data, 0644); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
wantErr: ErrCorruptChunk,
},
@@ -367,24 +437,34 @@ func TestBlockCorruption(t *testing.T) {
},
}
ulid, err := Flush(dataDir, flushData)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
blockDir := filepath.Join(dataDir, ulid)
r, err := Open(blockDir)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
series := r.Series()
require.Equal(t, 1, len(series))
if len(series) != 1 {
t.Fatalf("got %v, want %v", len(series), 1)
}
chunkRef := series[0].Chunks[0].Ref
r.Close()
tc.corruptFunc(t, blockDir, chunkRef)
r, err = Open(blockDir)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer r.Close()
_, err = r.ChunkIterator(chunkRef)
assert.Equal(t, tc.wantErr, err)
if err != tc.wantErr {
t.Errorf("got %v, want %v", err, tc.wantErr)
}
})
}
}
+51 -19
View File
@@ -8,8 +8,6 @@ import (
"testing"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestValidate(t *testing.T) {
@@ -33,7 +31,9 @@ func TestValidate(t *testing.T) {
},
}
ulid, err := Flush(dir, series)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return filepath.Join(dir, ulid)
},
wantErrors: 0,
@@ -53,7 +53,9 @@ func TestValidate(t *testing.T) {
},
}
ulid, err := Flush(dir, series)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
blockDir := filepath.Join(dir, ulid)
os.Remove(filepath.Join(blockDir, "meta.json"))
return blockDir
@@ -75,15 +77,21 @@ func TestValidate(t *testing.T) {
},
}
ulid, err := Flush(dir, series)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
blockDir := filepath.Join(dir, ulid)
// Corrupt a byte in the chunk data.
chunkPath := filepath.Join(blockDir, "chunks", "000001")
data, err := os.ReadFile(chunkPath)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
data[chunkHeaderLen+chunkEntryHeaderLen+2] ^= 0xFF
require.NoError(t, os.WriteFile(chunkPath, data, 0644))
if err := os.WriteFile(chunkPath, data, 0644); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return blockDir
},
wantErrors: 1,
@@ -103,14 +111,20 @@ func TestValidate(t *testing.T) {
},
}
ulid, err := Flush(dir, series)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
blockDir := filepath.Join(dir, ulid)
chunkPath := filepath.Join(blockDir, "chunks", "000001")
data, err := os.ReadFile(chunkPath)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
binary.BigEndian.PutUint32(data[:4], 0xDEADBEEF)
require.NoError(t, os.WriteFile(chunkPath, data, 0644))
if err := os.WriteFile(chunkPath, data, 0644); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return blockDir
},
wantErrors: 1,
@@ -122,7 +136,9 @@ func TestValidate(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
blockDir := tc.setup(t)
errs := Validate(blockDir)
assert.Equal(t, tc.wantErrors, len(errs), "error count: %v", errs)
if len(errs) != tc.wantErrors {
t.Errorf("error count: got %d, want %d: %v", len(errs), tc.wantErrors, errs)
}
// Concatenate all error strings; "" is contained in everything.
var combined strings.Builder
@@ -130,7 +146,9 @@ func TestValidate(t *testing.T) {
combined.WriteString(e.Error())
combined.WriteByte('\n')
}
assert.Contains(t, combined.String(), tc.wantMatch)
if !strings.Contains(combined.String(), tc.wantMatch) {
t.Errorf("got %q, want substring %q", combined.String(), tc.wantMatch)
}
})
}
}
@@ -159,7 +177,9 @@ func TestReadMeta(t *testing.T) {
},
},
})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return dir, ulid
},
wantULID: true,
@@ -175,12 +195,24 @@ func TestReadMeta(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
dir, ulid := tc.setup(t)
meta, err := ReadMeta(filepath.Join(dir, ulid))
assert.Equal(t, tc.wantErr, err)
assert.Equal(t, tc.wantULID, meta.ULID == ulid)
assert.Equal(t, tc.wantMinT, meta.MinTime)
assert.Equal(t, tc.wantMaxT, meta.MaxTime)
assert.Equal(t, tc.wantNSer, meta.Stats.NumSeries)
assert.Equal(t, tc.wantNChk, meta.Stats.NumChunks)
if err != tc.wantErr {
t.Errorf("error: got %v, want %v", err, tc.wantErr)
}
if (meta.ULID == ulid) != tc.wantULID {
t.Errorf("ULID match: got %v, want %v", meta.ULID == ulid, tc.wantULID)
}
if meta.MinTime != tc.wantMinT {
t.Errorf("MinTime: got %v, want %v", meta.MinTime, tc.wantMinT)
}
if meta.MaxTime != tc.wantMaxT {
t.Errorf("MaxTime: got %v, want %v", meta.MaxTime, tc.wantMaxT)
}
if meta.Stats.NumSeries != tc.wantNSer {
t.Errorf("NumSeries: got %v, want %v", meta.Stats.NumSeries, tc.wantNSer)
}
if meta.Stats.NumChunks != tc.wantNChk {
t.Errorf("NumChunks: got %v, want %v", meta.Stats.NumChunks, tc.wantNChk)
}
})
}
}
+6 -4
View File
@@ -4,8 +4,6 @@ import (
"fmt"
"math/rand"
"testing"
"github.com/stretchr/testify/assert"
)
// bstreamBytes writes (value, nbits) pairs and returns the encoded bytes.
@@ -144,8 +142,12 @@ func TestBstream(t *testing.T) {
r := newBReader(tc.data)
for i, op := range tc.reads {
got, err := r.readBits(op.nbits)
assert.Equal(t, op.wantVal, got, "op %d val (nbits=%d)", i, op.nbits)
assert.Equal(t, op.wantErr, err, "op %d err (nbits=%d)", i, op.nbits)
if got != op.wantVal {
t.Errorf("op %d val (nbits=%d): got %v, want %v", i, op.nbits, got, op.wantVal)
}
if err != op.wantErr {
t.Errorf("op %d err (nbits=%d): got %v, want %v", i, op.nbits, err, op.wantErr)
}
}
})
}
+46 -17
View File
@@ -6,9 +6,8 @@ import (
"fmt"
"math"
"math/rand"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
)
// encodeChunk builds a valid XOR chunk from samples and returns its bytes.
@@ -284,16 +283,28 @@ func TestXORChunk(t *testing.T) {
for i, r := range tc.reads {
next := it.Next()
gotT, gotV := it.At()
assert.Equal(t, r.wantNext, next, "read %d Next()", i)
assert.Equal(t, r.wantT, gotT, "read %d timestamp", i)
assert.Equal(t, r.wantVBits, math.Float64bits(gotV), "read %d value", i)
if next != r.wantNext {
t.Errorf("read %d Next(): got %v, want %v", i, next, r.wantNext)
}
if gotT != r.wantT {
t.Errorf("read %d timestamp: got %v, want %v", i, gotT, r.wantT)
}
if math.Float64bits(gotV) != r.wantVBits {
t.Errorf("read %d value: got %v, want %v", i, math.Float64bits(gotV), r.wantVBits)
}
}
if it.Err() != tc.wantIterErr {
t.Errorf("iter err: got %v, want %v", it.Err(), tc.wantIterErr)
}
assert.Equal(t, tc.wantIterErr, it.Err())
assert.LessOrEqual(t, len(tc.data), tc.maxBytes)
if !(len(tc.data) <= tc.maxBytes) {
t.Errorf("got %v bytes, want <= %v", len(tc.data), tc.maxBytes)
}
_, appErr := c.Appender()
assert.Equal(t, tc.wantAppenderErr, appErr)
if !reflect.DeepEqual(appErr, tc.wantAppenderErr) {
t.Errorf("appender err: got %v, want %v", appErr, tc.wantAppenderErr)
}
})
}
}
@@ -329,23 +340,39 @@ func TestXORChunkFromBytes(t *testing.T) {
reconstituted := XORChunkFromBytes(orig.Bytes())
assert.Equal(t, orig.NumSamples(), reconstituted.NumSamples())
assert.Equal(t, orig.Bytes(), reconstituted.Bytes())
if orig.NumSamples() != reconstituted.NumSamples() {
t.Errorf("NumSamples: got %v, want %v", reconstituted.NumSamples(), orig.NumSamples())
}
if !reflect.DeepEqual(orig.Bytes(), reconstituted.Bytes()) {
t.Errorf("Bytes mismatch")
}
// Verify iteration produces identical samples.
it := reconstituted.Iterator()
for i, s := range tc.samples {
assert.True(t, it.Next(), "sample %d", i)
if !it.Next() {
t.Errorf("sample %d: expected Next()=true", i)
}
gotT, gotV := it.At()
assert.Equal(t, int64(s[0]), gotT, "sample %d t", i)
assert.Equal(t, math.Float64bits(s[1]), math.Float64bits(gotV), "sample %d v", i)
if gotT != int64(s[0]) {
t.Errorf("sample %d t: got %v, want %v", i, gotT, int64(s[0]))
}
if math.Float64bits(gotV) != math.Float64bits(s[1]) {
t.Errorf("sample %d v: got %v, want %v", i, math.Float64bits(gotV), math.Float64bits(s[1]))
}
}
if it.Next() {
t.Errorf("expected Next()=false after all samples")
}
if it.Err() != nil {
t.Errorf("unexpected iter error: %v", it.Err())
}
assert.False(t, it.Next())
assert.NoError(t, it.Err())
// Appender on non-empty reconstituted chunk should fail.
_, err := reconstituted.Appender()
assert.Error(t, err)
if err == nil {
t.Errorf("expected error from Appender on non-empty chunk")
}
})
}
}
@@ -429,7 +456,9 @@ func TestBytesPerSample(t *testing.T) {
c := benchChunk(tc.gen)
bps := float64(len(c.Bytes())) / 120.0
t.Logf("%s: %.3f bytes/sample (%d bytes total)", tc.name, bps, len(c.Bytes()))
assert.LessOrEqual(t, bps, tc.max, "%s bytes/sample exceeds ceiling", tc.name)
if !(bps <= tc.max) {
t.Errorf("%s bytes/sample exceeds ceiling: got %v, want <= %v", tc.name, bps, tc.max)
}
})
}
}
+57 -21
View File
@@ -8,8 +8,6 @@ import (
"git.dvdt.dev/david/ingot/internal/block"
"git.dvdt.dev/david/ingot/internal/chunkenc"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
@@ -21,7 +19,9 @@ func makeChunk(t *testing.T, timestamps []int64, values []float64) []byte {
t.Helper()
c := chunkenc.NewXORChunk()
a, err := c.Appender()
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := range timestamps {
a.Append(timestamps[i], values[i])
}
@@ -38,9 +38,13 @@ func flushTestBlock(t *testing.T, dataDir string, series []block.SeriesFlush, le
} else {
ulid, err = block.FlushCompacted(dataDir, series, level, sources)
}
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
r, err := block.Open(filepath.Join(dataDir, ulid))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return r
}
@@ -48,13 +52,17 @@ func flushTestBlock(t *testing.T, dataDir string, series []block.SeriesFlush, le
func collectBlockSamples(t *testing.T, r *block.Reader, ref uint64) []sample {
t.Helper()
it, err := r.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var out []sample
for it.Next() {
ts, v := it.At()
out = append(out, sample{ts, v})
}
require.NoError(t, it.Err())
if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return out
}
@@ -162,12 +170,20 @@ func TestPlan(t *testing.T) {
group := c.Plan(blocks)
if !tc.wantGroup {
assert.Nil(t, group, "expected no compaction group")
if group != nil {
t.Errorf("expected no compaction group, got %v", group)
}
return
}
require.NotNil(t, group, "expected a compaction group")
assert.Equal(t, tc.wantLevel, group.Level, "compaction level")
assert.Equal(t, tc.wantCount, len(group.Sources), "source count")
if group == nil {
t.Fatalf("expected a compaction group")
}
if got, want := group.Level, tc.wantLevel; got != want {
t.Errorf("compaction level: got %v, want %v", got, want)
}
if got, want := len(group.Sources), tc.wantCount; got != want {
t.Errorf("source count: got %v, want %v", got, want)
}
})
}
}
@@ -263,8 +279,12 @@ func TestCompact(t *testing.T) {
// Compact.
newULID, err := c.Compact(sources)
require.NoError(t, err)
require.NotEmpty(t, newULID)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if newULID == "" {
t.Fatalf("expected non-empty ULID")
}
// Close source blocks.
for _, s := range sources {
@@ -273,24 +293,38 @@ func TestCompact(t *testing.T) {
// Open compacted block.
compacted, err := block.Open(filepath.Join(dataDir, newULID))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer compacted.Close()
// Verify compaction level.
assert.Equal(t, tc.wantLevel, compacted.Meta.Compaction.Level, "compaction level")
assert.Equal(t, len(tc.sourceBlocks), len(compacted.Meta.Compaction.Sources), "source count")
if got, want := compacted.Meta.Compaction.Level, tc.wantLevel; got != want {
t.Errorf("compaction level: got %v, want %v", got, want)
}
if got, want := len(compacted.Meta.Compaction.Sources), len(tc.sourceBlocks); got != want {
t.Errorf("source count: got %v, want %v", got, want)
}
// Verify series count.
assert.Equal(t, len(tc.wantSeriesRefs), compacted.Meta.Stats.NumSeries, "series count")
if got, want := compacted.Meta.Stats.NumSeries, len(tc.wantSeriesRefs); got != want {
t.Errorf("series count: got %v, want %v", got, want)
}
// Verify each series' samples.
for _, ref := range tc.wantSeriesRefs {
got := collectBlockSamples(t, compacted, ref)
want := tc.wantSamples[ref]
require.Equal(t, len(want), len(got), "sample count for ref %d", ref)
if len(got) != len(want) {
t.Fatalf("sample count for ref %d: got %v, want %v", ref, len(got), len(want))
}
for i := range want {
assert.Equal(t, want[i].t, got[i].t, "ref %d sample %d t", ref, i)
assert.Equal(t, want[i].v, got[i].v, "ref %d sample %d v", ref, i)
if got[i].t != want[i].t {
t.Errorf("ref %d sample %d t: got %v, want %v", ref, i, got[i].t, want[i].t)
}
if got[i].v != want[i].v {
t.Errorf("ref %d sample %d v: got %v, want %v", ref, i, got[i].v, want[i].v)
}
}
}
})
@@ -357,7 +391,9 @@ func TestExpired(t *testing.T) {
}
expired := c.Expired(blocks)
assert.Equal(t, tc.wantCount, len(expired), "expired block count")
if got, want := len(expired), tc.wantCount; got != want {
t.Errorf("expired block count: got %v, want %v", got, want)
}
})
}
}
+243 -83
View File
@@ -4,14 +4,13 @@ import (
"math"
"os"
"path/filepath"
"reflect"
"sync"
"testing"
"git.dvdt.dev/david/ingot/internal/block"
"git.dvdt.dev/david/ingot/internal/wal"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// sample is a convenience type for expected results.
@@ -31,7 +30,9 @@ func collectSamples(t *testing.T, h *Head, ref uint64, mint, maxt int64) []sampl
ts, v := it.At()
out = append(out, sample{ts, math.Float64bits(v)})
}
require.NoError(t, it.Err())
if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return out
}
@@ -61,7 +62,9 @@ func openHead(t *testing.T) *Head {
t.Helper()
dir := filepath.Join(t.TempDir(), "wal")
h, err := Open(dir, wal.Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
t.Cleanup(func() { h.Close() })
return h
}
@@ -285,22 +288,34 @@ func TestHead(t *testing.T) {
app = h.Appender()
case actAppend:
ref, err := app.Append(act.ref, act.labels, act.t, act.v)
assert.Equal(t, act.wantRef, ref, "action %d ref", i)
assert.Equal(t, act.wantErr, err, "action %d error", i)
if ref != act.wantRef {
t.Errorf("action %d ref: got %v, want %v", i, ref, act.wantRef)
}
if err != act.wantErr {
t.Errorf("action %d error: got %v, want %v", i, err, act.wantErr)
}
case actCommit:
err := app.Commit()
assert.Equal(t, act.wantErr, err, "action %d commit error", i)
if err != act.wantErr {
t.Errorf("action %d commit error: got %v, want %v", i, err, act.wantErr)
}
case actRollback:
err := app.Rollback()
assert.Equal(t, act.wantErr, err, "action %d rollback error", i)
if err != act.wantErr {
t.Errorf("action %d rollback error: got %v, want %v", i, err, act.wantErr)
}
}
}
for ref, wantSamples := range tc.wantSamples {
got := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64)
require.Equal(t, len(wantSamples), len(got), "ref %d sample count", ref)
if len(got) != len(wantSamples) {
t.Fatalf("ref %d sample count: got %v, want %v", ref, len(got), len(wantSamples))
}
for i, want := range wantSamples {
assert.Equal(t, want, got[i], "ref %d sample %d", ref, i)
if !reflect.DeepEqual(got[i], want) {
t.Errorf("ref %d sample %d: got %v, want %v", ref, i, got[i], want)
}
}
}
})
@@ -317,14 +332,24 @@ func TestWALReplay(t *testing.T) {
name: "basic_recovery",
setup: func(t *testing.T, dir string) {
h, err := Open(dir, wal.Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := h.Appender()
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 71.3)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(ref, nil, 1015, 71.4)
require.NoError(t, err)
require.NoError(t, app.Commit())
require.NoError(t, h.Close())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := h.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
wantSeries: map[uint64][]sample{
1: {s(1000, 71.3), s(1015, 71.4)},
@@ -334,14 +359,24 @@ func TestWALReplay(t *testing.T) {
name: "multi_series_recovery",
setup: func(t *testing.T, dir string) {
h, err := Open(dir, wal.Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := h.Appender()
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 71.3)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}}, 1000, 55.0)
require.NoError(t, err)
require.NoError(t, app.Commit())
require.NoError(t, h.Close())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := h.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
wantSeries: map[uint64][]sample{
1: {s(1000, 71.3)},
@@ -352,16 +387,26 @@ func TestWALReplay(t *testing.T) {
name: "chunk_sealing_recovery",
setup: func(t *testing.T, dir string) {
h, err := Open(dir, wal.Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := h.Appender()
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := h.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, app.Commit())
require.NoError(t, h.Close())
},
wantSeries: map[uint64][]sample{
1: func() []sample {
@@ -377,16 +422,28 @@ func TestWALReplay(t *testing.T) {
name: "multiple_commits_recovery",
setup: func(t *testing.T, dir string) {
h, err := Open(dir, wal.Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := h.Appender()
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 1.0)
require.NoError(t, err)
require.NoError(t, app.Commit())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
app = h.Appender()
_, err = app.Append(1, nil, 2000, 2.0)
require.NoError(t, err)
require.NoError(t, app.Commit())
require.NoError(t, h.Close())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := h.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
wantSeries: map[uint64][]sample{
1: {s(1000, 1.0), s(2000, 2.0)},
@@ -396,17 +453,29 @@ func TestWALReplay(t *testing.T) {
name: "rollback_not_recovered",
setup: func(t *testing.T, dir string) {
h, err := Open(dir, wal.Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := h.Appender()
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 1.0)
require.NoError(t, err)
require.NoError(t, app.Commit())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Second batch is rolled back — should not survive restart.
app = h.Appender()
_, err = app.Append(1, nil, 2000, 2.0)
require.NoError(t, err)
require.NoError(t, app.Rollback())
require.NoError(t, h.Close())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := app.Rollback(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := h.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
wantSeries: map[uint64][]sample{
1: {s(1000, 1.0)},
@@ -420,14 +489,20 @@ func TestWALReplay(t *testing.T) {
tc.setup(t, dir)
h, err := Open(dir, wal.Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer h.Close()
for ref, wantSamples := range tc.wantSeries {
got := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64)
require.Equal(t, len(wantSamples), len(got), "ref %d sample count", ref)
if len(got) != len(wantSamples) {
t.Fatalf("ref %d sample count: got %v, want %v", ref, len(got), len(wantSamples))
}
for i, want := range wantSamples {
assert.Equal(t, want, got[i], "ref %d sample %d", ref, i)
if !reflect.DeepEqual(got[i], want) {
t.Errorf("ref %d sample %d: got %v, want %v", ref, i, got[i], want)
}
}
}
})
@@ -493,49 +568,76 @@ func TestFlushOlderThan(t *testing.T) {
app := h.Appender()
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 1; i < tc.numSamples; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, app.Commit())
ulid, err := h.FlushOlderThan(tc.flushMaxT)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Post-flush appends.
if tc.postFlushAppend > 0 {
app = h.Appender()
for i := tc.numSamples; i < tc.numSamples+tc.postFlushAppend; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, app.Commit())
}
// Assert ULID presence.
assert.Equal(t, tc.wantULID, ulid != "", "block ULID presence")
gotULID := ulid != ""
if gotULID != tc.wantULID {
t.Errorf("block ULID presence: got %v, want %v", gotULID, tc.wantULID)
}
// Assert block contents.
blockCount := 0
if ulid != "" {
blockDir := filepath.Join(h.DataDir(), ulid)
br, err := block.Open(blockDir)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer br.Close()
assert.Equal(t, tc.wantBlockSeries, br.Meta.Stats.NumSeries, "block series count")
if br.Meta.Stats.NumSeries != tc.wantBlockSeries {
t.Errorf("block series count: got %v, want %v", br.Meta.Stats.NumSeries, tc.wantBlockSeries)
}
it, err := br.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for it.Next() {
blockCount++
}
require.NoError(t, it.Err())
if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if blockCount != tc.wantBlockCount {
t.Errorf("block sample count: got %v, want %v", blockCount, tc.wantBlockCount)
}
assert.Equal(t, tc.wantBlockCount, blockCount, "block sample count")
// Assert head still has data.
headSamples := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64)
assert.GreaterOrEqual(t, len(headSamples), tc.wantHeadCount, "head sample count")
if len(headSamples) < tc.wantHeadCount {
t.Errorf("head sample count: got %v, want >= %v", len(headSamples), tc.wantHeadCount)
}
})
}
}
@@ -555,41 +657,65 @@ func TestFlushWALTruncation(t *testing.T) {
walDir := filepath.Join(dir, "wal")
h, err := Open(walDir, wal.Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := h.Appender()
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_ = ref
for i := 1; i < tc.numSamples; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, app.Commit())
walEntries, err := os.ReadDir(walDir)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
segsBefore := len(walEntries)
_, err = h.FlushOlderThan(math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
walEntries, err = os.ReadDir(walDir)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
segsAfter := len(walEntries)
assert.LessOrEqual(t, segsAfter, segsBefore, "WAL should be truncated after flush")
if segsAfter > segsBefore {
t.Errorf("WAL should be truncated after flush: got %v segments after, had %v before", segsAfter, segsBefore)
}
require.NoError(t, h.Close())
if err := h.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Re-open: head should be functional.
h2, err := Open(walDir, wal.Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer h2.Close()
app = h2.Appender()
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "new_series"}}, 5000000, 42.0)
require.NoError(t, err)
require.NoError(t, app.Commit())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
})
}
}
@@ -600,12 +726,20 @@ func TestHeadPostings(t *testing.T) {
// Add three series.
app := h.Appender()
_, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}}, 1000, 1.0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "office"}}, 1000, 2.0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "kitchen"}}, 1000, 3.0)
require.NoError(t, err)
require.NoError(t, app.Commit())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
tests := []struct {
name string
@@ -622,7 +756,9 @@ func TestHeadPostings(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := h.Postings(tc.label, tc.value)
assert.Equal(t, tc.wantRefs, got)
if !reflect.DeepEqual(got, tc.wantRefs) {
t.Errorf("got %v, want %v", got, tc.wantRefs)
}
})
}
}
@@ -631,8 +767,8 @@ func TestHeadQueryMethods(t *testing.T) {
tests := []struct {
name string
series [][]labels.Label
wantLabelValues map[string][]string // label name -> expected values
wantLabels map[uint64][]labels.Label // ref -> expected labels
wantLabelValues map[string][]string // label name -> expected values
wantLabels map[uint64][]labels.Label // ref -> expected labels
wantAllPostings []uint64
}{
{
@@ -677,29 +813,43 @@ func TestHeadQueryMethods(t *testing.T) {
app := h.Appender()
for _, ls := range tc.series {
_, err := app.Append(0, ls, 1000, 1.0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, app.Commit())
// Assert LabelValues.
for name, wantVals := range tc.wantLabelValues {
got := h.LabelValues(name)
assert.Equal(t, wantVals, got, "LabelValues(%q)", name)
if !reflect.DeepEqual(got, wantVals) {
t.Errorf("LabelValues(%q): got %v, want %v", name, got, wantVals)
}
}
// Assert Labels by ref.
for ref, wantLabels := range tc.wantLabels {
ls, ok := h.Labels(ref)
assert.True(t, ok, "Labels(%d) should exist", ref)
assert.Equal(t, wantLabels, ls, "Labels(%d)", ref)
if !ok {
t.Errorf("Labels(%d) should exist", ref)
}
if !reflect.DeepEqual(ls, wantLabels) {
t.Errorf("Labels(%d): got %v, want %v", ref, ls, wantLabels)
}
}
// Unknown ref returns false.
_, ok := h.Labels(999)
assert.False(t, ok, "Labels(999) should not exist")
if ok {
t.Errorf("Labels(999) should not exist")
}
// Assert AllPostings.
assert.Equal(t, tc.wantAllPostings, h.AllPostings(), "AllPostings")
if !reflect.DeepEqual(h.AllPostings(), tc.wantAllPostings) {
t.Errorf("AllPostings: got %v, want %v", h.AllPostings(), tc.wantAllPostings)
}
})
}
}
@@ -732,20 +882,30 @@ func TestConcurrentAppend(t *testing.T) {
{Name: "goroutine", Value: string(rune('A' + g))},
}
ref, err := app.Append(0, ls, int64(g*1000000), 0)
require.NoError(t, err)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
refs[g] = ref
for i := 1; i < tc.samplesPerGoroutine; i++ {
_, err = app.Append(ref, nil, int64(g*1000000+i*1000), float64(i))
require.NoError(t, err)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
}
if err := app.Commit(); err != nil {
t.Errorf("unexpected error: %v", err)
}
require.NoError(t, app.Commit())
}(g)
}
wg.Wait()
for g := 0; g < tc.numGoroutines; g++ {
got := collectSamples(t, h, refs[g], math.MinInt64, math.MaxInt64)
assert.Equal(t, tc.samplesPerGoroutine, len(got), "goroutine %d sample count", g)
if len(got) != tc.samplesPerGoroutine {
t.Errorf("goroutine %d sample count: got %v, want %v", g, len(got), tc.samplesPerGoroutine)
}
}
})
}
+86 -28
View File
@@ -2,11 +2,10 @@ package index
import (
"bytes"
"reflect"
"testing"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func writeIndex(t *testing.T, entries []SeriesEntry) []byte {
@@ -17,7 +16,9 @@ func writeIndex(t *testing.T, entries []SeriesEntry) []byte {
w.AddSeries(e)
}
_, err := w.WriteTo()
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return buf.Bytes()
}
@@ -81,41 +82,74 @@ func TestIndexRoundTrip(t *testing.T) {
data := writeIndex(t, tc.entries)
r, err := NewReader(data)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Verify series count and content.
gotSeries := r.Series()
require.Equal(t, len(tc.entries), len(gotSeries))
if len(gotSeries) != len(tc.entries) {
t.Fatalf("got %v, want %v", len(gotSeries), len(tc.entries))
}
for i, want := range tc.entries {
got := gotSeries[i]
assert.Equal(t, want.Ref, got.Ref, "series %d ref", i)
assert.Equal(t, want.Labels, got.Labels, "series %d labels", i)
require.Equal(t, len(want.Chunks), len(got.Chunks), "series %d chunk count", i)
if got.Ref != want.Ref {
t.Errorf("series %d ref: got %v, want %v", i, got.Ref, want.Ref)
}
if !reflect.DeepEqual(got.Labels, want.Labels) {
t.Errorf("series %d labels: got %v, want %v", i, got.Labels, want.Labels)
}
if len(got.Chunks) != len(want.Chunks) {
t.Fatalf("series %d chunk count: got %v, want %v", i, len(got.Chunks), len(want.Chunks))
}
for j, wc := range want.Chunks {
assert.Equal(t, wc.MinT, got.Chunks[j].MinT, "series %d chunk %d minT", i, j)
assert.Equal(t, wc.MaxT, got.Chunks[j].MaxT, "series %d chunk %d maxT", i, j)
assert.Equal(t, wc.Ref, got.Chunks[j].Ref, "series %d chunk %d ref", i, j)
if got.Chunks[j].MinT != wc.MinT {
t.Errorf("series %d chunk %d minT: got %v, want %v", i, j, got.Chunks[j].MinT, wc.MinT)
}
if got.Chunks[j].MaxT != wc.MaxT {
t.Errorf("series %d chunk %d maxT: got %v, want %v", i, j, got.Chunks[j].MaxT, wc.MaxT)
}
if got.Chunks[j].Ref != wc.Ref {
t.Errorf("series %d chunk %d ref: got %v, want %v", i, j, got.Chunks[j].Ref, wc.Ref)
}
}
// SeriesByRef lookup.
byRef, ok := r.SeriesByRef(want.Ref)
assert.True(t, ok, "series %d lookup by ref", i)
assert.Equal(t, want.Ref, byRef.Ref)
if !ok {
t.Errorf("series %d lookup by ref: got false, want true", i)
}
if byRef.Ref != want.Ref {
t.Errorf("got %v, want %v", byRef.Ref, want.Ref)
}
}
// Verify postings.
for _, e := range tc.entries {
for _, l := range e.Labels {
refs := r.Postings(l.Name, l.Value)
assert.Contains(t, refs, e.Ref, "postings for %s=%s should contain ref %d", l.Name, l.Value, e.Ref)
found := false
for _, ref := range refs {
if ref == e.Ref {
found = true
break
}
}
if !found {
t.Errorf("postings for %s=%s should contain ref %d, got %v", l.Name, l.Value, e.Ref, refs)
}
}
}
// Verify missing lookups return empty/false.
_, ok := r.SeriesByRef(999999)
assert.False(t, ok)
assert.Nil(t, r.Postings("nonexistent", "value"))
if ok {
t.Errorf("got true, want false")
}
if refs := r.Postings("nonexistent", "value"); refs != nil {
t.Errorf("got %v, want nil", refs)
}
})
}
}
@@ -129,13 +163,23 @@ func TestIndexPostingsSorted(t *testing.T) {
data := writeIndex(t, entries)
r, err := NewReader(data)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
refs := r.Postings("room", "office")
require.Equal(t, 3, len(refs))
assert.Equal(t, uint64(2), refs[0])
assert.Equal(t, uint64(5), refs[1])
assert.Equal(t, uint64(8), refs[2])
if len(refs) != 3 {
t.Fatalf("got %v, want %v", len(refs), 3)
}
if refs[0] != uint64(2) {
t.Errorf("got %v, want %v", refs[0], uint64(2))
}
if refs[1] != uint64(5) {
t.Errorf("got %v, want %v", refs[1], uint64(5))
}
if refs[2] != uint64(8) {
t.Errorf("got %v, want %v", refs[2], uint64(8))
}
}
func TestIndexChunkRefEncoding(t *testing.T) {
@@ -153,8 +197,12 @@ func TestIndexChunkRefEncoding(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
ref := NewChunkRef(tc.segment, tc.offset)
assert.Equal(t, tc.segment, ref.Segment())
assert.Equal(t, tc.offset, ref.Offset())
if ref.Segment() != tc.segment {
t.Errorf("got %v, want %v", ref.Segment(), tc.segment)
}
if ref.Offset() != tc.offset {
t.Errorf("got %v, want %v", ref.Offset(), tc.offset)
}
})
}
}
@@ -168,7 +216,9 @@ func TestIndexLabelValues(t *testing.T) {
data := writeIndex(t, entries)
r, err := NewReader(data)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
tests := []struct {
name string
@@ -183,7 +233,9 @@ func TestIndexLabelValues(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := r.LabelValues(tc.label)
assert.Equal(t, tc.wantVals, got)
if !reflect.DeepEqual(got, tc.wantVals) {
t.Errorf("got %v, want %v", got, tc.wantVals)
}
})
}
}
@@ -197,10 +249,14 @@ func TestIndexAllPostings(t *testing.T) {
data := writeIndex(t, entries)
r, err := NewReader(data)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
refs := r.AllPostings()
assert.Equal(t, []uint64{2, 5, 8}, refs)
if !reflect.DeepEqual(refs, []uint64{2, 5, 8}) {
t.Errorf("got %v, want %v", refs, []uint64{2, 5, 8})
}
}
func TestIndexCorruptData(t *testing.T) {
@@ -246,7 +302,9 @@ func TestIndexCorruptData(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := NewReader(tc.data)
assert.Equal(t, tc.wantErr, err)
if err != tc.wantErr {
t.Errorf("got %v, want %v", err, tc.wantErr)
}
})
}
}
+10 -5
View File
@@ -1,9 +1,8 @@
package postings
import (
"reflect"
"testing"
"github.com/stretchr/testify/assert"
)
func TestIntersect(t *testing.T) {
@@ -25,7 +24,9 @@ func TestIntersect(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := Intersect(tc.lists...)
assert.Equal(t, tc.want, got)
if !reflect.DeepEqual(got, tc.want) {
t.Errorf("got %v, want %v", got, tc.want)
}
})
}
}
@@ -49,7 +50,9 @@ func TestUnion(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := Union(tc.lists...)
assert.Equal(t, tc.want, got)
if !reflect.DeepEqual(got, tc.want) {
t.Errorf("got %v, want %v", got, tc.want)
}
})
}
}
@@ -73,7 +76,9 @@ func TestWithout(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := Without(tc.full, tc.remove)
assert.Equal(t, tc.want, got)
if !reflect.DeepEqual(got, tc.want) {
t.Errorf("got %v, want %v", got, tc.want)
}
})
}
}
+16 -7
View File
@@ -3,11 +3,10 @@ package wal
import (
"encoding/binary"
"math"
"reflect"
"testing"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSeriesRecord(t *testing.T) {
@@ -140,8 +139,12 @@ func TestSeriesRecord(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
rec, err := DecodeSeriesRecord(tc.data)
assert.Equal(t, tc.want.rec, rec, "record")
assert.Equal(t, tc.want.err, err, "error")
if !reflect.DeepEqual(rec, tc.want.rec) {
t.Errorf("record: got %v, want %v", rec, tc.want.rec)
}
if err != tc.want.err {
t.Errorf("error: got %v, want %v", err, tc.want.err)
}
})
}
}
@@ -260,10 +263,16 @@ func TestSamplesRecord(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
samples, err := DecodeSamplesRecord(tc.data)
got := samplesToBits(samples)
assert.Equal(t, tc.want.err, err, "error")
require.Equal(t, len(tc.want.samples), len(got), "sample count")
if err != tc.want.err {
t.Errorf("error: got %v, want %v", err, tc.want.err)
}
if len(got) != len(tc.want.samples) {
t.Fatalf("sample count: got %d, want %d", len(got), len(tc.want.samples))
}
for i := range tc.want.samples {
assert.Equal(t, tc.want.samples[i], got[i], "sample %d", i)
if got[i] != tc.want.samples[i] {
t.Errorf("sample %d: got %v, want %v", i, got[i], tc.want.samples[i])
}
}
})
}
+28 -11
View File
@@ -3,9 +3,8 @@ package wal
import (
"encoding/binary"
"hash/crc32"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
)
func TestRecord(t *testing.T) {
@@ -138,10 +137,18 @@ func TestRecord(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
typ, payload, consumed, err := DecodeRecord(tc.data)
assert.Equal(t, tc.want.typ, typ, "type")
assert.Equal(t, tc.want.payload, payload, "payload")
assert.Equal(t, tc.want.consumed, consumed, "consumed")
assert.Equal(t, tc.want.err, err, "error")
if typ != tc.want.typ {
t.Errorf("type: got %v, want %v", typ, tc.want.typ)
}
if !reflect.DeepEqual(payload, tc.want.payload) {
t.Errorf("payload: got %v, want %v", payload, tc.want.payload)
}
if consumed != tc.want.consumed {
t.Errorf("consumed: got %v, want %v", consumed, tc.want.consumed)
}
if err != tc.want.err {
t.Errorf("error: got %v, want %v", err, tc.want.err)
}
})
}
}
@@ -149,11 +156,17 @@ func TestRecord(t *testing.T) {
func TestEncodeRecordAppendsToExisting(t *testing.T) {
prefix := []byte("existing")
result := EncodeRecord(prefix, RecordSeries, []byte{0x01})
assert.Equal(t, []byte("existing"), result[:8])
if !reflect.DeepEqual([]byte("existing"), result[:8]) {
t.Errorf("prefix: got %v, want %v", result[:8], []byte("existing"))
}
_, payload, _, err := DecodeRecord(result[8:])
assert.Equal(t, []byte{0x01}, payload)
assert.Equal(t, nil, err)
if !reflect.DeepEqual([]byte{0x01}, payload) {
t.Errorf("payload: got %v, want %v", payload, []byte{0x01})
}
if err != nil {
t.Errorf("error: got %v, want nil", err)
}
}
func TestRecordSize(t *testing.T) {
@@ -167,7 +180,9 @@ func TestRecordSize(t *testing.T) {
}
for _, tc := range tests {
assert.Equal(t, tc.want, RecordSize(tc.payloadLen))
if got := RecordSize(tc.payloadLen); got != tc.want {
t.Errorf("RecordSize(%d): got %v, want %v", tc.payloadLen, got, tc.want)
}
}
}
@@ -179,5 +194,7 @@ func TestEncodeCRCCoversHeaderAndPayload(t *testing.T) {
headerAndPayload := rec[:recordHeaderSize+len(payload)]
want := crc32.Checksum(headerAndPayload, castagnoliTable)
got := binary.BigEndian.Uint32(rec[recordHeaderSize+len(payload):])
assert.Equal(t, want, got)
if got != want {
t.Errorf("CRC: got %v, want %v", got, want)
}
}
+186 -69
View File
@@ -3,25 +3,28 @@ package wal
import (
"os"
"path/filepath"
"reflect"
"testing"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// collectRecords replays all records from a WAL directory.
func collectRecords(t *testing.T, dir string) []Record {
t.Helper()
r, err := NewReader(dir)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer r.Close()
var recs []Record
for r.Next() {
recs = append(recs, r.Record())
}
require.NoError(t, r.Err())
if r.Err() != nil {
t.Fatalf("unexpected error: %v", r.Err())
}
return recs
}
@@ -36,15 +39,23 @@ func TestWAL(t *testing.T) {
name: "write_and_replay",
setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{})
require.NoError(t, err)
require.NoError(t, w.LogSeries([]SeriesRecord{
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.LogSeries([]SeriesRecord{
{Ref: 1, Labels: []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}}},
}))
require.NoError(t, w.LogSamples([]RefSample{
}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.LogSamples([]RefSample{
{Ref: 1, T: 1000, V: 71.3},
{Ref: 1, T: 1015, V: 71.4},
}))
require.NoError(t, w.Close())
}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
wantRecords: 2,
wantMinSegs: 1,
@@ -53,8 +64,12 @@ func TestWAL(t *testing.T) {
name: "empty",
setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{})
require.NoError(t, err)
require.NoError(t, w.Close())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
wantRecords: 0,
wantMinSegs: 1,
@@ -63,14 +78,26 @@ func TestWAL(t *testing.T) {
name: "reopen_and_append",
setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{})
require.NoError(t, err)
require.NoError(t, w.LogSamples([]RefSample{{Ref: 1, T: 1000, V: 1.0}}))
require.NoError(t, w.Close())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.LogSamples([]RefSample{{Ref: 1, T: 1000, V: 1.0}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
w, err = Open(dir, Options{})
require.NoError(t, err)
require.NoError(t, w.LogSamples([]RefSample{{Ref: 2, T: 2000, V: 2.0}}))
require.NoError(t, w.Close())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.LogSamples([]RefSample{{Ref: 2, T: 2000, V: 2.0}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
wantRecords: 2,
wantMinSegs: 1,
@@ -79,11 +106,17 @@ func TestWAL(t *testing.T) {
name: "segment_rotation",
setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{SegmentMaxSize: 50})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 0; i < 10; i++ {
require.NoError(t, w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i * 1000), V: float64(i)}}))
if err := w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i * 1000), V: float64(i)}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, w.Close())
},
wantRecords: 10,
wantMinSegs: 2,
@@ -92,13 +125,21 @@ func TestWAL(t *testing.T) {
name: "truncate_old_segments",
setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{SegmentMaxSize: 50})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 0; i < 10; i++ {
require.NoError(t, w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i), V: float64(i)}}))
if err := w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i), V: float64(i)}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
lastSeg := w.LastSegment()
require.NoError(t, w.Truncate(lastSeg))
require.NoError(t, w.Close())
if err := w.Truncate(lastSeg); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
wantRecords: 1, // only the last segment's record(s) survive
wantMinSegs: 1,
@@ -107,25 +148,42 @@ func TestWAL(t *testing.T) {
name: "recovery_truncates_trailing_garbage",
setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 0; i < 3; i++ {
require.NoError(t, w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i), V: float64(i)}}))
if err := w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i), V: float64(i)}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, w.Close())
// Append garbage after valid records.
segs, err := listSegments(dir)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
f, err := os.OpenFile(segmentPath(dir, segs[0]), os.O_WRONLY|os.O_APPEND, 0644)
require.NoError(t, err)
_, err = f.Write([]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF})
require.NoError(t, err)
require.NoError(t, f.Close())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if _, err := f.Write([]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := f.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Reopen triggers recovery.
w, err = Open(dir, Options{})
require.NoError(t, err)
require.NoError(t, w.Close())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
wantRecords: 3,
wantMinSegs: 1,
@@ -134,25 +192,42 @@ func TestWAL(t *testing.T) {
name: "recovery_truncates_corrupt_mid_record",
setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 0; i < 3; i++ {
require.NoError(t, w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i), V: float64(i)}}))
if err := w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i), V: float64(i)}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, w.Close())
// Write a valid header but truncated payload (looks like a torn write).
segs, err := listSegments(dir)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
f, err := os.OpenFile(segmentPath(dir, segs[0]), os.O_WRONLY|os.O_APPEND, 0644)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// type=1, len=100 (big), but no payload follows.
_, err = f.Write([]byte{0x01, 0x00, 0x00, 0x00, 0x64})
require.NoError(t, err)
require.NoError(t, f.Close())
if _, err := f.Write([]byte{0x01, 0x00, 0x00, 0x00, 0x64}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := f.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
w, err = Open(dir, Options{})
require.NoError(t, err)
require.NoError(t, w.Close())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
wantRecords: 3,
wantMinSegs: 1,
@@ -165,11 +240,17 @@ func TestWAL(t *testing.T) {
tc.setup(t, dir)
recs := collectRecords(t, dir)
assert.Equal(t, tc.wantRecords, len(recs), "record count")
if len(recs) != tc.wantRecords {
t.Errorf("record count: got %v, want %v", len(recs), tc.wantRecords)
}
segs, err := listSegments(dir)
require.NoError(t, err)
assert.GreaterOrEqual(t, len(segs), tc.wantMinSegs, "segment count")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !(len(segs) >= tc.wantMinSegs) {
t.Errorf("segment count: got %v, want >= %v", len(segs), tc.wantMinSegs)
}
})
}
}
@@ -187,17 +268,25 @@ func TestTornWriteRecovery(t *testing.T) {
name: "single_segment_mixed_records",
opts: Options{},
recs: func(t *testing.T, w *WAL) {
require.NoError(t, w.LogSeries([]SeriesRecord{
if err := w.LogSeries([]SeriesRecord{
{Ref: 1, Labels: []labels.Label{{Name: "__name__", Value: "temp"}}},
}))
require.NoError(t, w.LogSamples([]RefSample{{Ref: 1, T: 1000, V: 71.3}}))
require.NoError(t, w.LogSeries([]SeriesRecord{
}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.LogSamples([]RefSample{{Ref: 1, T: 1000, V: 71.3}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.LogSeries([]SeriesRecord{
{Ref: 2, Labels: []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "lab"}}},
}))
require.NoError(t, w.LogSamples([]RefSample{
}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.LogSamples([]RefSample{
{Ref: 1, T: 1015, V: 71.4},
{Ref: 2, T: 1000, V: 55.0},
}))
}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
},
{
@@ -205,7 +294,9 @@ func TestTornWriteRecovery(t *testing.T) {
opts: Options{SegmentMaxSize: 50},
recs: func(t *testing.T, w *WAL) {
for i := 0; i < 10; i++ {
require.NoError(t, w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i * 1000), V: float64(i)}}))
if err := w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i * 1000), V: float64(i)}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
},
},
@@ -216,21 +307,31 @@ func TestTornWriteRecovery(t *testing.T) {
// Write the reference WAL.
srcDir := filepath.Join(t.TempDir(), "src")
w, err := Open(srcDir, tc.opts)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
tc.recs(t, w)
require.NoError(t, w.Close())
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
origRecs := collectRecords(t, srcDir)
require.Greater(t, len(origRecs), 0)
if !(len(origRecs) > 0) {
t.Fatalf("got %v records, want > 0", len(origRecs))
}
segs, err := listSegments(srcDir)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Read all segment data.
segData := make(map[int][]byte)
for _, idx := range segs {
data, err := os.ReadFile(segmentPath(srcDir, idx))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
segData[idx] = data
}
@@ -240,25 +341,41 @@ func TestTornWriteRecovery(t *testing.T) {
for cutoff := 0; cutoff <= len(lastData); cutoff++ {
walDir := filepath.Join(t.TempDir(), "wal")
require.NoError(t, os.MkdirAll(walDir, 0755))
if err := os.MkdirAll(walDir, 0755); err != nil {
t.Fatalf("cutoff=%d: unexpected error: %v", cutoff, err)
}
// Copy earlier segments intact.
for _, idx := range segs[:len(segs)-1] {
require.NoError(t, os.WriteFile(segmentPath(walDir, idx), segData[idx], 0644))
if err := os.WriteFile(segmentPath(walDir, idx), segData[idx], 0644); err != nil {
t.Fatalf("cutoff=%d: unexpected error: %v", cutoff, err)
}
}
// Write truncated last segment.
require.NoError(t, os.WriteFile(segmentPath(walDir, lastSeg), lastData[:cutoff], 0644))
if err := os.WriteFile(segmentPath(walDir, lastSeg), lastData[:cutoff], 0644); err != nil {
t.Fatalf("cutoff=%d: unexpected error: %v", cutoff, err)
}
w2, err := Open(walDir, tc.opts)
require.NoError(t, err, "cutoff=%d", cutoff)
if err != nil {
t.Fatalf("cutoff=%d: unexpected error: %v", cutoff, err)
}
recovered := collectRecords(t, walDir)
require.NoError(t, w2.Close(), "cutoff=%d", cutoff)
if err := w2.Close(); err != nil {
t.Fatalf("cutoff=%d: unexpected error: %v", cutoff, err)
}
// Must be a valid prefix.
assert.LessOrEqual(t, len(recovered), len(origRecs), "cutoff=%d count", cutoff)
if !(len(recovered) <= len(origRecs)) {
t.Errorf("cutoff=%d: got %d records, want <= %d", cutoff, len(recovered), len(origRecs))
}
for i, rec := range recovered {
assert.Equal(t, origRecs[i].Type, rec.Type, "cutoff=%d rec=%d type", cutoff, i)
assert.Equal(t, origRecs[i].Data, rec.Data, "cutoff=%d rec=%d data", cutoff, i)
if origRecs[i].Type != rec.Type {
t.Errorf("cutoff=%d rec=%d type: got %v, want %v", cutoff, i, rec.Type, origRecs[i].Type)
}
if !reflect.DeepEqual(origRecs[i].Data, rec.Data) {
t.Errorf("cutoff=%d rec=%d data: got %v, want %v", cutoff, i, rec.Data, origRecs[i].Data)
}
}
}
})
+38 -12
View File
@@ -1,10 +1,8 @@
package labels
import (
"reflect"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMatcherMatches(t *testing.T) {
@@ -45,8 +43,12 @@ func TestMatcherMatches(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
m, err := NewMatcher(tc.typ, "__name__", tc.pattern)
require.NoError(t, err)
assert.Equal(t, tc.want, m.Matches(tc.value))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := m.Matches(tc.value); got != tc.want {
t.Errorf("got %v, want %v", got, tc.want)
}
})
}
}
@@ -64,7 +66,9 @@ func TestNewMatcherErrors(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := NewMatcher(tc.typ, "__name__", tc.pattern)
assert.Error(t, err)
if err == nil {
t.Errorf("expected error")
}
})
}
}
@@ -81,9 +85,18 @@ func TestMustNewMatcherPanics(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
assert.Panics(t, func() {
panicked := false
func() {
defer func() {
if r := recover(); r != nil {
panicked = true
}
}()
MustNewMatcher(tc.typ, "__name__", tc.pattern)
})
}()
if !panicked {
t.Errorf("expected panic")
}
})
}
}
@@ -114,7 +127,9 @@ func TestFromStrings(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := FromStrings(tc.args...)
assert.Equal(t, tc.want, got)
if !reflect.DeepEqual(got, tc.want) {
t.Errorf("got %v, want %v", got, tc.want)
}
})
}
}
@@ -130,9 +145,18 @@ func TestFromStringsPanics(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
assert.Panics(t, func() {
panicked := false
func() {
defer func() {
if r := recover(); r != nil {
panicked = true
}
}()
FromStrings(tc.args...)
})
}()
if !panicked {
t.Errorf("expected panic")
}
})
}
}
@@ -150,7 +174,9 @@ func TestMatchTypeString(t *testing.T) {
for _, tc := range tests {
t.Run(tc.want, func(t *testing.T) {
assert.Equal(t, tc.want, tc.typ.String())
if got := tc.typ.String(); got != tc.want {
t.Errorf("got %v, want %v", got, tc.want)
}
})
}
}
+60 -22
View File
@@ -5,8 +5,6 @@ import (
"testing"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCollectMetrics(t *testing.T) {
@@ -35,10 +33,16 @@ func TestCollectMetrics(t *testing.T) {
setup: func(t *testing.T, db *DB) {
app := db.Appender()
_, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "a"), 1000, 1.0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(0, labels.FromStrings("__name__", "temp", "room", "b"), 1000, 2.0)
require.NoError(t, err)
require.NoError(t, app.Commit())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
// 2 user series + 5 metric series = 7
collectCalls: 2,
@@ -52,14 +56,22 @@ func TestCollectMetrics(t *testing.T) {
setup: func(t *testing.T, db *DB) {
app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
require.NoError(t, app.Commit())
_, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
},
collectCalls: 2,
wantHeadSeries: 6, // 1 user + 5 metric series
@@ -82,7 +94,9 @@ func TestCollectMetrics(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
clock := &testClock{now: 100_000}
db, err := Open(t.TempDir(), Options{Clock: clock.fn()})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
t.Cleanup(func() { db.Close() })
tc.setup(t, db)
@@ -95,7 +109,9 @@ func TestCollectMetrics(t *testing.T) {
// Query each expected metric.
lastValue := func(name string) (float64, bool) {
q, err := db.Querier(math.MinInt64, math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer q.Close()
ss := q.Select(labels.MustNewMatcher(labels.MatchEqual, "__name__", name))
found := false
@@ -105,36 +121,58 @@ func TestCollectMetrics(t *testing.T) {
for it.Next() {
_, last = it.At()
}
require.NoError(t, it.Err())
if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
found = true
}
require.NoError(t, ss.Err())
if err := ss.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return last, found
}
v, found := lastValue(MetricHeadSeries)
assert.True(t, found, "ingot_head_series not found")
assert.Equal(t, tc.wantHeadSeries, v, "ingot_head_series")
if !found {
t.Errorf("ingot_head_series not found")
}
if v != tc.wantHeadSeries {
t.Errorf("ingot_head_series: got %v, want %v", v, tc.wantHeadSeries)
}
v, found = lastValue(MetricBlocksTotal)
assert.True(t, found, "ingot_blocks_total not found")
assert.Equal(t, tc.wantBlocksTotal, v, "ingot_blocks_total")
if !found {
t.Errorf("ingot_blocks_total not found")
}
if v != tc.wantBlocksTotal {
t.Errorf("ingot_blocks_total: got %v, want %v", v, tc.wantBlocksTotal)
}
v, found = lastValue(MetricCompactionsTotal)
assert.True(t, found, "ingot_compactions_total not found")
assert.Equal(t, tc.wantCompactions, v, "ingot_compactions_total")
if !found {
t.Errorf("ingot_compactions_total not found")
}
if v != tc.wantCompactions {
t.Errorf("ingot_compactions_total: got %v, want %v", v, tc.wantCompactions)
}
// Count total ingot_* series.
q, err := db.Querier(math.MinInt64, math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer q.Close()
ss := q.Select(labels.MustNewMatcher(labels.MatchRegexp, "__name__", "ingot_.*"))
count := 0
for ss.Next() {
count++
}
require.NoError(t, ss.Err())
assert.Equal(t, tc.wantMetricCount, count, "metric series count")
if err := ss.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if count != tc.wantMetricCount {
t.Errorf("metric series count: got %v, want %v", count, tc.wantMetricCount)
}
})
}
}
+48 -20
View File
@@ -11,8 +11,6 @@ import (
"time"
"git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestSoak simulates 48h of ingestion at 15s intervals across 10k series,
@@ -48,7 +46,9 @@ func TestSoak(t *testing.T) {
BlockDuration: time.Duration(blockDuration) * time.Millisecond,
Clock: clock,
})
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer db.Close()
// Generate series labels.
@@ -81,7 +81,9 @@ func TestSoak(t *testing.T) {
ls = seriesLabels[i]
}
r, err := app.Append(refs[i], ls, ts, float64(ts+int64(i)))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
refs[i] = r
if i < canaryCount {
if ls != nil {
@@ -90,20 +92,26 @@ func TestSoak(t *testing.T) {
oracle.addSample(r, ts, float64(ts+int64(i)))
}
}
require.NoError(t, app.Commit())
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
totalSamples += int64(numSeries)
// Periodic flush.
elapsed := ts - startTime
if elapsed > 0 && elapsed%flushInterval == 0 {
_, err := db.FlushOlderThan(ts - int64(blockDuration))
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
// Periodic compaction + retention.
if elapsed > 0 && elapsed%compactInterval == 0 {
err := db.RunCompaction()
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
db.ApplyRetention()
}
@@ -128,14 +136,20 @@ func TestSoak(t *testing.T) {
t.Logf("Peak heap: %d MiB", maxHeapMB)
// Assert zero query errors.
assert.Equal(t, int64(0), atomic.LoadInt64(&queryErrors), "query errors during soak")
if got := atomic.LoadInt64(&queryErrors); got != int64(0) {
t.Errorf("query errors during soak: got %v, want %v", got, int64(0))
}
// Assert bounded memory (should stay well under 1 GiB with 10k series).
assert.Less(t, maxHeapMB, uint64(1024), "heap should stay under 1 GiB")
if !(maxHeapMB < uint64(1024)) {
t.Errorf("heap should stay under 1 GiB: got %v, want < %v", maxHeapMB, uint64(1024))
}
// Assert bounded disk: count block directories.
entries, err := os.ReadDir(dir)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
blockCount := 0
for _, e := range entries {
if e.IsDir() && e.Name() != "wal" {
@@ -147,7 +161,9 @@ func TestSoak(t *testing.T) {
t.Logf("Block count at end: %d", blockCount)
// With 24h retention and 2h blocks, expect roughly 12 raw + some compacted.
// Should be well under 50.
assert.Less(t, blockCount, 50, "block count should be bounded by retention")
if !(blockCount < 50) {
t.Errorf("block count should be bounded by retention: got %v, want < %v", blockCount, 50)
}
// Final canary validation.
finalNow := now.Load()
@@ -155,7 +171,9 @@ func TestSoak(t *testing.T) {
// Live query during compaction: start query, compact, finish query.
q, err := db.Querier(math.MinInt64, math.MaxInt64)
require.NoError(t, err)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
ss := q.Select(labels.MustNewMatcher(labels.MatchRegexp, "__name__", "metric_0.*"))
// Trigger compaction while query is open.
db.RunCompaction()
@@ -164,10 +182,16 @@ func TestSoak(t *testing.T) {
it := ss.At().Iterator()
for it.Next() {
}
assert.NoError(t, it.Err())
if err := it.Err(); err != nil {
t.Errorf("unexpected error: %v", err)
}
}
if err := ss.Err(); err != nil {
t.Errorf("unexpected error: %v", err)
}
if err := q.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
assert.NoError(t, ss.Err())
require.NoError(t, q.Close())
}
// validateCanaries queries each canary series and compares with the oracle.
@@ -193,7 +217,9 @@ func validateCanaries(t *testing.T, db *DB, o *oracle, now int64, retentionMs in
// Query.
q, err := db.Querier(mint, maxt)
require.NoError(t, err, "querier for canary ref %d", ref)
if err != nil {
t.Fatalf("querier for canary ref %d: unexpected error: %v", ref, err)
}
ss := q.Select(matchers...)
var gotSamples []sample
@@ -211,10 +237,12 @@ func validateCanaries(t *testing.T, db *DB, o *oracle, now int64, retentionMs in
q.Close()
// All assertions run unconditionally for every canary.
if !assert.NoError(t, iterErr, "canary ref %d: iterator error", ref) {
if iterErr != nil {
t.Errorf("canary ref %d: iterator error: %v", ref, iterErr)
pass = false
}
if !assert.NoError(t, ssErr, "canary ref %d: series set error", ref) {
if ssErr != nil {
t.Errorf("canary ref %d: series set error: %v", ref, ssErr)
pass = false
}
@@ -225,8 +253,8 @@ func validateCanaries(t *testing.T, db *DB, o *oracle, now int64, retentionMs in
wantSamples = s
}
if !assert.Equal(t, len(wantSamples), len(gotSamples),
"canary ref %d sample count (mint=%d maxt=%d)", ref, mint, maxt) {
if len(wantSamples) != len(gotSamples) {
t.Errorf("canary ref %d sample count (mint=%d maxt=%d): got %v, want %v", ref, mint, maxt, len(gotSamples), len(wantSamples))
pass = false
}
}