david/ingot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove testify dependency

Browse Source
This commit is contained in:
david
2026-07-04 18:51:08 -04:00
parent 199e555e3d
commit 4a8b1768a1
20 changed files with 1335 additions and 495 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+13 -5
View File
@@ -31,7 +31,7 @@ db.Close()
## Status ## Status
**Pre-alpha.** The API is frozen (M4) and the system survives a 48h soak test under sustained load (M5), but this hasn't seen production use yet. **Alpha.** The API is frozen (M4) and the system survives a 48h soak test under sustained load (M5), but this hasn't seen production use yet.
| Milestone | State | | Milestone | State |
|---|---| |---|---|
@@ -44,14 +44,24 @@ db.Close()
See [DESIGN.md](DESIGN.md) for architecture, on-disk format, and the non-goals table. See [ROADMAP.md](ROADMAP.md) for what's next. See [DESIGN.md](DESIGN.md) for architecture, on-disk format, and the non-goals table. See [ROADMAP.md](ROADMAP.md) for what's next.
## Why
I needed a library to store time-series data locally and the options out there didn't quite work for my case. Prometheus was too heavy for what I needed but I wanted that level of compression. tstorage was close but it didn't have the compression or label indexing.
## Install
```sh
go get git.dvdt.dev/david/ingot
```
## Features ## Features
- **Gorilla XOR compression** — ~1 byte/sample on regular metric data (see benchmarks below) - **Gorilla XOR compression** — ~1 byte/sample on regular metric data (see benchmarks below)
- **Crash-safe** — WAL with CRC32C records; kill -9 at any point loses at most uncommitted samples - **Crash-safe** — WAL with CRC32C records. Committed data is persisted
- **Query by label matchers** — equality, negation, regex, negative regex; merged across head and blocks - **Query by label matchers** — equality, negation, regex, negative regex; merged across head and blocks
- **Levelled compaction** — 2h → 8h → 32h blocks, background merging, retention-based expiry - **Levelled compaction** — 2h → 8h → 32h blocks, background merging, retention-based expiry
- **Self-instrumentation** — the DB records its own metrics (series/chunk counts, compactions, WAL fsync duration) through the normal write path, queryable like any other series - **Self-instrumentation** — the DB records its own metrics (series/chunk counts, compactions, WAL fsync duration) through the normal write path, queryable like any other series
- **Zero external dependencies** (except testify for tests) - **Zero external dependencies**
## Tools ## Tools
@@ -138,8 +148,6 @@ The chunk encoding comes from the Gorilla paper (Pelkonen et al., VLDB 2015) via
[tstorage](https://github.com/nakabonne/tstorage) is the closest existing embedded TSDB for Go. It doesn't do Gorilla compression or label-based indexing, which is most of why ingot exists. [tstorage](https://github.com/nakabonne/tstorage) is the closest existing embedded TSDB for Go. It doesn't do Gorilla compression or label-based indexing, which is most of why ingot exists.
I needed a library to store time-series data locally and the options out there didn't quite fit. Prometheus was too heavy for what I needed but I wanted that level of compression. tstorage was close but it didn't have the compression or label indexing.
## Non-goals ## Non-goals
Replication, query languages, non-float64 values, deletes, multi-process access, out-of-order ingestion, Windows (sorry not my thing). Check DESIGN.md for reasoning. Replication, query languages, non-float64 values, deletes, multi-process access, out-of-order ingestion, Windows (sorry not my thing). Check DESIGN.md for reasoning.
cmd/ingotctl/main_test.go
+42 -16
View File
@@ -9,15 +9,15 @@ import (
"git.dvdt.dev/david/ingot/internal/block" "git.dvdt.dev/david/ingot/internal/block"
"git.dvdt.dev/david/ingot/internal/chunkenc" "git.dvdt.dev/david/ingot/internal/chunkenc"
"git.dvdt.dev/david/ingot/labels" "git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func makeChunk(t *testing.T, samples []struct{ t int64; v float64 }) []byte { func makeChunk(t *testing.T, samples []struct{ t int64; v float64 }) []byte {
t.Helper() t.Helper()
c := chunkenc.NewXORChunk() c := chunkenc.NewXORChunk()
a, err := c.Appender() a, err := c.Appender()
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for _, s := range samples { for _, s := range samples {
a.Append(s.t, s.v) a.Append(s.t, s.v)
} }
@@ -51,7 +51,9 @@ func setupTestData(t *testing.T) string {
} }
_, err := block.Flush(dir, series) _, err := block.Flush(dir, series)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return dir return dir
} }
@@ -59,7 +61,9 @@ func setupTestData(t *testing.T) string {
func blockDir(t *testing.T, dataDir string) string { func blockDir(t *testing.T, dataDir string) string {
t.Helper() t.Helper()
entries, err := os.ReadDir(dataDir) entries, err := os.ReadDir(dataDir)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for _, e := range entries { for _, e := range entries {
if e.IsDir() && e.Name() != "wal" { if e.IsDir() && e.Name() != "wal" {
return filepath.Join(dataDir, e.Name()) return filepath.Join(dataDir, e.Name())
@@ -124,10 +128,16 @@ func TestCmdBlocks(t *testing.T) {
return cmdErr return cmdErr
}() }()
assert.Equal(t, tc.wantErr != "", err != nil, "error presence") if got := (err != nil); got != (tc.wantErr != "") {
assert.Contains(t, errString(err), tc.wantErr) t.Errorf("error presence: got %v, want %v", got, tc.wantErr != "")
}
if !strings.Contains(errString(err), tc.wantErr) {
t.Errorf("got %q, want substring %q", errString(err), tc.wantErr)
}
for _, want := range tc.wantOutputs { for _, want := range tc.wantOutputs {
assert.Contains(t, output, want) if !strings.Contains(output, want) {
t.Errorf("got %q, want substring %q", output, want)
}
} }
}) })
} }
@@ -166,10 +176,16 @@ func TestCmdInspect(t *testing.T) {
return cmdErr return cmdErr
}() }()
assert.Equal(t, tc.wantErr != "", err != nil, "error presence") if got := (err != nil); got != (tc.wantErr != "") {
assert.Contains(t, errString(err), tc.wantErr) t.Errorf("error presence: got %v, want %v", got, tc.wantErr != "")
}
if !strings.Contains(errString(err), tc.wantErr) {
t.Errorf("got %q, want substring %q", errString(err), tc.wantErr)
}
for _, want := range tc.wantOutputs { for _, want := range tc.wantOutputs {
assert.Contains(t, output, want) if !strings.Contains(output, want) {
t.Errorf("got %q, want substring %q", output, want)
}
} }
}) })
} }
@@ -220,10 +236,16 @@ func TestCmdChunks(t *testing.T) {
return cmdErr return cmdErr
}() }()
assert.Equal(t, tc.wantErr != "", err != nil, "error presence") if got := (err != nil); got != (tc.wantErr != "") {
assert.Contains(t, errString(err), tc.wantErr) t.Errorf("error presence: got %v, want %v", got, tc.wantErr != "")
}
if !strings.Contains(errString(err), tc.wantErr) {
t.Errorf("got %q, want substring %q", errString(err), tc.wantErr)
}
for _, want := range tc.wantOutputs { for _, want := range tc.wantOutputs {
assert.Contains(t, output, want) if !strings.Contains(output, want) {
t.Errorf("got %q, want substring %q", output, want)
}
} }
}) })
} }
@@ -263,8 +285,12 @@ func TestCmdFsck(t *testing.T) {
return cmdErr return cmdErr
}() }()
assert.Equal(t, tc.wantErr != "", err != nil, "error presence") if got := (err != nil); got != (tc.wantErr != "") {
assert.Contains(t, errString(err), tc.wantErr) t.Errorf("error presence: got %v, want %v", got, tc.wantErr != "")
}
if !strings.Contains(errString(err), tc.wantErr) {
t.Errorf("got %q, want substring %q", errString(err), tc.wantErr)
}
}) })
} }
} }
cmd/ingothttp/handler_test.go
+39 -15
View File
@@ -10,14 +10,14 @@ import (
"git.dvdt.dev/david/ingot" "git.dvdt.dev/david/ingot"
"git.dvdt.dev/david/ingot/labels" "git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func openTestDB(t *testing.T) *ingot.DB { func openTestDB(t *testing.T) *ingot.DB {
t.Helper() t.Helper()
db, err := ingot.Open(t.TempDir(), ingot.Options{}) db, err := ingot.Open(t.TempDir(), ingot.Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
t.Cleanup(func() { db.Close() }) t.Cleanup(func() { db.Close() })
return db return db
} }
@@ -26,12 +26,20 @@ func seedDB(t *testing.T, db *ingot.DB) {
t.Helper() t.Helper()
app := db.Appender() app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 1000, 71.3) ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 1000, 71.3)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(ref, nil, 2000, 71.4) _, err = app.Append(ref, nil, 2000, 71.4)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(0, labels.FromStrings("__name__", "humidity", "room", "office"), 1000, 55.0) _, err = app.Append(0, labels.FromStrings("__name__", "humidity", "room", "office"), 1000, 55.0)
require.NoError(t, err) if err != nil {
require.NoError(t, app.Commit()) t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
} }
func TestQueryRange(t *testing.T) { func TestQueryRange(t *testing.T) {
@@ -117,12 +125,16 @@ func TestQueryRange(t *testing.T) {
rec := httptest.NewRecorder() rec := httptest.NewRecorder()
h.queryRange(rec, req) h.queryRange(rec, req)
assert.Equal(t, tc.wantStatus, rec.Code) if rec.Code != tc.wantStatus {
t.Errorf("got %v, want %v", rec.Code, tc.wantStatus)
}
// Always decode — error responses produce zero-valued struct. // Always decode — error responses produce zero-valued struct.
var resp queryRangeResponse var resp queryRangeResponse
json.Unmarshal(rec.Body.Bytes(), &resp) json.Unmarshal(rec.Body.Bytes(), &resp)
assert.Equal(t, tc.wantCount, len(resp.Data.Result)) if len(resp.Data.Result) != tc.wantCount {
t.Errorf("got %v, want %v", len(resp.Data.Result), tc.wantCount)
}
}) })
} }
} }
@@ -200,13 +212,17 @@ func TestRead(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
body, err := json.Marshal(tc.request) body, err := json.Marshal(tc.request)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
req := httptest.NewRequest(tc.method, "/api/v1/read", bytes.NewReader(body)) req := httptest.NewRequest(tc.method, "/api/v1/read", bytes.NewReader(body))
rec := httptest.NewRecorder() rec := httptest.NewRecorder()
h.read(rec, req) h.read(rec, req)
assert.Equal(t, tc.wantStatus, rec.Code) if rec.Code != tc.wantStatus {
t.Errorf("got %v, want %v", rec.Code, tc.wantStatus)
}
// Always decode — error responses produce zero-valued struct. // Always decode — error responses produce zero-valued struct.
var resp ReadResponse var resp ReadResponse
@@ -215,7 +231,9 @@ func TestRead(t *testing.T) {
if len(resp.Results) > 0 { if len(resp.Results) > 0 {
firstResultLen = len(resp.Results[0].Timeseries) firstResultLen = len(resp.Results[0].Timeseries)
} }
assert.Equal(t, tc.wantSeries, firstResultLen) if firstResultLen != tc.wantSeries {
t.Errorf("got %v, want %v", firstResultLen, tc.wantSeries)
}
}) })
} }
} }
@@ -267,13 +285,19 @@ func TestStatus(t *testing.T) {
rec := httptest.NewRecorder() rec := httptest.NewRecorder()
h.status(rec, req) h.status(rec, req)
assert.Equal(t, tc.wantStatus, rec.Code) if rec.Code != tc.wantStatus {
t.Errorf("got %v, want %v", rec.Code, tc.wantStatus)
}
// Always decode — error responses produce zero-valued struct. // Always decode — error responses produce zero-valued struct.
var resp statusResponse var resp statusResponse
json.Unmarshal(rec.Body.Bytes(), &resp) json.Unmarshal(rec.Body.Bytes(), &resp)
assert.Equal(t, tc.wantHeadSeries, resp.HeadSeries) if resp.HeadSeries != tc.wantHeadSeries {
assert.Equal(t, tc.wantBlocks, resp.Blocks) t.Errorf("got %v, want %v", resp.HeadSeries, tc.wantHeadSeries)
}
if resp.Blocks != tc.wantBlocks {
t.Errorf("got %v, want %v", resp.Blocks, tc.wantBlocks)
}
}) })
} }
} }
go.mod
-8
View File
@@ -1,11 +1,3 @@
module git.dvdt.dev/david/ingot module git.dvdt.dev/david/ingot
go 1.26.1 go 1.26.1
require github.com/stretchr/testify v1.11.1
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
go.sum
-10
View File
@@ -1,10 +0,0 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
ingot_test.go
+247 -84
View File
@@ -3,12 +3,11 @@ package ingot
import ( import (
"math" "math"
"os" "os"
"reflect"
"testing" "testing"
"time" "time"
"git.dvdt.dev/david/ingot/labels" "git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
// sample is a test convenience type. // sample is a test convenience type.
@@ -92,13 +91,17 @@ func collectSeriesSet(t *testing.T, ss SeriesSet) map[uint64][]sample {
st, sv := it.At() st, sv := it.At()
samples = append(samples, sample{st, sv}) samples = append(samples, sample{st, sv})
} }
require.NoError(t, it.Err()) if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(samples) > 0 { if len(samples) > 0 {
h := labels.Hash(ls) h := labels.Hash(ls)
result[h] = samples result[h] = samples
} }
} }
require.NoError(t, ss.Err()) if err := ss.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return result return result
} }
@@ -117,7 +120,9 @@ func oracleByLabelHash(o *oracle, mint, maxt int64, matchers ...*labels.Matcher)
func openTestDB(t *testing.T) *DB { func openTestDB(t *testing.T) *DB {
t.Helper() t.Helper()
db, err := Open(t.TempDir(), Options{}) db, err := Open(t.TempDir(), Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
t.Cleanup(func() { db.Close() }) t.Cleanup(func() { db.Close() })
return db return db
} }
@@ -133,20 +138,28 @@ func TestQueryOracle(t *testing.T) {
setup: func(t *testing.T, db *DB, o *oracle) { setup: func(t *testing.T, db *DB, o *oracle) {
app := db.Appender() app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 1000, 71.3) ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 1000, 71.3)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref, labels.FromStrings("__name__", "temp", "room", "office")) o.addSeries(ref, labels.FromStrings("__name__", "temp", "room", "office"))
o.addSample(ref, 1000, 71.3) o.addSample(ref, 1000, 71.3)
_, err = app.Append(ref, nil, 2000, 71.4) _, err = app.Append(ref, nil, 2000, 71.4)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, 2000, 71.4) o.addSample(ref, 2000, 71.4)
ref2, err := app.Append(0, labels.FromStrings("__name__", "humidity", "room", "office"), 1000, 55.0) ref2, err := app.Append(0, labels.FromStrings("__name__", "humidity", "room", "office"), 1000, 55.0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref2, labels.FromStrings("__name__", "humidity", "room", "office")) o.addSeries(ref2, labels.FromStrings("__name__", "humidity", "room", "office"))
o.addSample(ref2, 1000, 55.0) o.addSample(ref2, 1000, 55.0)
require.NoError(t, app.Commit()) if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
queries: []queryCase{ queries: []queryCase{
{ {
@@ -199,19 +212,27 @@ func TestQueryOracle(t *testing.T) {
app := db.Appender() app := db.Appender()
// Write enough samples to seal chunks (need >120 for a sealed chunk). // Write enough samples to seal chunks (need >120 for a sealed chunk).
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0) ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref, labels.FromStrings("__name__", "temp")) o.addSeries(ref, labels.FromStrings("__name__", "temp"))
o.addSample(ref, 0, 0) o.addSample(ref, 0, 0)
for i := 1; i < 250; i++ { for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i)) _, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, int64(i*15000), float64(i)) o.addSample(ref, int64(i*15000), float64(i))
} }
require.NoError(t, app.Commit()) if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Flush to block. // Flush to block.
_, err = db.FlushOlderThan(math.MaxInt64) _, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
queries: []queryCase{ queries: []queryCase{
{ {
@@ -233,30 +254,42 @@ func TestQueryOracle(t *testing.T) {
setup: func(t *testing.T, db *DB, o *oracle) { setup: func(t *testing.T, db *DB, o *oracle) {
app := db.Appender() app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 0, 0) ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 0, 0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref, labels.FromStrings("__name__", "temp", "room", "office")) o.addSeries(ref, labels.FromStrings("__name__", "temp", "room", "office"))
o.addSample(ref, 0, 0) o.addSample(ref, 0, 0)
// Write 250 samples (2 sealed chunks + 10 active). // Write 250 samples (2 sealed chunks + 10 active).
for i := 1; i < 250; i++ { for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i)) _, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, int64(i*15000), float64(i)) o.addSample(ref, int64(i*15000), float64(i))
} }
require.NoError(t, app.Commit()) if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Flush sealed chunks to block. // Flush sealed chunks to block.
_, err = db.FlushOlderThan(math.MaxInt64) _, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Append more samples to head (after flush). // Append more samples to head (after flush).
app = db.Appender() app = db.Appender()
for i := 250; i < 260; i++ { for i := 250; i < 260; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i)) _, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, int64(i*15000), float64(i)) o.addSample(ref, int64(i*15000), float64(i))
} }
require.NoError(t, app.Commit()) if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
queries: []queryCase{ queries: []queryCase{
{ {
@@ -300,15 +333,21 @@ func TestQueryOracle(t *testing.T) {
app := db.Appender() app := db.Appender()
for _, s := range series { for _, s := range series {
ref, err := app.Append(0, s.ls, 1000, 1.0) ref, err := app.Append(0, s.ls, 1000, 1.0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref, s.ls) o.addSeries(ref, s.ls)
o.addSample(ref, 1000, 1.0) o.addSample(ref, 1000, 1.0)
_, err = app.Append(ref, nil, 2000, 2.0) _, err = app.Append(ref, nil, 2000, 2.0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, 2000, 2.0) o.addSample(ref, 2000, 2.0)
} }
require.NoError(t, app.Commit()) if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
queries: []queryCase{ queries: []queryCase{
{ {
@@ -357,39 +396,57 @@ func TestQueryOracle(t *testing.T) {
setup: func(t *testing.T, db *DB, o *oracle) { setup: func(t *testing.T, db *DB, o *oracle) {
app := db.Appender() app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0) ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref, labels.FromStrings("__name__", "temp")) o.addSeries(ref, labels.FromStrings("__name__", "temp"))
o.addSample(ref, 0, 0) o.addSample(ref, 0, 0)
for i := 1; i < 250; i++ { for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i)) _, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, int64(i*15000), float64(i)) o.addSample(ref, int64(i*15000), float64(i))
} }
require.NoError(t, app.Commit()) if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// First flush. // First flush.
_, err = db.FlushOlderThan(math.MaxInt64) _, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// More data -> second flush. // More data -> second flush.
app = db.Appender() app = db.Appender()
for i := 250; i < 500; i++ { for i := 250; i < 500; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i)) _, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, int64(i*15000), float64(i)) o.addSample(ref, int64(i*15000), float64(i))
} }
require.NoError(t, app.Commit()) if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = db.FlushOlderThan(math.MaxInt64) _, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Some data in head. // Some data in head.
app = db.Appender() app = db.Appender()
for i := 500; i < 510; i++ { for i := 500; i < 510; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i)) _, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSample(ref, int64(i*15000), float64(i)) o.addSample(ref, int64(i*15000), float64(i))
} }
require.NoError(t, app.Commit()) if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
queries: []queryCase{ queries: []queryCase{
{ {
@@ -411,10 +468,14 @@ func TestQueryOracle(t *testing.T) {
setup: func(t *testing.T, db *DB, o *oracle) { setup: func(t *testing.T, db *DB, o *oracle) {
app := db.Appender() app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 1000, 1.0) ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 1000, 1.0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
o.addSeries(ref, labels.FromStrings("__name__", "temp")) o.addSeries(ref, labels.FromStrings("__name__", "temp"))
o.addSample(ref, 1000, 1.0) o.addSample(ref, 1000, 1.0)
require.NoError(t, app.Commit()) if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
queries: []queryCase{ queries: []queryCase{
{ {
@@ -443,18 +504,26 @@ func TestQueryOracle(t *testing.T) {
for _, qc := range tc.queries { for _, qc := range tc.queries {
t.Run(qc.name, func(t *testing.T) { t.Run(qc.name, func(t *testing.T) {
q, err := db.Querier(qc.mint, qc.maxt) q, err := db.Querier(qc.mint, qc.maxt)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer q.Close() defer q.Close()
ss := q.Select(qc.matchers...) ss := q.Select(qc.matchers...)
got := collectSeriesSet(t, ss) got := collectSeriesSet(t, ss)
want := oracleByLabelHash(o, qc.mint, qc.maxt, qc.matchers...) want := oracleByLabelHash(o, qc.mint, qc.maxt, qc.matchers...)
assert.Equal(t, len(want), len(got), "series count mismatch") if len(got) != len(want) {
t.Errorf("series count mismatch: got %v, want %v", len(got), len(want))
}
for h, wantSamples := range want { for h, wantSamples := range want {
gotSamples, ok := got[h] gotSamples, ok := got[h]
assert.True(t, ok, "missing series with hash %d", h) if !ok {
assert.Equal(t, wantSamples, gotSamples, "sample mismatch for hash %d", h) t.Errorf("missing series with hash %d", h)
}
if !reflect.DeepEqual(gotSamples, wantSamples) {
t.Errorf("sample mismatch for hash %d: got %v, want %v", h, gotSamples, wantSamples)
}
} }
}) })
} }
@@ -483,13 +552,21 @@ func TestDBLifecycle(t *testing.T) {
name: "append_and_query_back", name: "append_and_query_back",
setup: func(t *testing.T, dir string) *DB { setup: func(t *testing.T, dir string) *DB {
db, err := Open(dir, Options{}) db, err := Open(dir, Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := db.Appender() app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 1000, 71.3) ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 1000, 71.3)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(ref, nil, 2000, 71.4) _, err = app.Append(ref, nil, 2000, 71.4)
require.NoError(t, err) if err != nil {
require.NoError(t, app.Commit()) t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return db return db
}, },
wantSampleCount: 2, wantSampleCount: 2,
@@ -502,21 +579,35 @@ func TestDBLifecycle(t *testing.T) {
name: "reopen_with_blocks", name: "reopen_with_blocks",
setup: func(t *testing.T, dir string) *DB { setup: func(t *testing.T, dir string) *DB {
db, err := Open(dir, Options{}) db, err := Open(dir, Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := db.Appender() app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0) ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 1; i < 250; i++ { for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i)) _, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, app.Commit())
_, err = db.FlushOlderThan(math.MaxInt64) _, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err) if err != nil {
require.NoError(t, db.Close()) t.Fatalf("unexpected error: %v", err)
}
if err := db.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
db2, err := Open(dir, Options{}) db2, err := Open(dir, Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return db2 return db2
}, },
wantSampleCount: 250, wantSampleCount: 250,
@@ -529,7 +620,9 @@ func TestDBLifecycle(t *testing.T) {
name: "compacted_blocks_queryable", name: "compacted_blocks_queryable",
setup: func(t *testing.T, dir string) *DB { setup: func(t *testing.T, dir string) *DB {
db, err := Open(dir, Options{}) db, err := Open(dir, Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
const samplesPerBlock = 130 const samplesPerBlock = 130
ref := uint64(0) ref := uint64(0)
for b := 0; b < 4; b++ { for b := 0; b < 4; b++ {
@@ -537,14 +630,22 @@ func TestDBLifecycle(t *testing.T) {
for i := 0; i < samplesPerBlock; i++ { for i := 0; i < samplesPerBlock; i++ {
ts := int64((b*samplesPerBlock + i) * 15000) ts := int64((b*samplesPerBlock + i) * 15000)
r, err := app.Append(ref, labels.FromStrings("__name__", "temp"), ts, float64(ts)) r, err := app.Append(ref, labels.FromStrings("__name__", "temp"), ts, float64(ts))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
ref = r ref = r
} }
require.NoError(t, app.Commit()) if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err := db.FlushOlderThan(math.MaxInt64) _, err := db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := db.RunCompaction(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, db.RunCompaction())
return db return db
}, },
wantSampleCount: 520, wantSampleCount: 520,
@@ -561,29 +662,47 @@ func TestDBLifecycle(t *testing.T) {
Retention: 24 * time.Hour, Retention: 24 * time.Hour,
Clock: func() int64 { return now }, Clock: func() int64 { return now },
}) })
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Old data (50 hours ago). // Old data (50 hours ago).
app := db.Appender() app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "old"), 50*3600*1000, 1.0) ref, err := app.Append(0, labels.FromStrings("__name__", "old"), 50*3600*1000, 1.0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 1; i < 250; i++ { for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(50*3600*1000+i*15000), float64(i)) _, err = app.Append(ref, nil, int64(50*3600*1000+i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, app.Commit())
_, err = db.FlushOlderThan(math.MaxInt64) _, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Recent data (1 hour ago). // Recent data (1 hour ago).
app = db.Appender() app = db.Appender()
ref2, err := app.Append(0, labels.FromStrings("__name__", "recent"), 99*3600*1000, 1.0) ref2, err := app.Append(0, labels.FromStrings("__name__", "recent"), 99*3600*1000, 1.0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 1; i < 250; i++ { for i := 1; i < 250; i++ {
_, err = app.Append(ref2, nil, int64(99*3600*1000+i*15000), float64(i)) _, err = app.Append(ref2, nil, int64(99*3600*1000+i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, app.Commit())
_, err = db.FlushOlderThan(math.MaxInt64) _, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
db.ApplyRetention() db.ApplyRetention()
return db return db
}, },
@@ -602,7 +721,9 @@ func TestDBLifecycle(t *testing.T) {
defer db.Close() defer db.Close()
q, err := db.Querier(tc.mint, tc.maxt) q, err := db.Querier(tc.mint, tc.maxt)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer q.Close() defer q.Close()
ss := q.Select(tc.matchers...) ss := q.Select(tc.matchers...)
@@ -614,11 +735,19 @@ func TestDBLifecycle(t *testing.T) {
for it.Next() { for it.Next() {
sampleCount++ sampleCount++
} }
require.NoError(t, it.Err()) if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := ss.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if seriesCount != tc.wantSeriesCount {
t.Errorf("series count: got %v, want %v", seriesCount, tc.wantSeriesCount)
}
if sampleCount != tc.wantSampleCount {
t.Errorf("sample count: got %v, want %v", sampleCount, tc.wantSampleCount)
} }
require.NoError(t, ss.Err())
assert.Equal(t, tc.wantSeriesCount, seriesCount, "series count")
assert.Equal(t, tc.wantSampleCount, sampleCount, "sample count")
}) })
} }
} }
@@ -645,7 +774,9 @@ func TestQueryDuringCompaction(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
db, err := Open(dir, Options{}) db, err := Open(dir, Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer db.Close() defer db.Close()
ref := uint64(0) ref := uint64(0)
@@ -654,12 +785,18 @@ func TestQueryDuringCompaction(t *testing.T) {
for i := 0; i < tc.samplesPerBlock; i++ { for i := 0; i < tc.samplesPerBlock; i++ {
ts := int64((b*tc.samplesPerBlock + i) * 15000) ts := int64((b*tc.samplesPerBlock + i) * 15000)
r, err := app.Append(ref, labels.FromStrings("__name__", "temp"), ts, float64(ts)) r, err := app.Append(ref, labels.FromStrings("__name__", "temp"), ts, float64(ts))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
ref = r ref = r
} }
require.NoError(t, app.Commit()) if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err := db.FlushOlderThan(math.MaxInt64) _, err := db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
} }
// Snapshot source block dirs. // Snapshot source block dirs.
@@ -672,21 +809,31 @@ func TestQueryDuringCompaction(t *testing.T) {
// Start query holding refs on all blocks. // Start query holding refs on all blocks.
q, err := db.Querier(math.MinInt64, math.MaxInt64) q, err := db.Querier(math.MinInt64, math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
ss := q.Select(labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")) ss := q.Select(labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp"))
require.True(t, ss.Next()) if !ss.Next() {
t.Fatalf("expected true")
}
it := ss.At().Iterator() it := ss.At().Iterator()
require.True(t, it.Next()) if !it.Next() {
t.Fatalf("expected true")
}
// Compact while query is open. // Compact while query is open.
err = db.RunCompaction() err = db.RunCompaction()
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Source dirs still exist (query holds refs). // Source dirs still exist (query holds refs).
for _, d := range sourceDirs { for _, d := range sourceDirs {
_, statErr := os.Stat(d) _, statErr := os.Stat(d)
assert.NoError(t, statErr, "source dir should exist while query holds ref") if statErr != nil {
t.Errorf("source dir should exist while query holds ref: %v", statErr)
}
} }
// Finish iterating — all data still readable. // Finish iterating — all data still readable.
@@ -694,19 +841,29 @@ func TestQueryDuringCompaction(t *testing.T) {
for it.Next() { for it.Next() {
count++ count++
} }
require.NoError(t, it.Err()) if err := it.Err(); err != nil {
assert.Equal(t, tc.samplesPerBlock*tc.numBlocks, count, "all samples readable during compaction") t.Fatalf("unexpected error: %v", err)
}
if count != tc.samplesPerBlock*tc.numBlocks {
t.Errorf("all samples readable during compaction: got %v, want %v", count, tc.samplesPerBlock*tc.numBlocks)
}
// Close querier — source dirs should be deleted. // Close querier — source dirs should be deleted.
require.NoError(t, q.Close()) if err := q.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
for _, d := range sourceDirs { for _, d := range sourceDirs {
_, statErr := os.Stat(d) _, statErr := os.Stat(d)
assert.True(t, os.IsNotExist(statErr), "source dir should be deleted after query close") if !os.IsNotExist(statErr) {
t.Errorf("source dir should be deleted after query close")
}
} }
// Compacted block still queryable. // Compacted block still queryable.
q2, err := db.Querier(math.MinInt64, math.MaxInt64) q2, err := db.Querier(math.MinInt64, math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer q2.Close() defer q2.Close()
ss2 := q2.Select(labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")) ss2 := q2.Select(labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp"))
count = 0 count = 0
@@ -715,10 +872,16 @@ func TestQueryDuringCompaction(t *testing.T) {
for it2.Next() { for it2.Next() {
count++ count++
} }
require.NoError(t, it2.Err()) if err := it2.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := ss2.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if count != tc.samplesPerBlock*tc.numBlocks {
t.Errorf("compacted block has all samples: got %v, want %v", count, tc.samplesPerBlock*tc.numBlocks)
} }
require.NoError(t, ss2.Err())
assert.Equal(t, tc.samplesPerBlock*tc.numBlocks, count, "compacted block has all samples")
}) })
} }
} }
internal/block/block_test.go
+119 -39
View File
@@ -5,13 +5,12 @@ import (
"math/rand" "math/rand"
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"testing" "testing"
"git.dvdt.dev/david/ingot/internal/chunkenc" "git.dvdt.dev/david/ingot/internal/chunkenc"
"git.dvdt.dev/david/ingot/internal/index" "git.dvdt.dev/david/ingot/internal/index"
"git.dvdt.dev/david/ingot/labels" "git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
type sample struct { type sample struct {
@@ -26,7 +25,9 @@ func makeChunk(t *testing.T, samples []sample) []byte {
t.Helper() t.Helper()
c := chunkenc.NewXORChunk() c := chunkenc.NewXORChunk()
a, err := c.Appender() a, err := c.Appender()
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for _, s := range samples { for _, s := range samples {
a.Append(s.t, math.Float64frombits(s.vBits)) a.Append(s.t, math.Float64frombits(s.vBits))
} }
@@ -41,7 +42,9 @@ func collectIterator(t *testing.T, it chunkenc.ChunkIterator) []sample {
ts, v := it.At() ts, v := it.At()
out = append(out, sample{ts, math.Float64bits(v)}) out = append(out, sample{ts, math.Float64bits(v)})
} }
require.NoError(t, it.Err()) if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return out return out
} }
@@ -137,30 +140,52 @@ func TestBlockRoundTrip(t *testing.T) {
// Write block. // Write block.
ulid, err := Flush(dataDir, flushData) ulid, err := Flush(dataDir, flushData)
require.NoError(t, err) if err != nil {
require.NotEmpty(t, ulid) t.Fatalf("unexpected error: %v", err)
}
if ulid == "" {
t.Fatalf("got empty ULID")
}
// Open block for reading. // Open block for reading.
blockDir := filepath.Join(dataDir, ulid) blockDir := filepath.Join(dataDir, ulid)
r, err := Open(blockDir) r, err := Open(blockDir)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer r.Close() defer r.Close()
// Verify meta. // Verify meta.
assert.Equal(t, ulid, r.Meta.ULID) if got, want := r.Meta.ULID, ulid; got != want {
assert.Equal(t, 1, r.Meta.Version) t.Errorf("got %v, want %v", got, want)
assert.Equal(t, len(tc.series), r.Meta.Stats.NumSeries) }
assert.Equal(t, len(tc.series), r.Meta.Stats.NumChunks) if got, want := r.Meta.Version, 1; got != want {
t.Errorf("got %v, want %v", got, want)
}
if got, want := r.Meta.Stats.NumSeries, len(tc.series); got != want {
t.Errorf("got %v, want %v", got, want)
}
if got, want := r.Meta.Stats.NumChunks, len(tc.series); got != want {
t.Errorf("got %v, want %v", got, want)
}
// Verify each series' data via iteration. // Verify each series' data via iteration.
for _, s := range tc.series { for _, s := range tc.series {
it, err := r.SeriesChunkIterator(s.ref, math.MinInt64, math.MaxInt64) it, err := r.SeriesChunkIterator(s.ref, math.MinInt64, math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
got := collectIterator(t, it) got := collectIterator(t, it)
require.Equal(t, len(s.samples), len(got), "ref %d sample count", s.ref) if len(got) != len(s.samples) {
t.Fatalf("ref %d sample count: got %v, want %v", s.ref, len(got), len(s.samples))
}
for i, want := range s.samples { for i, want := range s.samples {
assert.Equal(t, want.t, got[i].t, "ref %d sample %d t", s.ref, i) if got[i].t != want.t {
assert.Equal(t, want.vBits, got[i].vBits, "ref %d sample %d v", s.ref, i) t.Errorf("ref %d sample %d t: got %v, want %v", s.ref, i, got[i].t, want.t)
}
if got[i].vBits != want.vBits {
t.Errorf("ref %d sample %d v: got %v, want %v", s.ref, i, got[i].vBits, want.vBits)
}
} }
} }
@@ -168,15 +193,28 @@ func TestBlockRoundTrip(t *testing.T) {
for _, s := range tc.series { for _, s := range tc.series {
for _, l := range s.labels { for _, l := range s.labels {
refs := r.Postings(l.Name, l.Value) refs := r.Postings(l.Name, l.Value)
assert.Contains(t, refs, s.ref, "postings for %s=%s", l.Name, l.Value) found := false
for _, ref := range refs {
if ref == s.ref {
found = true
break
}
}
if !found {
t.Errorf("postings for %s=%s: %v does not contain %d", l.Name, l.Value, refs, s.ref)
}
} }
} }
// Verify labels lookup. // Verify labels lookup.
for _, s := range tc.series { for _, s := range tc.series {
ls, ok := r.Labels(s.ref) ls, ok := r.Labels(s.ref)
assert.True(t, ok) if !ok {
assert.Equal(t, s.labels, ls) t.Errorf("Labels(%d) returned ok=false", s.ref)
}
if !reflect.DeepEqual(ls, s.labels) {
t.Errorf("got %v, want %v", ls, s.labels)
}
} }
}) })
} }
@@ -236,20 +274,32 @@ func TestBlockSeriesChunkIterator(t *testing.T) {
}, },
} }
ulid, err := Flush(dataDir, flushData) ulid, err := Flush(dataDir, flushData)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
r, err := Open(filepath.Join(dataDir, ulid)) r, err := Open(filepath.Join(dataDir, ulid))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer r.Close() defer r.Close()
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
it, err := r.SeriesChunkIterator(tc.ref, tc.mint, tc.maxt) it, err := r.SeriesChunkIterator(tc.ref, tc.mint, tc.maxt)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
got := collectIterator(t, it) got := collectIterator(t, it)
assert.Equal(t, len(tc.wantSamples), len(got), "sample count") if len(got) != len(tc.wantSamples) {
t.Errorf("sample count: got %v, want %v", len(got), len(tc.wantSamples))
}
for i, want := range tc.wantSamples { for i, want := range tc.wantSamples {
assert.Equal(t, want.t, got[i].t, "sample %d t", i) if got[i].t != want.t {
assert.Equal(t, want.vBits, got[i].vBits, "sample %d v", i) t.Errorf("sample %d t: got %v, want %v", i, got[i].t, want.t)
}
if got[i].vBits != want.vBits {
t.Errorf("sample %d v: got %v, want %v", i, got[i].vBits, want.vBits)
}
} }
}) })
} }
@@ -296,13 +346,21 @@ func TestBlockMetaTimeBounds(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
dataDir := t.TempDir() dataDir := t.TempDir()
ulid, err := Flush(dataDir, tc.series) ulid, err := Flush(dataDir, tc.series)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
r, err := Open(filepath.Join(dataDir, ulid)) r, err := Open(filepath.Join(dataDir, ulid))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer r.Close() defer r.Close()
assert.Equal(t, tc.wantMinT, r.Meta.MinTime, "MinTime") if got, want := r.Meta.MinTime, tc.wantMinT; got != want {
assert.Equal(t, tc.wantMaxT, r.Meta.MaxTime, "MaxTime") t.Errorf("MinTime: got %v, want %v", got, want)
}
if got, want := r.Meta.MaxTime, tc.wantMaxT; got != want {
t.Errorf("MaxTime: got %v, want %v", got, want)
}
}) })
} }
} }
@@ -310,11 +368,15 @@ func TestBlockMetaTimeBounds(t *testing.T) {
func TestULIDRoundTrip(t *testing.T) { func TestULIDRoundTrip(t *testing.T) {
for i := 0; i < 100; i++ { for i := 0; i < 100; i++ {
u := newULID() u := newULID()
assert.Equal(t, 26, len(u), "ULID length") if len(u) != 26 {
t.Errorf("ULID length: got %v, want %v", len(u), 26)
}
// Should parse without error. // Should parse without error.
_, err := parseULID(u) _, err := parseULID(u)
assert.NoError(t, err, "parse ULID %q", u) if err != nil {
t.Errorf("parse ULID %q: %v", u, err)
}
} }
} }
@@ -329,10 +391,14 @@ func TestBlockCorruption(t *testing.T) {
corruptFunc: func(t *testing.T, blockDir string, chunkRef index.ChunkRef) { corruptFunc: func(t *testing.T, blockDir string, chunkRef index.ChunkRef) {
chunkPath := filepath.Join(blockDir, chunksDirName, segmentName(int(chunkRef.Segment()))) chunkPath := filepath.Join(blockDir, chunksDirName, segmentName(int(chunkRef.Segment())))
data, err := os.ReadFile(chunkPath) data, err := os.ReadFile(chunkPath)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
off := int(chunkRef.Offset()) + chunkEntryHeaderLen + 1 off := int(chunkRef.Offset()) + chunkEntryHeaderLen + 1
data[off] ^= 0xFF data[off] ^= 0xFF
require.NoError(t, os.WriteFile(chunkPath, data, 0644)) if err := os.WriteFile(chunkPath, data, 0644); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
wantErr: ErrCorruptChunk, wantErr: ErrCorruptChunk,
}, },
@@ -341,7 +407,9 @@ func TestBlockCorruption(t *testing.T) {
corruptFunc: func(t *testing.T, blockDir string, chunkRef index.ChunkRef) { corruptFunc: func(t *testing.T, blockDir string, chunkRef index.ChunkRef) {
chunkPath := filepath.Join(blockDir, chunksDirName, segmentName(int(chunkRef.Segment()))) chunkPath := filepath.Join(blockDir, chunksDirName, segmentName(int(chunkRef.Segment())))
data, err := os.ReadFile(chunkPath) data, err := os.ReadFile(chunkPath)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Corrupt the last byte of the CRC. // Corrupt the last byte of the CRC.
off := int(chunkRef.Offset()) + chunkEntryHeaderLen off := int(chunkRef.Offset()) + chunkEntryHeaderLen
// Read dataLen to find CRC position. // Read dataLen to find CRC position.
@@ -349,7 +417,9 @@ func TestBlockCorruption(t *testing.T) {
int(data[chunkRef.Offset()+2])*256 + int(data[chunkRef.Offset()+3]) int(data[chunkRef.Offset()+2])*256 + int(data[chunkRef.Offset()+3])
crcOff := off + dataLen + 3 // last byte of CRC crcOff := off + dataLen + 3 // last byte of CRC
data[crcOff] ^= 0xFF data[crcOff] ^= 0xFF
require.NoError(t, os.WriteFile(chunkPath, data, 0644)) if err := os.WriteFile(chunkPath, data, 0644); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
wantErr: ErrCorruptChunk, wantErr: ErrCorruptChunk,
}, },
@@ -367,24 +437,34 @@ func TestBlockCorruption(t *testing.T) {
}, },
} }
ulid, err := Flush(dataDir, flushData) ulid, err := Flush(dataDir, flushData)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
blockDir := filepath.Join(dataDir, ulid) blockDir := filepath.Join(dataDir, ulid)
r, err := Open(blockDir) r, err := Open(blockDir)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
series := r.Series() series := r.Series()
require.Equal(t, 1, len(series)) if len(series) != 1 {
t.Fatalf("got %v, want %v", len(series), 1)
}
chunkRef := series[0].Chunks[0].Ref chunkRef := series[0].Chunks[0].Ref
r.Close() r.Close()
tc.corruptFunc(t, blockDir, chunkRef) tc.corruptFunc(t, blockDir, chunkRef)
r, err = Open(blockDir) r, err = Open(blockDir)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer r.Close() defer r.Close()
_, err = r.ChunkIterator(chunkRef) _, err = r.ChunkIterator(chunkRef)
assert.Equal(t, tc.wantErr, err) if err != tc.wantErr {
t.Errorf("got %v, want %v", err, tc.wantErr)
}
}) })
} }
} }
internal/block/validate_test.go
+51 -19
View File
@@ -8,8 +8,6 @@ import (
"testing" "testing"
"git.dvdt.dev/david/ingot/labels" "git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func TestValidate(t *testing.T) { func TestValidate(t *testing.T) {
@@ -33,7 +31,9 @@ func TestValidate(t *testing.T) {
}, },
} }
ulid, err := Flush(dir, series) ulid, err := Flush(dir, series)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return filepath.Join(dir, ulid) return filepath.Join(dir, ulid)
}, },
wantErrors: 0, wantErrors: 0,
@@ -53,7 +53,9 @@ func TestValidate(t *testing.T) {
}, },
} }
ulid, err := Flush(dir, series) ulid, err := Flush(dir, series)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
blockDir := filepath.Join(dir, ulid) blockDir := filepath.Join(dir, ulid)
os.Remove(filepath.Join(blockDir, "meta.json")) os.Remove(filepath.Join(blockDir, "meta.json"))
return blockDir return blockDir
@@ -75,15 +77,21 @@ func TestValidate(t *testing.T) {
}, },
} }
ulid, err := Flush(dir, series) ulid, err := Flush(dir, series)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
blockDir := filepath.Join(dir, ulid) blockDir := filepath.Join(dir, ulid)
// Corrupt a byte in the chunk data. // Corrupt a byte in the chunk data.
chunkPath := filepath.Join(blockDir, "chunks", "000001") chunkPath := filepath.Join(blockDir, "chunks", "000001")
data, err := os.ReadFile(chunkPath) data, err := os.ReadFile(chunkPath)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
data[chunkHeaderLen+chunkEntryHeaderLen+2] ^= 0xFF data[chunkHeaderLen+chunkEntryHeaderLen+2] ^= 0xFF
require.NoError(t, os.WriteFile(chunkPath, data, 0644)) if err := os.WriteFile(chunkPath, data, 0644); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return blockDir return blockDir
}, },
wantErrors: 1, wantErrors: 1,
@@ -103,14 +111,20 @@ func TestValidate(t *testing.T) {
}, },
} }
ulid, err := Flush(dir, series) ulid, err := Flush(dir, series)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
blockDir := filepath.Join(dir, ulid) blockDir := filepath.Join(dir, ulid)
chunkPath := filepath.Join(blockDir, "chunks", "000001") chunkPath := filepath.Join(blockDir, "chunks", "000001")
data, err := os.ReadFile(chunkPath) data, err := os.ReadFile(chunkPath)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
binary.BigEndian.PutUint32(data[:4], 0xDEADBEEF) binary.BigEndian.PutUint32(data[:4], 0xDEADBEEF)
require.NoError(t, os.WriteFile(chunkPath, data, 0644)) if err := os.WriteFile(chunkPath, data, 0644); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return blockDir return blockDir
}, },
wantErrors: 1, wantErrors: 1,
@@ -122,7 +136,9 @@ func TestValidate(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
blockDir := tc.setup(t) blockDir := tc.setup(t)
errs := Validate(blockDir) errs := Validate(blockDir)
assert.Equal(t, tc.wantErrors, len(errs), "error count: %v", errs) if len(errs) != tc.wantErrors {
t.Errorf("error count: got %d, want %d: %v", len(errs), tc.wantErrors, errs)
}
// Concatenate all error strings; "" is contained in everything. // Concatenate all error strings; "" is contained in everything.
var combined strings.Builder var combined strings.Builder
@@ -130,7 +146,9 @@ func TestValidate(t *testing.T) {
combined.WriteString(e.Error()) combined.WriteString(e.Error())
combined.WriteByte('\n') combined.WriteByte('\n')
} }
assert.Contains(t, combined.String(), tc.wantMatch) if !strings.Contains(combined.String(), tc.wantMatch) {
t.Errorf("got %q, want substring %q", combined.String(), tc.wantMatch)
}
}) })
} }
} }
@@ -159,7 +177,9 @@ func TestReadMeta(t *testing.T) {
}, },
}, },
}) })
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return dir, ulid return dir, ulid
}, },
wantULID: true, wantULID: true,
@@ -175,12 +195,24 @@ func TestReadMeta(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
dir, ulid := tc.setup(t) dir, ulid := tc.setup(t)
meta, err := ReadMeta(filepath.Join(dir, ulid)) meta, err := ReadMeta(filepath.Join(dir, ulid))
assert.Equal(t, tc.wantErr, err) if err != tc.wantErr {
assert.Equal(t, tc.wantULID, meta.ULID == ulid) t.Errorf("error: got %v, want %v", err, tc.wantErr)
assert.Equal(t, tc.wantMinT, meta.MinTime) }
assert.Equal(t, tc.wantMaxT, meta.MaxTime) if (meta.ULID == ulid) != tc.wantULID {
assert.Equal(t, tc.wantNSer, meta.Stats.NumSeries) t.Errorf("ULID match: got %v, want %v", meta.ULID == ulid, tc.wantULID)
assert.Equal(t, tc.wantNChk, meta.Stats.NumChunks) }
if meta.MinTime != tc.wantMinT {
t.Errorf("MinTime: got %v, want %v", meta.MinTime, tc.wantMinT)
}
if meta.MaxTime != tc.wantMaxT {
t.Errorf("MaxTime: got %v, want %v", meta.MaxTime, tc.wantMaxT)
}
if meta.Stats.NumSeries != tc.wantNSer {
t.Errorf("NumSeries: got %v, want %v", meta.Stats.NumSeries, tc.wantNSer)
}
if meta.Stats.NumChunks != tc.wantNChk {
t.Errorf("NumChunks: got %v, want %v", meta.Stats.NumChunks, tc.wantNChk)
}
}) })
} }
} }
internal/chunkenc/bstream_test.go
+6 -4
View File
@@ -4,8 +4,6 @@ import (
"fmt" "fmt"
"math/rand" "math/rand"
"testing" "testing"
"github.com/stretchr/testify/assert"
) )
// bstreamBytes writes (value, nbits) pairs and returns the encoded bytes. // bstreamBytes writes (value, nbits) pairs and returns the encoded bytes.
@@ -144,8 +142,12 @@ func TestBstream(t *testing.T) {
r := newBReader(tc.data) r := newBReader(tc.data)
for i, op := range tc.reads { for i, op := range tc.reads {
got, err := r.readBits(op.nbits) got, err := r.readBits(op.nbits)
assert.Equal(t, op.wantVal, got, "op %d val (nbits=%d)", i, op.nbits) if got != op.wantVal {
assert.Equal(t, op.wantErr, err, "op %d err (nbits=%d)", i, op.nbits) t.Errorf("op %d val (nbits=%d): got %v, want %v", i, op.nbits, got, op.wantVal)
}
if err != op.wantErr {
t.Errorf("op %d err (nbits=%d): got %v, want %v", i, op.nbits, err, op.wantErr)
}
} }
}) })
} }
internal/chunkenc/xor_test.go
+46 -17
View File
@@ -6,9 +6,8 @@ import (
"fmt" "fmt"
"math" "math"
"math/rand" "math/rand"
"reflect"
"testing" "testing"
"github.com/stretchr/testify/assert"
) )
// encodeChunk builds a valid XOR chunk from samples and returns its bytes. // encodeChunk builds a valid XOR chunk from samples and returns its bytes.
@@ -284,16 +283,28 @@ func TestXORChunk(t *testing.T) {
for i, r := range tc.reads { for i, r := range tc.reads {
next := it.Next() next := it.Next()
gotT, gotV := it.At() gotT, gotV := it.At()
assert.Equal(t, r.wantNext, next, "read %d Next()", i) if next != r.wantNext {
assert.Equal(t, r.wantT, gotT, "read %d timestamp", i) t.Errorf("read %d Next(): got %v, want %v", i, next, r.wantNext)
assert.Equal(t, r.wantVBits, math.Float64bits(gotV), "read %d value", i) }
if gotT != r.wantT {
t.Errorf("read %d timestamp: got %v, want %v", i, gotT, r.wantT)
}
if math.Float64bits(gotV) != r.wantVBits {
t.Errorf("read %d value: got %v, want %v", i, math.Float64bits(gotV), r.wantVBits)
}
}
if it.Err() != tc.wantIterErr {
t.Errorf("iter err: got %v, want %v", it.Err(), tc.wantIterErr)
} }
assert.Equal(t, tc.wantIterErr, it.Err())
assert.LessOrEqual(t, len(tc.data), tc.maxBytes) if !(len(tc.data) <= tc.maxBytes) {
t.Errorf("got %v bytes, want <= %v", len(tc.data), tc.maxBytes)
}
_, appErr := c.Appender() _, appErr := c.Appender()
assert.Equal(t, tc.wantAppenderErr, appErr) if !reflect.DeepEqual(appErr, tc.wantAppenderErr) {
t.Errorf("appender err: got %v, want %v", appErr, tc.wantAppenderErr)
}
}) })
} }
} }
@@ -329,23 +340,39 @@ func TestXORChunkFromBytes(t *testing.T) {
reconstituted := XORChunkFromBytes(orig.Bytes()) reconstituted := XORChunkFromBytes(orig.Bytes())
assert.Equal(t, orig.NumSamples(), reconstituted.NumSamples()) if orig.NumSamples() != reconstituted.NumSamples() {
assert.Equal(t, orig.Bytes(), reconstituted.Bytes()) t.Errorf("NumSamples: got %v, want %v", reconstituted.NumSamples(), orig.NumSamples())
}
if !reflect.DeepEqual(orig.Bytes(), reconstituted.Bytes()) {
t.Errorf("Bytes mismatch")
}
// Verify iteration produces identical samples. // Verify iteration produces identical samples.
it := reconstituted.Iterator() it := reconstituted.Iterator()
for i, s := range tc.samples { for i, s := range tc.samples {
assert.True(t, it.Next(), "sample %d", i) if !it.Next() {
t.Errorf("sample %d: expected Next()=true", i)
}
gotT, gotV := it.At() gotT, gotV := it.At()
assert.Equal(t, int64(s[0]), gotT, "sample %d t", i) if gotT != int64(s[0]) {
assert.Equal(t, math.Float64bits(s[1]), math.Float64bits(gotV), "sample %d v", i) t.Errorf("sample %d t: got %v, want %v", i, gotT, int64(s[0]))
}
if math.Float64bits(gotV) != math.Float64bits(s[1]) {
t.Errorf("sample %d v: got %v, want %v", i, math.Float64bits(gotV), math.Float64bits(s[1]))
}
}
if it.Next() {
t.Errorf("expected Next()=false after all samples")
}
if it.Err() != nil {
t.Errorf("unexpected iter error: %v", it.Err())
} }
assert.False(t, it.Next())
assert.NoError(t, it.Err())
// Appender on non-empty reconstituted chunk should fail. // Appender on non-empty reconstituted chunk should fail.
_, err := reconstituted.Appender() _, err := reconstituted.Appender()
assert.Error(t, err) if err == nil {
t.Errorf("expected error from Appender on non-empty chunk")
}
}) })
} }
} }
@@ -429,7 +456,9 @@ func TestBytesPerSample(t *testing.T) {
c := benchChunk(tc.gen) c := benchChunk(tc.gen)
bps := float64(len(c.Bytes())) / 120.0 bps := float64(len(c.Bytes())) / 120.0
t.Logf("%s: %.3f bytes/sample (%d bytes total)", tc.name, bps, len(c.Bytes())) t.Logf("%s: %.3f bytes/sample (%d bytes total)", tc.name, bps, len(c.Bytes()))
assert.LessOrEqual(t, bps, tc.max, "%s bytes/sample exceeds ceiling", tc.name) if !(bps <= tc.max) {
t.Errorf("%s bytes/sample exceeds ceiling: got %v, want <= %v", tc.name, bps, tc.max)
}
}) })
} }
} }
internal/compact/compact_test.go
+57 -21
View File
@@ -8,8 +8,6 @@ import (
"git.dvdt.dev/david/ingot/internal/block" "git.dvdt.dev/david/ingot/internal/block"
"git.dvdt.dev/david/ingot/internal/chunkenc" "git.dvdt.dev/david/ingot/internal/chunkenc"
"git.dvdt.dev/david/ingot/labels" "git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
const ( const (
@@ -21,7 +19,9 @@ func makeChunk(t *testing.T, timestamps []int64, values []float64) []byte {
t.Helper() t.Helper()
c := chunkenc.NewXORChunk() c := chunkenc.NewXORChunk()
a, err := c.Appender() a, err := c.Appender()
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := range timestamps { for i := range timestamps {
a.Append(timestamps[i], values[i]) a.Append(timestamps[i], values[i])
} }
@@ -38,9 +38,13 @@ func flushTestBlock(t *testing.T, dataDir string, series []block.SeriesFlush, le
} else { } else {
ulid, err = block.FlushCompacted(dataDir, series, level, sources) ulid, err = block.FlushCompacted(dataDir, series, level, sources)
} }
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
r, err := block.Open(filepath.Join(dataDir, ulid)) r, err := block.Open(filepath.Join(dataDir, ulid))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return r return r
} }
@@ -48,13 +52,17 @@ func flushTestBlock(t *testing.T, dataDir string, series []block.SeriesFlush, le
func collectBlockSamples(t *testing.T, r *block.Reader, ref uint64) []sample { func collectBlockSamples(t *testing.T, r *block.Reader, ref uint64) []sample {
t.Helper() t.Helper()
it, err := r.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64) it, err := r.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var out []sample var out []sample
for it.Next() { for it.Next() {
ts, v := it.At() ts, v := it.At()
out = append(out, sample{ts, v}) out = append(out, sample{ts, v})
} }
require.NoError(t, it.Err()) if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return out return out
} }
@@ -162,12 +170,20 @@ func TestPlan(t *testing.T) {
group := c.Plan(blocks) group := c.Plan(blocks)
if !tc.wantGroup { if !tc.wantGroup {
assert.Nil(t, group, "expected no compaction group") if group != nil {
t.Errorf("expected no compaction group, got %v", group)
}
return return
} }
require.NotNil(t, group, "expected a compaction group") if group == nil {
assert.Equal(t, tc.wantLevel, group.Level, "compaction level") t.Fatalf("expected a compaction group")
assert.Equal(t, tc.wantCount, len(group.Sources), "source count") }
if got, want := group.Level, tc.wantLevel; got != want {
t.Errorf("compaction level: got %v, want %v", got, want)
}
if got, want := len(group.Sources), tc.wantCount; got != want {
t.Errorf("source count: got %v, want %v", got, want)
}
}) })
} }
} }
@@ -263,8 +279,12 @@ func TestCompact(t *testing.T) {
// Compact. // Compact.
newULID, err := c.Compact(sources) newULID, err := c.Compact(sources)
require.NoError(t, err) if err != nil {
require.NotEmpty(t, newULID) t.Fatalf("unexpected error: %v", err)
}
if newULID == "" {
t.Fatalf("expected non-empty ULID")
}
// Close source blocks. // Close source blocks.
for _, s := range sources { for _, s := range sources {
@@ -273,24 +293,38 @@ func TestCompact(t *testing.T) {
// Open compacted block. // Open compacted block.
compacted, err := block.Open(filepath.Join(dataDir, newULID)) compacted, err := block.Open(filepath.Join(dataDir, newULID))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer compacted.Close() defer compacted.Close()
// Verify compaction level. // Verify compaction level.
assert.Equal(t, tc.wantLevel, compacted.Meta.Compaction.Level, "compaction level") if got, want := compacted.Meta.Compaction.Level, tc.wantLevel; got != want {
assert.Equal(t, len(tc.sourceBlocks), len(compacted.Meta.Compaction.Sources), "source count") t.Errorf("compaction level: got %v, want %v", got, want)
}
if got, want := len(compacted.Meta.Compaction.Sources), len(tc.sourceBlocks); got != want {
t.Errorf("source count: got %v, want %v", got, want)
}
// Verify series count. // Verify series count.
assert.Equal(t, len(tc.wantSeriesRefs), compacted.Meta.Stats.NumSeries, "series count") if got, want := compacted.Meta.Stats.NumSeries, len(tc.wantSeriesRefs); got != want {
t.Errorf("series count: got %v, want %v", got, want)
}
// Verify each series' samples. // Verify each series' samples.
for _, ref := range tc.wantSeriesRefs { for _, ref := range tc.wantSeriesRefs {
got := collectBlockSamples(t, compacted, ref) got := collectBlockSamples(t, compacted, ref)
want := tc.wantSamples[ref] want := tc.wantSamples[ref]
require.Equal(t, len(want), len(got), "sample count for ref %d", ref) if len(got) != len(want) {
t.Fatalf("sample count for ref %d: got %v, want %v", ref, len(got), len(want))
}
for i := range want { for i := range want {
assert.Equal(t, want[i].t, got[i].t, "ref %d sample %d t", ref, i) if got[i].t != want[i].t {
assert.Equal(t, want[i].v, got[i].v, "ref %d sample %d v", ref, i) t.Errorf("ref %d sample %d t: got %v, want %v", ref, i, got[i].t, want[i].t)
}
if got[i].v != want[i].v {
t.Errorf("ref %d sample %d v: got %v, want %v", ref, i, got[i].v, want[i].v)
}
} }
} }
}) })
@@ -357,7 +391,9 @@ func TestExpired(t *testing.T) {
} }
expired := c.Expired(blocks) expired := c.Expired(blocks)
assert.Equal(t, tc.wantCount, len(expired), "expired block count") if got, want := len(expired), tc.wantCount; got != want {
t.Errorf("expired block count: got %v, want %v", got, want)
}
}) })
} }
} }
internal/head/head_test.go
+241 -81
View File
@@ -4,14 +4,13 @@ import (
"math" "math"
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"sync" "sync"
"testing" "testing"
"git.dvdt.dev/david/ingot/internal/block" "git.dvdt.dev/david/ingot/internal/block"
"git.dvdt.dev/david/ingot/internal/wal" "git.dvdt.dev/david/ingot/internal/wal"
"git.dvdt.dev/david/ingot/labels" "git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
// sample is a convenience type for expected results. // sample is a convenience type for expected results.
@@ -31,7 +30,9 @@ func collectSamples(t *testing.T, h *Head, ref uint64, mint, maxt int64) []sampl
ts, v := it.At() ts, v := it.At()
out = append(out, sample{ts, math.Float64bits(v)}) out = append(out, sample{ts, math.Float64bits(v)})
} }
require.NoError(t, it.Err()) if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return out return out
} }
@@ -61,7 +62,9 @@ func openHead(t *testing.T) *Head {
t.Helper() t.Helper()
dir := filepath.Join(t.TempDir(), "wal") dir := filepath.Join(t.TempDir(), "wal")
h, err := Open(dir, wal.Options{}) h, err := Open(dir, wal.Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
t.Cleanup(func() { h.Close() }) t.Cleanup(func() { h.Close() })
return h return h
} }
@@ -285,22 +288,34 @@ func TestHead(t *testing.T) {
app = h.Appender() app = h.Appender()
case actAppend: case actAppend:
ref, err := app.Append(act.ref, act.labels, act.t, act.v) ref, err := app.Append(act.ref, act.labels, act.t, act.v)
assert.Equal(t, act.wantRef, ref, "action %d ref", i) if ref != act.wantRef {
assert.Equal(t, act.wantErr, err, "action %d error", i) t.Errorf("action %d ref: got %v, want %v", i, ref, act.wantRef)
}
if err != act.wantErr {
t.Errorf("action %d error: got %v, want %v", i, err, act.wantErr)
}
case actCommit: case actCommit:
err := app.Commit() err := app.Commit()
assert.Equal(t, act.wantErr, err, "action %d commit error", i) if err != act.wantErr {
t.Errorf("action %d commit error: got %v, want %v", i, err, act.wantErr)
}
case actRollback: case actRollback:
err := app.Rollback() err := app.Rollback()
assert.Equal(t, act.wantErr, err, "action %d rollback error", i) if err != act.wantErr {
t.Errorf("action %d rollback error: got %v, want %v", i, err, act.wantErr)
}
} }
} }
for ref, wantSamples := range tc.wantSamples { for ref, wantSamples := range tc.wantSamples {
got := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64) got := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64)
require.Equal(t, len(wantSamples), len(got), "ref %d sample count", ref) if len(got) != len(wantSamples) {
t.Fatalf("ref %d sample count: got %v, want %v", ref, len(got), len(wantSamples))
}
for i, want := range wantSamples { for i, want := range wantSamples {
assert.Equal(t, want, got[i], "ref %d sample %d", ref, i) if !reflect.DeepEqual(got[i], want) {
t.Errorf("ref %d sample %d: got %v, want %v", ref, i, got[i], want)
}
} }
} }
}) })
@@ -317,14 +332,24 @@ func TestWALReplay(t *testing.T) {
name: "basic_recovery", name: "basic_recovery",
setup: func(t *testing.T, dir string) { setup: func(t *testing.T, dir string) {
h, err := Open(dir, wal.Options{}) h, err := Open(dir, wal.Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := h.Appender() app := h.Appender()
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 71.3) ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 71.3)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(ref, nil, 1015, 71.4) _, err = app.Append(ref, nil, 1015, 71.4)
require.NoError(t, err) if err != nil {
require.NoError(t, app.Commit()) t.Fatalf("unexpected error: %v", err)
require.NoError(t, h.Close()) }
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := h.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
wantSeries: map[uint64][]sample{ wantSeries: map[uint64][]sample{
1: {s(1000, 71.3), s(1015, 71.4)}, 1: {s(1000, 71.3), s(1015, 71.4)},
@@ -334,14 +359,24 @@ func TestWALReplay(t *testing.T) {
name: "multi_series_recovery", name: "multi_series_recovery",
setup: func(t *testing.T, dir string) { setup: func(t *testing.T, dir string) {
h, err := Open(dir, wal.Options{}) h, err := Open(dir, wal.Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := h.Appender() app := h.Appender()
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 71.3) _, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 71.3)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}}, 1000, 55.0) _, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}}, 1000, 55.0)
require.NoError(t, err) if err != nil {
require.NoError(t, app.Commit()) t.Fatalf("unexpected error: %v", err)
require.NoError(t, h.Close()) }
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := h.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
wantSeries: map[uint64][]sample{ wantSeries: map[uint64][]sample{
1: {s(1000, 71.3)}, 1: {s(1000, 71.3)},
@@ -352,16 +387,26 @@ func TestWALReplay(t *testing.T) {
name: "chunk_sealing_recovery", name: "chunk_sealing_recovery",
setup: func(t *testing.T, dir string) { setup: func(t *testing.T, dir string) {
h, err := Open(dir, wal.Options{}) h, err := Open(dir, wal.Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := h.Appender() app := h.Appender()
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0) ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 1; i < 250; i++ { for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i)) _, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := h.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, app.Commit())
require.NoError(t, h.Close())
}, },
wantSeries: map[uint64][]sample{ wantSeries: map[uint64][]sample{
1: func() []sample { 1: func() []sample {
@@ -377,16 +422,28 @@ func TestWALReplay(t *testing.T) {
name: "multiple_commits_recovery", name: "multiple_commits_recovery",
setup: func(t *testing.T, dir string) { setup: func(t *testing.T, dir string) {
h, err := Open(dir, wal.Options{}) h, err := Open(dir, wal.Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := h.Appender() app := h.Appender()
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 1.0) _, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 1.0)
require.NoError(t, err) if err != nil {
require.NoError(t, app.Commit()) t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
app = h.Appender() app = h.Appender()
_, err = app.Append(1, nil, 2000, 2.0) _, err = app.Append(1, nil, 2000, 2.0)
require.NoError(t, err) if err != nil {
require.NoError(t, app.Commit()) t.Fatalf("unexpected error: %v", err)
require.NoError(t, h.Close()) }
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := h.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
wantSeries: map[uint64][]sample{ wantSeries: map[uint64][]sample{
1: {s(1000, 1.0), s(2000, 2.0)}, 1: {s(1000, 1.0), s(2000, 2.0)},
@@ -396,17 +453,29 @@ func TestWALReplay(t *testing.T) {
name: "rollback_not_recovered", name: "rollback_not_recovered",
setup: func(t *testing.T, dir string) { setup: func(t *testing.T, dir string) {
h, err := Open(dir, wal.Options{}) h, err := Open(dir, wal.Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := h.Appender() app := h.Appender()
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 1.0) _, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 1.0)
require.NoError(t, err) if err != nil {
require.NoError(t, app.Commit()) t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Second batch is rolled back — should not survive restart. // Second batch is rolled back — should not survive restart.
app = h.Appender() app = h.Appender()
_, err = app.Append(1, nil, 2000, 2.0) _, err = app.Append(1, nil, 2000, 2.0)
require.NoError(t, err) if err != nil {
require.NoError(t, app.Rollback()) t.Fatalf("unexpected error: %v", err)
require.NoError(t, h.Close()) }
if err := app.Rollback(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := h.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
wantSeries: map[uint64][]sample{ wantSeries: map[uint64][]sample{
1: {s(1000, 1.0)}, 1: {s(1000, 1.0)},
@@ -420,14 +489,20 @@ func TestWALReplay(t *testing.T) {
tc.setup(t, dir) tc.setup(t, dir)
h, err := Open(dir, wal.Options{}) h, err := Open(dir, wal.Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer h.Close() defer h.Close()
for ref, wantSamples := range tc.wantSeries { for ref, wantSamples := range tc.wantSeries {
got := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64) got := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64)
require.Equal(t, len(wantSamples), len(got), "ref %d sample count", ref) if len(got) != len(wantSamples) {
t.Fatalf("ref %d sample count: got %v, want %v", ref, len(got), len(wantSamples))
}
for i, want := range wantSamples { for i, want := range wantSamples {
assert.Equal(t, want, got[i], "ref %d sample %d", ref, i) if !reflect.DeepEqual(got[i], want) {
t.Errorf("ref %d sample %d: got %v, want %v", ref, i, got[i], want)
}
} }
} }
}) })
@@ -493,49 +568,76 @@ func TestFlushOlderThan(t *testing.T) {
app := h.Appender() app := h.Appender()
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0) ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 1; i < tc.numSamples; i++ { for i := 1; i < tc.numSamples; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i)) _, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, app.Commit())
ulid, err := h.FlushOlderThan(tc.flushMaxT) ulid, err := h.FlushOlderThan(tc.flushMaxT)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Post-flush appends. // Post-flush appends.
if tc.postFlushAppend > 0 { if tc.postFlushAppend > 0 {
app = h.Appender() app = h.Appender()
for i := tc.numSamples; i < tc.numSamples+tc.postFlushAppend; i++ { for i := tc.numSamples; i < tc.numSamples+tc.postFlushAppend; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i)) _, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, app.Commit())
} }
// Assert ULID presence. // Assert ULID presence.
assert.Equal(t, tc.wantULID, ulid != "", "block ULID presence") gotULID := ulid != ""
if gotULID != tc.wantULID {
t.Errorf("block ULID presence: got %v, want %v", gotULID, tc.wantULID)
}
// Assert block contents. // Assert block contents.
blockCount := 0 blockCount := 0
if ulid != "" { if ulid != "" {
blockDir := filepath.Join(h.DataDir(), ulid) blockDir := filepath.Join(h.DataDir(), ulid)
br, err := block.Open(blockDir) br, err := block.Open(blockDir)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer br.Close() defer br.Close()
assert.Equal(t, tc.wantBlockSeries, br.Meta.Stats.NumSeries, "block series count") if br.Meta.Stats.NumSeries != tc.wantBlockSeries {
t.Errorf("block series count: got %v, want %v", br.Meta.Stats.NumSeries, tc.wantBlockSeries)
}
it, err := br.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64) it, err := br.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for it.Next() { for it.Next() {
blockCount++ blockCount++
} }
require.NoError(t, it.Err()) if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if blockCount != tc.wantBlockCount {
t.Errorf("block sample count: got %v, want %v", blockCount, tc.wantBlockCount)
} }
assert.Equal(t, tc.wantBlockCount, blockCount, "block sample count")
// Assert head still has data. // Assert head still has data.
headSamples := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64) headSamples := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64)
assert.GreaterOrEqual(t, len(headSamples), tc.wantHeadCount, "head sample count") if len(headSamples) < tc.wantHeadCount {
t.Errorf("head sample count: got %v, want >= %v", len(headSamples), tc.wantHeadCount)
}
}) })
} }
} }
@@ -555,41 +657,65 @@ func TestFlushWALTruncation(t *testing.T) {
walDir := filepath.Join(dir, "wal") walDir := filepath.Join(dir, "wal")
h, err := Open(walDir, wal.Options{}) h, err := Open(walDir, wal.Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
app := h.Appender() app := h.Appender()
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0) ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_ = ref _ = ref
for i := 1; i < tc.numSamples; i++ { for i := 1; i < tc.numSamples; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i)) _, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, app.Commit())
walEntries, err := os.ReadDir(walDir) walEntries, err := os.ReadDir(walDir)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
segsBefore := len(walEntries) segsBefore := len(walEntries)
_, err = h.FlushOlderThan(math.MaxInt64) _, err = h.FlushOlderThan(math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
walEntries, err = os.ReadDir(walDir) walEntries, err = os.ReadDir(walDir)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
segsAfter := len(walEntries) segsAfter := len(walEntries)
assert.LessOrEqual(t, segsAfter, segsBefore, "WAL should be truncated after flush") if segsAfter > segsBefore {
t.Errorf("WAL should be truncated after flush: got %v segments after, had %v before", segsAfter, segsBefore)
}
require.NoError(t, h.Close()) if err := h.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Re-open: head should be functional. // Re-open: head should be functional.
h2, err := Open(walDir, wal.Options{}) h2, err := Open(walDir, wal.Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer h2.Close() defer h2.Close()
app = h2.Appender() app = h2.Appender()
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "new_series"}}, 5000000, 42.0) _, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "new_series"}}, 5000000, 42.0)
require.NoError(t, err) if err != nil {
require.NoError(t, app.Commit()) t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}) })
} }
} }
@@ -600,12 +726,20 @@ func TestHeadPostings(t *testing.T) {
// Add three series. // Add three series.
app := h.Appender() app := h.Appender()
_, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}}, 1000, 1.0) _, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}}, 1000, 1.0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "office"}}, 1000, 2.0) _, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "office"}}, 1000, 2.0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "kitchen"}}, 1000, 3.0) _, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "kitchen"}}, 1000, 3.0)
require.NoError(t, err) if err != nil {
require.NoError(t, app.Commit()) t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
tests := []struct { tests := []struct {
name string name string
@@ -622,7 +756,9 @@ func TestHeadPostings(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
got := h.Postings(tc.label, tc.value) got := h.Postings(tc.label, tc.value)
assert.Equal(t, tc.wantRefs, got) if !reflect.DeepEqual(got, tc.wantRefs) {
t.Errorf("got %v, want %v", got, tc.wantRefs)
}
}) })
} }
} }
@@ -677,29 +813,43 @@ func TestHeadQueryMethods(t *testing.T) {
app := h.Appender() app := h.Appender()
for _, ls := range tc.series { for _, ls := range tc.series {
_, err := app.Append(0, ls, 1000, 1.0) _, err := app.Append(0, ls, 1000, 1.0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, app.Commit())
// Assert LabelValues. // Assert LabelValues.
for name, wantVals := range tc.wantLabelValues { for name, wantVals := range tc.wantLabelValues {
got := h.LabelValues(name) got := h.LabelValues(name)
assert.Equal(t, wantVals, got, "LabelValues(%q)", name) if !reflect.DeepEqual(got, wantVals) {
t.Errorf("LabelValues(%q): got %v, want %v", name, got, wantVals)
}
} }
// Assert Labels by ref. // Assert Labels by ref.
for ref, wantLabels := range tc.wantLabels { for ref, wantLabels := range tc.wantLabels {
ls, ok := h.Labels(ref) ls, ok := h.Labels(ref)
assert.True(t, ok, "Labels(%d) should exist", ref) if !ok {
assert.Equal(t, wantLabels, ls, "Labels(%d)", ref) t.Errorf("Labels(%d) should exist", ref)
}
if !reflect.DeepEqual(ls, wantLabels) {
t.Errorf("Labels(%d): got %v, want %v", ref, ls, wantLabels)
}
} }
// Unknown ref returns false. // Unknown ref returns false.
_, ok := h.Labels(999) _, ok := h.Labels(999)
assert.False(t, ok, "Labels(999) should not exist") if ok {
t.Errorf("Labels(999) should not exist")
}
// Assert AllPostings. // Assert AllPostings.
assert.Equal(t, tc.wantAllPostings, h.AllPostings(), "AllPostings") if !reflect.DeepEqual(h.AllPostings(), tc.wantAllPostings) {
t.Errorf("AllPostings: got %v, want %v", h.AllPostings(), tc.wantAllPostings)
}
}) })
} }
} }
@@ -732,20 +882,30 @@ func TestConcurrentAppend(t *testing.T) {
{Name: "goroutine", Value: string(rune('A' + g))}, {Name: "goroutine", Value: string(rune('A' + g))},
} }
ref, err := app.Append(0, ls, int64(g*1000000), 0) ref, err := app.Append(0, ls, int64(g*1000000), 0)
require.NoError(t, err) if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
refs[g] = ref refs[g] = ref
for i := 1; i < tc.samplesPerGoroutine; i++ { for i := 1; i < tc.samplesPerGoroutine; i++ {
_, err = app.Append(ref, nil, int64(g*1000000+i*1000), float64(i)) _, err = app.Append(ref, nil, int64(g*1000000+i*1000), float64(i))
require.NoError(t, err) if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
}
if err := app.Commit(); err != nil {
t.Errorf("unexpected error: %v", err)
} }
require.NoError(t, app.Commit())
}(g) }(g)
} }
wg.Wait() wg.Wait()
for g := 0; g < tc.numGoroutines; g++ { for g := 0; g < tc.numGoroutines; g++ {
got := collectSamples(t, h, refs[g], math.MinInt64, math.MaxInt64) got := collectSamples(t, h, refs[g], math.MinInt64, math.MaxInt64)
assert.Equal(t, tc.samplesPerGoroutine, len(got), "goroutine %d sample count", g) if len(got) != tc.samplesPerGoroutine {
t.Errorf("goroutine %d sample count: got %v, want %v", g, len(got), tc.samplesPerGoroutine)
}
} }
}) })
} }
internal/index/index_test.go
+86 -28
View File
@@ -2,11 +2,10 @@ package index
import ( import (
"bytes" "bytes"
"reflect"
"testing" "testing"
"git.dvdt.dev/david/ingot/labels" "git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func writeIndex(t *testing.T, entries []SeriesEntry) []byte { func writeIndex(t *testing.T, entries []SeriesEntry) []byte {
@@ -17,7 +16,9 @@ func writeIndex(t *testing.T, entries []SeriesEntry) []byte {
w.AddSeries(e) w.AddSeries(e)
} }
_, err := w.WriteTo() _, err := w.WriteTo()
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
return buf.Bytes() return buf.Bytes()
} }
@@ -81,41 +82,74 @@ func TestIndexRoundTrip(t *testing.T) {
data := writeIndex(t, tc.entries) data := writeIndex(t, tc.entries)
r, err := NewReader(data) r, err := NewReader(data)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Verify series count and content. // Verify series count and content.
gotSeries := r.Series() gotSeries := r.Series()
require.Equal(t, len(tc.entries), len(gotSeries)) if len(gotSeries) != len(tc.entries) {
t.Fatalf("got %v, want %v", len(gotSeries), len(tc.entries))
}
for i, want := range tc.entries { for i, want := range tc.entries {
got := gotSeries[i] got := gotSeries[i]
assert.Equal(t, want.Ref, got.Ref, "series %d ref", i) if got.Ref != want.Ref {
assert.Equal(t, want.Labels, got.Labels, "series %d labels", i) t.Errorf("series %d ref: got %v, want %v", i, got.Ref, want.Ref)
require.Equal(t, len(want.Chunks), len(got.Chunks), "series %d chunk count", i) }
if !reflect.DeepEqual(got.Labels, want.Labels) {
t.Errorf("series %d labels: got %v, want %v", i, got.Labels, want.Labels)
}
if len(got.Chunks) != len(want.Chunks) {
t.Fatalf("series %d chunk count: got %v, want %v", i, len(got.Chunks), len(want.Chunks))
}
for j, wc := range want.Chunks { for j, wc := range want.Chunks {
assert.Equal(t, wc.MinT, got.Chunks[j].MinT, "series %d chunk %d minT", i, j) if got.Chunks[j].MinT != wc.MinT {
assert.Equal(t, wc.MaxT, got.Chunks[j].MaxT, "series %d chunk %d maxT", i, j) t.Errorf("series %d chunk %d minT: got %v, want %v", i, j, got.Chunks[j].MinT, wc.MinT)
assert.Equal(t, wc.Ref, got.Chunks[j].Ref, "series %d chunk %d ref", i, j) }
if got.Chunks[j].MaxT != wc.MaxT {
t.Errorf("series %d chunk %d maxT: got %v, want %v", i, j, got.Chunks[j].MaxT, wc.MaxT)
}
if got.Chunks[j].Ref != wc.Ref {
t.Errorf("series %d chunk %d ref: got %v, want %v", i, j, got.Chunks[j].Ref, wc.Ref)
}
} }
// SeriesByRef lookup. // SeriesByRef lookup.
byRef, ok := r.SeriesByRef(want.Ref) byRef, ok := r.SeriesByRef(want.Ref)
assert.True(t, ok, "series %d lookup by ref", i) if !ok {
assert.Equal(t, want.Ref, byRef.Ref) t.Errorf("series %d lookup by ref: got false, want true", i)
}
if byRef.Ref != want.Ref {
t.Errorf("got %v, want %v", byRef.Ref, want.Ref)
}
} }
// Verify postings. // Verify postings.
for _, e := range tc.entries { for _, e := range tc.entries {
for _, l := range e.Labels { for _, l := range e.Labels {
refs := r.Postings(l.Name, l.Value) refs := r.Postings(l.Name, l.Value)
assert.Contains(t, refs, e.Ref, "postings for %s=%s should contain ref %d", l.Name, l.Value, e.Ref) found := false
for _, ref := range refs {
if ref == e.Ref {
found = true
break
}
}
if !found {
t.Errorf("postings for %s=%s should contain ref %d, got %v", l.Name, l.Value, e.Ref, refs)
}
} }
} }
// Verify missing lookups return empty/false. // Verify missing lookups return empty/false.
_, ok := r.SeriesByRef(999999) _, ok := r.SeriesByRef(999999)
assert.False(t, ok) if ok {
assert.Nil(t, r.Postings("nonexistent", "value")) t.Errorf("got true, want false")
}
if refs := r.Postings("nonexistent", "value"); refs != nil {
t.Errorf("got %v, want nil", refs)
}
}) })
} }
} }
@@ -129,13 +163,23 @@ func TestIndexPostingsSorted(t *testing.T) {
data := writeIndex(t, entries) data := writeIndex(t, entries)
r, err := NewReader(data) r, err := NewReader(data)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
refs := r.Postings("room", "office") refs := r.Postings("room", "office")
require.Equal(t, 3, len(refs)) if len(refs) != 3 {
assert.Equal(t, uint64(2), refs[0]) t.Fatalf("got %v, want %v", len(refs), 3)
assert.Equal(t, uint64(5), refs[1]) }
assert.Equal(t, uint64(8), refs[2]) if refs[0] != uint64(2) {
t.Errorf("got %v, want %v", refs[0], uint64(2))
}
if refs[1] != uint64(5) {
t.Errorf("got %v, want %v", refs[1], uint64(5))
}
if refs[2] != uint64(8) {
t.Errorf("got %v, want %v", refs[2], uint64(8))
}
} }
func TestIndexChunkRefEncoding(t *testing.T) { func TestIndexChunkRefEncoding(t *testing.T) {
@@ -153,8 +197,12 @@ func TestIndexChunkRefEncoding(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
ref := NewChunkRef(tc.segment, tc.offset) ref := NewChunkRef(tc.segment, tc.offset)
assert.Equal(t, tc.segment, ref.Segment()) if ref.Segment() != tc.segment {
assert.Equal(t, tc.offset, ref.Offset()) t.Errorf("got %v, want %v", ref.Segment(), tc.segment)
}
if ref.Offset() != tc.offset {
t.Errorf("got %v, want %v", ref.Offset(), tc.offset)
}
}) })
} }
} }
@@ -168,7 +216,9 @@ func TestIndexLabelValues(t *testing.T) {
data := writeIndex(t, entries) data := writeIndex(t, entries)
r, err := NewReader(data) r, err := NewReader(data)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
tests := []struct { tests := []struct {
name string name string
@@ -183,7 +233,9 @@ func TestIndexLabelValues(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
got := r.LabelValues(tc.label) got := r.LabelValues(tc.label)
assert.Equal(t, tc.wantVals, got) if !reflect.DeepEqual(got, tc.wantVals) {
t.Errorf("got %v, want %v", got, tc.wantVals)
}
}) })
} }
} }
@@ -197,10 +249,14 @@ func TestIndexAllPostings(t *testing.T) {
data := writeIndex(t, entries) data := writeIndex(t, entries)
r, err := NewReader(data) r, err := NewReader(data)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
refs := r.AllPostings() refs := r.AllPostings()
assert.Equal(t, []uint64{2, 5, 8}, refs) if !reflect.DeepEqual(refs, []uint64{2, 5, 8}) {
t.Errorf("got %v, want %v", refs, []uint64{2, 5, 8})
}
} }
func TestIndexCorruptData(t *testing.T) { func TestIndexCorruptData(t *testing.T) {
@@ -246,7 +302,9 @@ func TestIndexCorruptData(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
_, err := NewReader(tc.data) _, err := NewReader(tc.data)
assert.Equal(t, tc.wantErr, err) if err != tc.wantErr {
t.Errorf("got %v, want %v", err, tc.wantErr)
}
}) })
} }
} }
internal/postings/postings_test.go
+10 -5
View File
@@ -1,9 +1,8 @@
package postings package postings
import ( import (
"reflect"
"testing" "testing"
"github.com/stretchr/testify/assert"
) )
func TestIntersect(t *testing.T) { func TestIntersect(t *testing.T) {
@@ -25,7 +24,9 @@ func TestIntersect(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
got := Intersect(tc.lists...) got := Intersect(tc.lists...)
assert.Equal(t, tc.want, got) if !reflect.DeepEqual(got, tc.want) {
t.Errorf("got %v, want %v", got, tc.want)
}
}) })
} }
} }
@@ -49,7 +50,9 @@ func TestUnion(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
got := Union(tc.lists...) got := Union(tc.lists...)
assert.Equal(t, tc.want, got) if !reflect.DeepEqual(got, tc.want) {
t.Errorf("got %v, want %v", got, tc.want)
}
}) })
} }
} }
@@ -73,7 +76,9 @@ func TestWithout(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
got := Without(tc.full, tc.remove) got := Without(tc.full, tc.remove)
assert.Equal(t, tc.want, got) if !reflect.DeepEqual(got, tc.want) {
t.Errorf("got %v, want %v", got, tc.want)
}
}) })
} }
} }
internal/wal/codec_test.go
+16 -7
View File
@@ -3,11 +3,10 @@ package wal
import ( import (
"encoding/binary" "encoding/binary"
"math" "math"
"reflect"
"testing" "testing"
"git.dvdt.dev/david/ingot/labels" "git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func TestSeriesRecord(t *testing.T) { func TestSeriesRecord(t *testing.T) {
@@ -140,8 +139,12 @@ func TestSeriesRecord(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
rec, err := DecodeSeriesRecord(tc.data) rec, err := DecodeSeriesRecord(tc.data)
assert.Equal(t, tc.want.rec, rec, "record") if !reflect.DeepEqual(rec, tc.want.rec) {
assert.Equal(t, tc.want.err, err, "error") t.Errorf("record: got %v, want %v", rec, tc.want.rec)
}
if err != tc.want.err {
t.Errorf("error: got %v, want %v", err, tc.want.err)
}
}) })
} }
} }
@@ -260,10 +263,16 @@ func TestSamplesRecord(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
samples, err := DecodeSamplesRecord(tc.data) samples, err := DecodeSamplesRecord(tc.data)
got := samplesToBits(samples) got := samplesToBits(samples)
assert.Equal(t, tc.want.err, err, "error") if err != tc.want.err {
require.Equal(t, len(tc.want.samples), len(got), "sample count") t.Errorf("error: got %v, want %v", err, tc.want.err)
}
if len(got) != len(tc.want.samples) {
t.Fatalf("sample count: got %d, want %d", len(got), len(tc.want.samples))
}
for i := range tc.want.samples { for i := range tc.want.samples {
assert.Equal(t, tc.want.samples[i], got[i], "sample %d", i) if got[i] != tc.want.samples[i] {
t.Errorf("sample %d: got %v, want %v", i, got[i], tc.want.samples[i])
}
} }
}) })
} }
internal/wal/record_test.go
+28 -11
View File
@@ -3,9 +3,8 @@ package wal
import ( import (
"encoding/binary" "encoding/binary"
"hash/crc32" "hash/crc32"
"reflect"
"testing" "testing"
"github.com/stretchr/testify/assert"
) )
func TestRecord(t *testing.T) { func TestRecord(t *testing.T) {
@@ -138,10 +137,18 @@ func TestRecord(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
typ, payload, consumed, err := DecodeRecord(tc.data) typ, payload, consumed, err := DecodeRecord(tc.data)
assert.Equal(t, tc.want.typ, typ, "type") if typ != tc.want.typ {
assert.Equal(t, tc.want.payload, payload, "payload") t.Errorf("type: got %v, want %v", typ, tc.want.typ)
assert.Equal(t, tc.want.consumed, consumed, "consumed") }
assert.Equal(t, tc.want.err, err, "error") if !reflect.DeepEqual(payload, tc.want.payload) {
t.Errorf("payload: got %v, want %v", payload, tc.want.payload)
}
if consumed != tc.want.consumed {
t.Errorf("consumed: got %v, want %v", consumed, tc.want.consumed)
}
if err != tc.want.err {
t.Errorf("error: got %v, want %v", err, tc.want.err)
}
}) })
} }
} }
@@ -149,11 +156,17 @@ func TestRecord(t *testing.T) {
func TestEncodeRecordAppendsToExisting(t *testing.T) { func TestEncodeRecordAppendsToExisting(t *testing.T) {
prefix := []byte("existing") prefix := []byte("existing")
result := EncodeRecord(prefix, RecordSeries, []byte{0x01}) result := EncodeRecord(prefix, RecordSeries, []byte{0x01})
assert.Equal(t, []byte("existing"), result[:8]) if !reflect.DeepEqual([]byte("existing"), result[:8]) {
t.Errorf("prefix: got %v, want %v", result[:8], []byte("existing"))
}
_, payload, _, err := DecodeRecord(result[8:]) _, payload, _, err := DecodeRecord(result[8:])
assert.Equal(t, []byte{0x01}, payload) if !reflect.DeepEqual([]byte{0x01}, payload) {
assert.Equal(t, nil, err) t.Errorf("payload: got %v, want %v", payload, []byte{0x01})
}
if err != nil {
t.Errorf("error: got %v, want nil", err)
}
} }
func TestRecordSize(t *testing.T) { func TestRecordSize(t *testing.T) {
@@ -167,7 +180,9 @@ func TestRecordSize(t *testing.T) {
} }
for _, tc := range tests { for _, tc := range tests {
assert.Equal(t, tc.want, RecordSize(tc.payloadLen)) if got := RecordSize(tc.payloadLen); got != tc.want {
t.Errorf("RecordSize(%d): got %v, want %v", tc.payloadLen, got, tc.want)
}
} }
} }
@@ -179,5 +194,7 @@ func TestEncodeCRCCoversHeaderAndPayload(t *testing.T) {
headerAndPayload := rec[:recordHeaderSize+len(payload)] headerAndPayload := rec[:recordHeaderSize+len(payload)]
want := crc32.Checksum(headerAndPayload, castagnoliTable) want := crc32.Checksum(headerAndPayload, castagnoliTable)
got := binary.BigEndian.Uint32(rec[recordHeaderSize+len(payload):]) got := binary.BigEndian.Uint32(rec[recordHeaderSize+len(payload):])
assert.Equal(t, want, got) if got != want {
t.Errorf("CRC: got %v, want %v", got, want)
}
} }
internal/wal/wal_test.go
+186 -69
View File
@@ -3,25 +3,28 @@ package wal
import ( import (
"os" "os"
"path/filepath" "path/filepath"
"reflect"
"testing" "testing"
"git.dvdt.dev/david/ingot/labels" "git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
// collectRecords replays all records from a WAL directory. // collectRecords replays all records from a WAL directory.
func collectRecords(t *testing.T, dir string) []Record { func collectRecords(t *testing.T, dir string) []Record {
t.Helper() t.Helper()
r, err := NewReader(dir) r, err := NewReader(dir)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer r.Close() defer r.Close()
var recs []Record var recs []Record
for r.Next() { for r.Next() {
recs = append(recs, r.Record()) recs = append(recs, r.Record())
} }
require.NoError(t, r.Err()) if r.Err() != nil {
t.Fatalf("unexpected error: %v", r.Err())
}
return recs return recs
} }
@@ -36,15 +39,23 @@ func TestWAL(t *testing.T) {
name: "write_and_replay", name: "write_and_replay",
setup: func(t *testing.T, dir string) { setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{}) w, err := Open(dir, Options{})
require.NoError(t, err) if err != nil {
require.NoError(t, w.LogSeries([]SeriesRecord{ t.Fatalf("unexpected error: %v", err)
}
if err := w.LogSeries([]SeriesRecord{
{Ref: 1, Labels: []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}}}, {Ref: 1, Labels: []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}}},
})) }); err != nil {
require.NoError(t, w.LogSamples([]RefSample{ t.Fatalf("unexpected error: %v", err)
}
if err := w.LogSamples([]RefSample{
{Ref: 1, T: 1000, V: 71.3}, {Ref: 1, T: 1000, V: 71.3},
{Ref: 1, T: 1015, V: 71.4}, {Ref: 1, T: 1015, V: 71.4},
})) }); err != nil {
require.NoError(t, w.Close()) t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
wantRecords: 2, wantRecords: 2,
wantMinSegs: 1, wantMinSegs: 1,
@@ -53,8 +64,12 @@ func TestWAL(t *testing.T) {
name: "empty", name: "empty",
setup: func(t *testing.T, dir string) { setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{}) w, err := Open(dir, Options{})
require.NoError(t, err) if err != nil {
require.NoError(t, w.Close()) t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
wantRecords: 0, wantRecords: 0,
wantMinSegs: 1, wantMinSegs: 1,
@@ -63,14 +78,26 @@ func TestWAL(t *testing.T) {
name: "reopen_and_append", name: "reopen_and_append",
setup: func(t *testing.T, dir string) { setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{}) w, err := Open(dir, Options{})
require.NoError(t, err) if err != nil {
require.NoError(t, w.LogSamples([]RefSample{{Ref: 1, T: 1000, V: 1.0}})) t.Fatalf("unexpected error: %v", err)
require.NoError(t, w.Close()) }
if err := w.LogSamples([]RefSample{{Ref: 1, T: 1000, V: 1.0}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
w, err = Open(dir, Options{}) w, err = Open(dir, Options{})
require.NoError(t, err) if err != nil {
require.NoError(t, w.LogSamples([]RefSample{{Ref: 2, T: 2000, V: 2.0}})) t.Fatalf("unexpected error: %v", err)
require.NoError(t, w.Close()) }
if err := w.LogSamples([]RefSample{{Ref: 2, T: 2000, V: 2.0}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
wantRecords: 2, wantRecords: 2,
wantMinSegs: 1, wantMinSegs: 1,
@@ -79,11 +106,17 @@ func TestWAL(t *testing.T) {
name: "segment_rotation", name: "segment_rotation",
setup: func(t *testing.T, dir string) { setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{SegmentMaxSize: 50}) w, err := Open(dir, Options{SegmentMaxSize: 50})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
require.NoError(t, w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i * 1000), V: float64(i)}})) if err := w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i * 1000), V: float64(i)}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, w.Close())
}, },
wantRecords: 10, wantRecords: 10,
wantMinSegs: 2, wantMinSegs: 2,
@@ -92,13 +125,21 @@ func TestWAL(t *testing.T) {
name: "truncate_old_segments", name: "truncate_old_segments",
setup: func(t *testing.T, dir string) { setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{SegmentMaxSize: 50}) w, err := Open(dir, Options{SegmentMaxSize: 50})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
require.NoError(t, w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i), V: float64(i)}})) if err := w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i), V: float64(i)}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
} }
lastSeg := w.LastSegment() lastSeg := w.LastSegment()
require.NoError(t, w.Truncate(lastSeg)) if err := w.Truncate(lastSeg); err != nil {
require.NoError(t, w.Close()) t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
wantRecords: 1, // only the last segment's record(s) survive wantRecords: 1, // only the last segment's record(s) survive
wantMinSegs: 1, wantMinSegs: 1,
@@ -107,25 +148,42 @@ func TestWAL(t *testing.T) {
name: "recovery_truncates_trailing_garbage", name: "recovery_truncates_trailing_garbage",
setup: func(t *testing.T, dir string) { setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{}) w, err := Open(dir, Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 0; i < 3; i++ { for i := 0; i < 3; i++ {
require.NoError(t, w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i), V: float64(i)}})) if err := w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i), V: float64(i)}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, w.Close())
// Append garbage after valid records. // Append garbage after valid records.
segs, err := listSegments(dir) segs, err := listSegments(dir)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
f, err := os.OpenFile(segmentPath(dir, segs[0]), os.O_WRONLY|os.O_APPEND, 0644) f, err := os.OpenFile(segmentPath(dir, segs[0]), os.O_WRONLY|os.O_APPEND, 0644)
require.NoError(t, err) if err != nil {
_, err = f.Write([]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}) t.Fatalf("unexpected error: %v", err)
require.NoError(t, err) }
require.NoError(t, f.Close()) if _, err := f.Write([]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := f.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Reopen triggers recovery. // Reopen triggers recovery.
w, err = Open(dir, Options{}) w, err = Open(dir, Options{})
require.NoError(t, err) if err != nil {
require.NoError(t, w.Close()) t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
wantRecords: 3, wantRecords: 3,
wantMinSegs: 1, wantMinSegs: 1,
@@ -134,25 +192,42 @@ func TestWAL(t *testing.T) {
name: "recovery_truncates_corrupt_mid_record", name: "recovery_truncates_corrupt_mid_record",
setup: func(t *testing.T, dir string) { setup: func(t *testing.T, dir string) {
w, err := Open(dir, Options{}) w, err := Open(dir, Options{})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 0; i < 3; i++ { for i := 0; i < 3; i++ {
require.NoError(t, w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i), V: float64(i)}})) if err := w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i), V: float64(i)}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, w.Close())
// Write a valid header but truncated payload (looks like a torn write). // Write a valid header but truncated payload (looks like a torn write).
segs, err := listSegments(dir) segs, err := listSegments(dir)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
f, err := os.OpenFile(segmentPath(dir, segs[0]), os.O_WRONLY|os.O_APPEND, 0644) f, err := os.OpenFile(segmentPath(dir, segs[0]), os.O_WRONLY|os.O_APPEND, 0644)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// type=1, len=100 (big), but no payload follows. // type=1, len=100 (big), but no payload follows.
_, err = f.Write([]byte{0x01, 0x00, 0x00, 0x00, 0x64}) if _, err := f.Write([]byte{0x01, 0x00, 0x00, 0x00, 0x64}); err != nil {
require.NoError(t, err) t.Fatalf("unexpected error: %v", err)
require.NoError(t, f.Close()) }
if err := f.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
w, err = Open(dir, Options{}) w, err = Open(dir, Options{})
require.NoError(t, err) if err != nil {
require.NoError(t, w.Close()) t.Fatalf("unexpected error: %v", err)
}
if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
wantRecords: 3, wantRecords: 3,
wantMinSegs: 1, wantMinSegs: 1,
@@ -165,11 +240,17 @@ func TestWAL(t *testing.T) {
tc.setup(t, dir) tc.setup(t, dir)
recs := collectRecords(t, dir) recs := collectRecords(t, dir)
assert.Equal(t, tc.wantRecords, len(recs), "record count") if len(recs) != tc.wantRecords {
t.Errorf("record count: got %v, want %v", len(recs), tc.wantRecords)
}
segs, err := listSegments(dir) segs, err := listSegments(dir)
require.NoError(t, err) if err != nil {
assert.GreaterOrEqual(t, len(segs), tc.wantMinSegs, "segment count") t.Fatalf("unexpected error: %v", err)
}
if !(len(segs) >= tc.wantMinSegs) {
t.Errorf("segment count: got %v, want >= %v", len(segs), tc.wantMinSegs)
}
}) })
} }
} }
@@ -187,17 +268,25 @@ func TestTornWriteRecovery(t *testing.T) {
name: "single_segment_mixed_records", name: "single_segment_mixed_records",
opts: Options{}, opts: Options{},
recs: func(t *testing.T, w *WAL) { recs: func(t *testing.T, w *WAL) {
require.NoError(t, w.LogSeries([]SeriesRecord{ if err := w.LogSeries([]SeriesRecord{
{Ref: 1, Labels: []labels.Label{{Name: "__name__", Value: "temp"}}}, {Ref: 1, Labels: []labels.Label{{Name: "__name__", Value: "temp"}}},
})) }); err != nil {
require.NoError(t, w.LogSamples([]RefSample{{Ref: 1, T: 1000, V: 71.3}})) t.Fatalf("unexpected error: %v", err)
require.NoError(t, w.LogSeries([]SeriesRecord{ }
if err := w.LogSamples([]RefSample{{Ref: 1, T: 1000, V: 71.3}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if err := w.LogSeries([]SeriesRecord{
{Ref: 2, Labels: []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "lab"}}}, {Ref: 2, Labels: []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "lab"}}},
})) }); err != nil {
require.NoError(t, w.LogSamples([]RefSample{ t.Fatalf("unexpected error: %v", err)
}
if err := w.LogSamples([]RefSample{
{Ref: 1, T: 1015, V: 71.4}, {Ref: 1, T: 1015, V: 71.4},
{Ref: 2, T: 1000, V: 55.0}, {Ref: 2, T: 1000, V: 55.0},
})) }); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
}, },
{ {
@@ -205,7 +294,9 @@ func TestTornWriteRecovery(t *testing.T) {
opts: Options{SegmentMaxSize: 50}, opts: Options{SegmentMaxSize: 50},
recs: func(t *testing.T, w *WAL) { recs: func(t *testing.T, w *WAL) {
for i := 0; i < 10; i++ { for i := 0; i < 10; i++ {
require.NoError(t, w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i * 1000), V: float64(i)}})) if err := w.LogSamples([]RefSample{{Ref: uint64(i), T: int64(i * 1000), V: float64(i)}}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
} }
}, },
}, },
@@ -216,21 +307,31 @@ func TestTornWriteRecovery(t *testing.T) {
// Write the reference WAL. // Write the reference WAL.
srcDir := filepath.Join(t.TempDir(), "src") srcDir := filepath.Join(t.TempDir(), "src")
w, err := Open(srcDir, tc.opts) w, err := Open(srcDir, tc.opts)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
tc.recs(t, w) tc.recs(t, w)
require.NoError(t, w.Close()) if err := w.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
origRecs := collectRecords(t, srcDir) origRecs := collectRecords(t, srcDir)
require.Greater(t, len(origRecs), 0) if !(len(origRecs) > 0) {
t.Fatalf("got %v records, want > 0", len(origRecs))
}
segs, err := listSegments(srcDir) segs, err := listSegments(srcDir)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Read all segment data. // Read all segment data.
segData := make(map[int][]byte) segData := make(map[int][]byte)
for _, idx := range segs { for _, idx := range segs {
data, err := os.ReadFile(segmentPath(srcDir, idx)) data, err := os.ReadFile(segmentPath(srcDir, idx))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
segData[idx] = data segData[idx] = data
} }
@@ -240,25 +341,41 @@ func TestTornWriteRecovery(t *testing.T) {
for cutoff := 0; cutoff <= len(lastData); cutoff++ { for cutoff := 0; cutoff <= len(lastData); cutoff++ {
walDir := filepath.Join(t.TempDir(), "wal") walDir := filepath.Join(t.TempDir(), "wal")
require.NoError(t, os.MkdirAll(walDir, 0755)) if err := os.MkdirAll(walDir, 0755); err != nil {
t.Fatalf("cutoff=%d: unexpected error: %v", cutoff, err)
}
// Copy earlier segments intact. // Copy earlier segments intact.
for _, idx := range segs[:len(segs)-1] { for _, idx := range segs[:len(segs)-1] {
require.NoError(t, os.WriteFile(segmentPath(walDir, idx), segData[idx], 0644)) if err := os.WriteFile(segmentPath(walDir, idx), segData[idx], 0644); err != nil {
t.Fatalf("cutoff=%d: unexpected error: %v", cutoff, err)
}
} }
// Write truncated last segment. // Write truncated last segment.
require.NoError(t, os.WriteFile(segmentPath(walDir, lastSeg), lastData[:cutoff], 0644)) if err := os.WriteFile(segmentPath(walDir, lastSeg), lastData[:cutoff], 0644); err != nil {
t.Fatalf("cutoff=%d: unexpected error: %v", cutoff, err)
}
w2, err := Open(walDir, tc.opts) w2, err := Open(walDir, tc.opts)
require.NoError(t, err, "cutoff=%d", cutoff) if err != nil {
t.Fatalf("cutoff=%d: unexpected error: %v", cutoff, err)
}
recovered := collectRecords(t, walDir) recovered := collectRecords(t, walDir)
require.NoError(t, w2.Close(), "cutoff=%d", cutoff) if err := w2.Close(); err != nil {
t.Fatalf("cutoff=%d: unexpected error: %v", cutoff, err)
}
// Must be a valid prefix. // Must be a valid prefix.
assert.LessOrEqual(t, len(recovered), len(origRecs), "cutoff=%d count", cutoff) if !(len(recovered) <= len(origRecs)) {
t.Errorf("cutoff=%d: got %d records, want <= %d", cutoff, len(recovered), len(origRecs))
}
for i, rec := range recovered { for i, rec := range recovered {
assert.Equal(t, origRecs[i].Type, rec.Type, "cutoff=%d rec=%d type", cutoff, i) if origRecs[i].Type != rec.Type {
assert.Equal(t, origRecs[i].Data, rec.Data, "cutoff=%d rec=%d data", cutoff, i) t.Errorf("cutoff=%d rec=%d type: got %v, want %v", cutoff, i, rec.Type, origRecs[i].Type)
}
if !reflect.DeepEqual(origRecs[i].Data, rec.Data) {
t.Errorf("cutoff=%d rec=%d data: got %v, want %v", cutoff, i, rec.Data, origRecs[i].Data)
}
} }
} }
}) })
labels/matchers_test.go
+38 -12
View File
@@ -1,10 +1,8 @@
package labels package labels
import ( import (
"reflect"
"testing" "testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func TestMatcherMatches(t *testing.T) { func TestMatcherMatches(t *testing.T) {
@@ -45,8 +43,12 @@ func TestMatcherMatches(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
m, err := NewMatcher(tc.typ, "__name__", tc.pattern) m, err := NewMatcher(tc.typ, "__name__", tc.pattern)
require.NoError(t, err) if err != nil {
assert.Equal(t, tc.want, m.Matches(tc.value)) t.Fatalf("unexpected error: %v", err)
}
if got := m.Matches(tc.value); got != tc.want {
t.Errorf("got %v, want %v", got, tc.want)
}
}) })
} }
} }
@@ -64,7 +66,9 @@ func TestNewMatcherErrors(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
_, err := NewMatcher(tc.typ, "__name__", tc.pattern) _, err := NewMatcher(tc.typ, "__name__", tc.pattern)
assert.Error(t, err) if err == nil {
t.Errorf("expected error")
}
}) })
} }
} }
@@ -81,9 +85,18 @@ func TestMustNewMatcherPanics(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
assert.Panics(t, func() { panicked := false
func() {
defer func() {
if r := recover(); r != nil {
panicked = true
}
}()
MustNewMatcher(tc.typ, "__name__", tc.pattern) MustNewMatcher(tc.typ, "__name__", tc.pattern)
}) }()
if !panicked {
t.Errorf("expected panic")
}
}) })
} }
} }
@@ -114,7 +127,9 @@ func TestFromStrings(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
got := FromStrings(tc.args...) got := FromStrings(tc.args...)
assert.Equal(t, tc.want, got) if !reflect.DeepEqual(got, tc.want) {
t.Errorf("got %v, want %v", got, tc.want)
}
}) })
} }
} }
@@ -130,9 +145,18 @@ func TestFromStringsPanics(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
assert.Panics(t, func() { panicked := false
func() {
defer func() {
if r := recover(); r != nil {
panicked = true
}
}()
FromStrings(tc.args...) FromStrings(tc.args...)
}) }()
if !panicked {
t.Errorf("expected panic")
}
}) })
} }
} }
@@ -150,7 +174,9 @@ func TestMatchTypeString(t *testing.T) {
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.want, func(t *testing.T) { t.Run(tc.want, func(t *testing.T) {
assert.Equal(t, tc.want, tc.typ.String()) if got := tc.typ.String(); got != tc.want {
t.Errorf("got %v, want %v", got, tc.want)
}
}) })
} }
} }
metrics_test.go
+60 -22
View File
@@ -5,8 +5,6 @@ import (
"testing" "testing"
"git.dvdt.dev/david/ingot/labels" "git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func TestCollectMetrics(t *testing.T) { func TestCollectMetrics(t *testing.T) {
@@ -35,10 +33,16 @@ func TestCollectMetrics(t *testing.T) {
setup: func(t *testing.T, db *DB) { setup: func(t *testing.T, db *DB) {
app := db.Appender() app := db.Appender()
_, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "a"), 1000, 1.0) _, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "a"), 1000, 1.0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = app.Append(0, labels.FromStrings("__name__", "temp", "room", "b"), 1000, 2.0) _, err = app.Append(0, labels.FromStrings("__name__", "temp", "room", "b"), 1000, 2.0)
require.NoError(t, err) if err != nil {
require.NoError(t, app.Commit()) t.Fatalf("unexpected error: %v", err)
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
// 2 user series + 5 metric series = 7 // 2 user series + 5 metric series = 7
collectCalls: 2, collectCalls: 2,
@@ -52,14 +56,22 @@ func TestCollectMetrics(t *testing.T) {
setup: func(t *testing.T, db *DB) { setup: func(t *testing.T, db *DB) {
app := db.Appender() app := db.Appender()
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0) ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for i := 1; i < 250; i++ { for i := 1; i < 250; i++ {
_, err = app.Append(ref, nil, int64(i*15000), float64(i)) _, err = app.Append(ref, nil, int64(i*15000), float64(i))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
require.NoError(t, app.Commit())
_, err = db.FlushOlderThan(math.MaxInt64) _, err = db.FlushOlderThan(math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}, },
collectCalls: 2, collectCalls: 2,
wantHeadSeries: 6, // 1 user + 5 metric series wantHeadSeries: 6, // 1 user + 5 metric series
@@ -82,7 +94,9 @@ func TestCollectMetrics(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
clock := &testClock{now: 100_000} clock := &testClock{now: 100_000}
db, err := Open(t.TempDir(), Options{Clock: clock.fn()}) db, err := Open(t.TempDir(), Options{Clock: clock.fn()})
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
t.Cleanup(func() { db.Close() }) t.Cleanup(func() { db.Close() })
tc.setup(t, db) tc.setup(t, db)
@@ -95,7 +109,9 @@ func TestCollectMetrics(t *testing.T) {
// Query each expected metric. // Query each expected metric.
lastValue := func(name string) (float64, bool) { lastValue := func(name string) (float64, bool) {
q, err := db.Querier(math.MinInt64, math.MaxInt64) q, err := db.Querier(math.MinInt64, math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer q.Close() defer q.Close()
ss := q.Select(labels.MustNewMatcher(labels.MatchEqual, "__name__", name)) ss := q.Select(labels.MustNewMatcher(labels.MatchEqual, "__name__", name))
found := false found := false
@@ -105,36 +121,58 @@ func TestCollectMetrics(t *testing.T) {
for it.Next() { for it.Next() {
_, last = it.At() _, last = it.At()
} }
require.NoError(t, it.Err()) if err := it.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
found = true found = true
} }
require.NoError(t, ss.Err()) if err := ss.Err(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
return last, found return last, found
} }
v, found := lastValue(MetricHeadSeries) v, found := lastValue(MetricHeadSeries)
assert.True(t, found, "ingot_head_series not found") if !found {
assert.Equal(t, tc.wantHeadSeries, v, "ingot_head_series") t.Errorf("ingot_head_series not found")
}
if v != tc.wantHeadSeries {
t.Errorf("ingot_head_series: got %v, want %v", v, tc.wantHeadSeries)
}
v, found = lastValue(MetricBlocksTotal) v, found = lastValue(MetricBlocksTotal)
assert.True(t, found, "ingot_blocks_total not found") if !found {
assert.Equal(t, tc.wantBlocksTotal, v, "ingot_blocks_total") t.Errorf("ingot_blocks_total not found")
}
if v != tc.wantBlocksTotal {
t.Errorf("ingot_blocks_total: got %v, want %v", v, tc.wantBlocksTotal)
}
v, found = lastValue(MetricCompactionsTotal) v, found = lastValue(MetricCompactionsTotal)
assert.True(t, found, "ingot_compactions_total not found") if !found {
assert.Equal(t, tc.wantCompactions, v, "ingot_compactions_total") t.Errorf("ingot_compactions_total not found")
}
if v != tc.wantCompactions {
t.Errorf("ingot_compactions_total: got %v, want %v", v, tc.wantCompactions)
}
// Count total ingot_* series. // Count total ingot_* series.
q, err := db.Querier(math.MinInt64, math.MaxInt64) q, err := db.Querier(math.MinInt64, math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer q.Close() defer q.Close()
ss := q.Select(labels.MustNewMatcher(labels.MatchRegexp, "__name__", "ingot_.*")) ss := q.Select(labels.MustNewMatcher(labels.MatchRegexp, "__name__", "ingot_.*"))
count := 0 count := 0
for ss.Next() { for ss.Next() {
count++ count++
} }
require.NoError(t, ss.Err()) if err := ss.Err(); err != nil {
assert.Equal(t, tc.wantMetricCount, count, "metric series count") t.Fatalf("unexpected error: %v", err)
}
if count != tc.wantMetricCount {
t.Errorf("metric series count: got %v, want %v", count, tc.wantMetricCount)
}
}) })
} }
} }
soak_test.go
+48 -20
View File
@@ -11,8 +11,6 @@ import (
"time" "time"
"git.dvdt.dev/david/ingot/labels" "git.dvdt.dev/david/ingot/labels"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
// TestSoak simulates 48h of ingestion at 15s intervals across 10k series, // TestSoak simulates 48h of ingestion at 15s intervals across 10k series,
@@ -48,7 +46,9 @@ func TestSoak(t *testing.T) {
BlockDuration: time.Duration(blockDuration) * time.Millisecond, BlockDuration: time.Duration(blockDuration) * time.Millisecond,
Clock: clock, Clock: clock,
}) })
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer db.Close() defer db.Close()
// Generate series labels. // Generate series labels.
@@ -81,7 +81,9 @@ func TestSoak(t *testing.T) {
ls = seriesLabels[i] ls = seriesLabels[i]
} }
r, err := app.Append(refs[i], ls, ts, float64(ts+int64(i))) r, err := app.Append(refs[i], ls, ts, float64(ts+int64(i)))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
refs[i] = r refs[i] = r
if i < canaryCount { if i < canaryCount {
if ls != nil { if ls != nil {
@@ -90,20 +92,26 @@ func TestSoak(t *testing.T) {
oracle.addSample(r, ts, float64(ts+int64(i))) oracle.addSample(r, ts, float64(ts+int64(i)))
} }
} }
require.NoError(t, app.Commit()) if err := app.Commit(); err != nil {
t.Fatalf("unexpected error: %v", err)
}
totalSamples += int64(numSeries) totalSamples += int64(numSeries)
// Periodic flush. // Periodic flush.
elapsed := ts - startTime elapsed := ts - startTime
if elapsed > 0 && elapsed%flushInterval == 0 { if elapsed > 0 && elapsed%flushInterval == 0 {
_, err := db.FlushOlderThan(ts - int64(blockDuration)) _, err := db.FlushOlderThan(ts - int64(blockDuration))
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
} }
// Periodic compaction + retention. // Periodic compaction + retention.
if elapsed > 0 && elapsed%compactInterval == 0 { if elapsed > 0 && elapsed%compactInterval == 0 {
err := db.RunCompaction() err := db.RunCompaction()
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
db.ApplyRetention() db.ApplyRetention()
} }
@@ -128,14 +136,20 @@ func TestSoak(t *testing.T) {
t.Logf("Peak heap: %d MiB", maxHeapMB) t.Logf("Peak heap: %d MiB", maxHeapMB)
// Assert zero query errors. // Assert zero query errors.
assert.Equal(t, int64(0), atomic.LoadInt64(&queryErrors), "query errors during soak") if got := atomic.LoadInt64(&queryErrors); got != int64(0) {
t.Errorf("query errors during soak: got %v, want %v", got, int64(0))
}
// Assert bounded memory (should stay well under 1 GiB with 10k series). // Assert bounded memory (should stay well under 1 GiB with 10k series).
assert.Less(t, maxHeapMB, uint64(1024), "heap should stay under 1 GiB") if !(maxHeapMB < uint64(1024)) {
t.Errorf("heap should stay under 1 GiB: got %v, want < %v", maxHeapMB, uint64(1024))
}
// Assert bounded disk: count block directories. // Assert bounded disk: count block directories.
entries, err := os.ReadDir(dir) entries, err := os.ReadDir(dir)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
blockCount := 0 blockCount := 0
for _, e := range entries { for _, e := range entries {
if e.IsDir() && e.Name() != "wal" { if e.IsDir() && e.Name() != "wal" {
@@ -147,7 +161,9 @@ func TestSoak(t *testing.T) {
t.Logf("Block count at end: %d", blockCount) t.Logf("Block count at end: %d", blockCount)
// With 24h retention and 2h blocks, expect roughly 12 raw + some compacted. // With 24h retention and 2h blocks, expect roughly 12 raw + some compacted.
// Should be well under 50. // Should be well under 50.
assert.Less(t, blockCount, 50, "block count should be bounded by retention") if !(blockCount < 50) {
t.Errorf("block count should be bounded by retention: got %v, want < %v", blockCount, 50)
}
// Final canary validation. // Final canary validation.
finalNow := now.Load() finalNow := now.Load()
@@ -155,7 +171,9 @@ func TestSoak(t *testing.T) {
// Live query during compaction: start query, compact, finish query. // Live query during compaction: start query, compact, finish query.
q, err := db.Querier(math.MinInt64, math.MaxInt64) q, err := db.Querier(math.MinInt64, math.MaxInt64)
require.NoError(t, err) if err != nil {
t.Fatalf("unexpected error: %v", err)
}
ss := q.Select(labels.MustNewMatcher(labels.MatchRegexp, "__name__", "metric_0.*")) ss := q.Select(labels.MustNewMatcher(labels.MatchRegexp, "__name__", "metric_0.*"))
// Trigger compaction while query is open. // Trigger compaction while query is open.
db.RunCompaction() db.RunCompaction()
@@ -164,10 +182,16 @@ func TestSoak(t *testing.T) {
it := ss.At().Iterator() it := ss.At().Iterator()
for it.Next() { for it.Next() {
} }
assert.NoError(t, it.Err()) if err := it.Err(); err != nil {
t.Errorf("unexpected error: %v", err)
}
}
if err := ss.Err(); err != nil {
t.Errorf("unexpected error: %v", err)
}
if err := q.Close(); err != nil {
t.Fatalf("unexpected error: %v", err)
} }
assert.NoError(t, ss.Err())
require.NoError(t, q.Close())
} }
// validateCanaries queries each canary series and compares with the oracle. // validateCanaries queries each canary series and compares with the oracle.
@@ -193,7 +217,9 @@ func validateCanaries(t *testing.T, db *DB, o *oracle, now int64, retentionMs in
// Query. // Query.
q, err := db.Querier(mint, maxt) q, err := db.Querier(mint, maxt)
require.NoError(t, err, "querier for canary ref %d", ref) if err != nil {
t.Fatalf("querier for canary ref %d: unexpected error: %v", ref, err)
}
ss := q.Select(matchers...) ss := q.Select(matchers...)
var gotSamples []sample var gotSamples []sample
@@ -211,10 +237,12 @@ func validateCanaries(t *testing.T, db *DB, o *oracle, now int64, retentionMs in
q.Close() q.Close()
// All assertions run unconditionally for every canary. // All assertions run unconditionally for every canary.
if !assert.NoError(t, iterErr, "canary ref %d: iterator error", ref) { if iterErr != nil {
t.Errorf("canary ref %d: iterator error: %v", ref, iterErr)
pass = false pass = false
} }
if !assert.NoError(t, ssErr, "canary ref %d: series set error", ref) { if ssErr != nil {
t.Errorf("canary ref %d: series set error: %v", ref, ssErr)
pass = false pass = false
} }
@@ -225,8 +253,8 @@ func validateCanaries(t *testing.T, db *DB, o *oracle, now int64, retentionMs in
wantSamples = s wantSamples = s
} }
if !assert.Equal(t, len(wantSamples), len(gotSamples), if len(wantSamples) != len(gotSamples) {
"canary ref %d sample count (mint=%d maxt=%d)", ref, mint, maxt) { t.Errorf("canary ref %d sample count (mint=%d maxt=%d): got %v, want %v", ref, mint, maxt, len(gotSamples), len(wantSamples))
pass = false pass = false
} }
} }