david
323a6f2951
Label matchers (equal, not-equal, regexp, not-regexp) in labels/, postings intersection/union/difference in internal/postings/, LabelValues and AllPostings on index, block, and head readers. ingot.go wired to real DB backed by head + block readers. Querier snapshots overlapping blocks, Select resolves matchers to postings, merged iterator chains blocks then head with block-wins dedup. Oracle tests compare every query against a naive []sample reference across head-only, block-only, head+block merge seam, multiple blocks, all four matcher types, and time range filtering.
731 lines
21 KiB
Go
731 lines
21 KiB
Go
package head
|
|
|
|
import (
|
|
"math"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
"testing"
|
|
|
|
"git.dvdt.dev/david/ingot/internal/block"
|
|
"git.dvdt.dev/david/ingot/internal/wal"
|
|
"git.dvdt.dev/david/ingot/labels"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// sample is a convenience type for expected results.
|
|
type sample struct {
|
|
t int64
|
|
vBits uint64
|
|
}
|
|
|
|
func s(t int64, v float64) sample { return sample{t, math.Float64bits(v)} }
|
|
|
|
// collectSamples reads all samples from a head series iterator.
|
|
func collectSamples(t *testing.T, h *Head, ref uint64, mint, maxt int64) []sample {
|
|
t.Helper()
|
|
it := h.SeriesIterator(ref, mint, maxt)
|
|
var out []sample
|
|
for it.Next() {
|
|
ts, v := it.At()
|
|
out = append(out, sample{ts, math.Float64bits(v)})
|
|
}
|
|
require.NoError(t, it.Err())
|
|
return out
|
|
}
|
|
|
|
// Action kinds for the appender lifecycle table.
|
|
const (
|
|
actNew = iota // create a new Appender
|
|
actAppend // call Append
|
|
actCommit // call Commit
|
|
actRollback // call Rollback
|
|
)
|
|
|
|
type action struct {
|
|
kind int
|
|
|
|
// actAppend fields.
|
|
ref uint64
|
|
labels []labels.Label
|
|
t int64
|
|
v float64
|
|
|
|
// Expected results (actAppend: ref+err, actCommit/actRollback: err).
|
|
wantRef uint64
|
|
wantErr error
|
|
}
|
|
|
|
func openHead(t *testing.T) *Head {
|
|
t.Helper()
|
|
dir := filepath.Join(t.TempDir(), "wal")
|
|
h, err := Open(dir, wal.Options{})
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() { h.Close() })
|
|
return h
|
|
}
|
|
|
|
func TestHead(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
actions []action
|
|
wantSamples map[uint64][]sample // ref -> expected samples after all actions
|
|
}{
|
|
{
|
|
name: "single_series_single_sample",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actAppend, labels: []labels.Label{{Name: "__name__", Value: "temp"}}, t: 1000, v: 71.3, wantRef: 1, wantErr: nil},
|
|
{kind: actCommit, wantErr: nil},
|
|
},
|
|
wantSamples: map[uint64][]sample{
|
|
1: {s(1000, 71.3)},
|
|
},
|
|
},
|
|
{
|
|
name: "single_series_multiple_samples",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actAppend, labels: []labels.Label{{Name: "__name__", Value: "temp"}}, t: 1000, v: 1.0, wantRef: 1, wantErr: nil},
|
|
{kind: actAppend, ref: 1, t: 1015, v: 2.0, wantRef: 1, wantErr: nil},
|
|
{kind: actAppend, ref: 1, t: 1030, v: 3.0, wantRef: 1, wantErr: nil},
|
|
{kind: actCommit, wantErr: nil},
|
|
},
|
|
wantSamples: map[uint64][]sample{
|
|
1: {s(1000, 1.0), s(1015, 2.0), s(1030, 3.0)},
|
|
},
|
|
},
|
|
{
|
|
name: "multiple_series",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actAppend, labels: []labels.Label{{Name: "__name__", Value: "temp"}}, t: 1000, v: 71.3, wantRef: 1, wantErr: nil},
|
|
{kind: actAppend, labels: []labels.Label{{Name: "__name__", Value: "humidity"}}, t: 1000, v: 55.0, wantRef: 2, wantErr: nil},
|
|
{kind: actAppend, ref: 1, t: 1015, v: 71.4, wantRef: 1, wantErr: nil},
|
|
{kind: actAppend, ref: 2, t: 1015, v: 54.0, wantRef: 2, wantErr: nil},
|
|
{kind: actCommit, wantErr: nil},
|
|
},
|
|
wantSamples: map[uint64][]sample{
|
|
1: {s(1000, 71.3), s(1015, 71.4)},
|
|
2: {s(1000, 55.0), s(1015, 54.0)},
|
|
},
|
|
},
|
|
{
|
|
name: "out_of_order_rejected",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actAppend, labels: []labels.Label{{Name: "__name__", Value: "temp"}}, t: 1000, v: 1.0, wantRef: 1, wantErr: nil},
|
|
{kind: actAppend, ref: 1, t: 999, v: 2.0, wantRef: 0, wantErr: ErrOutOfOrder},
|
|
{kind: actCommit, wantErr: nil},
|
|
},
|
|
wantSamples: map[uint64][]sample{
|
|
1: {s(1000, 1.0)},
|
|
},
|
|
},
|
|
{
|
|
name: "duplicate_timestamp_rejected",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actAppend, labels: []labels.Label{{Name: "__name__", Value: "temp"}}, t: 1000, v: 1.0, wantRef: 1, wantErr: nil},
|
|
{kind: actAppend, ref: 1, t: 1000, v: 2.0, wantRef: 0, wantErr: ErrOutOfOrder},
|
|
{kind: actCommit, wantErr: nil},
|
|
},
|
|
wantSamples: map[uint64][]sample{
|
|
1: {s(1000, 1.0)},
|
|
},
|
|
},
|
|
{
|
|
name: "out_of_order_across_commits",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actAppend, labels: []labels.Label{{Name: "__name__", Value: "temp"}}, t: 1000, v: 1.0, wantRef: 1, wantErr: nil},
|
|
{kind: actCommit, wantErr: nil},
|
|
{kind: actNew},
|
|
{kind: actAppend, ref: 1, t: 500, v: 2.0, wantRef: 0, wantErr: ErrOutOfOrder},
|
|
{kind: actCommit, wantErr: nil},
|
|
},
|
|
wantSamples: map[uint64][]sample{
|
|
1: {s(1000, 1.0)},
|
|
},
|
|
},
|
|
{
|
|
name: "unknown_ref_rejected",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actAppend, ref: 999, t: 1000, v: 1.0, wantRef: 0, wantErr: ErrSeriesNotFound},
|
|
{kind: actCommit, wantErr: nil},
|
|
},
|
|
wantSamples: map[uint64][]sample{},
|
|
},
|
|
{
|
|
name: "empty_label_name_rejected",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actAppend, labels: []labels.Label{{Name: "", Value: "temp"}}, t: 1000, v: 1.0, wantRef: 0, wantErr: labels.ErrEmptyName},
|
|
{kind: actCommit, wantErr: nil},
|
|
},
|
|
wantSamples: map[uint64][]sample{},
|
|
},
|
|
{
|
|
name: "unsorted_labels_normalized",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actAppend, labels: []labels.Label{{Name: "room", Value: "office"}, {Name: "__name__", Value: "temp"}}, t: 1000, v: 71.3, wantRef: 1, wantErr: nil},
|
|
{kind: actCommit, wantErr: nil},
|
|
},
|
|
wantSamples: map[uint64][]sample{
|
|
1: {s(1000, 71.3)},
|
|
},
|
|
},
|
|
{
|
|
name: "chunk_sealing",
|
|
actions: func() []action {
|
|
acts := []action{{kind: actNew}}
|
|
acts = append(acts, action{kind: actAppend, labels: []labels.Label{{Name: "__name__", Value: "temp"}}, t: 0, v: 0, wantRef: 1})
|
|
for i := 1; i < 250; i++ {
|
|
acts = append(acts, action{kind: actAppend, ref: 1, t: int64(i * 15000), v: float64(i), wantRef: 1})
|
|
}
|
|
acts = append(acts, action{kind: actCommit})
|
|
return acts
|
|
}(),
|
|
wantSamples: map[uint64][]sample{
|
|
1: func() []sample {
|
|
out := make([]sample, 250)
|
|
for i := range out {
|
|
out[i] = s(int64(i*15000), float64(i))
|
|
}
|
|
return out
|
|
}(),
|
|
},
|
|
},
|
|
{
|
|
name: "query_out_of_range_returns_empty",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actAppend, labels: []labels.Label{{Name: "__name__", Value: "temp"}}, t: 1000, v: 1.0, wantRef: 1, wantErr: nil},
|
|
{kind: actCommit, wantErr: nil},
|
|
},
|
|
// wantSamples checks ref=1 over all time; a separate query assertion below.
|
|
wantSamples: map[uint64][]sample{
|
|
1: {s(1000, 1.0)},
|
|
},
|
|
},
|
|
|
|
// --- Appender lifecycle error paths ---
|
|
{
|
|
name: "rollback_discards_samples",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actAppend, labels: []labels.Label{{Name: "__name__", Value: "temp"}}, t: 1000, v: 1.0, wantRef: 1, wantErr: nil},
|
|
{kind: actCommit, wantErr: nil},
|
|
{kind: actNew},
|
|
{kind: actAppend, ref: 1, t: 2000, v: 2.0, wantRef: 1, wantErr: nil},
|
|
{kind: actRollback, wantErr: nil},
|
|
},
|
|
wantSamples: map[uint64][]sample{
|
|
1: {s(1000, 1.0)},
|
|
},
|
|
},
|
|
{
|
|
name: "rollback_removes_new_series",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actAppend, labels: []labels.Label{{Name: "__name__", Value: "temp"}}, t: 1000, v: 1.0, wantRef: 1, wantErr: nil},
|
|
{kind: actRollback, wantErr: nil},
|
|
},
|
|
wantSamples: map[uint64][]sample{},
|
|
},
|
|
{
|
|
name: "commit_after_commit",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actCommit, wantErr: nil},
|
|
{kind: actCommit, wantErr: ErrAppenderClosed},
|
|
},
|
|
wantSamples: map[uint64][]sample{},
|
|
},
|
|
{
|
|
name: "rollback_after_commit",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actCommit, wantErr: nil},
|
|
{kind: actRollback, wantErr: ErrAppenderClosed},
|
|
},
|
|
wantSamples: map[uint64][]sample{},
|
|
},
|
|
{
|
|
name: "commit_after_rollback",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actRollback, wantErr: nil},
|
|
{kind: actCommit, wantErr: ErrAppenderClosed},
|
|
},
|
|
wantSamples: map[uint64][]sample{},
|
|
},
|
|
{
|
|
name: "rollback_after_rollback",
|
|
actions: []action{
|
|
{kind: actNew},
|
|
{kind: actRollback, wantErr: nil},
|
|
{kind: actRollback, wantErr: ErrAppenderClosed},
|
|
},
|
|
wantSamples: map[uint64][]sample{},
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
h := openHead(t)
|
|
|
|
var app *Appender
|
|
for i, act := range tc.actions {
|
|
switch act.kind {
|
|
case actNew:
|
|
app = h.Appender()
|
|
case actAppend:
|
|
ref, err := app.Append(act.ref, act.labels, act.t, act.v)
|
|
assert.Equal(t, act.wantRef, ref, "action %d ref", i)
|
|
assert.Equal(t, act.wantErr, err, "action %d error", i)
|
|
case actCommit:
|
|
err := app.Commit()
|
|
assert.Equal(t, act.wantErr, err, "action %d commit error", i)
|
|
case actRollback:
|
|
err := app.Rollback()
|
|
assert.Equal(t, act.wantErr, err, "action %d rollback error", i)
|
|
}
|
|
}
|
|
|
|
for ref, wantSamples := range tc.wantSamples {
|
|
got := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64)
|
|
require.Equal(t, len(wantSamples), len(got), "ref %d sample count", ref)
|
|
for i, want := range wantSamples {
|
|
assert.Equal(t, want, got[i], "ref %d sample %d", ref, i)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestWALReplay(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
setup func(t *testing.T, dir string)
|
|
wantSeries map[uint64][]sample
|
|
}{
|
|
{
|
|
name: "basic_recovery",
|
|
setup: func(t *testing.T, dir string) {
|
|
h, err := Open(dir, wal.Options{})
|
|
require.NoError(t, err)
|
|
app := h.Appender()
|
|
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 71.3)
|
|
require.NoError(t, err)
|
|
_, err = app.Append(ref, nil, 1015, 71.4)
|
|
require.NoError(t, err)
|
|
require.NoError(t, app.Commit())
|
|
require.NoError(t, h.Close())
|
|
},
|
|
wantSeries: map[uint64][]sample{
|
|
1: {s(1000, 71.3), s(1015, 71.4)},
|
|
},
|
|
},
|
|
{
|
|
name: "multi_series_recovery",
|
|
setup: func(t *testing.T, dir string) {
|
|
h, err := Open(dir, wal.Options{})
|
|
require.NoError(t, err)
|
|
app := h.Appender()
|
|
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 71.3)
|
|
require.NoError(t, err)
|
|
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}}, 1000, 55.0)
|
|
require.NoError(t, err)
|
|
require.NoError(t, app.Commit())
|
|
require.NoError(t, h.Close())
|
|
},
|
|
wantSeries: map[uint64][]sample{
|
|
1: {s(1000, 71.3)},
|
|
2: {s(1000, 55.0)},
|
|
},
|
|
},
|
|
{
|
|
name: "chunk_sealing_recovery",
|
|
setup: func(t *testing.T, dir string) {
|
|
h, err := Open(dir, wal.Options{})
|
|
require.NoError(t, err)
|
|
app := h.Appender()
|
|
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
|
|
require.NoError(t, err)
|
|
for i := 1; i < 250; i++ {
|
|
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
|
|
require.NoError(t, err)
|
|
}
|
|
require.NoError(t, app.Commit())
|
|
require.NoError(t, h.Close())
|
|
},
|
|
wantSeries: map[uint64][]sample{
|
|
1: func() []sample {
|
|
out := make([]sample, 250)
|
|
for i := range out {
|
|
out[i] = s(int64(i*15000), float64(i))
|
|
}
|
|
return out
|
|
}(),
|
|
},
|
|
},
|
|
{
|
|
name: "multiple_commits_recovery",
|
|
setup: func(t *testing.T, dir string) {
|
|
h, err := Open(dir, wal.Options{})
|
|
require.NoError(t, err)
|
|
app := h.Appender()
|
|
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 1.0)
|
|
require.NoError(t, err)
|
|
require.NoError(t, app.Commit())
|
|
app = h.Appender()
|
|
_, err = app.Append(1, nil, 2000, 2.0)
|
|
require.NoError(t, err)
|
|
require.NoError(t, app.Commit())
|
|
require.NoError(t, h.Close())
|
|
},
|
|
wantSeries: map[uint64][]sample{
|
|
1: {s(1000, 1.0), s(2000, 2.0)},
|
|
},
|
|
},
|
|
{
|
|
name: "rollback_not_recovered",
|
|
setup: func(t *testing.T, dir string) {
|
|
h, err := Open(dir, wal.Options{})
|
|
require.NoError(t, err)
|
|
app := h.Appender()
|
|
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 1.0)
|
|
require.NoError(t, err)
|
|
require.NoError(t, app.Commit())
|
|
// Second batch is rolled back — should not survive restart.
|
|
app = h.Appender()
|
|
_, err = app.Append(1, nil, 2000, 2.0)
|
|
require.NoError(t, err)
|
|
require.NoError(t, app.Rollback())
|
|
require.NoError(t, h.Close())
|
|
},
|
|
wantSeries: map[uint64][]sample{
|
|
1: {s(1000, 1.0)},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
dir := filepath.Join(t.TempDir(), "wal")
|
|
tc.setup(t, dir)
|
|
|
|
h, err := Open(dir, wal.Options{})
|
|
require.NoError(t, err)
|
|
defer h.Close()
|
|
|
|
for ref, wantSamples := range tc.wantSeries {
|
|
got := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64)
|
|
require.Equal(t, len(wantSamples), len(got), "ref %d sample count", ref)
|
|
for i, want := range wantSamples {
|
|
assert.Equal(t, want, got[i], "ref %d sample %d", ref, i)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFlushOlderThan(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
numSamples int // samples per series (each at 15s intervals)
|
|
flushMaxT int64
|
|
wantBlockSeries int // number of series in the block
|
|
wantBlockExists bool
|
|
}{
|
|
{
|
|
name: "flush_sealed_chunks",
|
|
numSamples: 250, // 2 sealed chunks (120 each) + 10 active
|
|
flushMaxT: math.MaxInt64,
|
|
wantBlockSeries: 1,
|
|
wantBlockExists: true,
|
|
},
|
|
{
|
|
name: "nothing_to_flush",
|
|
numSamples: 50, // only active chunk, no sealed
|
|
flushMaxT: math.MaxInt64,
|
|
wantBlockSeries: 0,
|
|
wantBlockExists: false,
|
|
},
|
|
{
|
|
name: "partial_flush_by_time",
|
|
numSamples: 250,
|
|
flushMaxT: 120 * 15000, // only flush first sealed chunk
|
|
wantBlockSeries: 1,
|
|
wantBlockExists: true,
|
|
},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
h := openHead(t)
|
|
|
|
// Append samples.
|
|
app := h.Appender()
|
|
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
|
|
require.NoError(t, err)
|
|
for i := 1; i < tc.numSamples; i++ {
|
|
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
|
|
require.NoError(t, err)
|
|
}
|
|
require.NoError(t, app.Commit())
|
|
|
|
// Flush.
|
|
ulid, err := h.FlushOlderThan(tc.flushMaxT)
|
|
require.NoError(t, err)
|
|
|
|
if !tc.wantBlockExists {
|
|
assert.Empty(t, ulid)
|
|
return
|
|
}
|
|
|
|
assert.NotEmpty(t, ulid)
|
|
|
|
// Verify block exists and is readable.
|
|
blockDir := filepath.Join(h.DataDir(), ulid)
|
|
br, err := block.Open(blockDir)
|
|
require.NoError(t, err)
|
|
defer br.Close()
|
|
|
|
assert.Equal(t, tc.wantBlockSeries, br.Meta.Stats.NumSeries)
|
|
|
|
// Verify block data is correct by iterating.
|
|
if tc.wantBlockSeries > 0 {
|
|
it, err := br.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64)
|
|
require.NoError(t, err)
|
|
count := 0
|
|
for it.Next() {
|
|
count++
|
|
}
|
|
require.NoError(t, it.Err())
|
|
assert.Greater(t, count, 0, "block should contain samples")
|
|
}
|
|
|
|
// Head should still have its active chunk data.
|
|
allSamples := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64)
|
|
assert.Greater(t, len(allSamples), 0, "head should still have active chunk")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFlushThenContinueAppending(t *testing.T) {
|
|
h := openHead(t)
|
|
|
|
// Append enough to seal two chunks (240 samples), plus a few more.
|
|
app := h.Appender()
|
|
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
|
|
require.NoError(t, err)
|
|
for i := 1; i < 250; i++ {
|
|
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
|
|
require.NoError(t, err)
|
|
}
|
|
require.NoError(t, app.Commit())
|
|
|
|
// Flush sealed chunks.
|
|
ulid, err := h.FlushOlderThan(math.MaxInt64)
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, ulid)
|
|
|
|
// Continue appending after flush.
|
|
app = h.Appender()
|
|
for i := 250; i < 260; i++ {
|
|
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
|
|
require.NoError(t, err)
|
|
}
|
|
require.NoError(t, app.Commit())
|
|
|
|
// Head should have the active chunk data (unflushed).
|
|
allSamples := collectSamples(t, h, ref, math.MinInt64, math.MaxInt64)
|
|
assert.Greater(t, len(allSamples), 0)
|
|
|
|
// Block should have the flushed data.
|
|
blockDir := filepath.Join(h.DataDir(), ulid)
|
|
br, err := block.Open(blockDir)
|
|
require.NoError(t, err)
|
|
defer br.Close()
|
|
|
|
it, err := br.SeriesChunkIterator(ref, math.MinInt64, math.MaxInt64)
|
|
require.NoError(t, err)
|
|
blockCount := 0
|
|
for it.Next() {
|
|
blockCount++
|
|
}
|
|
require.NoError(t, it.Err())
|
|
assert.Equal(t, 240, blockCount, "block should contain 2 sealed chunks of 120 samples each")
|
|
}
|
|
|
|
func TestFlushWALTruncation(t *testing.T) {
|
|
dir := t.TempDir()
|
|
walDir := filepath.Join(dir, "wal")
|
|
|
|
h, err := Open(walDir, wal.Options{})
|
|
require.NoError(t, err)
|
|
|
|
// Append enough to seal chunks.
|
|
app := h.Appender()
|
|
ref, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 0, 0)
|
|
require.NoError(t, err)
|
|
for i := 1; i < 250; i++ {
|
|
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
|
|
require.NoError(t, err)
|
|
}
|
|
require.NoError(t, app.Commit())
|
|
|
|
// Count WAL segments before flush.
|
|
walEntries, err := os.ReadDir(walDir)
|
|
require.NoError(t, err)
|
|
segsBefore := len(walEntries)
|
|
|
|
// Flush.
|
|
_, err = h.FlushOlderThan(math.MaxInt64)
|
|
require.NoError(t, err)
|
|
|
|
// WAL segments should have been truncated (or at least not grown).
|
|
walEntries, err = os.ReadDir(walDir)
|
|
require.NoError(t, err)
|
|
segsAfter := len(walEntries)
|
|
assert.LessOrEqual(t, segsAfter, segsBefore, "WAL should be truncated after flush")
|
|
|
|
require.NoError(t, h.Close())
|
|
|
|
// Re-open: head should recover from WAL (only unflushed data).
|
|
h2, err := Open(walDir, wal.Options{})
|
|
require.NoError(t, err)
|
|
defer h2.Close()
|
|
|
|
// The re-opened head should be functional.
|
|
app = h2.Appender()
|
|
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "new_series"}}, 5000000, 42.0)
|
|
require.NoError(t, err)
|
|
require.NoError(t, app.Commit())
|
|
}
|
|
|
|
func TestHeadPostings(t *testing.T) {
|
|
h := openHead(t)
|
|
|
|
// Add three series.
|
|
app := h.Appender()
|
|
_, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}}, 1000, 1.0)
|
|
require.NoError(t, err)
|
|
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "office"}}, 1000, 2.0)
|
|
require.NoError(t, err)
|
|
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "kitchen"}}, 1000, 3.0)
|
|
require.NoError(t, err)
|
|
require.NoError(t, app.Commit())
|
|
|
|
tests := []struct {
|
|
name string
|
|
label string
|
|
value string
|
|
wantRefs []uint64
|
|
}{
|
|
{name: "match_name_temp", label: "__name__", value: "temp", wantRefs: []uint64{1, 3}},
|
|
{name: "match_room_office", label: "room", value: "office", wantRefs: []uint64{1, 2}},
|
|
{name: "match_name_humidity", label: "__name__", value: "humidity", wantRefs: []uint64{2}},
|
|
{name: "no_match", label: "__name__", value: "pressure", wantRefs: nil},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got := h.Postings(tc.label, tc.value)
|
|
assert.Equal(t, tc.wantRefs, got)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHeadLabelValues(t *testing.T) {
|
|
h := openHead(t)
|
|
|
|
app := h.Appender()
|
|
_, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}}, 1000, 1.0)
|
|
require.NoError(t, err)
|
|
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "office"}}, 1000, 2.0)
|
|
require.NoError(t, err)
|
|
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "kitchen"}}, 1000, 3.0)
|
|
require.NoError(t, err)
|
|
require.NoError(t, app.Commit())
|
|
|
|
assert.Equal(t, []string{"humidity", "temp"}, h.LabelValues("__name__"))
|
|
assert.Equal(t, []string{"kitchen", "office"}, h.LabelValues("room"))
|
|
assert.Equal(t, []string{}, h.LabelValues("nonexistent"))
|
|
}
|
|
|
|
func TestHeadLabelsAndAllPostings(t *testing.T) {
|
|
h := openHead(t)
|
|
|
|
app := h.Appender()
|
|
_, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 1.0)
|
|
require.NoError(t, err)
|
|
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}}, 1000, 2.0)
|
|
require.NoError(t, err)
|
|
require.NoError(t, app.Commit())
|
|
|
|
// Labels
|
|
ls, ok := h.Labels(1)
|
|
assert.True(t, ok)
|
|
assert.Equal(t, []labels.Label{{Name: "__name__", Value: "temp"}}, ls)
|
|
|
|
_, ok = h.Labels(999)
|
|
assert.False(t, ok)
|
|
|
|
// AllPostings
|
|
refs := h.AllPostings()
|
|
assert.Equal(t, []uint64{1, 2}, refs)
|
|
}
|
|
|
|
func TestConcurrentAppend(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
numGoroutines int
|
|
samplesPerGoroutine int
|
|
}{
|
|
{name: "8_goroutines_100_samples", numGoroutines: 8, samplesPerGoroutine: 100},
|
|
{name: "1_goroutine_1000_samples", numGoroutines: 1, samplesPerGoroutine: 1000},
|
|
{name: "32_goroutines_10_samples", numGoroutines: 32, samplesPerGoroutine: 10},
|
|
}
|
|
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
h := openHead(t)
|
|
|
|
refs := make([]uint64, tc.numGoroutines)
|
|
var wg sync.WaitGroup
|
|
|
|
for g := 0; g < tc.numGoroutines; g++ {
|
|
wg.Add(1)
|
|
go func(g int) {
|
|
defer wg.Done()
|
|
app := h.Appender()
|
|
ls := []labels.Label{
|
|
{Name: "__name__", Value: "metric"},
|
|
{Name: "goroutine", Value: string(rune('A' + g))},
|
|
}
|
|
ref, err := app.Append(0, ls, int64(g*1000000), 0)
|
|
require.NoError(t, err)
|
|
refs[g] = ref
|
|
for i := 1; i < tc.samplesPerGoroutine; i++ {
|
|
_, err = app.Append(ref, nil, int64(g*1000000+i*1000), float64(i))
|
|
require.NoError(t, err)
|
|
}
|
|
require.NoError(t, app.Commit())
|
|
}(g)
|
|
}
|
|
wg.Wait()
|
|
|
|
for g := 0; g < tc.numGoroutines; g++ {
|
|
got := collectSamples(t, h, refs[g], math.MinInt64, math.MaxInt64)
|
|
assert.Equal(t, tc.samplesPerGoroutine, len(got), "goroutine %d sample count", g)
|
|
}
|
|
})
|
|
}
|
|
}
|