Query oracle and API freeze
Label matchers (equal, not-equal, regexp, not-regexp) in labels/, postings intersection/union/difference in internal/postings/, LabelValues and AllPostings on index, block, and head readers. ingot.go wired to real DB backed by head + block readers. Querier snapshots overlapping blocks, Select resolves matchers to postings, merged iterator chains blocks then head with block-wins dedup. Oracle tests compare every query against a naive []sample reference across head-only, block-only, head+block merge seam, multiple blocks, all four matcher types, and time range filtering.
This commit is contained in:
+547
@@ -0,0 +1,547 @@
|
||||
package ingot
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// sample is a test convenience type.
|
||||
type sample struct {
|
||||
t int64
|
||||
v float64
|
||||
}
|
||||
|
||||
// oracle is a naive reference implementation for query comparison.
|
||||
type oracle struct {
|
||||
series map[uint64][]sample // ref -> samples in order
|
||||
labels map[uint64][]labels.Label // ref -> labels
|
||||
}
|
||||
|
||||
func newOracle() *oracle {
|
||||
return &oracle{
|
||||
series: make(map[uint64][]sample),
|
||||
labels: make(map[uint64][]labels.Label),
|
||||
}
|
||||
}
|
||||
|
||||
func (o *oracle) addSeries(ref uint64, ls []labels.Label) {
|
||||
if _, ok := o.labels[ref]; !ok {
|
||||
o.labels[ref] = ls
|
||||
}
|
||||
}
|
||||
|
||||
func (o *oracle) addSample(ref uint64, t int64, v float64) {
|
||||
o.series[ref] = append(o.series[ref], sample{t, v})
|
||||
}
|
||||
|
||||
func (o *oracle) query(mint, maxt int64, matchers ...*labels.Matcher) map[uint64][]sample {
|
||||
result := make(map[uint64][]sample)
|
||||
for ref, ls := range o.labels {
|
||||
if !matchesAll(ls, matchers) {
|
||||
continue
|
||||
}
|
||||
var filtered []sample
|
||||
for _, s := range o.series[ref] {
|
||||
if s.t >= mint && s.t <= maxt {
|
||||
filtered = append(filtered, s)
|
||||
}
|
||||
}
|
||||
if len(filtered) > 0 {
|
||||
result[ref] = filtered
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func matchesAll(ls []labels.Label, matchers []*labels.Matcher) bool {
|
||||
for _, m := range matchers {
|
||||
val := ""
|
||||
for _, l := range ls {
|
||||
if l.Name == m.Name {
|
||||
val = l.Value
|
||||
break
|
||||
}
|
||||
}
|
||||
if !m.Matches(val) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// collectSeriesSet drains a SeriesSet into a map of ref -> samples.
|
||||
func collectSeriesSet(t *testing.T, ss SeriesSet) map[uint64][]sample {
|
||||
t.Helper()
|
||||
result := make(map[uint64][]sample)
|
||||
for ss.Next() {
|
||||
s := ss.At()
|
||||
ls := s.Labels()
|
||||
// Determine ref by looking at labels hash — we need a stable key.
|
||||
// Use the series ref which is the map key from the querier.
|
||||
// Actually, we need to match by labels since the oracle uses refs.
|
||||
// Let's collect by label hash.
|
||||
it := s.Iterator()
|
||||
var samples []sample
|
||||
for it.Next() {
|
||||
st, sv := it.At()
|
||||
samples = append(samples, sample{st, sv})
|
||||
}
|
||||
require.NoError(t, it.Err())
|
||||
if len(samples) > 0 {
|
||||
h := labels.Hash(ls)
|
||||
result[h] = samples
|
||||
}
|
||||
}
|
||||
require.NoError(t, ss.Err())
|
||||
return result
|
||||
}
|
||||
|
||||
// oracleByLabelHash re-keys oracle results by label hash for comparison.
|
||||
func oracleByLabelHash(o *oracle, mint, maxt int64, matchers ...*labels.Matcher) map[uint64][]sample {
|
||||
byRef := o.query(mint, maxt, matchers...)
|
||||
result := make(map[uint64][]sample, len(byRef))
|
||||
for ref, samples := range byRef {
|
||||
ls := o.labels[ref]
|
||||
h := labels.Hash(ls)
|
||||
result[h] = samples
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func openTestDB(t *testing.T) *DB {
|
||||
t.Helper()
|
||||
db, err := Open(t.TempDir(), Options{})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db
|
||||
}
|
||||
|
||||
func TestQueryOracle(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(t *testing.T, db *DB, o *oracle)
|
||||
queries []queryCase
|
||||
}{
|
||||
{
|
||||
name: "head_only",
|
||||
setup: func(t *testing.T, db *DB, o *oracle) {
|
||||
app := db.Appender()
|
||||
ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 1000, 71.3)
|
||||
require.NoError(t, err)
|
||||
o.addSeries(ref, labels.FromStrings("__name__", "temp", "room", "office"))
|
||||
o.addSample(ref, 1000, 71.3)
|
||||
|
||||
_, err = app.Append(ref, nil, 2000, 71.4)
|
||||
require.NoError(t, err)
|
||||
o.addSample(ref, 2000, 71.4)
|
||||
|
||||
ref2, err := app.Append(0, labels.FromStrings("__name__", "humidity", "room", "office"), 1000, 55.0)
|
||||
require.NoError(t, err)
|
||||
o.addSeries(ref2, labels.FromStrings("__name__", "humidity", "room", "office"))
|
||||
o.addSample(ref2, 1000, 55.0)
|
||||
|
||||
require.NoError(t, app.Commit())
|
||||
},
|
||||
queries: []queryCase{
|
||||
{
|
||||
name: "match_all_by_room",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "room", "office")},
|
||||
},
|
||||
{
|
||||
name: "match_temp_only",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")},
|
||||
},
|
||||
{
|
||||
name: "match_not_equal",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotEqual, "__name__", "temp")},
|
||||
},
|
||||
{
|
||||
name: "match_regex",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "__name__", "te.*")},
|
||||
},
|
||||
{
|
||||
name: "match_not_regex",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotRegexp, "__name__", "te.*")},
|
||||
},
|
||||
{
|
||||
name: "time_range_filter",
|
||||
mint: 1500,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")},
|
||||
},
|
||||
{
|
||||
name: "no_match",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "pressure")},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "block_only",
|
||||
setup: func(t *testing.T, db *DB, o *oracle) {
|
||||
app := db.Appender()
|
||||
// Write enough samples to seal chunks (need >120 for a sealed chunk).
|
||||
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0)
|
||||
require.NoError(t, err)
|
||||
o.addSeries(ref, labels.FromStrings("__name__", "temp"))
|
||||
o.addSample(ref, 0, 0)
|
||||
for i := 1; i < 250; i++ {
|
||||
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
|
||||
require.NoError(t, err)
|
||||
o.addSample(ref, int64(i*15000), float64(i))
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
// Flush to block.
|
||||
_, err = db.FlushOlderThan(math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
},
|
||||
queries: []queryCase{
|
||||
{
|
||||
name: "query_all",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")},
|
||||
},
|
||||
{
|
||||
name: "query_subset",
|
||||
mint: 100000,
|
||||
maxt: 200000,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "head_block_merge_seam",
|
||||
setup: func(t *testing.T, db *DB, o *oracle) {
|
||||
app := db.Appender()
|
||||
ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 0, 0)
|
||||
require.NoError(t, err)
|
||||
o.addSeries(ref, labels.FromStrings("__name__", "temp", "room", "office"))
|
||||
o.addSample(ref, 0, 0)
|
||||
|
||||
// Write 250 samples (2 sealed chunks + 10 active).
|
||||
for i := 1; i < 250; i++ {
|
||||
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
|
||||
require.NoError(t, err)
|
||||
o.addSample(ref, int64(i*15000), float64(i))
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
// Flush sealed chunks to block.
|
||||
_, err = db.FlushOlderThan(math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Append more samples to head (after flush).
|
||||
app = db.Appender()
|
||||
for i := 250; i < 260; i++ {
|
||||
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
|
||||
require.NoError(t, err)
|
||||
o.addSample(ref, int64(i*15000), float64(i))
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
},
|
||||
queries: []queryCase{
|
||||
{
|
||||
name: "full_range",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")},
|
||||
},
|
||||
{
|
||||
name: "block_portion_only",
|
||||
mint: 0,
|
||||
maxt: 120 * 15000,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")},
|
||||
},
|
||||
{
|
||||
name: "head_portion_only",
|
||||
mint: 250 * 15000,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")},
|
||||
},
|
||||
{
|
||||
name: "overlap_seam",
|
||||
mint: 230 * 15000,
|
||||
maxt: 255 * 15000,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "room", "office")},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple_series_with_matchers",
|
||||
setup: func(t *testing.T, db *DB, o *oracle) {
|
||||
series := []struct {
|
||||
ls []labels.Label
|
||||
}{
|
||||
{ls: labels.FromStrings("__name__", "temp", "room", "office")},
|
||||
{ls: labels.FromStrings("__name__", "temp", "room", "kitchen")},
|
||||
{ls: labels.FromStrings("__name__", "humidity", "room", "office")},
|
||||
{ls: labels.FromStrings("__name__", "pressure", "room", "lab")},
|
||||
}
|
||||
|
||||
app := db.Appender()
|
||||
for _, s := range series {
|
||||
ref, err := app.Append(0, s.ls, 1000, 1.0)
|
||||
require.NoError(t, err)
|
||||
o.addSeries(ref, s.ls)
|
||||
o.addSample(ref, 1000, 1.0)
|
||||
|
||||
_, err = app.Append(ref, nil, 2000, 2.0)
|
||||
require.NoError(t, err)
|
||||
o.addSample(ref, 2000, 2.0)
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
},
|
||||
queries: []queryCase{
|
||||
{
|
||||
name: "equal_name_temp",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")},
|
||||
},
|
||||
{
|
||||
name: "equal_room_office",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "room", "office")},
|
||||
},
|
||||
{
|
||||
name: "combined_matchers",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{
|
||||
labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp"),
|
||||
labels.MustNewMatcher(labels.MatchEqual, "room", "office"),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "regex_name",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchRegexp, "__name__", "temp|humidity")},
|
||||
},
|
||||
{
|
||||
name: "not_equal_room",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotEqual, "room", "office")},
|
||||
},
|
||||
{
|
||||
name: "not_regex_room",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchNotRegexp, "room", "off.*")},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple_blocks",
|
||||
setup: func(t *testing.T, db *DB, o *oracle) {
|
||||
app := db.Appender()
|
||||
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0)
|
||||
require.NoError(t, err)
|
||||
o.addSeries(ref, labels.FromStrings("__name__", "temp"))
|
||||
o.addSample(ref, 0, 0)
|
||||
for i := 1; i < 250; i++ {
|
||||
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
|
||||
require.NoError(t, err)
|
||||
o.addSample(ref, int64(i*15000), float64(i))
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
// First flush.
|
||||
_, err = db.FlushOlderThan(math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
|
||||
// More data -> second flush.
|
||||
app = db.Appender()
|
||||
for i := 250; i < 500; i++ {
|
||||
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
|
||||
require.NoError(t, err)
|
||||
o.addSample(ref, int64(i*15000), float64(i))
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
_, err = db.FlushOlderThan(math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Some data in head.
|
||||
app = db.Appender()
|
||||
for i := 500; i < 510; i++ {
|
||||
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
|
||||
require.NoError(t, err)
|
||||
o.addSample(ref, int64(i*15000), float64(i))
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
},
|
||||
queries: []queryCase{
|
||||
{
|
||||
name: "full_range",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")},
|
||||
},
|
||||
{
|
||||
name: "second_block_range",
|
||||
mint: 300 * 15000,
|
||||
maxt: 400 * 15000,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty_results",
|
||||
setup: func(t *testing.T, db *DB, o *oracle) {
|
||||
app := db.Appender()
|
||||
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 1000, 1.0)
|
||||
require.NoError(t, err)
|
||||
o.addSeries(ref, labels.FromStrings("__name__", "temp"))
|
||||
o.addSample(ref, 1000, 1.0)
|
||||
require.NoError(t, app.Commit())
|
||||
},
|
||||
queries: []queryCase{
|
||||
{
|
||||
name: "no_matching_series",
|
||||
mint: math.MinInt64,
|
||||
maxt: math.MaxInt64,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "nonexistent")},
|
||||
},
|
||||
{
|
||||
name: "time_range_miss",
|
||||
mint: 5000,
|
||||
maxt: 6000,
|
||||
matchers: []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp")},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
db := openTestDB(t)
|
||||
o := newOracle()
|
||||
|
||||
tc.setup(t, db, o)
|
||||
|
||||
for _, qc := range tc.queries {
|
||||
t.Run(qc.name, func(t *testing.T) {
|
||||
q, err := db.Querier(qc.mint, qc.maxt)
|
||||
require.NoError(t, err)
|
||||
defer q.Close()
|
||||
|
||||
ss := q.Select(qc.matchers...)
|
||||
got := collectSeriesSet(t, ss)
|
||||
want := oracleByLabelHash(o, qc.mint, qc.maxt, qc.matchers...)
|
||||
|
||||
assert.Equal(t, len(want), len(got), "series count mismatch")
|
||||
for h, wantSamples := range want {
|
||||
gotSamples, ok := got[h]
|
||||
assert.True(t, ok, "missing series with hash %d", h)
|
||||
assert.Equal(t, wantSamples, gotSamples, "sample mismatch for hash %d", h)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type queryCase struct {
|
||||
name string
|
||||
mint int64
|
||||
maxt int64
|
||||
matchers []*labels.Matcher
|
||||
}
|
||||
|
||||
func TestDBAppenderAPI(t *testing.T) {
|
||||
db := openTestDB(t)
|
||||
|
||||
app := db.Appender()
|
||||
ref, err := app.Append(0, labels.FromStrings("__name__", "temp", "room", "office"), 1000, 71.3)
|
||||
require.NoError(t, err)
|
||||
assert.NotZero(t, ref)
|
||||
|
||||
_, err = app.Append(ref, nil, 2000, 71.4)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
// Query back.
|
||||
q, err := db.Querier(math.MinInt64, math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
defer q.Close()
|
||||
|
||||
ss := q.Select(labels.MustNewMatcher(labels.MatchEqual, "room", "office"))
|
||||
require.True(t, ss.Next())
|
||||
s := ss.At()
|
||||
assert.Equal(t, labels.FromStrings("__name__", "temp", "room", "office"), s.Labels())
|
||||
|
||||
it := s.Iterator()
|
||||
require.True(t, it.Next())
|
||||
st, sv := it.At()
|
||||
assert.Equal(t, int64(1000), st)
|
||||
assert.Equal(t, 71.3, sv)
|
||||
|
||||
require.True(t, it.Next())
|
||||
st, sv = it.At()
|
||||
assert.Equal(t, int64(2000), st)
|
||||
assert.Equal(t, 71.4, sv)
|
||||
|
||||
assert.False(t, it.Next())
|
||||
assert.False(t, ss.Next())
|
||||
}
|
||||
|
||||
func TestDBReopenWithBlocks(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
// Write data, flush, close.
|
||||
db, err := Open(dir, Options{})
|
||||
require.NoError(t, err)
|
||||
|
||||
app := db.Appender()
|
||||
ref, err := app.Append(0, labels.FromStrings("__name__", "temp"), 0, 0)
|
||||
require.NoError(t, err)
|
||||
for i := 1; i < 250; i++ {
|
||||
_, err = app.Append(ref, nil, int64(i*15000), float64(i))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
_, err = db.FlushOlderThan(math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
// Reopen.
|
||||
db2, err := Open(dir, Options{})
|
||||
require.NoError(t, err)
|
||||
defer db2.Close()
|
||||
|
||||
// Should find data in block.
|
||||
q, err := db2.Querier(math.MinInt64, math.MaxInt64)
|
||||
require.NoError(t, err)
|
||||
defer q.Close()
|
||||
|
||||
ss := q.Select(labels.MustNewMatcher(labels.MatchEqual, "__name__", "temp"))
|
||||
require.True(t, ss.Next())
|
||||
it := ss.At().Iterator()
|
||||
count := 0
|
||||
for it.Next() {
|
||||
count++
|
||||
}
|
||||
require.NoError(t, it.Err())
|
||||
assert.Equal(t, 250, count, "should find all samples: 240 from block + 10 from head WAL replay")
|
||||
}
|
||||
Reference in New Issue
Block a user