Query oracle and API freeze
Label matchers (equal, not-equal, regexp, not-regexp) in labels/, postings intersection/union/difference in internal/postings/, LabelValues and AllPostings on index, block, and head readers. ingot.go wired to real DB backed by head + block readers. Querier snapshots overlapping blocks, Select resolves matchers to postings, merged iterator chains blocks then head with block-wins dedup. Oracle tests compare every query against a naive []sample reference across head-only, block-only, head+block merge seam, multiple blocks, all four matcher types, and time range filtering.
This commit is contained in:
@@ -105,6 +105,16 @@ func (r *Reader) Labels(ref uint64) ([]labels.Label, bool) {
|
||||
return entry.Labels, true
|
||||
}
|
||||
|
||||
// LabelValues returns sorted unique values for the given label name.
|
||||
func (r *Reader) LabelValues(name string) []string {
|
||||
return r.idx.LabelValues(name)
|
||||
}
|
||||
|
||||
// AllPostings returns sorted refs for all series in the block.
|
||||
func (r *Reader) AllPostings() []uint64 {
|
||||
return r.idx.AllPostings()
|
||||
}
|
||||
|
||||
// Close releases all resources (munmaps chunk files).
|
||||
func (r *Reader) Close() error {
|
||||
return r.chunks.close()
|
||||
|
||||
@@ -5,6 +5,7 @@ package head
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync/atomic"
|
||||
|
||||
"git.dvdt.dev/david/ingot/internal/block"
|
||||
@@ -214,6 +215,59 @@ func (h *Head) FlushOlderThan(maxT int64) (string, error) {
|
||||
return ulid, nil
|
||||
}
|
||||
|
||||
// Postings returns sorted series refs where the series has label name=value.
|
||||
func (h *Head) Postings(name, value string) []uint64 {
|
||||
var refs []uint64
|
||||
h.series.forEach(func(s *memSeries) {
|
||||
for _, l := range s.labels {
|
||||
if l.Name == name && l.Value == value {
|
||||
refs = append(refs, s.ref)
|
||||
break
|
||||
}
|
||||
}
|
||||
})
|
||||
sort.Slice(refs, func(i, j int) bool { return refs[i] < refs[j] })
|
||||
return refs
|
||||
}
|
||||
|
||||
// LabelValues returns sorted unique values for the given label name.
|
||||
func (h *Head) LabelValues(name string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
h.series.forEach(func(s *memSeries) {
|
||||
for _, l := range s.labels {
|
||||
if l.Name == name {
|
||||
seen[l.Value] = struct{}{}
|
||||
break
|
||||
}
|
||||
}
|
||||
})
|
||||
vals := make([]string, 0, len(seen))
|
||||
for v := range seen {
|
||||
vals = append(vals, v)
|
||||
}
|
||||
sort.Strings(vals)
|
||||
return vals
|
||||
}
|
||||
|
||||
// Labels returns the labels for a series by ref.
|
||||
func (h *Head) Labels(ref uint64) ([]labels.Label, bool) {
|
||||
s := h.series.getByRef(ref)
|
||||
if s == nil {
|
||||
return nil, false
|
||||
}
|
||||
return s.labels, true
|
||||
}
|
||||
|
||||
// AllPostings returns sorted refs for all series in the head.
|
||||
func (h *Head) AllPostings() []uint64 {
|
||||
var refs []uint64
|
||||
h.series.forEach(func(s *memSeries) {
|
||||
refs = append(refs, s.ref)
|
||||
})
|
||||
sort.Slice(refs, func(i, j int) bool { return refs[i] < refs[j] })
|
||||
return refs
|
||||
}
|
||||
|
||||
// DataDir returns the data directory (parent of WAL dir).
|
||||
func (h *Head) DataDir() string {
|
||||
return h.dataDir
|
||||
|
||||
@@ -609,6 +609,79 @@ func TestFlushWALTruncation(t *testing.T) {
|
||||
require.NoError(t, app.Commit())
|
||||
}
|
||||
|
||||
func TestHeadPostings(t *testing.T) {
|
||||
h := openHead(t)
|
||||
|
||||
// Add three series.
|
||||
app := h.Appender()
|
||||
_, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}}, 1000, 1.0)
|
||||
require.NoError(t, err)
|
||||
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "office"}}, 1000, 2.0)
|
||||
require.NoError(t, err)
|
||||
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "kitchen"}}, 1000, 3.0)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
label string
|
||||
value string
|
||||
wantRefs []uint64
|
||||
}{
|
||||
{name: "match_name_temp", label: "__name__", value: "temp", wantRefs: []uint64{1, 3}},
|
||||
{name: "match_room_office", label: "room", value: "office", wantRefs: []uint64{1, 2}},
|
||||
{name: "match_name_humidity", label: "__name__", value: "humidity", wantRefs: []uint64{2}},
|
||||
{name: "no_match", label: "__name__", value: "pressure", wantRefs: nil},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := h.Postings(tc.label, tc.value)
|
||||
assert.Equal(t, tc.wantRefs, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeadLabelValues(t *testing.T) {
|
||||
h := openHead(t)
|
||||
|
||||
app := h.Appender()
|
||||
_, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}}, 1000, 1.0)
|
||||
require.NoError(t, err)
|
||||
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "office"}}, 1000, 2.0)
|
||||
require.NoError(t, err)
|
||||
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "kitchen"}}, 1000, 3.0)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
assert.Equal(t, []string{"humidity", "temp"}, h.LabelValues("__name__"))
|
||||
assert.Equal(t, []string{"kitchen", "office"}, h.LabelValues("room"))
|
||||
assert.Equal(t, []string{}, h.LabelValues("nonexistent"))
|
||||
}
|
||||
|
||||
func TestHeadLabelsAndAllPostings(t *testing.T) {
|
||||
h := openHead(t)
|
||||
|
||||
app := h.Appender()
|
||||
_, err := app.Append(0, []labels.Label{{Name: "__name__", Value: "temp"}}, 1000, 1.0)
|
||||
require.NoError(t, err)
|
||||
_, err = app.Append(0, []labels.Label{{Name: "__name__", Value: "humidity"}}, 1000, 2.0)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, app.Commit())
|
||||
|
||||
// Labels
|
||||
ls, ok := h.Labels(1)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, []labels.Label{{Name: "__name__", Value: "temp"}}, ls)
|
||||
|
||||
_, ok = h.Labels(999)
|
||||
assert.False(t, ok)
|
||||
|
||||
// AllPostings
|
||||
refs := h.AllPostings()
|
||||
assert.Equal(t, []uint64{1, 2}, refs)
|
||||
}
|
||||
|
||||
func TestConcurrentAppend(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -159,6 +159,50 @@ func TestIndexChunkRefEncoding(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexLabelValues(t *testing.T) {
|
||||
entries := []SeriesEntry{
|
||||
{Ref: 1, Labels: []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}}, Chunks: []ChunkMeta{{MinT: 0, MaxT: 1, Ref: 0}}},
|
||||
{Ref: 2, Labels: []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "office"}}, Chunks: []ChunkMeta{{MinT: 0, MaxT: 1, Ref: 0}}},
|
||||
{Ref: 3, Labels: []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "kitchen"}}, Chunks: []ChunkMeta{{MinT: 0, MaxT: 1, Ref: 0}}},
|
||||
}
|
||||
|
||||
data := writeIndex(t, entries)
|
||||
r, err := NewReader(data)
|
||||
require.NoError(t, err)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
label string
|
||||
wantVals []string
|
||||
}{
|
||||
{name: "name_values", label: "__name__", wantVals: []string{"humidity", "temp"}},
|
||||
{name: "room_values", label: "room", wantVals: []string{"kitchen", "office"}},
|
||||
{name: "missing_label", label: "nonexistent", wantVals: []string{}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := r.LabelValues(tc.label)
|
||||
assert.Equal(t, tc.wantVals, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexAllPostings(t *testing.T) {
|
||||
entries := []SeriesEntry{
|
||||
{Ref: 5, Labels: []labels.Label{{Name: "__name__", Value: "a"}}, Chunks: []ChunkMeta{{MinT: 0, MaxT: 1, Ref: 0}}},
|
||||
{Ref: 2, Labels: []labels.Label{{Name: "__name__", Value: "b"}}, Chunks: []ChunkMeta{{MinT: 0, MaxT: 1, Ref: 0}}},
|
||||
{Ref: 8, Labels: []labels.Label{{Name: "__name__", Value: "c"}}, Chunks: []ChunkMeta{{MinT: 0, MaxT: 1, Ref: 0}}},
|
||||
}
|
||||
|
||||
data := writeIndex(t, entries)
|
||||
r, err := NewReader(data)
|
||||
require.NoError(t, err)
|
||||
|
||||
refs := r.AllPostings()
|
||||
assert.Equal(t, []uint64{2, 5, 8}, refs)
|
||||
}
|
||||
|
||||
func TestIndexCorruptData(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -3,6 +3,7 @@ package index
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"sort"
|
||||
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
)
|
||||
@@ -213,3 +214,29 @@ func (r *Reader) SeriesByRef(ref uint64) (SeriesEntry, bool) {
|
||||
func (r *Reader) Postings(name, value string) []uint64 {
|
||||
return r.postings[labelPair{name, value}]
|
||||
}
|
||||
|
||||
// LabelValues returns sorted unique values for the given label name.
|
||||
func (r *Reader) LabelValues(name string) []string {
|
||||
seen := make(map[string]struct{})
|
||||
for key := range r.postings {
|
||||
if key.name == name {
|
||||
seen[key.value] = struct{}{}
|
||||
}
|
||||
}
|
||||
vals := make([]string, 0, len(seen))
|
||||
for v := range seen {
|
||||
vals = append(vals, v)
|
||||
}
|
||||
sort.Strings(vals)
|
||||
return vals
|
||||
}
|
||||
|
||||
// AllPostings returns sorted refs for all series in the index.
|
||||
func (r *Reader) AllPostings() []uint64 {
|
||||
refs := make([]uint64, len(r.series))
|
||||
for i, s := range r.series {
|
||||
refs[i] = s.Ref
|
||||
}
|
||||
sort.Slice(refs, func(i, j int) bool { return refs[i] < refs[j] })
|
||||
return refs
|
||||
}
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
// Package postings provides set operations on sorted uint64 slices,
|
||||
// used for combining postings lists from index lookups.
|
||||
package postings
|
||||
|
||||
// Intersect returns the sorted intersection of all input lists.
|
||||
// An empty input returns nil.
|
||||
func Intersect(lists ...[]uint64) []uint64 {
|
||||
if len(lists) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(lists) == 1 {
|
||||
return lists[0]
|
||||
}
|
||||
result := lists[0]
|
||||
for _, b := range lists[1:] {
|
||||
result = intersectTwo(result, b)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func intersectTwo(a, b []uint64) []uint64 {
|
||||
var out []uint64
|
||||
i, j := 0, 0
|
||||
for i < len(a) && j < len(b) {
|
||||
switch {
|
||||
case a[i] < b[j]:
|
||||
i++
|
||||
case a[i] > b[j]:
|
||||
j++
|
||||
default:
|
||||
out = append(out, a[i])
|
||||
i++
|
||||
j++
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Union returns the sorted union of all input lists.
|
||||
func Union(lists ...[]uint64) []uint64 {
|
||||
if len(lists) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(lists) == 1 {
|
||||
return lists[0]
|
||||
}
|
||||
result := lists[0]
|
||||
for _, b := range lists[1:] {
|
||||
result = unionTwo(result, b)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func unionTwo(a, b []uint64) []uint64 {
|
||||
out := make([]uint64, 0, len(a)+len(b))
|
||||
i, j := 0, 0
|
||||
for i < len(a) && j < len(b) {
|
||||
switch {
|
||||
case a[i] < b[j]:
|
||||
out = append(out, a[i])
|
||||
i++
|
||||
case a[i] > b[j]:
|
||||
out = append(out, b[j])
|
||||
j++
|
||||
default:
|
||||
out = append(out, a[i])
|
||||
i++
|
||||
j++
|
||||
}
|
||||
}
|
||||
out = append(out, a[i:]...)
|
||||
out = append(out, b[j:]...)
|
||||
return out
|
||||
}
|
||||
|
||||
// Without returns elements in full that are not in remove.
|
||||
// Both inputs must be sorted.
|
||||
func Without(full, remove []uint64) []uint64 {
|
||||
var out []uint64
|
||||
j := 0
|
||||
for _, v := range full {
|
||||
for j < len(remove) && remove[j] < v {
|
||||
j++
|
||||
}
|
||||
if j < len(remove) && remove[j] == v {
|
||||
continue
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package postings
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIntersect(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
lists [][]uint64
|
||||
want []uint64
|
||||
}{
|
||||
{name: "empty_input", lists: nil, want: nil},
|
||||
{name: "single_list", lists: [][]uint64{{1, 2, 3}}, want: []uint64{1, 2, 3}},
|
||||
{name: "full_overlap", lists: [][]uint64{{1, 2, 3}, {1, 2, 3}}, want: []uint64{1, 2, 3}},
|
||||
{name: "partial_overlap", lists: [][]uint64{{1, 2, 3, 4}, {2, 3, 5}}, want: []uint64{2, 3}},
|
||||
{name: "no_overlap", lists: [][]uint64{{1, 2}, {3, 4}}, want: nil},
|
||||
{name: "one_empty", lists: [][]uint64{{1, 2, 3}, {}}, want: nil},
|
||||
{name: "three_lists", lists: [][]uint64{{1, 2, 3, 4, 5}, {2, 3, 4}, {3, 4, 5}}, want: []uint64{3, 4}},
|
||||
{name: "single_element", lists: [][]uint64{{5}, {5}}, want: []uint64{5}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := Intersect(tc.lists...)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
lists [][]uint64
|
||||
want []uint64
|
||||
}{
|
||||
{name: "empty_input", lists: nil, want: nil},
|
||||
{name: "single_list", lists: [][]uint64{{1, 2, 3}}, want: []uint64{1, 2, 3}},
|
||||
{name: "full_overlap", lists: [][]uint64{{1, 2, 3}, {1, 2, 3}}, want: []uint64{1, 2, 3}},
|
||||
{name: "no_overlap", lists: [][]uint64{{1, 2}, {3, 4}}, want: []uint64{1, 2, 3, 4}},
|
||||
{name: "partial_overlap", lists: [][]uint64{{1, 3, 5}, {2, 3, 4}}, want: []uint64{1, 2, 3, 4, 5}},
|
||||
{name: "one_empty", lists: [][]uint64{{1, 2}, {}}, want: []uint64{1, 2}},
|
||||
{name: "both_empty", lists: [][]uint64{{}, {}}, want: []uint64{}},
|
||||
{name: "three_lists", lists: [][]uint64{{1}, {2}, {3}}, want: []uint64{1, 2, 3}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := Union(tc.lists...)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithout(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
full []uint64
|
||||
remove []uint64
|
||||
want []uint64
|
||||
}{
|
||||
{name: "empty_full", full: nil, remove: []uint64{1, 2}, want: nil},
|
||||
{name: "empty_remove", full: []uint64{1, 2, 3}, remove: nil, want: []uint64{1, 2, 3}},
|
||||
{name: "both_empty", full: nil, remove: nil, want: nil},
|
||||
{name: "remove_subset", full: []uint64{1, 2, 3, 4, 5}, remove: []uint64{2, 4}, want: []uint64{1, 3, 5}},
|
||||
{name: "remove_all", full: []uint64{1, 2, 3}, remove: []uint64{1, 2, 3}, want: nil},
|
||||
{name: "remove_none", full: []uint64{1, 2, 3}, remove: []uint64{4, 5}, want: []uint64{1, 2, 3}},
|
||||
{name: "remove_superset", full: []uint64{2, 3}, remove: []uint64{1, 2, 3, 4}, want: nil},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := Without(tc.full, tc.remove)
|
||||
assert.Equal(t, tc.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user