Immutable blocks with mmap reads
Flush sealed head chunks to ULID-named block directories on disk. Each block contains CRC'd chunk segment files (mmap'd for reads), a binary index (symbol table, series, postings with TOC), and a meta.json written last as the immutability gate. WAL is truncated after block fsync, preserving the crash-safety ording invariant.
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
// Package index implements the binary index file format for ingot blocks.
|
||||
//
|
||||
// An index file contains four sections:
|
||||
//
|
||||
// [Header 5B] [Symbol Table] [Series] [Postings] [TOC 28B]
|
||||
//
|
||||
// The TOC at the end stores offsets to each section. Readers seek to the
|
||||
// end, read the TOC, and use the offsets to locate each section.
|
||||
package index
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
)
|
||||
|
||||
const (
|
||||
indexMagic uint32 = 0x0FACEDB1
|
||||
indexVersion byte = 1
|
||||
headerLen = 5 // 4-byte magic + 1-byte version
|
||||
tocLen = 28 // 3 offsets (24) + CRC (4)
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidMagic = errors.New("index: invalid magic number")
|
||||
ErrInvalidVersion = errors.New("index: unsupported version")
|
||||
ErrCorruptTOC = errors.New("index: corrupt TOC (CRC mismatch)")
|
||||
ErrCorruptIndex = errors.New("index: corrupt index data")
|
||||
ErrTooShort = errors.New("index: file too short")
|
||||
)
|
||||
|
||||
// ChunkRef encodes a chunk's location as (segment << 32 | offset).
|
||||
type ChunkRef uint64
|
||||
|
||||
func NewChunkRef(segment, offset uint32) ChunkRef {
|
||||
return ChunkRef(uint64(segment)<<32 | uint64(offset))
|
||||
}
|
||||
|
||||
func (r ChunkRef) Segment() uint32 { return uint32(r >> 32) }
|
||||
func (r ChunkRef) Offset() uint32 { return uint32(r) }
|
||||
|
||||
// ChunkMeta describes a chunk's time range and location in a chunk file.
|
||||
type ChunkMeta struct {
|
||||
MinT int64
|
||||
MaxT int64
|
||||
Ref ChunkRef
|
||||
}
|
||||
|
||||
// SeriesEntry is the input for writing and the output for reading a series.
|
||||
type SeriesEntry struct {
|
||||
Ref uint64
|
||||
Labels []labels.Label
|
||||
Chunks []ChunkMeta
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func writeIndex(t *testing.T, entries []SeriesEntry) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
w := NewWriter(&buf)
|
||||
for _, e := range entries {
|
||||
w.AddSeries(e)
|
||||
}
|
||||
_, err := w.WriteTo()
|
||||
require.NoError(t, err)
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestIndexRoundTrip(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
entries []SeriesEntry
|
||||
}{
|
||||
{
|
||||
name: "single_series_single_chunk",
|
||||
entries: []SeriesEntry{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "temp"}},
|
||||
Chunks: []ChunkMeta{{MinT: 1000, MaxT: 2000, Ref: NewChunkRef(1, 0)}},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "single_series_multiple_chunks",
|
||||
entries: []SeriesEntry{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}},
|
||||
Chunks: []ChunkMeta{
|
||||
{MinT: 1000, MaxT: 2000, Ref: NewChunkRef(1, 0)},
|
||||
{MinT: 2001, MaxT: 3000, Ref: NewChunkRef(1, 500)},
|
||||
{MinT: 3001, MaxT: 4000, Ref: NewChunkRef(1, 1000)},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple_series",
|
||||
entries: []SeriesEntry{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "office"}},
|
||||
Chunks: []ChunkMeta{{MinT: 1000, MaxT: 2000, Ref: NewChunkRef(1, 0)}},
|
||||
},
|
||||
{
|
||||
Ref: 2,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "humidity"}, {Name: "room", Value: "office"}},
|
||||
Chunks: []ChunkMeta{{MinT: 1000, MaxT: 2000, Ref: NewChunkRef(1, 200)}},
|
||||
},
|
||||
{
|
||||
Ref: 3,
|
||||
Labels: []labels.Label{{Name: "__name__", Value: "temp"}, {Name: "room", Value: "kitchen"}},
|
||||
Chunks: []ChunkMeta{{MinT: 1000, MaxT: 2000, Ref: NewChunkRef(1, 400)}},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
entries: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
data := writeIndex(t, tc.entries)
|
||||
|
||||
r, err := NewReader(data)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify series count and content.
|
||||
gotSeries := r.Series()
|
||||
require.Equal(t, len(tc.entries), len(gotSeries))
|
||||
|
||||
for i, want := range tc.entries {
|
||||
got := gotSeries[i]
|
||||
assert.Equal(t, want.Ref, got.Ref, "series %d ref", i)
|
||||
assert.Equal(t, want.Labels, got.Labels, "series %d labels", i)
|
||||
require.Equal(t, len(want.Chunks), len(got.Chunks), "series %d chunk count", i)
|
||||
for j, wc := range want.Chunks {
|
||||
assert.Equal(t, wc.MinT, got.Chunks[j].MinT, "series %d chunk %d minT", i, j)
|
||||
assert.Equal(t, wc.MaxT, got.Chunks[j].MaxT, "series %d chunk %d maxT", i, j)
|
||||
assert.Equal(t, wc.Ref, got.Chunks[j].Ref, "series %d chunk %d ref", i, j)
|
||||
}
|
||||
|
||||
// SeriesByRef lookup.
|
||||
byRef, ok := r.SeriesByRef(want.Ref)
|
||||
assert.True(t, ok, "series %d lookup by ref", i)
|
||||
assert.Equal(t, want.Ref, byRef.Ref)
|
||||
}
|
||||
|
||||
// Verify postings.
|
||||
for _, e := range tc.entries {
|
||||
for _, l := range e.Labels {
|
||||
refs := r.Postings(l.Name, l.Value)
|
||||
assert.Contains(t, refs, e.Ref, "postings for %s=%s should contain ref %d", l.Name, l.Value, e.Ref)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify missing lookups return empty/false.
|
||||
_, ok := r.SeriesByRef(999999)
|
||||
assert.False(t, ok)
|
||||
assert.Nil(t, r.Postings("nonexistent", "value"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexPostingsSorted(t *testing.T) {
|
||||
entries := []SeriesEntry{
|
||||
{Ref: 5, Labels: []labels.Label{{Name: "room", Value: "office"}}, Chunks: []ChunkMeta{{MinT: 0, MaxT: 1, Ref: 0}}},
|
||||
{Ref: 2, Labels: []labels.Label{{Name: "room", Value: "office"}}, Chunks: []ChunkMeta{{MinT: 0, MaxT: 1, Ref: 0}}},
|
||||
{Ref: 8, Labels: []labels.Label{{Name: "room", Value: "office"}}, Chunks: []ChunkMeta{{MinT: 0, MaxT: 1, Ref: 0}}},
|
||||
}
|
||||
|
||||
data := writeIndex(t, entries)
|
||||
r, err := NewReader(data)
|
||||
require.NoError(t, err)
|
||||
|
||||
refs := r.Postings("room", "office")
|
||||
require.Equal(t, 3, len(refs))
|
||||
assert.Equal(t, uint64(2), refs[0])
|
||||
assert.Equal(t, uint64(5), refs[1])
|
||||
assert.Equal(t, uint64(8), refs[2])
|
||||
}
|
||||
|
||||
func TestIndexChunkRefEncoding(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
segment uint32
|
||||
offset uint32
|
||||
}{
|
||||
{"zero", 0, 0},
|
||||
{"first_segment", 1, 0},
|
||||
{"with_offset", 1, 12345},
|
||||
{"large_values", 100, 536870912},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ref := NewChunkRef(tc.segment, tc.offset)
|
||||
assert.Equal(t, tc.segment, ref.Segment())
|
||||
assert.Equal(t, tc.offset, ref.Offset())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexCorruptData(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
wantErr error
|
||||
}{
|
||||
{
|
||||
name: "too_short",
|
||||
data: []byte{1, 2, 3},
|
||||
wantErr: ErrTooShort,
|
||||
},
|
||||
{
|
||||
name: "bad_magic",
|
||||
data: func() []byte {
|
||||
d := writeIndex(t, nil)
|
||||
d[0] = 0xFF
|
||||
return d
|
||||
}(),
|
||||
wantErr: ErrInvalidMagic,
|
||||
},
|
||||
{
|
||||
name: "bad_version",
|
||||
data: func() []byte {
|
||||
d := writeIndex(t, nil)
|
||||
d[4] = 99
|
||||
return d
|
||||
}(),
|
||||
wantErr: ErrInvalidVersion,
|
||||
},
|
||||
{
|
||||
name: "corrupt_toc_crc",
|
||||
data: func() []byte {
|
||||
d := writeIndex(t, nil)
|
||||
d[len(d)-1] ^= 0xFF // flip CRC bits
|
||||
return d
|
||||
}(),
|
||||
wantErr: ErrCorruptTOC,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := NewReader(tc.data)
|
||||
assert.Equal(t, tc.wantErr, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
)
|
||||
|
||||
// Reader reads an index from a byte slice (typically mmap'd).
|
||||
type Reader struct {
|
||||
data []byte
|
||||
|
||||
symbols []string
|
||||
series []SeriesEntry
|
||||
seriesByRef map[uint64]int // ref -> index into series
|
||||
postings map[labelPair][]uint64 // label pair -> sorted refs
|
||||
}
|
||||
|
||||
// NewReader parses an index from data.
|
||||
func NewReader(data []byte) (*Reader, error) {
|
||||
if len(data) < headerLen+tocLen {
|
||||
return nil, ErrTooShort
|
||||
}
|
||||
|
||||
// Validate header.
|
||||
magic := binary.BigEndian.Uint32(data[:4])
|
||||
if magic != indexMagic {
|
||||
return nil, ErrInvalidMagic
|
||||
}
|
||||
if data[4] != indexVersion {
|
||||
return nil, ErrInvalidVersion
|
||||
}
|
||||
|
||||
// Read TOC from the last 28 bytes.
|
||||
tocStart := len(data) - tocLen
|
||||
toc := data[tocStart:]
|
||||
|
||||
// Validate TOC CRC.
|
||||
wantCRC := binary.BigEndian.Uint32(toc[24:28])
|
||||
gotCRC := crc32.Checksum(toc[:24], castagnoliTable)
|
||||
if gotCRC != wantCRC {
|
||||
return nil, ErrCorruptTOC
|
||||
}
|
||||
|
||||
symbolsOff := int(binary.BigEndian.Uint64(toc[0:8]))
|
||||
seriesOff := int(binary.BigEndian.Uint64(toc[8:16]))
|
||||
postingsOff := int(binary.BigEndian.Uint64(toc[16:24]))
|
||||
|
||||
r := &Reader{
|
||||
data: data,
|
||||
seriesByRef: make(map[uint64]int),
|
||||
postings: make(map[labelPair][]uint64),
|
||||
}
|
||||
|
||||
if err := r.readSymbols(symbolsOff); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := r.readSeries(seriesOff); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := r.readPostings(postingsOff, tocStart); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func (r *Reader) readSymbols(off int) error {
|
||||
if off+4 > len(r.data) {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
numSymbols := int(binary.BigEndian.Uint32(r.data[off : off+4]))
|
||||
off += 4
|
||||
|
||||
r.symbols = make([]string, 0, numSymbols)
|
||||
for i := 0; i < numSymbols; i++ {
|
||||
if off+2 > len(r.data) {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
slen := int(binary.BigEndian.Uint16(r.data[off : off+2]))
|
||||
off += 2
|
||||
if off+slen > len(r.data) {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
r.symbols = append(r.symbols, string(r.data[off:off+slen]))
|
||||
off += slen
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) readSeries(off int) error {
|
||||
if off+4 > len(r.data) {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
numSeries := int(binary.BigEndian.Uint32(r.data[off : off+4]))
|
||||
off += 4
|
||||
|
||||
r.series = make([]SeriesEntry, 0, numSeries)
|
||||
for i := 0; i < numSeries; i++ {
|
||||
if off+8 > len(r.data) {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
ref := binary.BigEndian.Uint64(r.data[off : off+8])
|
||||
off += 8
|
||||
|
||||
if off+2 > len(r.data) {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
numLabels := int(binary.BigEndian.Uint16(r.data[off : off+2]))
|
||||
off += 2
|
||||
|
||||
ls := make([]labels.Label, numLabels)
|
||||
for j := 0; j < numLabels; j++ {
|
||||
if off+8 > len(r.data) {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
nameIdx := int(binary.BigEndian.Uint32(r.data[off : off+4]))
|
||||
off += 4
|
||||
valueIdx := int(binary.BigEndian.Uint32(r.data[off : off+4]))
|
||||
off += 4
|
||||
|
||||
if nameIdx >= len(r.symbols) || valueIdx >= len(r.symbols) {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
ls[j] = labels.Label{Name: r.symbols[nameIdx], Value: r.symbols[valueIdx]}
|
||||
}
|
||||
|
||||
if off+4 > len(r.data) {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
numChunks := int(binary.BigEndian.Uint32(r.data[off : off+4]))
|
||||
off += 4
|
||||
|
||||
chunks := make([]ChunkMeta, numChunks)
|
||||
for j := 0; j < numChunks; j++ {
|
||||
if off+24 > len(r.data) {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
chunks[j].MinT = int64(binary.BigEndian.Uint64(r.data[off : off+8]))
|
||||
off += 8
|
||||
chunks[j].MaxT = int64(binary.BigEndian.Uint64(r.data[off : off+8]))
|
||||
off += 8
|
||||
chunks[j].Ref = ChunkRef(binary.BigEndian.Uint64(r.data[off : off+8]))
|
||||
off += 8
|
||||
}
|
||||
|
||||
entry := SeriesEntry{Ref: ref, Labels: ls, Chunks: chunks}
|
||||
r.seriesByRef[ref] = len(r.series)
|
||||
r.series = append(r.series, entry)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) readPostings(off, limit int) error {
|
||||
if off+4 > limit {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
numEntries := int(binary.BigEndian.Uint32(r.data[off : off+4]))
|
||||
off += 4
|
||||
|
||||
for i := 0; i < numEntries; i++ {
|
||||
if off+12 > limit {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
nameIdx := int(binary.BigEndian.Uint32(r.data[off : off+4]))
|
||||
off += 4
|
||||
valueIdx := int(binary.BigEndian.Uint32(r.data[off : off+4]))
|
||||
off += 4
|
||||
numRefs := int(binary.BigEndian.Uint32(r.data[off : off+4]))
|
||||
off += 4
|
||||
|
||||
if nameIdx >= len(r.symbols) || valueIdx >= len(r.symbols) {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
|
||||
if off+numRefs*8 > limit {
|
||||
return ErrCorruptIndex
|
||||
}
|
||||
|
||||
refs := make([]uint64, numRefs)
|
||||
for j := 0; j < numRefs; j++ {
|
||||
refs[j] = binary.BigEndian.Uint64(r.data[off : off+8])
|
||||
off += 8
|
||||
}
|
||||
|
||||
key := labelPair{r.symbols[nameIdx], r.symbols[valueIdx]}
|
||||
r.postings[key] = refs
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Symbols returns all symbols in the index.
|
||||
func (r *Reader) Symbols() []string {
|
||||
return r.symbols
|
||||
}
|
||||
|
||||
// Series returns all series entries.
|
||||
func (r *Reader) Series() []SeriesEntry {
|
||||
return r.series
|
||||
}
|
||||
|
||||
// SeriesByRef looks up a series by its ref.
|
||||
func (r *Reader) SeriesByRef(ref uint64) (SeriesEntry, bool) {
|
||||
idx, ok := r.seriesByRef[ref]
|
||||
if !ok {
|
||||
return SeriesEntry{}, false
|
||||
}
|
||||
return r.series[idx], true
|
||||
}
|
||||
|
||||
// Postings returns sorted series refs for the given label pair.
|
||||
func (r *Reader) Postings(name, value string) []uint64 {
|
||||
return r.postings[labelPair{name, value}]
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"sort"
|
||||
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
)
|
||||
|
||||
var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
|
||||
|
||||
// Writer builds an index file from series data.
|
||||
type Writer struct {
|
||||
w io.Writer
|
||||
off int // bytes written so far
|
||||
|
||||
symbols []string // ordered symbol list
|
||||
symbolIdx map[string]int // string -> index in symbols
|
||||
series []SeriesEntry
|
||||
postings map[labelPair][]uint64 // label pair -> sorted series refs
|
||||
}
|
||||
|
||||
type labelPair struct {
|
||||
name, value string
|
||||
}
|
||||
|
||||
// NewWriter creates a Writer that writes to w.
|
||||
func NewWriter(w io.Writer) *Writer {
|
||||
return &Writer{
|
||||
w: w,
|
||||
symbolIdx: make(map[string]int),
|
||||
postings: make(map[labelPair][]uint64),
|
||||
}
|
||||
}
|
||||
|
||||
// AddSeries adds a series to the index. Series must be added before calling
|
||||
// WriteTo. Labels must be sorted.
|
||||
func (iw *Writer) AddSeries(entry SeriesEntry) {
|
||||
for _, l := range entry.Labels {
|
||||
iw.addSymbol(l.Name)
|
||||
iw.addSymbol(l.Value)
|
||||
}
|
||||
iw.series = append(iw.series, entry)
|
||||
|
||||
for _, l := range entry.Labels {
|
||||
key := labelPair{l.Name, l.Value}
|
||||
iw.postings[key] = append(iw.postings[key], entry.Ref)
|
||||
}
|
||||
}
|
||||
|
||||
func (iw *Writer) addSymbol(s string) {
|
||||
if _, ok := iw.symbolIdx[s]; ok {
|
||||
return
|
||||
}
|
||||
idx := len(iw.symbols)
|
||||
iw.symbols = append(iw.symbols, s)
|
||||
iw.symbolIdx[s] = idx
|
||||
}
|
||||
|
||||
// WriteTo writes the complete index file. Returns bytes written and any error.
|
||||
func (iw *Writer) WriteTo() (int, error) {
|
||||
// Sort symbols for deterministic output.
|
||||
sort.Strings(iw.symbols)
|
||||
iw.symbolIdx = make(map[string]int, len(iw.symbols))
|
||||
for i, s := range iw.symbols {
|
||||
iw.symbolIdx[s] = i
|
||||
}
|
||||
|
||||
iw.off = 0
|
||||
|
||||
// Header.
|
||||
if err := iw.writeHeader(); err != nil {
|
||||
return iw.off, err
|
||||
}
|
||||
|
||||
// Symbol table.
|
||||
symbolsOff := iw.off
|
||||
if err := iw.writeSymbols(); err != nil {
|
||||
return iw.off, err
|
||||
}
|
||||
|
||||
// Series.
|
||||
seriesOff := iw.off
|
||||
if err := iw.writeSeries(); err != nil {
|
||||
return iw.off, err
|
||||
}
|
||||
|
||||
// Postings.
|
||||
postingsOff := iw.off
|
||||
if err := iw.writePostings(); err != nil {
|
||||
return iw.off, err
|
||||
}
|
||||
|
||||
// TOC.
|
||||
if err := iw.writeTOC(symbolsOff, seriesOff, postingsOff); err != nil {
|
||||
return iw.off, err
|
||||
}
|
||||
|
||||
return iw.off, nil
|
||||
}
|
||||
|
||||
func (iw *Writer) write(b []byte) error {
|
||||
n, err := iw.w.Write(b)
|
||||
iw.off += n
|
||||
return err
|
||||
}
|
||||
|
||||
func (iw *Writer) writeHeader() error {
|
||||
var buf [headerLen]byte
|
||||
binary.BigEndian.PutUint32(buf[:4], indexMagic)
|
||||
buf[4] = indexVersion
|
||||
return iw.write(buf[:])
|
||||
}
|
||||
|
||||
func (iw *Writer) writeSymbols() error {
|
||||
var buf [4]byte
|
||||
binary.BigEndian.PutUint32(buf[:], uint32(len(iw.symbols)))
|
||||
if err := iw.write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var lenbuf [2]byte
|
||||
for _, s := range iw.symbols {
|
||||
binary.BigEndian.PutUint16(lenbuf[:], uint16(len(s)))
|
||||
if err := iw.write(lenbuf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := iw.write([]byte(s)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (iw *Writer) writeSeries() error {
|
||||
var buf [8]byte
|
||||
|
||||
// numSeries
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(len(iw.series)))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, s := range iw.series {
|
||||
// ref
|
||||
binary.BigEndian.PutUint64(buf[:8], s.Ref)
|
||||
if err := iw.write(buf[:8]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// numLabels
|
||||
binary.BigEndian.PutUint16(buf[:2], uint16(len(s.Labels)))
|
||||
if err := iw.write(buf[:2]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// labels as symbol refs
|
||||
for _, l := range s.Labels {
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(iw.symbolIdx[l.Name]))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(iw.symbolIdx[l.Value]))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// numChunks
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(len(s.Chunks)))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// chunks
|
||||
for _, cm := range s.Chunks {
|
||||
binary.BigEndian.PutUint64(buf[:8], uint64(cm.MinT))
|
||||
if err := iw.write(buf[:8]); err != nil {
|
||||
return err
|
||||
}
|
||||
binary.BigEndian.PutUint64(buf[:8], uint64(cm.MaxT))
|
||||
if err := iw.write(buf[:8]); err != nil {
|
||||
return err
|
||||
}
|
||||
binary.BigEndian.PutUint64(buf[:8], uint64(cm.Ref))
|
||||
if err := iw.write(buf[:8]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (iw *Writer) writePostings() error {
|
||||
// Sort postings keys for deterministic output.
|
||||
keys := make([]labelPair, 0, len(iw.postings))
|
||||
for k := range iw.postings {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].name != keys[j].name {
|
||||
return keys[i].name < keys[j].name
|
||||
}
|
||||
return keys[i].value < keys[j].value
|
||||
})
|
||||
|
||||
var buf [8]byte
|
||||
|
||||
// numEntries
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(len(keys)))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, key := range keys {
|
||||
refs := iw.postings[key]
|
||||
|
||||
// nameSymIdx
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(iw.symbolIdx[key.name]))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
// valueSymIdx
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(iw.symbolIdx[key.value]))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sort refs.
|
||||
sort.Slice(refs, func(i, j int) bool { return refs[i] < refs[j] })
|
||||
|
||||
// numRefs
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(len(refs)))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, ref := range refs {
|
||||
binary.BigEndian.PutUint64(buf[:8], ref)
|
||||
if err := iw.write(buf[:8]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (iw *Writer) writeTOC(symbolsOff, seriesOff, postingsOff int) error {
|
||||
var buf [tocLen]byte
|
||||
binary.BigEndian.PutUint64(buf[0:8], uint64(symbolsOff))
|
||||
binary.BigEndian.PutUint64(buf[8:16], uint64(seriesOff))
|
||||
binary.BigEndian.PutUint64(buf[16:24], uint64(postingsOff))
|
||||
crc := crc32.Checksum(buf[:24], castagnoliTable)
|
||||
binary.BigEndian.PutUint32(buf[24:28], crc)
|
||||
return iw.write(buf[:])
|
||||
}
|
||||
|
||||
// SymbolIndex returns the index of a symbol string, for use in lookups.
|
||||
func (iw *Writer) SymbolIndex(s string) (int, bool) {
|
||||
idx, ok := iw.symbolIdx[s]
|
||||
return idx, ok
|
||||
}
|
||||
|
||||
// labelValues extracts the sorted label set for a series, resolving symbol refs.
|
||||
func resolveLabelNames(ls []labels.Label) []labelPair {
|
||||
pairs := make([]labelPair, len(ls))
|
||||
for i, l := range ls {
|
||||
pairs[i] = labelPair{l.Name, l.Value}
|
||||
}
|
||||
return pairs
|
||||
}
|
||||
Reference in New Issue
Block a user