Immutable blocks with mmap reads
Flush sealed head chunks to ULID-named block directories on disk. Each block contains CRC'd chunk segment files (mmap'd for reads), a binary index (symbol table, series, postings with TOC), and a meta.json written last as the immutability gate. WAL is truncated after block fsync, preserving the crash-safety ording invariant.
This commit is contained in:
@@ -0,0 +1,273 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"sort"
|
||||
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
)
|
||||
|
||||
var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
|
||||
|
||||
// Writer builds an index file from series data.
|
||||
type Writer struct {
|
||||
w io.Writer
|
||||
off int // bytes written so far
|
||||
|
||||
symbols []string // ordered symbol list
|
||||
symbolIdx map[string]int // string -> index in symbols
|
||||
series []SeriesEntry
|
||||
postings map[labelPair][]uint64 // label pair -> sorted series refs
|
||||
}
|
||||
|
||||
type labelPair struct {
|
||||
name, value string
|
||||
}
|
||||
|
||||
// NewWriter creates a Writer that writes to w.
|
||||
func NewWriter(w io.Writer) *Writer {
|
||||
return &Writer{
|
||||
w: w,
|
||||
symbolIdx: make(map[string]int),
|
||||
postings: make(map[labelPair][]uint64),
|
||||
}
|
||||
}
|
||||
|
||||
// AddSeries adds a series to the index. Series must be added before calling
|
||||
// WriteTo. Labels must be sorted.
|
||||
func (iw *Writer) AddSeries(entry SeriesEntry) {
|
||||
for _, l := range entry.Labels {
|
||||
iw.addSymbol(l.Name)
|
||||
iw.addSymbol(l.Value)
|
||||
}
|
||||
iw.series = append(iw.series, entry)
|
||||
|
||||
for _, l := range entry.Labels {
|
||||
key := labelPair{l.Name, l.Value}
|
||||
iw.postings[key] = append(iw.postings[key], entry.Ref)
|
||||
}
|
||||
}
|
||||
|
||||
func (iw *Writer) addSymbol(s string) {
|
||||
if _, ok := iw.symbolIdx[s]; ok {
|
||||
return
|
||||
}
|
||||
idx := len(iw.symbols)
|
||||
iw.symbols = append(iw.symbols, s)
|
||||
iw.symbolIdx[s] = idx
|
||||
}
|
||||
|
||||
// WriteTo writes the complete index file. Returns bytes written and any error.
|
||||
func (iw *Writer) WriteTo() (int, error) {
|
||||
// Sort symbols for deterministic output.
|
||||
sort.Strings(iw.symbols)
|
||||
iw.symbolIdx = make(map[string]int, len(iw.symbols))
|
||||
for i, s := range iw.symbols {
|
||||
iw.symbolIdx[s] = i
|
||||
}
|
||||
|
||||
iw.off = 0
|
||||
|
||||
// Header.
|
||||
if err := iw.writeHeader(); err != nil {
|
||||
return iw.off, err
|
||||
}
|
||||
|
||||
// Symbol table.
|
||||
symbolsOff := iw.off
|
||||
if err := iw.writeSymbols(); err != nil {
|
||||
return iw.off, err
|
||||
}
|
||||
|
||||
// Series.
|
||||
seriesOff := iw.off
|
||||
if err := iw.writeSeries(); err != nil {
|
||||
return iw.off, err
|
||||
}
|
||||
|
||||
// Postings.
|
||||
postingsOff := iw.off
|
||||
if err := iw.writePostings(); err != nil {
|
||||
return iw.off, err
|
||||
}
|
||||
|
||||
// TOC.
|
||||
if err := iw.writeTOC(symbolsOff, seriesOff, postingsOff); err != nil {
|
||||
return iw.off, err
|
||||
}
|
||||
|
||||
return iw.off, nil
|
||||
}
|
||||
|
||||
func (iw *Writer) write(b []byte) error {
|
||||
n, err := iw.w.Write(b)
|
||||
iw.off += n
|
||||
return err
|
||||
}
|
||||
|
||||
func (iw *Writer) writeHeader() error {
|
||||
var buf [headerLen]byte
|
||||
binary.BigEndian.PutUint32(buf[:4], indexMagic)
|
||||
buf[4] = indexVersion
|
||||
return iw.write(buf[:])
|
||||
}
|
||||
|
||||
func (iw *Writer) writeSymbols() error {
|
||||
var buf [4]byte
|
||||
binary.BigEndian.PutUint32(buf[:], uint32(len(iw.symbols)))
|
||||
if err := iw.write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var lenbuf [2]byte
|
||||
for _, s := range iw.symbols {
|
||||
binary.BigEndian.PutUint16(lenbuf[:], uint16(len(s)))
|
||||
if err := iw.write(lenbuf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := iw.write([]byte(s)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (iw *Writer) writeSeries() error {
|
||||
var buf [8]byte
|
||||
|
||||
// numSeries
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(len(iw.series)))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, s := range iw.series {
|
||||
// ref
|
||||
binary.BigEndian.PutUint64(buf[:8], s.Ref)
|
||||
if err := iw.write(buf[:8]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// numLabels
|
||||
binary.BigEndian.PutUint16(buf[:2], uint16(len(s.Labels)))
|
||||
if err := iw.write(buf[:2]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// labels as symbol refs
|
||||
for _, l := range s.Labels {
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(iw.symbolIdx[l.Name]))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(iw.symbolIdx[l.Value]))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// numChunks
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(len(s.Chunks)))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// chunks
|
||||
for _, cm := range s.Chunks {
|
||||
binary.BigEndian.PutUint64(buf[:8], uint64(cm.MinT))
|
||||
if err := iw.write(buf[:8]); err != nil {
|
||||
return err
|
||||
}
|
||||
binary.BigEndian.PutUint64(buf[:8], uint64(cm.MaxT))
|
||||
if err := iw.write(buf[:8]); err != nil {
|
||||
return err
|
||||
}
|
||||
binary.BigEndian.PutUint64(buf[:8], uint64(cm.Ref))
|
||||
if err := iw.write(buf[:8]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (iw *Writer) writePostings() error {
|
||||
// Sort postings keys for deterministic output.
|
||||
keys := make([]labelPair, 0, len(iw.postings))
|
||||
for k := range iw.postings {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].name != keys[j].name {
|
||||
return keys[i].name < keys[j].name
|
||||
}
|
||||
return keys[i].value < keys[j].value
|
||||
})
|
||||
|
||||
var buf [8]byte
|
||||
|
||||
// numEntries
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(len(keys)))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, key := range keys {
|
||||
refs := iw.postings[key]
|
||||
|
||||
// nameSymIdx
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(iw.symbolIdx[key.name]))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
// valueSymIdx
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(iw.symbolIdx[key.value]))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sort refs.
|
||||
sort.Slice(refs, func(i, j int) bool { return refs[i] < refs[j] })
|
||||
|
||||
// numRefs
|
||||
binary.BigEndian.PutUint32(buf[:4], uint32(len(refs)))
|
||||
if err := iw.write(buf[:4]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, ref := range refs {
|
||||
binary.BigEndian.PutUint64(buf[:8], ref)
|
||||
if err := iw.write(buf[:8]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (iw *Writer) writeTOC(symbolsOff, seriesOff, postingsOff int) error {
|
||||
var buf [tocLen]byte
|
||||
binary.BigEndian.PutUint64(buf[0:8], uint64(symbolsOff))
|
||||
binary.BigEndian.PutUint64(buf[8:16], uint64(seriesOff))
|
||||
binary.BigEndian.PutUint64(buf[16:24], uint64(postingsOff))
|
||||
crc := crc32.Checksum(buf[:24], castagnoliTable)
|
||||
binary.BigEndian.PutUint32(buf[24:28], crc)
|
||||
return iw.write(buf[:])
|
||||
}
|
||||
|
||||
// SymbolIndex returns the index of a symbol string, for use in lookups.
|
||||
func (iw *Writer) SymbolIndex(s string) (int, bool) {
|
||||
idx, ok := iw.symbolIdx[s]
|
||||
return idx, ok
|
||||
}
|
||||
|
||||
// labelValues extracts the sorted label set for a series, resolving symbol refs.
|
||||
func resolveLabelNames(ls []labels.Label) []labelPair {
|
||||
pairs := make([]labelPair, len(ls))
|
||||
for i, l := range ls {
|
||||
pairs[i] = labelPair{l.Name, l.Value}
|
||||
}
|
||||
return pairs
|
||||
}
|
||||
Reference in New Issue
Block a user