ingotctl, HTTP layer, and self-instrumentation
Add cmd/ingotctl with four subcommands: blocks (list with stats), inspect (series/postings dump), chunks (decode raw samples by ref), and fsck (CRC and index integrity validation across all blocks). Add cmd/ingothttp with JSON query endpoints: /api/v1/query_range (Prometheus-style matrix response), /api/v1/read (matcher-based read requests), and /epi/v1/status (DB stats snapshot). Uses JSON instead of protobuf to maintain zero dependencies. Add self-instrumentation via the normal Appender path so metrics are queryable with the same API: ingot_head_series, ingot_head_chunks_active, ingot_blocks_total, ingot_compactions_total, ingot_wal_fsync_duration_seconds. Supporting changes: block.Validate() and block.ReadMeta() exports, Head.Stats() for series/chunk counts, WAL.LastSyncDuration() with timed fsync tracking, DB.Stats() for the HTTP status endpoint.
This commit is contained in:
@@ -0,0 +1,148 @@
|
||||
package block
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.dvdt.dev/david/ingot/internal/index"
|
||||
)
|
||||
|
||||
// ValidationError describes a single integrity issue found during validation.
|
||||
type ValidationError struct {
|
||||
Block string // ULID or directory name
|
||||
Section string // "meta", "index", "chunks"
|
||||
Detail string
|
||||
}
|
||||
|
||||
func (e ValidationError) Error() string {
|
||||
return fmt.Sprintf("%s: %s: %s", e.Block, e.Section, e.Detail)
|
||||
}
|
||||
|
||||
// ReadMeta reads and returns the BlockMeta for a block directory.
|
||||
// Exported for use by ingotctl.
|
||||
func ReadMeta(dir string) (BlockMeta, error) {
|
||||
return readMeta(dir)
|
||||
}
|
||||
|
||||
// Validate performs integrity checks on a single block directory.
|
||||
// It checks meta.json consistency, index integrity (magic, version, TOC CRC),
|
||||
// and CRC validation on every chunk entry.
|
||||
func Validate(dir string) []ValidationError {
|
||||
var errs []ValidationError
|
||||
blockName := filepath.Base(dir)
|
||||
|
||||
// 1. Validate meta.json.
|
||||
meta, err := readMeta(dir)
|
||||
if err != nil {
|
||||
errs = append(errs, ValidationError{blockName, "meta", err.Error()})
|
||||
return errs
|
||||
}
|
||||
if meta.Version != 1 {
|
||||
errs = append(errs, ValidationError{blockName, "meta",
|
||||
fmt.Sprintf("unsupported version %d", meta.Version)})
|
||||
}
|
||||
if meta.MinTime > meta.MaxTime {
|
||||
errs = append(errs, ValidationError{blockName, "meta",
|
||||
fmt.Sprintf("minTime %d > maxTime %d", meta.MinTime, meta.MaxTime)})
|
||||
}
|
||||
|
||||
// 2. Validate index file.
|
||||
indexData, err := os.ReadFile(filepath.Join(dir, "index"))
|
||||
if err != nil {
|
||||
errs = append(errs, ValidationError{blockName, "index", err.Error()})
|
||||
} else {
|
||||
_, err := index.NewReader(indexData)
|
||||
if err != nil {
|
||||
errs = append(errs, ValidationError{blockName, "index", err.Error()})
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Validate all chunk segment files — iterate every entry and check CRC.
|
||||
chunksDir := filepath.Join(dir, chunksDirName)
|
||||
entries, err := os.ReadDir(chunksDir)
|
||||
if err != nil {
|
||||
errs = append(errs, ValidationError{blockName, "chunks", err.Error()})
|
||||
return errs
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
if e.IsDir() {
|
||||
continue
|
||||
}
|
||||
segIdx := parseSegmentName(e.Name())
|
||||
if segIdx < 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(chunksDir, e.Name()))
|
||||
if err != nil {
|
||||
errs = append(errs, ValidationError{blockName, "chunks",
|
||||
fmt.Sprintf("segment %s: %s", e.Name(), err)})
|
||||
continue
|
||||
}
|
||||
|
||||
segErrs := validateChunkSegment(data, e.Name())
|
||||
for _, se := range segErrs {
|
||||
errs = append(errs, ValidationError{blockName, "chunks", se})
|
||||
}
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
// validateChunkSegment checks every chunk entry in a segment file for CRC integrity.
|
||||
func validateChunkSegment(data []byte, name string) []string {
|
||||
var errs []string
|
||||
|
||||
if len(data) < chunkHeaderLen {
|
||||
return []string{fmt.Sprintf("segment %s: too short for header", name)}
|
||||
}
|
||||
|
||||
magic := binary.BigEndian.Uint32(data[:4])
|
||||
if magic != chunkMagic {
|
||||
return []string{fmt.Sprintf("segment %s: invalid magic %#x", name, magic)}
|
||||
}
|
||||
if data[4] != chunkVersion {
|
||||
return []string{fmt.Sprintf("segment %s: unsupported version %d", name, data[4])}
|
||||
}
|
||||
|
||||
off := chunkHeaderLen
|
||||
entryIdx := 0
|
||||
for off < len(data) {
|
||||
if off+chunkEntryHeaderLen > len(data) {
|
||||
errs = append(errs, fmt.Sprintf("segment %s entry %d at offset %d: truncated header",
|
||||
name, entryIdx, off))
|
||||
break
|
||||
}
|
||||
|
||||
dataLen := int(binary.BigEndian.Uint32(data[off : off+4]))
|
||||
encoding := data[off+4]
|
||||
off += chunkEntryHeaderLen
|
||||
|
||||
end := off + dataLen + chunkEntryCRCLen
|
||||
if end > len(data) {
|
||||
errs = append(errs, fmt.Sprintf("segment %s entry %d: truncated data (need %d bytes, have %d)",
|
||||
name, entryIdx, dataLen+chunkEntryCRCLen, len(data)-off))
|
||||
break
|
||||
}
|
||||
|
||||
chunkBytes := data[off : off+dataLen]
|
||||
off += dataLen
|
||||
|
||||
wantCRC := binary.BigEndian.Uint32(data[off : off+4])
|
||||
crc := crc32.New(castagnoliTable)
|
||||
crc.Write([]byte{encoding})
|
||||
crc.Write(chunkBytes)
|
||||
if crc.Sum32() != wantCRC {
|
||||
errs = append(errs, fmt.Sprintf("segment %s entry %d at offset %d: CRC mismatch",
|
||||
name, entryIdx, off-dataLen-chunkEntryHeaderLen))
|
||||
}
|
||||
off += chunkEntryCRCLen
|
||||
entryIdx++
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package block
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(t *testing.T) string // returns block dir
|
||||
wantErrors int
|
||||
wantMatch string // substring to find in concatenated errors; "" matches everything
|
||||
}{
|
||||
{
|
||||
name: "valid_block",
|
||||
setup: func(t *testing.T) string {
|
||||
dir := t.TempDir()
|
||||
series := []SeriesFlush{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: labels.FromStrings("__name__", "temp"),
|
||||
Chunks: []ChunkData{
|
||||
{MinT: 0, MaxT: 1000, Data: makeTestChunk(t)},
|
||||
},
|
||||
},
|
||||
}
|
||||
ulid, err := Flush(dir, series)
|
||||
require.NoError(t, err)
|
||||
return filepath.Join(dir, ulid)
|
||||
},
|
||||
wantErrors: 0,
|
||||
wantMatch: "",
|
||||
},
|
||||
{
|
||||
name: "missing_meta",
|
||||
setup: func(t *testing.T) string {
|
||||
dir := t.TempDir()
|
||||
series := []SeriesFlush{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: labels.FromStrings("__name__", "temp"),
|
||||
Chunks: []ChunkData{
|
||||
{MinT: 0, MaxT: 1000, Data: makeTestChunk(t)},
|
||||
},
|
||||
},
|
||||
}
|
||||
ulid, err := Flush(dir, series)
|
||||
require.NoError(t, err)
|
||||
blockDir := filepath.Join(dir, ulid)
|
||||
os.Remove(filepath.Join(blockDir, "meta.json"))
|
||||
return blockDir
|
||||
},
|
||||
wantErrors: 1,
|
||||
wantMatch: "meta",
|
||||
},
|
||||
{
|
||||
name: "corrupt_chunk_crc",
|
||||
setup: func(t *testing.T) string {
|
||||
dir := t.TempDir()
|
||||
series := []SeriesFlush{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: labels.FromStrings("__name__", "temp"),
|
||||
Chunks: []ChunkData{
|
||||
{MinT: 0, MaxT: 1000, Data: makeTestChunk(t)},
|
||||
},
|
||||
},
|
||||
}
|
||||
ulid, err := Flush(dir, series)
|
||||
require.NoError(t, err)
|
||||
blockDir := filepath.Join(dir, ulid)
|
||||
|
||||
// Corrupt a byte in the chunk data.
|
||||
chunkPath := filepath.Join(blockDir, "chunks", "000001")
|
||||
data, err := os.ReadFile(chunkPath)
|
||||
require.NoError(t, err)
|
||||
data[chunkHeaderLen+chunkEntryHeaderLen+2] ^= 0xFF
|
||||
require.NoError(t, os.WriteFile(chunkPath, data, 0644))
|
||||
return blockDir
|
||||
},
|
||||
wantErrors: 1,
|
||||
wantMatch: "CRC mismatch",
|
||||
},
|
||||
{
|
||||
name: "corrupt_chunk_magic",
|
||||
setup: func(t *testing.T) string {
|
||||
dir := t.TempDir()
|
||||
series := []SeriesFlush{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: labels.FromStrings("__name__", "temp"),
|
||||
Chunks: []ChunkData{
|
||||
{MinT: 0, MaxT: 1000, Data: makeTestChunk(t)},
|
||||
},
|
||||
},
|
||||
}
|
||||
ulid, err := Flush(dir, series)
|
||||
require.NoError(t, err)
|
||||
blockDir := filepath.Join(dir, ulid)
|
||||
|
||||
chunkPath := filepath.Join(blockDir, "chunks", "000001")
|
||||
data, err := os.ReadFile(chunkPath)
|
||||
require.NoError(t, err)
|
||||
binary.BigEndian.PutUint32(data[:4], 0xDEADBEEF)
|
||||
require.NoError(t, os.WriteFile(chunkPath, data, 0644))
|
||||
return blockDir
|
||||
},
|
||||
wantErrors: 1,
|
||||
wantMatch: "invalid magic",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
blockDir := tc.setup(t)
|
||||
errs := Validate(blockDir)
|
||||
assert.Equal(t, tc.wantErrors, len(errs), "error count: %v", errs)
|
||||
|
||||
// Concatenate all error strings; "" is contained in everything.
|
||||
var combined strings.Builder
|
||||
for _, e := range errs {
|
||||
combined.WriteString(e.Error())
|
||||
combined.WriteByte('\n')
|
||||
}
|
||||
assert.Contains(t, combined.String(), tc.wantMatch)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadMeta(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setup func(t *testing.T) (string, string) // returns (dir, ulid)
|
||||
wantULID bool // true = ULID should match
|
||||
wantMinT int64
|
||||
wantMaxT int64
|
||||
wantNSer int
|
||||
wantNChk int
|
||||
wantErr error
|
||||
}{
|
||||
{
|
||||
name: "valid_block",
|
||||
setup: func(t *testing.T) (string, string) {
|
||||
dir := t.TempDir()
|
||||
ulid, err := Flush(dir, []SeriesFlush{
|
||||
{
|
||||
Ref: 1,
|
||||
Labels: labels.FromStrings("__name__", "test"),
|
||||
Chunks: []ChunkData{
|
||||
{MinT: 100, MaxT: 200, Data: makeTestChunk(t)},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return dir, ulid
|
||||
},
|
||||
wantULID: true,
|
||||
wantMinT: 100,
|
||||
wantMaxT: 200,
|
||||
wantNSer: 1,
|
||||
wantNChk: 1,
|
||||
wantErr: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
dir, ulid := tc.setup(t)
|
||||
meta, err := ReadMeta(filepath.Join(dir, ulid))
|
||||
assert.Equal(t, tc.wantErr, err)
|
||||
assert.Equal(t, tc.wantULID, meta.ULID == ulid)
|
||||
assert.Equal(t, tc.wantMinT, meta.MinTime)
|
||||
assert.Equal(t, tc.wantMaxT, meta.MaxTime)
|
||||
assert.Equal(t, tc.wantNSer, meta.Stats.NumSeries)
|
||||
assert.Equal(t, tc.wantNChk, meta.Stats.NumChunks)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func makeTestChunk(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
return makeChunk(t, []sample{s(0, 1.0), s(15000, 2.0), s(30000, 3.0)})
|
||||
}
|
||||
Reference in New Issue
Block a user