ingotctl, HTTP layer, and self-instrumentation

Add cmd/ingotctl with four subcommands: blocks (list with stats), inspect (series/postings dump), chunks (decode raw samples by ref), and fsck (CRC and index integrity validation across all blocks). Add cmd/ingothttp with JSON query endpoints: /api/v1/query_range (Prometheus-style matrix response), /api/v1/read (matcher-based read requests), and /epi/v1/status (DB stats snapshot). Uses JSON instead of protobuf to maintain zero dependencies. Add self-instrumentation via the normal Appender path so metrics are queryable with the same API: ingot_head_series, ingot_head_chunks_active, ingot_blocks_total, ingot_compactions_total, ingot_wal_fsync_duration_seconds. Supporting changes: block.Validate() and block.ReadMeta() exports, Head.Stats() for series/chunk counts, WAL.LastSyncDuration() with timed fsync tracking, DB.Stats() for the HTTP status endpoint.
2026-07-04 17:40:50 -04:00
parent 0356f2e082
commit 30a93a868e
12 changed files with 1878 additions and 2 deletions
@@ -0,0 +1,357 @@
+// Command ingotctl provides CLI tools for inspecting and validating ingot blocks.
+//
+// Usage:
+//
+//	ingotctl blocks <datadir>              — list blocks with metadata
+//	ingotctl inspect <blockdir>            — detailed block dump
+//	ingotctl chunks <blockdir> <series-ref> — decode and print raw samples
+//	ingotctl fsck <datadir>                — validate all blocks
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"git.dvdt.dev/david/ingot/internal/block"
+	"git.dvdt.dev/david/ingot/internal/index"
+	"git.dvdt.dev/david/ingot/labels"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		usage()
+		os.Exit(1)
+	}
+
+	var err error
+	switch os.Args[1] {
+	case "blocks":
+		err = cmdBlocks(os.Args[2:])
+	case "inspect":
+		err = cmdInspect(os.Args[2:])
+	case "chunks":
+		err = cmdChunks(os.Args[2:])
+	case "fsck":
+		err = cmdFsck(os.Args[2:])
+	case "help", "-h", "--help":
+		usage()
+	default:
+		fmt.Fprintf(os.Stderr, "unknown command: %s\n\n", os.Args[1])
+		usage()
+		os.Exit(1)
+	}
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func usage() {
+	fmt.Fprintln(os.Stderr, `Usage: ingotctl <command> [args]
+
+Commands:
+  blocks <datadir>                List blocks with ULID, time range, level, stats
+  inspect <blockdir>              Detailed block dump: series, chunks, postings
+  chunks <blockdir> <series-ref>  Decode and print raw samples for a series ref
+  fsck <datadir>                  Validate all blocks: CRC checks, index integrity`)
+}
+
+// cmdBlocks lists all blocks in a data directory.
+func cmdBlocks(args []string) error {
+	if len(args) < 1 {
+		return fmt.Errorf("usage: ingotctl blocks <datadir>")
+	}
+	dataDir := args[0]
+
+	entries, err := os.ReadDir(dataDir)
+	if err != nil {
+		return err
+	}
+
+	type blockInfo struct {
+		meta block.BlockMeta
+		dir  string
+	}
+	var blocks []blockInfo
+
+	for _, e := range entries {
+		if !e.IsDir() || e.Name() == "wal" {
+			continue
+		}
+		dir := filepath.Join(dataDir, e.Name())
+		if _, err := os.Stat(filepath.Join(dir, "meta.json")); err != nil {
+			continue
+		}
+		meta, err := block.ReadMeta(dir)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "warning: %s: %v\n", e.Name(), err)
+			continue
+		}
+		blocks = append(blocks, blockInfo{meta: meta, dir: dir})
+	}
+
+	sort.Slice(blocks, func(i, j int) bool {
+		return blocks[i].meta.MinTime < blocks[j].meta.MinTime
+	})
+
+	if len(blocks) == 0 {
+		fmt.Println("no blocks found")
+		return nil
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0)
+	fmt.Fprintln(w, "ULID\tMIN TIME\tMAX TIME\tDURATION\tLEVEL\tSERIES\tSAMPLES\tCHUNKS")
+	for _, b := range blocks {
+		m := b.meta
+		dur := time.Duration(m.MaxTime-m.MinTime) * time.Millisecond
+		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%d\t%d\t%d\n",
+			m.ULID,
+			formatTimestamp(m.MinTime),
+			formatTimestamp(m.MaxTime),
+			dur.Truncate(time.Second),
+			m.Compaction.Level,
+			m.Stats.NumSeries,
+			m.Stats.NumSamples,
+			m.Stats.NumChunks,
+		)
+	}
+	w.Flush()
+
+	fmt.Printf("\n%d block(s) total\n", len(blocks))
+	return nil
+}
+
+// cmdInspect dumps detailed information about a single block.
+func cmdInspect(args []string) error {
+	if len(args) < 1 {
+		return fmt.Errorf("usage: ingotctl inspect <blockdir>")
+	}
+	blockDir := args[0]
+
+	meta, err := block.ReadMeta(blockDir)
+	if err != nil {
+		return fmt.Errorf("read meta: %w", err)
+	}
+
+	// Print meta.
+	fmt.Println("=== Block Meta ===")
+	metaJSON, _ := json.MarshalIndent(meta, "", "  ")
+	fmt.Println(string(metaJSON))
+
+	// Open block for index inspection.
+	br, err := block.Open(blockDir)
+	if err != nil {
+		return fmt.Errorf("open block: %w", err)
+	}
+	defer br.Close()
+
+	series := br.Series()
+
+	fmt.Printf("\n=== Series (%d) ===\n", len(series))
+	w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0)
+	fmt.Fprintln(w, "REF\tLABELS\tCHUNKS\tMIN TIME\tMAX TIME")
+	for _, s := range series {
+		minT, maxT := seriesTimeRange(s.Chunks)
+		fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\n",
+			s.Ref,
+			formatLabels(s.Labels),
+			len(s.Chunks),
+			formatTimestamp(minT),
+			formatTimestamp(maxT),
+		)
+	}
+	w.Flush()
+
+	// Postings stats.
+	fmt.Printf("\n=== Postings ===\n")
+	postingsStats := collectPostingsStats(br, series)
+	pw := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0)
+	fmt.Fprintln(pw, "LABEL\tVALUES\tPOSTINGS")
+	for _, ps := range postingsStats {
+		fmt.Fprintf(pw, "%s\t%d\t%d\n", ps.name, ps.numValues, ps.totalPostings)
+	}
+	pw.Flush()
+
+	return nil
+}
+
+// cmdChunks decodes and prints raw samples for a series ref in a block.
+func cmdChunks(args []string) error {
+	if len(args) < 2 {
+		return fmt.Errorf("usage: ingotctl chunks <blockdir> <series-ref>")
+	}
+	blockDir := args[0]
+	ref, err := strconv.ParseUint(args[1], 10, 64)
+	if err != nil {
+		return fmt.Errorf("invalid series ref %q: %w", args[1], err)
+	}
+
+	br, err := block.Open(blockDir)
+	if err != nil {
+		return fmt.Errorf("open block: %w", err)
+	}
+	defer br.Close()
+
+	entry, ok := br.SeriesByRef(ref)
+	if !ok {
+		return fmt.Errorf("series ref %d not found in block", ref)
+	}
+
+	fmt.Printf("Series %d: %s\n", ref, formatLabels(entry.Labels))
+	fmt.Printf("Chunks: %d\n\n", len(entry.Chunks))
+
+	for i, cm := range entry.Chunks {
+		fmt.Printf("--- Chunk %d [%s .. %s] segment=%d offset=%d ---\n",
+			i, formatTimestamp(cm.MinT), formatTimestamp(cm.MaxT),
+			cm.Ref.Segment(), cm.Ref.Offset())
+
+		it, err := br.ChunkIterator(cm.Ref)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "  error reading chunk: %v\n", err)
+			continue
+		}
+
+		w := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0)
+		fmt.Fprintln(w, "  TIMESTAMP\tVALUE")
+		count := 0
+		for it.Next() {
+			t, v := it.At()
+			fmt.Fprintf(w, "  %s\t%g\n", formatTimestamp(t), v)
+			count++
+		}
+		w.Flush()
+		if it.Err() != nil {
+			fmt.Fprintf(os.Stderr, "  iterator error: %v\n", it.Err())
+		}
+		fmt.Printf("  (%d samples)\n\n", count)
+	}
+
+	return nil
+}
+
+// cmdFsck validates all blocks in a data directory.
+func cmdFsck(args []string) error {
+	if len(args) < 1 {
+		return fmt.Errorf("usage: ingotctl fsck <datadir>")
+	}
+	dataDir := args[0]
+
+	entries, err := os.ReadDir(dataDir)
+	if err != nil {
+		return err
+	}
+
+	var blockDirs []string
+	for _, e := range entries {
+		if !e.IsDir() || e.Name() == "wal" {
+			continue
+		}
+		dir := filepath.Join(dataDir, e.Name())
+		if _, err := os.Stat(filepath.Join(dir, "meta.json")); err != nil {
+			continue
+		}
+		blockDirs = append(blockDirs, dir)
+	}
+
+	if len(blockDirs) == 0 {
+		fmt.Println("no blocks found")
+		return nil
+	}
+
+	totalErrors := 0
+	for _, dir := range blockDirs {
+		name := filepath.Base(dir)
+		errs := block.Validate(dir)
+		if len(errs) == 0 {
+			fmt.Printf("%s: ok\n", name)
+		} else {
+			for _, e := range errs {
+				fmt.Printf("%s\n", e.Error())
+				totalErrors++
+			}
+		}
+	}
+
+	fmt.Printf("\n%d block(s) checked, %d error(s)\n", len(blockDirs), totalErrors)
+	if totalErrors > 0 {
+		return fmt.Errorf("%d integrity error(s) found", totalErrors)
+	}
+	return nil
+}
+
+// --- helpers ---
+
+func formatTimestamp(ms int64) string {
+	t := time.UnixMilli(ms)
+	return t.UTC().Format("2006-01-02T15:04:05Z")
+}
+
+func formatLabels(ls []labels.Label) string {
+	var parts []string
+	for _, l := range ls {
+		parts = append(parts, l.Name+"="+strconv.Quote(l.Value))
+	}
+	return "{" + strings.Join(parts, ", ") + "}"
+}
+
+func seriesTimeRange(chunks []index.ChunkMeta) (int64, int64) {
+	if len(chunks) == 0 {
+		return 0, 0
+	}
+	minT := chunks[0].MinT
+	maxT := chunks[0].MaxT
+	for _, c := range chunks[1:] {
+		if c.MinT < minT {
+			minT = c.MinT
+		}
+		if c.MaxT > maxT {
+			maxT = c.MaxT
+		}
+	}
+	return minT, maxT
+}
+
+type postingsStat struct {
+	name          string
+	numValues     int
+	totalPostings int
+}
+
+func collectPostingsStats(br *block.Reader, series []index.SeriesEntry) []postingsStat {
+	// Collect all label names.
+	nameSet := make(map[string]struct{})
+	for _, s := range series {
+		for _, l := range s.Labels {
+			nameSet[l.Name] = struct{}{}
+		}
+	}
+
+	names := make([]string, 0, len(nameSet))
+	for n := range nameSet {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+
+	var stats []postingsStat
+	for _, name := range names {
+		values := br.LabelValues(name)
+		total := 0
+		for _, v := range values {
+			total += len(br.Postings(name, v))
+		}
+		stats = append(stats, postingsStat{
+			name:          name,
+			numValues:     len(values),
+			totalPostings: total,
+		})
+	}
+	return stats
+}
@@ -0,0 +1,278 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"git.dvdt.dev/david/ingot/internal/block"
+	"git.dvdt.dev/david/ingot/internal/chunkenc"
+	"git.dvdt.dev/david/ingot/labels"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func makeChunk(t *testing.T, samples []struct{ t int64; v float64 }) []byte {
+	t.Helper()
+	c := chunkenc.NewXORChunk()
+	a, err := c.Appender()
+	require.NoError(t, err)
+	for _, s := range samples {
+		a.Append(s.t, s.v)
+	}
+	return append([]byte(nil), c.Bytes()...)
+}
+
+func setupTestData(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+
+	samples := []struct{ t int64; v float64 }{
+		{0, 1.0}, {15000, 2.0}, {30000, 3.0},
+	}
+	chunk := makeChunk(t, samples)
+
+	series := []block.SeriesFlush{
+		{
+			Ref:    1,
+			Labels: labels.FromStrings("__name__", "temp", "room", "office"),
+			Chunks: []block.ChunkData{
+				{MinT: 0, MaxT: 30000, Data: chunk},
+			},
+		},
+		{
+			Ref:    2,
+			Labels: labels.FromStrings("__name__", "humidity", "room", "office"),
+			Chunks: []block.ChunkData{
+				{MinT: 0, MaxT: 30000, Data: chunk},
+			},
+		},
+	}
+
+	_, err := block.Flush(dir, series)
+	require.NoError(t, err)
+	return dir
+}
+
+// blockDir returns the first block directory inside dataDir.
+func blockDir(t *testing.T, dataDir string) string {
+	t.Helper()
+	entries, err := os.ReadDir(dataDir)
+	require.NoError(t, err)
+	for _, e := range entries {
+		if e.IsDir() && e.Name() != "wal" {
+			return filepath.Join(dataDir, e.Name())
+		}
+	}
+	t.Fatal("no block directory found")
+	return ""
+}
+
+// captureStdout calls fn with stdout redirected and returns the output.
+func captureStdout(fn func()) string {
+	old := os.Stdout
+	r, w, _ := os.Pipe()
+	os.Stdout = w
+	fn()
+	w.Close()
+	os.Stdout = old
+	out := make([]byte, 16384)
+	n, _ := r.Read(out)
+	return string(out[:n])
+}
+
+func TestCmdBlocks(t *testing.T) {
+	tests := []struct {
+		name        string
+		args        []string
+		wantErr     string // "" = no error expected
+		wantOutputs []string
+	}{
+		{
+			name:        "list_blocks",
+			args:        nil, // replaced in loop with setupTestData result
+			wantErr:     "",
+			wantOutputs: []string{"ULID", "1 block(s) total"},
+		},
+		{
+			name:        "no_blocks",
+			args:        nil, // replaced with empty TempDir
+			wantErr:     "",
+			wantOutputs: []string{"no blocks found"},
+		},
+		{
+			name:        "missing_args",
+			args:        []string{},
+			wantErr:     "usage",
+			wantOutputs: nil,
+		},
+	}
+
+	// Set up args that need dynamic dirs.
+	dataDir := setupTestData(t)
+	emptyDir := t.TempDir()
+	tests[0].args = []string{dataDir}
+	tests[1].args = []string{emptyDir}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var output string
+			err := func() error {
+				var cmdErr error
+				output = captureStdout(func() { cmdErr = cmdBlocks(tc.args) })
+				return cmdErr
+			}()
+
+			assert.Equal(t, tc.wantErr != "", err != nil, "error presence")
+			assert.Contains(t, errString(err), tc.wantErr)
+			for _, want := range tc.wantOutputs {
+				assert.Contains(t, output, want)
+			}
+		})
+	}
+}
+
+func TestCmdInspect(t *testing.T) {
+	dataDir := setupTestData(t)
+	bd := blockDir(t, dataDir)
+
+	tests := []struct {
+		name        string
+		args        []string
+		wantErr     string
+		wantOutputs []string
+	}{
+		{
+			name:        "valid_block",
+			args:        []string{bd},
+			wantErr:     "",
+			wantOutputs: []string{"Block Meta", "__name__", "Postings"},
+		},
+		{
+			name:        "missing_args",
+			args:        []string{},
+			wantErr:     "usage",
+			wantOutputs: nil,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var output string
+			err := func() error {
+				var cmdErr error
+				output = captureStdout(func() { cmdErr = cmdInspect(tc.args) })
+				return cmdErr
+			}()
+
+			assert.Equal(t, tc.wantErr != "", err != nil, "error presence")
+			assert.Contains(t, errString(err), tc.wantErr)
+			for _, want := range tc.wantOutputs {
+				assert.Contains(t, output, want)
+			}
+		})
+	}
+}
+
+func TestCmdChunks(t *testing.T) {
+	dataDir := setupTestData(t)
+	bd := blockDir(t, dataDir)
+
+	tests := []struct {
+		name        string
+		args        []string
+		wantErr     string
+		wantOutputs []string
+	}{
+		{
+			name:        "valid_ref",
+			args:        []string{bd, "1"},
+			wantErr:     "",
+			wantOutputs: []string{"Series 1", "(3 samples)"},
+		},
+		{
+			name:        "ref_not_found",
+			args:        []string{bd, "999"},
+			wantErr:     "not found",
+			wantOutputs: nil,
+		},
+		{
+			name:        "missing_ref_arg",
+			args:        []string{bd},
+			wantErr:     "usage",
+			wantOutputs: nil,
+		},
+		{
+			name:        "missing_all_args",
+			args:        []string{},
+			wantErr:     "usage",
+			wantOutputs: nil,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			var output string
+			err := func() error {
+				var cmdErr error
+				output = captureStdout(func() { cmdErr = cmdChunks(tc.args) })
+				return cmdErr
+			}()
+
+			assert.Equal(t, tc.wantErr != "", err != nil, "error presence")
+			assert.Contains(t, errString(err), tc.wantErr)
+			for _, want := range tc.wantOutputs {
+				assert.Contains(t, output, want)
+			}
+		})
+	}
+}
+
+func TestCmdFsck(t *testing.T) {
+	dataDir := setupTestData(t)
+	emptyDir := t.TempDir()
+
+	tests := []struct {
+		name    string
+		args    []string
+		wantErr string
+	}{
+		{
+			name:    "valid_data",
+			args:    []string{dataDir},
+			wantErr: "",
+		},
+		{
+			name:    "no_blocks",
+			args:    []string{emptyDir},
+			wantErr: "",
+		},
+		{
+			name:    "missing_args",
+			args:    []string{},
+			wantErr: "usage",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			err := func() error {
+				var cmdErr error
+				captureStdout(func() { cmdErr = cmdFsck(tc.args) })
+				return cmdErr
+			}()
+
+			assert.Equal(t, tc.wantErr != "", err != nil, "error presence")
+			assert.Contains(t, errString(err), tc.wantErr)
+		})
+	}
+}
+
+// errString returns the error message or "" for nil.
+func errString(err error) string {
+	if err == nil {
+		return ""
+	}
+	return strings.ToLower(err.Error())
+}