ingot/soak_test.go

package ingot

import (
	"fmt"
	"math"
	"os"
	"path/filepath"
	"runtime"
	"sync/atomic"
	"testing"
	"time"

	"git.dvdt.dev/david/ingot/labels"
)

// TestSoak simulates 48h of ingestion at 15s intervals across 10k series,
// validating that compaction, retention, and concurrent queries all work
// correctly under sustained load.
//
// Skipped with -short. Expected runtime: 1-5 minutes.
func TestSoak(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping soak test in -short mode")
	}

	const (
		numSeries      = 10_000
		canaryCount    = 10 // series tracked in the oracle for query validation
		scrapeInterval = 15_000 // 15s in ms
		simDuration    = 48 * 3600 * 1000 // 48h in ms
		blockDuration  = 2 * 3600 * 1000 // 2h in ms
		retentionMs    = 24 * 3600 * 1000 // 24h in ms
		flushInterval  = 5 * 60 * 1000 // flush every 5 simulated minutes
		compactInterval = 30 * 60 * 1000 // compact every 30 simulated minutes
		queryInterval  = 10 * 60 * 1000 // validate queries every 10 simulated minutes
	)

	var now atomic.Int64
	startTime := int64(1_000_000_000) // arbitrary start: ~11.5 days in ms
	now.Store(startTime)
	clock := func() int64 { return now.Load() }

	dir := t.TempDir()
	db, err := Open(dir, Options{
		Retention:     time.Duration(retentionMs) * time.Millisecond,
		BlockDuration: time.Duration(blockDuration) * time.Millisecond,
		Clock:         clock,
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	defer db.Close()

	// Generate series labels.
	seriesLabels := make([][]labels.Label, numSeries)
	for i := 0; i < numSeries; i++ {
		seriesLabels[i] = labels.FromStrings(
			"__name__", fmt.Sprintf("metric_%d", i/100),
			"instance", fmt.Sprintf("inst_%d", i%100),
		)
	}

	// Oracle tracks canary series only.
	oracle := newOracle()
	refs := make([]uint64, numSeries)

	// Track stats.
	var totalSamples int64
	var queryErrors int64
	var maxHeapMB uint64

	endTime := startTime + simDuration
	for ts := startTime; ts < endTime; ts += scrapeInterval {
		now.Store(ts)

		// Append samples for all series.
		app := db.Appender()
		for i := 0; i < numSeries; i++ {
			var ls []labels.Label
			if refs[i] == 0 {
				ls = seriesLabels[i]
			}
			r, err := app.Append(refs[i], ls, ts, float64(ts+int64(i)))
			if err != nil {
				t.Fatalf("unexpected error: %v", err)
			}
			refs[i] = r
			if i < canaryCount {
				if ls != nil {
					oracle.addSeries(r, seriesLabels[i])
				}
				oracle.addSample(r, ts, float64(ts+int64(i)))
			}
		}
		if err := app.Commit(); err != nil {
			t.Fatalf("unexpected error: %v", err)
		}
		totalSamples += int64(numSeries)

		// Periodic flush.
		elapsed := ts - startTime
		if elapsed > 0 && elapsed%flushInterval == 0 {
			_, err := db.FlushOlderThan(ts - int64(blockDuration))
			if err != nil {
				t.Fatalf("unexpected error: %v", err)
			}
		}

		// Periodic compaction + retention.
		if elapsed > 0 && elapsed%compactInterval == 0 {
			err := db.RunCompaction()
			if err != nil {
				t.Fatalf("unexpected error: %v", err)
			}
			db.ApplyRetention()
		}

		// Periodic query validation.
		if elapsed > 0 && elapsed%queryInterval == 0 {
			if !validateCanaries(t, db, oracle, ts, retentionMs, clock) {
				atomic.AddInt64(&queryErrors, 1)
			}

			// Check heap allocation.
			var ms runtime.MemStats
			runtime.ReadMemStats(&ms)
			heapMB := ms.HeapAlloc / (1024 * 1024)
			if heapMB > maxHeapMB {
				maxHeapMB = heapMB
			}
		}
	}

	// Final validation.
	t.Logf("Total samples written: %d", totalSamples)
	t.Logf("Peak heap: %d MiB", maxHeapMB)

	// Assert zero query errors.
	if got := atomic.LoadInt64(&queryErrors); got != int64(0) {
		t.Errorf("query errors during soak: got %v, want %v", got, int64(0))
	}

	// Assert bounded memory (should stay well under 1 GiB with 10k series).
	if !(maxHeapMB < uint64(1024)) {
		t.Errorf("heap should stay under 1 GiB: got %v, want < %v", maxHeapMB, uint64(1024))
	}

	// Assert bounded disk: count block directories.
	entries, err := os.ReadDir(dir)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	blockCount := 0
	for _, e := range entries {
		if e.IsDir() && e.Name() != "wal" {
			if _, err := os.Stat(filepath.Join(dir, e.Name(), "meta.json")); err == nil {
				blockCount++
			}
		}
	}
	t.Logf("Block count at end: %d", blockCount)
	// With 24h retention and 2h blocks, expect roughly 12 raw + some compacted.
	// Should be well under 50.
	if !(blockCount < 50) {
		t.Errorf("block count should be bounded by retention: got %v, want < %v", blockCount, 50)
	}

	// Final canary validation.
	finalNow := now.Load()
	validateCanaries(t, db, oracle, finalNow, retentionMs, clock)

	// Live query during compaction: start query, compact, finish query.
	q, err := db.Querier(math.MinInt64, math.MaxInt64)
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	ss := q.Select(labels.MustNewMatcher(labels.MatchRegexp, "__name__", "metric_0.*"))
	// Trigger compaction while query is open.
	db.RunCompaction()
	// Drain the query — should not error.
	for ss.Next() {
		it := ss.At().Iterator()
		for it.Next() {
		}
		if err := it.Err(); err != nil {
			t.Errorf("unexpected error: %v", err)
		}
	}
	if err := ss.Err(); err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if err := q.Close(); err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
}

// validateCanaries queries each canary series and compares with the oracle.
// Returns true if all validations pass. Every assertion runs for every canary.
func validateCanaries(t *testing.T, db *DB, o *oracle, now int64, retentionMs int64, clock func() int64) bool {
	t.Helper()
	pass := true

	// Query window: last 1h or from retention cutoff, whichever is later.
	maxt := now
	mint := now - 3600*1000
	retentionCutoff := clock() - retentionMs
	if mint < retentionCutoff {
		mint = retentionCutoff
	}

	for ref, ls := range o.labels {
		// Build matchers from labels.
		var matchers []*labels.Matcher
		for _, l := range ls {
			matchers = append(matchers, labels.MustNewMatcher(labels.MatchEqual, l.Name, l.Value))
		}

		// Query.
		q, err := db.Querier(mint, maxt)
		if err != nil {
			t.Fatalf("querier for canary ref %d: unexpected error: %v", ref, err)
		}

		ss := q.Select(matchers...)
		var gotSamples []sample
		var iterErr error
		var ssErr error
		for ss.Next() {
			it := ss.At().Iterator()
			for it.Next() {
				st, sv := it.At()
				gotSamples = append(gotSamples, sample{st, sv})
			}
			iterErr = it.Err()
		}
		ssErr = ss.Err()
		q.Close()

		// All assertions run unconditionally for every canary.
		if iterErr != nil {
			t.Errorf("canary ref %d: iterator error: %v", ref, iterErr)
			pass = false
		}
		if ssErr != nil {
			t.Errorf("canary ref %d: series set error: %v", ref, ssErr)
			pass = false
		}

		// Compare with oracle.
		wantByRef := o.query(mint, maxt, matchers...)
		var wantSamples []sample
		if s, ok := wantByRef[ref]; ok {
			wantSamples = s
		}

		if len(wantSamples) != len(gotSamples) {
			t.Errorf("canary ref %d sample count (mint=%d maxt=%d): got %v, want %v", ref, mint, maxt, len(gotSamples), len(wantSamples))
			pass = false
		}
	}
	return pass
}