Compaction, retention, and soak testing
Block refcounting (atomic refs + condemned flag) lets live queries survive concurrent compaction and retention. Levelled compactor merges 2h->8h->32h blocks via raw chunk passthrough. Retention drops blocks older than the configured window. Background goroutine drives flush/compact/retain cycles; exported RunCompaction/ApplyRetention allow deterministic test control via injectable clock. Soak test: 10k series x 48h simulated at 15s intervals (115M samples). Validates flat memory (158 MiB peak), bounded disk (13 blocks), and zero query errors during compaction. All existing tests refactored to table-driven with uniform assertions.
This commit is contained in:
@@ -0,0 +1,206 @@
|
||||
// Package compact implements levelled compaction and retention for ingot blocks.
|
||||
package compact
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.dvdt.dev/david/ingot/internal/block"
|
||||
"git.dvdt.dev/david/ingot/labels"
|
||||
)
|
||||
|
||||
// Clock returns the current time in milliseconds since epoch.
|
||||
type Clock func() int64
|
||||
|
||||
// Compactor manages levelled compaction and retention.
|
||||
type Compactor struct {
|
||||
dataDir string
|
||||
levels []int64 // level durations in ms, e.g. [2h, 8h, 32h]
|
||||
retention int64 // retention window in ms (0 = disabled)
|
||||
clock Clock
|
||||
}
|
||||
|
||||
// New creates a Compactor. levels are the compaction level durations in
|
||||
// ascending order (e.g. 2h, 8h, 32h in milliseconds). retention is the
|
||||
// maximum age of data in milliseconds (0 disables retention).
|
||||
func New(dataDir string, levels []int64, retention int64, clock Clock) *Compactor {
|
||||
return &Compactor{
|
||||
dataDir: dataDir,
|
||||
levels: levels,
|
||||
retention: retention,
|
||||
clock: clock,
|
||||
}
|
||||
}
|
||||
|
||||
// CompactionGroup describes a set of source blocks to compact.
|
||||
type CompactionGroup struct {
|
||||
Sources []*block.Reader
|
||||
Level int // resulting compaction level
|
||||
}
|
||||
|
||||
// Plan returns the first eligible compaction group, or nil if no compaction
|
||||
// is needed. Lower levels are prioritized. A group requires at least 2
|
||||
// blocks at the same compaction level whose combined time span fits within
|
||||
// the next level's duration.
|
||||
func (c *Compactor) Plan(blocks []*block.Reader) *CompactionGroup {
|
||||
if len(blocks) < 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Group blocks by compaction level.
|
||||
byLevel := make(map[int][]*block.Reader)
|
||||
for _, b := range blocks {
|
||||
lvl := b.Meta.Compaction.Level
|
||||
byLevel[lvl] = append(byLevel[lvl], b)
|
||||
}
|
||||
|
||||
// Sort levels ascending.
|
||||
var levels []int
|
||||
for lvl := range byLevel {
|
||||
levels = append(levels, lvl)
|
||||
}
|
||||
sort.Ints(levels)
|
||||
|
||||
for _, lvl := range levels {
|
||||
group := c.planLevel(byLevel[lvl], lvl)
|
||||
if group != nil {
|
||||
return group
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// planLevel finds a compactable group within blocks at the same level.
|
||||
func (c *Compactor) planLevel(blocks []*block.Reader, level int) *CompactionGroup {
|
||||
if len(blocks) < 2 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sort by MinTime.
|
||||
sorted := make([]*block.Reader, len(blocks))
|
||||
copy(sorted, blocks)
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
return sorted[i].Meta.MinTime < sorted[j].Meta.MinTime
|
||||
})
|
||||
|
||||
// Determine the max span for the next level.
|
||||
var maxSpan int64
|
||||
if level-1 < len(c.levels) && level >= 1 {
|
||||
// Find the next level's duration. Level 1 blocks should compact
|
||||
// into level 2 when they span up to levels[1] (8h), etc.
|
||||
if level < len(c.levels) {
|
||||
maxSpan = c.levels[level]
|
||||
}
|
||||
}
|
||||
if maxSpan == 0 {
|
||||
// No higher level defined, or level 0 — use the second level duration.
|
||||
if level < len(c.levels) {
|
||||
maxSpan = c.levels[level]
|
||||
} else {
|
||||
return nil // already at max level
|
||||
}
|
||||
}
|
||||
|
||||
// Find the first group of consecutive blocks whose span fits maxSpan.
|
||||
for i := 0; i < len(sorted)-1; i++ {
|
||||
group := []*block.Reader{sorted[i]}
|
||||
for j := i + 1; j < len(sorted); j++ {
|
||||
span := sorted[j].Meta.MaxTime - sorted[i].Meta.MinTime
|
||||
if span > maxSpan {
|
||||
break
|
||||
}
|
||||
group = append(group, sorted[j])
|
||||
}
|
||||
if len(group) >= 2 {
|
||||
return &CompactionGroup{
|
||||
Sources: group,
|
||||
Level: level + 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Compact merges source blocks into a single new block. Returns the new
|
||||
// block's ULID. The caller is responsible for swapping the block set and
|
||||
// releasing source blocks.
|
||||
func (c *Compactor) Compact(sources []*block.Reader) (string, error) {
|
||||
if len(sources) == 0 {
|
||||
return "", fmt.Errorf("compact: no source blocks")
|
||||
}
|
||||
|
||||
merged := make(map[uint64]*mergedEntry)
|
||||
|
||||
for _, src := range sources {
|
||||
for _, entry := range src.Series() {
|
||||
me, ok := merged[entry.Ref]
|
||||
if !ok {
|
||||
me = &mergedEntry{
|
||||
ref: entry.Ref,
|
||||
labels: entry.Labels,
|
||||
}
|
||||
merged[entry.Ref] = me
|
||||
}
|
||||
for _, cm := range entry.Chunks {
|
||||
raw, err := src.RawChunkData(cm.Ref)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("compact: read chunk ref %v from %s: %w",
|
||||
cm.Ref, src.Meta.ULID, err)
|
||||
}
|
||||
// Copy the raw bytes since the source may be munmapped later.
|
||||
data := make([]byte, len(raw))
|
||||
copy(data, raw)
|
||||
me.chunks = append(me.chunks, block.ChunkData{
|
||||
MinT: cm.MinT,
|
||||
MaxT: cm.MaxT,
|
||||
Data: data,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build flush data sorted by ref for deterministic output.
|
||||
flushData := make([]block.SeriesFlush, 0, len(merged))
|
||||
for _, me := range merged {
|
||||
flushData = append(flushData, block.SeriesFlush{
|
||||
Ref: me.ref,
|
||||
Labels: me.labels,
|
||||
Chunks: me.chunks,
|
||||
})
|
||||
}
|
||||
sort.Slice(flushData, func(i, j int) bool { return flushData[i].Ref < flushData[j].Ref })
|
||||
|
||||
// Determine new compaction level and collect source ULIDs.
|
||||
maxLevel := 0
|
||||
sourceULIDs := make([]string, 0, len(sources))
|
||||
for _, src := range sources {
|
||||
if src.Meta.Compaction.Level > maxLevel {
|
||||
maxLevel = src.Meta.Compaction.Level
|
||||
}
|
||||
sourceULIDs = append(sourceULIDs, src.Meta.ULID)
|
||||
}
|
||||
|
||||
return block.FlushCompacted(c.dataDir, flushData, maxLevel+1, sourceULIDs)
|
||||
}
|
||||
|
||||
// Expired returns blocks whose MaxTime is older than the retention window.
|
||||
// Returns nil if retention is disabled (zero).
|
||||
func (c *Compactor) Expired(blocks []*block.Reader) []*block.Reader {
|
||||
if c.retention == 0 {
|
||||
return nil
|
||||
}
|
||||
cutoff := c.clock() - c.retention
|
||||
var expired []*block.Reader
|
||||
for _, b := range blocks {
|
||||
if b.Meta.MaxTime < cutoff {
|
||||
expired = append(expired, b)
|
||||
}
|
||||
}
|
||||
return expired
|
||||
}
|
||||
|
||||
type mergedEntry struct {
|
||||
ref uint64
|
||||
labels []labels.Label
|
||||
chunks []block.ChunkData
|
||||
}
|
||||
Reference in New Issue
Block a user