Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 7 additions & 15 deletions internal/engine/duck.go
Original file line number Diff line number Diff line change
Expand Up @@ -997,10 +997,6 @@ func runMaintenanceSchedule(ctx context.Context, every time.Duration, run func()

func (d *Duck) runParquetCompactionCycle(ctx context.Context) (int, error) {
started := time.Now()
if err := d.repository.RecoverParquet(ctx, d); err != nil {
metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, metrics.TelemetryError, time.Since(started).Seconds())
return 0, err
}
compacted, err := d.repository.CompactParquetPass(ctx, d, parquetMaintenanceBatchLimit, parquetCompactionCycleBudget)
// Retired inputs wait only for readers that captured them, which query
// deadlines bound, so each cycle deletes what earlier cycles retired.
Expand All @@ -1027,25 +1023,21 @@ func (d *Duck) runParquetCompactionCycle(ctx context.Context) (int, error) {

func (d *Duck) runRepositoryMaintenance(ctx context.Context) error {
start := time.Now()
recoveryErr := d.repository.RecoverParquet(ctx, d)
compactStart := time.Now()
var cleanupErr, parquetErr, compactErr error
compacted := 0
if recoveryErr == nil {
cleanupErr = d.repository.CleanupParquet()
if d.cfg.RetentionDays > 0 {
_, parquetErr = d.repository.PruneParquetPass(ctx, d, time.Now().Add(-time.Duration(d.cfg.RetentionDays)*24*time.Hour).UnixNano(), parquetMaintenanceBatchLimit, parquetMaintenancePhaseBudget)
}
compacted, compactErr = d.repository.CompactParquetPass(ctx, d, parquetMaintenanceBatchLimit, parquetMaintenancePhaseBudget)
var parquetErr error
cleanupErr := d.repository.CleanupParquet()
if d.cfg.RetentionDays > 0 {
_, parquetErr = d.repository.PruneParquetPass(ctx, d, time.Now().Add(-time.Duration(d.cfg.RetentionDays)*24*time.Hour).UnixNano(), parquetMaintenanceBatchLimit, parquetMaintenancePhaseBudget)
}
compacted, compactErr := d.repository.CompactParquetPass(ctx, d, parquetMaintenanceBatchLimit, parquetMaintenancePhaseBudget)
compactResult := metrics.TelemetryNoop
if recoveryErr != nil || compactErr != nil {
if compactErr != nil {
compactResult = metrics.TelemetryError
} else if compacted > 0 {
compactResult = metrics.TelemetrySuccess
}
metrics.RecordTelemetryOperation(metrics.TelemetryCompaction, compactResult, time.Since(compactStart).Seconds())
pruneErr := errors.Join(recoveryErr, cleanupErr, parquetErr, compactErr)
pruneErr := errors.Join(cleanupErr, parquetErr, compactErr)
// Retention DELETEs on the rollup-cache tables are maintenance work: they
// reclaim storage, so a failure here still degrades maintenance readiness.
// They hold both write gates and the background slot, as the checkpoint
Expand Down
2 changes: 1 addition & 1 deletion internal/engine/duck_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ func TestMaintenanceRetriesRetiredDirectoryCleanup(t *testing.T) {
t.Fatal(err)
}
defer repository.Close()
retired := filepath.Join(repository.Parquet.BatchesDir(), "stale.retired-old")
retired := filepath.Join(repository.Parquet.BatchesDir(), "stale.retired")
if err := os.Mkdir(retired, 0o755); err != nil {
t.Fatal(err)
}
Expand Down
13 changes: 6 additions & 7 deletions internal/telemetry/format.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,17 @@ func validateBatchFormats(dir string) error {
}
for _, entry := range entries {
name := entry.Name()
// Quarantine is outside the authoritative set. Retired inputs may still
// be needed for compaction recovery and must be checked before cleanup.
// Quarantine is outside the authoritative set. A retired directory is
// retention's interrupted delete and keeps its format until cleanup
// reclaims it, so it is checked before cleanup runs.
active := strings.HasSuffix(name, BatchSuffix)
retired := strings.HasSuffix(name, ".retired") || strings.Contains(name, ".retired-")
if !entry.IsDir() || name == SchemaBatch || (!active && (!retired || strings.Contains(name, ".quarantined-"))) {
retired := strings.HasSuffix(name, ".retired")
if !entry.IsDir() || name == SchemaBatch || (!active && !retired) {
continue
}
if _, err := readBatchMetadata(filepath.Join(dir, name, "metadata.json")); err != nil {
// A retired directory without metadata was partly removed by
// retention or compaction cleanup and has no format left to
// preserve. Cleanup reclaims it, or compaction recovery reports the
// missing input through its runbook.
// retention and has no format left to preserve; cleanup reclaims it.
if retired && errors.Is(err, os.ErrNotExist) {
continue
}
Expand Down
4 changes: 2 additions & 2 deletions internal/telemetry/metric_format_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ func TestMetricPhysicalRowKeepsFidelityColumns(t *testing.T) {
}

func TestFormatThreeIsRefusedBeforeCleanup(t *testing.T) {
for _, suffix := range []string{BatchSuffix, ".retired", ".retired-replacement"} {
for _, suffix := range []string{BatchSuffix, ".retired"} {
t.Run(suffix, func(t *testing.T) {
root := t.TempDir()
store, err := OpenParquetStore(root)
Expand Down Expand Up @@ -161,7 +161,7 @@ func TestFormatFiveRequiresMetricPhysicalSchema(t *testing.T) {
}

func TestPartiallyRemovedRetiredBatchDoesNotStopOpen(t *testing.T) {
for _, suffix := range []string{".retired", ".retired-replacement"} {
for _, suffix := range []string{".retired"} {
for _, missing := range []string{"spans.parquet", "logs.parquet", "metrics.parquet", "trace.fidx", "metadata.json"} {
t.Run(suffix+"/"+missing, func(t *testing.T) {
root := t.TempDir()
Expand Down
88 changes: 0 additions & 88 deletions internal/telemetry/parquet.go
Original file line number Diff line number Diff line change
Expand Up @@ -432,94 +432,6 @@ func (p *ParquetStore) lockCommitPublish(ctx context.Context) error {

func (p *ParquetStore) unlockPublish() { p.publishGate <- struct{}{} }

// RestoreRetiredInputs rolls back a compaction whose durable output vanished.
// Readers see the inputs only once the catalog installs them.
func (p *ParquetStore) RestoreRetiredInputs(inputs []string, replacementID string, publish func(func(context.Context) error) error) error {
if err := ValidateBatchID(replacementID); err != nil {
return err
}
for _, id := range inputs {
if err := ValidateBatchID(id); err != nil {
return err
}
}
type restoredInput struct {
id string
active string
retired string
batch *storedBatch
move bool
}
prepared := make([]restoredInput, 0, len(inputs))
for _, id := range inputs {
active := p.BatchPath(id)
path := active
move := false
if _, err := os.Stat(active); errors.Is(err, os.ErrNotExist) {
path = filepath.Join(p.batchesDir, id+".retired-"+replacementID)
move = true
} else if err != nil {
return err
}
batch, err := loadStoredBatch(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("input %s is missing", id)
}
return fmt.Errorf("load input %s: %w", id, err)
}
batch.dir = active
if batch.metadata.Spans > 0 {
batch.traces.path = filepath.Join(active, "trace.fidx")
}
prepared = append(prepared, restoredInput{id: id, active: active, retired: path, batch: batch, move: move})
}
return publish(func(ctx context.Context) error {
if err := p.lockPublish(ctx); err != nil {
return err
}
defer p.unlockPublish()
installActive := func() {
p.mu.Lock()
defer p.mu.Unlock()
for _, input := range prepared {
if _, err := os.Stat(input.active); err == nil {
// The replacement never reached the namespace, so its
// replaces list no longer covers the restored input.
if lister, ok := p.replacedBy[input.id]; ok && lister == replacementID {
delete(p.replacedBy, input.id)
}
p.setBatchLocked(input.id, input.batch)
} else {
p.setBatchLocked(input.id, nil)
}
}
}

moved := make([]restoredInput, 0, len(prepared))
for _, input := range prepared {
if !input.move {
continue
}
if err := os.Rename(input.retired, input.active); err != nil {
var rollbackErr error
for i := len(moved) - 1; i >= 0; i-- {
rollbackErr = errors.Join(rollbackErr, os.Rename(moved[i].active, moved[i].retired))
}
installActive()
return errors.Join(err, rollbackErr, syncDirectory(p.batchesDir))
}
moved = append(moved, input)
}
var syncErr error
if len(moved) > 0 {
syncErr = syncDirectory(p.batchesDir)
}
installActive()
return syncErr
})
}

// TraceTotals describes the whole trace, not the page a limit admitted.
//
// These come free: readIndexedTrace already decodes every row of the trace and
Expand Down
Loading
Loading