obs(api): add expiration index size and evictor sweep duration metrics

Closes #3605.

Two new histograms:

  api.redis_storage.expiration_index.size ({entry})
    ZCARD of sandbox:storage:global:expiration sampled once per heal
    pass (every 5 min). Use max/p99 to detect unbounded growth caused
    by stale ZSET entries that are not being swept.

  api.redis_storage.expiration_index.sweep_duration (ms)
    Wall-clock duration of each ExpiredItems call (ZRANGEBYSCORE +
    MGET pipeline). Rising p99 alongside rising index size confirms
    the O(log N + K) cost growth described in #3605.

Collection points:
- heal.go: ZCARD after each healExpirationIndex pass — one extra
  Redis round-trip per 5 min, negligible next to forEachSandboxBatch.
- items.go: defer-based timer wrapping the full ExpiredItems body,
  recorded on every evictor tick.

Author: adababys

packages/api/internal/sandbox/storage/redis/heal.go

@@ -83,6 +83,15 @@ func (s *Storage) healExpirationIndex(ctx context.Context) (int, error) {
 		return healed, err
 	}
 
+	// Sample the ZSET cardinality once per heal pass. A single ZCARD piggybacked
+	// here costs one Redis round-trip every 5 min — negligible next to the
+	// forEachSandboxBatch scan that precedes it.
+	if n, zErr := s.redisClient.ZCard(ctx, globalExpirationSet).Result(); zErr == nil {
+		s.metrics.indexSize.Record(ctx, n)
+	} else {
+		logger.L().Warn(ctx, "Failed to sample expiration index size", zap.Error(zErr))
+	}
+
 	return healed, nil
 }
 

packages/api/internal/sandbox/storage/redis/items.go

@@ -18,7 +18,12 @@ const expiredItemsBatchSize = 256
 // ExpiredItems returns running sandboxes whose EndTime has passed.
 // It bounds per-cycle work via LIMIT and cleans up orphaned ZSET entries.
 func (s *Storage) ExpiredItems(ctx context.Context) ([]sandboxtypes.Sandbox, error) {
-	now := time.Now()
+	sweepStart := time.Now()
+	defer func() {
+		s.metrics.sweepDuration.Record(ctx, time.Since(sweepStart).Milliseconds())
+	}()
+
+	now := sweepStart
 	nowMs := float64(now.UnixMilli())
 
 	// Fetch members whose score (EndTime in ms) is <= now, bounded to 256 per cycle.

packages/api/internal/sandbox/storage/redis/main.go

@@ -53,6 +53,11 @@ type expirationIndexMetrics struct {
 	sweptOrphan        metric.MeasurementOption
 	sweptDeadExecution metric.MeasurementOption
 	sweptInvalid       metric.MeasurementOption
+
+	// indexSize records the ZSET cardinality once per heal pass (every 5 min).
+	indexSize metric.Int64Histogram
+	// sweepDuration records wall-clock time of each ExpiredItems call.
+	sweepDuration metric.Int64Histogram
 }
 
 const sweptReasonAttr = "reason"
@@ -73,13 +78,25 @@ func newExpirationIndexMetrics(meter metric.Meter) (expirationIndexMetrics, erro
 		return expirationIndexMetrics{}, fmt.Errorf("expiration index swept counter: %w", err)
 	}
 
+	indexSize, err := telemetry.GetHistogram(meter, telemetry.ApiRedisStorageExpirationIndexSize)
+	if err != nil {
+		return expirationIndexMetrics{}, fmt.Errorf("expiration index size histogram: %w", err)
+	}
+
+	sweepDuration, err := telemetry.GetHistogram(meter, telemetry.ApiRedisStorageExpirationIndexSweepDuration)
+	if err != nil {
+		return expirationIndexMetrics{}, fmt.Errorf("expiration index sweep duration histogram: %w", err)
+	}
+
 	return expirationIndexMetrics{
 		indexHealed:        healed,
 		indexRescored:      rescored,
 		indexSwept:         swept,
 		sweptOrphan:        metric.WithAttributeSet(attribute.NewSet(attribute.String(sweptReasonAttr, "orphan"))),
 		sweptDeadExecution: metric.WithAttributeSet(attribute.NewSet(attribute.String(sweptReasonAttr, "dead_execution"))),
 		sweptInvalid:       metric.WithAttributeSet(attribute.NewSet(attribute.String(sweptReasonAttr, "invalid"))),
+		indexSize:          indexSize,
+		sweepDuration:      sweepDuration,
 	}, nil
 }
 

packages/shared/pkg/telemetry/meters.go

@@ -279,6 +279,18 @@ const (
 )
 
 const (
+	// ApiRedisStorageExpirationIndexSize records the cardinality of the global
+	// expiration ZSET (sandbox:storage:global:expiration) once per heal pass.
+	// Use the max/p99 to detect unbounded growth; a rising trend without a
+	// corresponding rise in live sandbox count indicates stale entries are
+	// accumulating faster than they are swept.
+	ApiRedisStorageExpirationIndexSize HistogramType = "api.redis_storage.expiration_index.size"
+
+	// ApiRedisStorageExpirationIndexSweepDuration records the wall-clock time
+	// of each ExpiredItems call (ZRANGEBYSCORE + MGET pipeline). Rising p99
+	// alongside a rising expiration_index.size confirms O(log N + K) growth.
+	ApiRedisStorageExpirationIndexSweepDuration HistogramType = "api.redis_storage.expiration_index.sweep_duration"
+
 	ApiRedisStoragePublisherPublishDuration HistogramType = "api.redis_storage.publisher.publish.duration"
 
 	// Firecracker net histograms — per-sandbox distribution per metrics flush, no sandbox_id.
@@ -560,7 +572,9 @@ func GetGaugeInt(meter metric.Meter, name GaugeIntType) (metric.Int64ObservableG
 }
 
 var histogramDesc = map[HistogramType]string{
-	ApiRedisStoragePublisherPublishDuration: "Duration of a single Redis PUBLISH round-trip from the storage publisher",
+	ApiRedisStorageExpirationIndexSize:          "Cardinality of sandbox:storage:global:expiration ZSET at the time of each heal pass; use max/p99 to detect unbounded growth",
+	ApiRedisStorageExpirationIndexSweepDuration: "Wall-clock duration of one ExpiredItems call (ZRANGEBYSCORE + MGET pipeline); rising p99 alongside rising index size confirms O(log N) growth",
+	ApiRedisStoragePublisherPublishDuration:     "Duration of a single Redis PUBLISH round-trip from the storage publisher",
 
 	BuildDurationHistogramName:                 "Time taken to build a template",
 	BuildPhaseDurationHistogramName:            "Time taken to build each phase of a template",
@@ -629,7 +643,9 @@ var histogramDesc = map[HistogramType]string{
 }
 
 var histogramUnits = map[HistogramType]string{
-	ApiRedisStoragePublisherPublishDuration: "ms",
+	ApiRedisStorageExpirationIndexSize:          "{entry}",
+	ApiRedisStorageExpirationIndexSweepDuration: "ms",
+	ApiRedisStoragePublisherPublishDuration:     "ms",
 
 	BuildDurationHistogramName:                    "ms",
 	BuildPhaseDurationHistogramName:               "ms",