From 3f97e1bcdcae78e5fa5fe80336e2244def8eabf1 Mon Sep 17 00:00:00 2001 From: 98001yash Date: Tue, 13 Oct 2026 20:40:36 +0530 Subject: [PATCH] Add Redis readiness and startup resilience --- .../config/GlobalCapacityBootstrap.java | 68 ++++++++++++++- src/main/resources/application.properties | 10 +++ .../config/GlobalCapacityBootstrapTest.java | 85 +++++++++++++++++++ 3 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 src/test/java/com/yashdotdev/distributed_traffic_control/config/GlobalCapacityBootstrapTest.java diff --git a/src/main/java/com/yashdotdev/distributed_traffic_control/config/GlobalCapacityBootstrap.java b/src/main/java/com/yashdotdev/distributed_traffic_control/config/GlobalCapacityBootstrap.java index 0712a5d..419f699 100644 --- a/src/main/java/com/yashdotdev/distributed_traffic_control/config/GlobalCapacityBootstrap.java +++ b/src/main/java/com/yashdotdev/distributed_traffic_control/config/GlobalCapacityBootstrap.java @@ -1,8 +1,12 @@ + package com.yashdotdev.distributed_traffic_control.config; import com.yashdotdev.distributed_traffic_control.lease.GlobalCapacityKey; import com.yashdotdev.distributed_traffic_control.lease.LeaseCoordinator; import com.yashdotdev.distributed_traffic_control.policy.TrafficPolicy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; import org.springframework.stereotype.Component; @@ -10,19 +14,75 @@ @Component public class GlobalCapacityBootstrap implements ApplicationRunner { + private static final Logger logger = + LoggerFactory.getLogger(GlobalCapacityBootstrap.class); + private static final String DEFAULT_RESOURCE = "/api/orders"; private final LeaseCoordinator leaseCoordinator; private final TrafficPolicy defaultTrafficPolicy; + private final int maxAttempts; + private final long retryDelayMs; + + public GlobalCapacityBootstrap( + LeaseCoordinator leaseCoordinator, + TrafficPolicy defaultTrafficPolicy, + @Value("${allocation.bootstrap.max-attempts:5}") int maxAttempts, + @Value("${allocation.bootstrap.retry-delay-ms:1000}") long retryDelayMs) { + + if (maxAttempts < 1) { + throw new IllegalArgumentException( + "allocation.bootstrap.max-attempts must be at least 1"); + } + if (retryDelayMs < 0) { + throw new IllegalArgumentException( + "allocation.bootstrap.retry-delay-ms must not be negative"); + } - public GlobalCapacityBootstrap(LeaseCoordinator leaseCoordinator, TrafficPolicy defaultTrafficPolicy) { this.leaseCoordinator = leaseCoordinator; this.defaultTrafficPolicy = defaultTrafficPolicy; + this.maxAttempts = maxAttempts; + this.retryDelayMs = retryDelayMs; } @Override public void run(ApplicationArguments args) { - GlobalCapacityKey capacityKey = new GlobalCapacityKey(defaultTrafficPolicy.getPolicyId(), DEFAULT_RESOURCE); - leaseCoordinator.registerCapacity(capacityKey, defaultTrafficPolicy.getCapacity()); + GlobalCapacityKey capacityKey = new GlobalCapacityKey( + defaultTrafficPolicy.getPolicyId(), DEFAULT_RESOURCE); + + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try { + leaseCoordinator.registerCapacity( + capacityKey, defaultTrafficPolicy.getCapacity()); + + logger.info( + "Registered global capacity for {} on attempt {}", + DEFAULT_RESOURCE, attempt); + return; + } + catch (RuntimeException exception) { + if (attempt == maxAttempts) { + throw new IllegalStateException( + "Failed to register global capacity after " + + maxAttempts + " attempts", + exception); + } + + logger.warn( + "Failed to register global capacity (attempt {}/{}). " + + "Retrying in {} ms", + attempt, maxAttempts, retryDelayMs, exception); + + try { + Thread.sleep(retryDelayMs); + } + catch (InterruptedException interruptedException) { + Thread.currentThread().interrupt(); + throw new IllegalStateException( + "Interrupted while retrying global capacity registration", + interruptedException); + } + } + } } -} \ No newline at end of file +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 93748b5..48aa4a9 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -19,3 +19,13 @@ management.tracing.sampling.probability=${TRACING_SAMPLING_PROBABILITY:1.0} # Do not export application metrics to Jaeger management.otlp.metrics.export.enabled=false + +# Redis-aware readiness +management.endpoint.health.probes.enabled=true +management.endpoint.health.group.readiness.include=readinessState,redis +management.endpoint.health.show-details=never + + +# Global capacity bootstrap resilience +allocation.bootstrap.max-attempts=${ALLOCATION_BOOTSTRAP_MAX_ATTEMPTS:5} +allocation.bootstrap.retry-delay-ms=${ALLOCATION_BOOTSTRAP_RETRY_DELAY_MS:1000} \ No newline at end of file diff --git a/src/test/java/com/yashdotdev/distributed_traffic_control/config/GlobalCapacityBootstrapTest.java b/src/test/java/com/yashdotdev/distributed_traffic_control/config/GlobalCapacityBootstrapTest.java new file mode 100644 index 0000000..4157fac --- /dev/null +++ b/src/test/java/com/yashdotdev/distributed_traffic_control/config/GlobalCapacityBootstrapTest.java @@ -0,0 +1,85 @@ +package com.yashdotdev.distributed_traffic_control.config; + +import com.yashdotdev.distributed_traffic_control.lease.LeaseCoordinator; +import com.yashdotdev.distributed_traffic_control.policy.TrafficPolicy; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.*; + +class GlobalCapacityBootstrapTest { + + @Test + void shouldRegisterCapacityOnFirstAttempt() { + LeaseCoordinator coordinator = mock(LeaseCoordinator.class); + TrafficPolicy policy = mock(TrafficPolicy.class); + + when(policy.getPolicyId()).thenReturn("default-policy"); + when(policy.getCapacity()).thenReturn(100L); + + GlobalCapacityBootstrap bootstrap = + new GlobalCapacityBootstrap(coordinator, policy, 3, 0); + + assertDoesNotThrow(() -> bootstrap.run(null)); + + verify(coordinator, times(1)).registerCapacity(any(), eq(100L)); + } + + @Test + void shouldRetryWhenRegistrationInitiallyFails() { + LeaseCoordinator coordinator = mock(LeaseCoordinator.class); + TrafficPolicy policy = mock(TrafficPolicy.class); + + when(policy.getPolicyId()).thenReturn("default-policy"); + when(policy.getCapacity()).thenReturn(100L); + + doThrow(new IllegalStateException("Redis unavailable")) + .doNothing() + .when(coordinator) + .registerCapacity(any(), eq(100L)); + + GlobalCapacityBootstrap bootstrap = + new GlobalCapacityBootstrap(coordinator, policy, 3, 0); + + assertDoesNotThrow(() -> bootstrap.run(null)); + + verify(coordinator, times(2)).registerCapacity(any(), eq(100L)); + } + + @Test + void shouldFailAfterExhaustingRetries() { + LeaseCoordinator coordinator = mock(LeaseCoordinator.class); + TrafficPolicy policy = mock(TrafficPolicy.class); + + when(policy.getPolicyId()).thenReturn("default-policy"); + when(policy.getCapacity()).thenReturn(100L); + + doThrow(new IllegalStateException("Redis unavailable")) + .when(coordinator) + .registerCapacity(any(), eq(100L)); + + GlobalCapacityBootstrap bootstrap = + new GlobalCapacityBootstrap(coordinator, policy, 3, 0); + + assertThrows( + IllegalStateException.class, + () -> bootstrap.run(null)); + + verify(coordinator, times(3)).registerCapacity(any(), eq(100L)); + } + + @Test + void shouldRejectInvalidRetryConfiguration() { + LeaseCoordinator coordinator = mock(LeaseCoordinator.class); + TrafficPolicy policy = mock(TrafficPolicy.class); + + assertThrows( + IllegalArgumentException.class, + () -> new GlobalCapacityBootstrap(coordinator, policy, 0, 0)); + + assertThrows( + IllegalArgumentException.class, + () -> new GlobalCapacityBootstrap(coordinator, policy, 3, -1)); + } +}