From 3286ec2908b8ca7207bfaada9a6f62c9513eee85 Mon Sep 17 00:00:00 2001 From: ulrichschulte Date: Fri, 23 Oct 2026 10:09:51 +0200 Subject: [PATCH 1/2] fix(server): skip endpoint probing when unreachable A transport error on the actuator index query (connection refused, reset, timeout) made ChainingStrategy fall back to probing, firing one OPTIONS request per configured endpoint that could never succeed. The empty result then overwrote previously detected endpoints. Transport errors now propagate instead of yielding empty, and EndpointDetector keeps existing endpoints when detection fails. Closes #5778 --- .../server/services/EndpointDetector.java | 7 ++++- .../endpoints/QueryIndexEndpointStrategy.java | 11 ++++++++ .../server/services/EndpointDetectorTest.java | 23 ++++++++++++++++ .../endpoints/ChainingStrategyTest.java | 18 +++++++++++++ .../QueryIndexEndpointStrategyTest.java | 27 +++++++++++++++++-- 5 files changed, 83 insertions(+), 3 deletions(-) diff --git a/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/EndpointDetector.java b/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/EndpointDetector.java index 5e5870a7650..5716aefae71 100644 --- a/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/EndpointDetector.java +++ b/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/EndpointDetector.java @@ -52,7 +52,12 @@ private Mono doDetectEndpoints(Instance instance) { return Mono.empty(); } log.debug("Detect endpoints for {}", instance); - return strategy.detectEndpoints(instance).map(instance::withEndpoints); + return strategy.detectEndpoints(instance).map(instance::withEndpoints).onErrorResume((e) -> { + log.warn("Endpoint detection for instance {} failed: {}. Keeping previously detected endpoints.", + instance.getId(), e.getMessage()); + log.debug("Endpoint detection for instance {} failed.", instance.getId(), e); + return Mono.empty(); + }); } } diff --git a/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategy.java b/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategy.java index 602c32a02c9..9e37198a295 100644 --- a/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategy.java +++ b/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategy.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.TimeoutException; import java.util.function.Function; import com.fasterxml.jackson.annotation.JsonCreator; @@ -28,6 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.reactive.function.client.ClientResponse; +import org.springframework.web.reactive.function.client.WebClientRequestException; import reactor.core.publisher.Mono; import de.codecentric.boot.admin.server.domain.entities.Instance; @@ -65,6 +67,11 @@ public Mono detectEndpoints(Instance instance) { .uri(managementUrl) .exchangeToMono(this.convert(instance, managementUrl)) .onErrorResume((e) -> { + if (isTransportError(e)) { + // instance unreachable: probing would fail as well, so don't fall + // back + return Mono.error(e); + } log.warn("Querying actuator-index for instance {} on '{}' failed: {}", instance.getId(), managementUrl, e.getMessage()); log.debug("Querying actuator-index for instance {} on '{}' failed.", instance.getId(), managementUrl, @@ -123,6 +130,10 @@ protected Mono convertResponse(Response response) { return endpoints.isEmpty() ? Mono.empty() : Mono.just(Endpoints.of(endpoints)); } + private static boolean isTransportError(Throwable e) { + return e instanceof WebClientRequestException || e instanceof TimeoutException; + } + @Data protected static class Response { diff --git a/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/EndpointDetectorTest.java b/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/EndpointDetectorTest.java index d1fcb8e824b..8145eef37e8 100644 --- a/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/EndpointDetectorTest.java +++ b/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/EndpointDetectorTest.java @@ -104,4 +104,27 @@ void should_update_endpoints() { .verifyComplete(); } + @Test + void should_keep_endpoints_on_error() { + // given + Registration registration = Registration.create("foo", "https://health").managementUrl("https://mgmt").build(); + Endpoints originalEndpoints = Endpoints.single("info", "https://mgmt/info") + .withEndpoint("health", "https://health"); + Instance instance = Instance.create(InstanceId.of("id")) + .register(registration) + .withEndpoints(originalEndpoints) + .withStatusInfo(StatusInfo.ofUp()); + StepVerifier.create(repository.save(instance)).expectNextCount(1).verifyComplete(); + + when(strategy.detectEndpoints(any(Instance.class))).thenReturn(Mono.error(new IllegalStateException("x"))); + + // when + StepVerifier.create(detector.detectEndpoints(instance.getId())).verifyComplete(); + + // then + StepVerifier.create(repository.find(instance.getId())) + .assertNext((app) -> assertThat(app.getEndpoints()).isEqualTo(originalEndpoints)) + .verifyComplete(); + } + } diff --git a/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/endpoints/ChainingStrategyTest.java b/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/endpoints/ChainingStrategyTest.java index 3c3ccb10585..388720fde16 100644 --- a/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/endpoints/ChainingStrategyTest.java +++ b/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/endpoints/ChainingStrategyTest.java @@ -16,6 +16,8 @@ package de.codecentric.boot.admin.server.services.endpoints; +import java.util.concurrent.atomic.AtomicBoolean; + import org.junit.jupiter.api.Test; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; @@ -24,6 +26,7 @@ import de.codecentric.boot.admin.server.domain.values.Endpoints; import de.codecentric.boot.admin.server.domain.values.InstanceId; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; class ChainingStrategyTest { @@ -59,4 +62,19 @@ void should_return_empty_endpoints_when_all_empty() { StepVerifier.create(strategy.detectEndpoints(instance)).expectNext(Endpoints.empty()).verifyComplete(); } + @Test + void should_not_chain_on_error() { + AtomicBoolean subscribed = new AtomicBoolean(false); + Instance instance = Instance.create(InstanceId.of("id")); + ChainingStrategy strategy = new ChainingStrategy((a) -> Mono.error(new IllegalStateException("x")), + (a) -> Mono.fromCallable(() -> { + subscribed.set(true); + return Endpoints.single("id", "path"); + })); + + StepVerifier.create(strategy.detectEndpoints(instance)).verifyError(IllegalStateException.class); + + assertThat(subscribed).isFalse(); + } + } diff --git a/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategyTest.java b/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategyTest.java index b76622a6bde..13f67f49b56 100644 --- a/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategyTest.java +++ b/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategyTest.java @@ -31,6 +31,7 @@ import org.springframework.http.MediaType; import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.WebClientRequestException; import reactor.netty.http.client.HttpClient; import reactor.test.StepVerifier; @@ -177,7 +178,7 @@ void should_return_empty_on_not_found() { } @Test - void should_return_empty_on_error() { + void should_return_error_on_transport_error() { // given Instance instance = Instance.create(InstanceId.of("id")) .register(Registration.create("test", this.wireMock.url("/mgmt/health")) @@ -192,7 +193,7 @@ void should_return_empty_on_error() { // when StepVerifier.create(strategy.detectEndpoints(instance)) // then - .verifyComplete(); + .verifyError(WebClientRequestException.class); } @Test @@ -261,6 +262,28 @@ void should_retry() { .verifyComplete(); } + @Test + void should_return_error_on_connection_refused() { + // given + WireMockServer closedServer = new WireMockServer(wireMockConfig().dynamicPort()); + closedServer.start(); + int port = closedServer.port(); + closedServer.stop(); + + Instance instance = Instance.create(InstanceId.of("id")) + .register(Registration.create("test", "http://localhost:" + port + "/mgmt/health") + .managementUrl("http://localhost:" + port + "/mgmt") + .build()); + + QueryIndexEndpointStrategy strategy = new QueryIndexEndpointStrategy(this.instanceWebClient, + this.apiMediaTypeHandler); + + // when + StepVerifier.create(strategy.detectEndpoints(instance)) + // then + .verifyError(WebClientRequestException.class); + } + private ReactorClientHttpConnector httpConnector() { HttpClient client = HttpClient.create().secure((ssl) -> { try { From e6443a9f08213913f216c31631aee755cd228832 Mon Sep 17 00:00:00 2001 From: ulrichschulte Date: Fri, 23 Oct 2026 10:55:24 +0200 Subject: [PATCH 2/2] fix(server): propagate detection errors to trigger Catching every error in EndpointDetector hid real bugs behind a WARN with no stack trace. The trigger already logs errors with the full stack, and nothing is saved on error, so existing endpoints are kept either way. Treat any IOException in the cause chain as a transport error, so a reset during the body read no longer falls back to probing. Codec errors still fall back because the instance is reachable. --- .../services/EndpointDetectionTrigger.java | 9 ++- .../server/services/EndpointDetector.java | 7 +- .../endpoints/QueryIndexEndpointStrategy.java | 32 ++++++--- .../server/services/EndpointDetectorTest.java | 4 +- .../QueryIndexEndpointStrategyTest.java | 71 +++++++++++++++++-- 5 files changed, 100 insertions(+), 23 deletions(-) diff --git a/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/EndpointDetectionTrigger.java b/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/EndpointDetectionTrigger.java index 53132b71a8d..f7a7b944450 100644 --- a/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/EndpointDetectionTrigger.java +++ b/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/EndpointDetectionTrigger.java @@ -25,6 +25,7 @@ import de.codecentric.boot.admin.server.domain.events.InstanceEvent; import de.codecentric.boot.admin.server.domain.events.InstanceRegistrationUpdatedEvent; import de.codecentric.boot.admin.server.domain.events.InstanceStatusChangedEvent; +import de.codecentric.boot.admin.server.services.endpoints.QueryIndexEndpointStrategy; public class EndpointDetectionTrigger extends AbstractEventHandler { @@ -47,7 +48,13 @@ protected Publisher handle(Flux publisher) { protected Mono detectEndpoints(InstanceEvent event) { return this.endpointDetector.detectEndpoints(event.getInstance()).onErrorResume((e) -> { - log.warn("Unexpected error while detecting endpoints for {}", event.getInstance(), e); + if (QueryIndexEndpointStrategy.isTransportError(e)) { + log.warn("Endpoint detection for {} failed, instance unreachable: {}", event.getInstance(), + e.toString()); + } + else { + log.warn("Unexpected error while detecting endpoints for {}", event.getInstance(), e); + } return Mono.empty(); }); } diff --git a/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/EndpointDetector.java b/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/EndpointDetector.java index 5716aefae71..5e5870a7650 100644 --- a/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/EndpointDetector.java +++ b/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/EndpointDetector.java @@ -52,12 +52,7 @@ private Mono doDetectEndpoints(Instance instance) { return Mono.empty(); } log.debug("Detect endpoints for {}", instance); - return strategy.detectEndpoints(instance).map(instance::withEndpoints).onErrorResume((e) -> { - log.warn("Endpoint detection for instance {} failed: {}. Keeping previously detected endpoints.", - instance.getId(), e.getMessage()); - log.debug("Endpoint detection for instance {} failed.", instance.getId(), e); - return Mono.empty(); - }); + return strategy.detectEndpoints(instance).map(instance::withEndpoints); } } diff --git a/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategy.java b/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategy.java index 9e37198a295..8b684d1c804 100644 --- a/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategy.java +++ b/spring-boot-admin-server/src/main/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategy.java @@ -16,10 +16,14 @@ package de.codecentric.boot.admin.server.services.endpoints; +import java.io.IOException; +import java.util.Collections; import java.util.HashMap; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.TimeoutException; import java.util.function.Function; @@ -28,6 +32,7 @@ import lombok.Data; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.core.codec.CodecException; import org.springframework.web.reactive.function.client.ClientResponse; import org.springframework.web.reactive.function.client.WebClientRequestException; import reactor.core.publisher.Mono; @@ -66,14 +71,12 @@ public Mono detectEndpoints(Instance instance) { .get() .uri(managementUrl) .exchangeToMono(this.convert(instance, managementUrl)) - .onErrorResume((e) -> { - if (isTransportError(e)) { - // instance unreachable: probing would fail as well, so don't fall - // back - return Mono.error(e); - } + // shortcut: failed detection is not retried until the next + // status/registration event, add scheduled re-detection if instances stay + // without endpoints + .onErrorResume((e) -> !isTransportError(e), (e) -> { log.warn("Querying actuator-index for instance {} on '{}' failed: {}", instance.getId(), managementUrl, - e.getMessage()); + e.toString()); log.debug("Querying actuator-index for instance {} on '{}' failed.", instance.getId(), managementUrl, e); return Mono.empty(); @@ -130,8 +133,19 @@ protected Mono convertResponse(Response response) { return endpoints.isEmpty() ? Mono.empty() : Mono.just(Endpoints.of(endpoints)); } - private static boolean isTransportError(Throwable e) { - return e instanceof WebClientRequestException || e instanceof TimeoutException; + // transport errors propagate instead of falling back: instance is unreachable, so + // probing would fail as well + public static boolean isTransportError(Throwable e) { + Set seen = Collections.newSetFromMap(new IdentityHashMap<>()); + for (Throwable t = e; t != null && seen.add(t); t = t.getCause()) { + if (t instanceof CodecException) { + return false; // malformed body, instance reachable -> allow fallback + } + if (t instanceof WebClientRequestException || t instanceof TimeoutException || t instanceof IOException) { + return true; + } + } + return false; } @Data diff --git a/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/EndpointDetectorTest.java b/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/EndpointDetectorTest.java index 8145eef37e8..ed53c0ddf3b 100644 --- a/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/EndpointDetectorTest.java +++ b/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/EndpointDetectorTest.java @@ -105,7 +105,7 @@ void should_update_endpoints() { } @Test - void should_keep_endpoints_on_error() { + void should_propagate_error_and_keep_endpoints() { // given Registration registration = Registration.create("foo", "https://health").managementUrl("https://mgmt").build(); Endpoints originalEndpoints = Endpoints.single("info", "https://mgmt/info") @@ -119,7 +119,7 @@ void should_keep_endpoints_on_error() { when(strategy.detectEndpoints(any(Instance.class))).thenReturn(Mono.error(new IllegalStateException("x"))); // when - StepVerifier.create(detector.detectEndpoints(instance.getId())).verifyComplete(); + StepVerifier.create(detector.detectEndpoints(instance.getId())).verifyError(IllegalStateException.class); // then StepVerifier.create(repository.find(instance.getId())) diff --git a/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategyTest.java b/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategyTest.java index 13f67f49b56..4f13b4f7587 100644 --- a/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategyTest.java +++ b/spring-boot-admin-server/src/test/java/de/codecentric/boot/admin/server/services/endpoints/QueryIndexEndpointStrategyTest.java @@ -16,7 +16,11 @@ package de.codecentric.boot.admin.server.services.endpoints; +import java.io.IOException; +import java.net.ServerSocket; +import java.net.SocketException; import java.time.Duration; +import java.util.concurrent.TimeoutException; import javax.net.ssl.SSLException; @@ -27,7 +31,9 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.springframework.boot.actuate.endpoint.ApiVersion; +import org.springframework.core.codec.DecodingException; import org.springframework.http.MediaType; import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.web.reactive.function.client.WebClient; @@ -57,6 +63,7 @@ import static de.codecentric.boot.admin.server.web.client.InstanceExchangeFilterFunctions.timeout; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; +import static org.assertj.core.api.Assertions.assertThat; class QueryIndexEndpointStrategyTest { @@ -263,12 +270,12 @@ void should_retry() { } @Test - void should_return_error_on_connection_refused() { + void should_return_error_on_connection_refused() throws IOException { // given - WireMockServer closedServer = new WireMockServer(wireMockConfig().dynamicPort()); - closedServer.start(); - int port = closedServer.port(); - closedServer.stop(); + int port; + try (ServerSocket socket = new ServerSocket(0)) { + port = socket.getLocalPort(); + } Instance instance = Instance.create(InstanceId.of("id")) .register(Registration.create("test", "http://localhost:" + port + "/mgmt/health") @@ -284,6 +291,60 @@ void should_return_error_on_connection_refused() { .verifyError(WebClientRequestException.class); } + @Test + void should_return_error_on_timeout() { + // given + Instance instance = Instance.create(InstanceId.of("id")) + .register(Registration.create("test", this.wireMock.url("/mgmt/health")) + .managementUrl(this.wireMock.url("/mgmt")) + .build()); + + this.wireMock.stubFor(get("/mgmt").willReturn(okJson("{}").withFixedDelay(2000))); + + QueryIndexEndpointStrategy strategy = new QueryIndexEndpointStrategy(this.instanceWebClient, + this.apiMediaTypeHandler); + + // when + StepVerifier.create(strategy.detectEndpoints(instance)) + // then + .verifyError(TimeoutException.class); + } + + @Test + void should_return_empty_on_invalid_body() { + // given + Instance instance = Instance.create(InstanceId.of("id")) + .register(Registration.create("test", this.wireMock.url("/mgmt/health")) + .managementUrl(this.wireMock.url("/mgmt")) + .build()); + + this.wireMock.stubFor(get("/mgmt") + .willReturn(ok("{invalid").withHeader("Content-Type", ApiVersion.LATEST.getProducedMimeType().toString()))); + + QueryIndexEndpointStrategy strategy = new QueryIndexEndpointStrategy(this.instanceWebClient, + this.apiMediaTypeHandler); + + // when + StepVerifier.create(strategy.detectEndpoints(instance)) + // then + .verifyComplete(); + } + + @Test + @Timeout(5) + void should_detect_transport_errors() { + assertThat(QueryIndexEndpointStrategy.isTransportError(new TimeoutException())).isTrue(); + assertThat(QueryIndexEndpointStrategy.isTransportError(new RuntimeException(new SocketException("reset")))) + .isTrue(); + assertThat(QueryIndexEndpointStrategy.isTransportError(new DecodingException("bad", new IOException()))) + .isFalse(); + assertThat(QueryIndexEndpointStrategy.isTransportError(new IllegalStateException())).isFalse(); + RuntimeException a = new RuntimeException("a"); + RuntimeException b = new RuntimeException("b", a); + a.initCause(b); + assertThat(QueryIndexEndpointStrategy.isTransportError(a)).isFalse(); + } + private ReactorClientHttpConnector httpConnector() { HttpClient client = HttpClient.create().secure((ssl) -> { try {