diff --git a/ice-rest-catalog/pom.xml b/ice-rest-catalog/pom.xml index 5cbd7be9..b7778882 100644 --- a/ice-rest-catalog/pom.xml +++ b/ice-rest-catalog/pom.xml @@ -598,6 +598,7 @@ **/DockerScenarioBasedIT.java **/DockerLocalFileIOClickHouseIT.java **/DockerLocalFileIOClickHouseAllTypesIT.java + **/DockerSparkClientIdIT.java diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java index 180730ce..42c2d8f9 100644 --- a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/Main.java @@ -38,7 +38,9 @@ import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogMiddlewareCredentials; import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogMiddlewareTableConfig; import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogMiddlewareTableCredentials; +import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogMiddlewareTracing; import com.altinity.ice.rest.catalog.internal.rest.RESTCatalogServlet; +import com.altinity.ice.rest.catalog.internal.rest.RequestTracing; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.net.HostAndPort; import io.prometheus.metrics.instrumentation.jvm.JvmMetrics; @@ -324,6 +326,14 @@ private static Server createBaseServer( } } + var tracingConfig = config.tracing(); + if (tracingConfig.enabledOrDefault() && tracingConfig.advertiseClientIdOrDefault()) { + restCatalogAdapter = + new RESTCatalogMiddlewareTracing( + restCatalogAdapter, tracingConfig.clientIdHeaderOrDefault()); + } + RequestTracing tracing = new RequestTracing(tracingConfig); + logger.info( "Commit retry config: numRetries={} minWaitMs={} maxWaitMs={} totalTimeoutMs={}", config.commitRetry().numRetries(), @@ -336,7 +346,7 @@ private static Server createBaseServer( config.commitLock().leaseTtlSeconds(), config.commitLock().acquireTimeoutMs()); - var h = new ServletHolder(new RESTCatalogServlet(restCatalogAdapter)); + var h = new ServletHolder(new RESTCatalogServlet(restCatalogAdapter, tracing)); mux.addServlet(h, "/*"); if (registerAdminServlet) { diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/Config.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/Config.java index cb98cded..1dafe7ba 100644 --- a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/Config.java +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/Config.java @@ -68,7 +68,10 @@ public record Config( @JsonPropertyDescription( "(experimental) Iceberg properties (see https://iceberg.apache.org/javadoc/1.8.1/" + "; e.g. https://iceberg.apache.org/javadoc/1.8.1/org/apache/iceberg/aws/s3/S3FileIOProperties.html)") - Map icebergProperties) { + Map icebergProperties, + @JsonPropertyDescription( + "OpenTelemetry tracing and request/client correlation config (disabled by default)") + TracingConfig tracing) { private static final String DEFAULT_ADDR = "0.0.0.0:5000"; private static final String DEFAULT_DEBUG_ADDR = "0.0.0.0:5001"; @@ -90,7 +93,8 @@ public Config( CommitRetryConfig commitRetry, CommitLockConfig commitLock, Map loadTableProperties, - @JsonProperty("iceberg") Map icebergProperties) { + @JsonProperty("iceberg") Map icebergProperties, + TracingConfig tracing) { this.addr = Strings.orDefault(addr, DEFAULT_ADDR); this.debugAddr = Strings.orDefault(debugAddr, DEFAULT_DEBUG_ADDR); this.adminAddr = Strings.orDefault(adminAddr, System.getenv("ICE_REST_CATALOG_ADMIN_ADDR")); @@ -110,6 +114,7 @@ public Config( this.commitLock = Objects.requireNonNullElse(commitLock, CommitLockConfig.defaults()); this.loadTableProperties = Objects.requireNonNullElse(loadTableProperties, Map.of()); this.icebergProperties = Objects.requireNonNullElse(icebergProperties, Map.of()); + this.tracing = Objects.requireNonNullElse(tracing, TracingConfig.defaults()); } public record S3( diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/TracingConfig.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/TracingConfig.java new file mode 100644 index 00000000..78ff22b5 --- /dev/null +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/config/TracingConfig.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.rest.catalog.internal.config; + +import com.altinity.ice.internal.strings.Strings; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; + +/** + * Per-request / per-client correlation configuration. + * + *

Adds a stable {@code clientId} (grouping all calls from one Iceberg client) and a per-request + * {@code requestId} to logs (via MDC) and response headers, so a whole client session can be + * grouped from the logs alone. All settings are optional; the feature is on by default and has no + * external dependencies. + */ +public record TracingConfig( + @JsonPropertyDescription( + "Enable request/client correlation ids in logs and response headers (true by default)") + Boolean enabled, + @JsonPropertyDescription( + "Header carrying the per-client id echoed by Iceberg clients (X-Ice-Client-Id)") + String clientIdHeader, + @JsonPropertyDescription("Header carrying the per-request id (X-Request-Id)") + String requestIdHeader, + @JsonPropertyDescription( + "Advertise a generated client id via the /v1/config response so Iceberg Java/PyIceberg clients echo it on every request (true by default)") + Boolean advertiseClientId) { + + public static final String DEFAULT_CLIENT_ID_HEADER = "X-Ice-Client-Id"; + public static final String DEFAULT_REQUEST_ID_HEADER = "X-Request-Id"; + + public static TracingConfig defaults() { + return new TracingConfig(null, null, null, null); + } + + public boolean enabledOrDefault() { + return enabled == null || enabled; + } + + public String clientIdHeaderOrDefault() { + return !Strings.isNullOrEmpty(clientIdHeader) ? clientIdHeader : DEFAULT_CLIENT_ID_HEADER; + } + + public String requestIdHeaderOrDefault() { + return !Strings.isNullOrEmpty(requestIdHeader) ? requestIdHeader : DEFAULT_REQUEST_ID_HEADER; + } + + public boolean advertiseClientIdOrDefault() { + return advertiseClientId == null || advertiseClientId; + } +} diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogMiddlewareTracing.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogMiddlewareTracing.java new file mode 100644 index 00000000..c343d07c --- /dev/null +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogMiddlewareTracing.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.rest.catalog.internal.rest; + +import com.altinity.ice.rest.catalog.internal.auth.Session; +import java.util.Map; +import org.apache.iceberg.rest.RESTResponse; +import org.apache.iceberg.rest.responses.ConfigResponse; + +/** + * Advertises a per-client id to Iceberg clients via the {@code /v1/config} handshake. + * + *

The id is returned as a {@code header.} override. Iceberg Java and PyIceberg + * merge config overrides and re-send matching {@code header.*} properties on every subsequent + * request, so all calls from that client instance share one {@code clientId}. Clients that ignore + * config overrides (e.g. ClickHouse) simply fall back to the server-side fingerprint. + */ +public class RESTCatalogMiddlewareTracing extends RESTCatalogMiddleware { + + private static final String HEADER_PREFIX = "header."; + + private final String clientIdHeader; + + public RESTCatalogMiddlewareTracing(RESTCatalogHandler next, String clientIdHeader) { + super(next); + this.clientIdHeader = clientIdHeader; + } + + @Override + public T handle( + Session session, + Route route, + Map vars, + Object requestBody, + Class responseType) { + T restResponse = this.next.handle(session, route, vars, requestBody, responseType); + if (restResponse instanceof ConfigResponse configResponse) { + configResponse + .overrides() + .putIfAbsent(HEADER_PREFIX + clientIdHeader, RequestTracing.newClientId()); + } + return restResponse; + } +} diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogServlet.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogServlet.java index 65926cca..413b9518 100644 --- a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogServlet.java +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogServlet.java @@ -76,10 +76,12 @@ public class RESTCatalogServlet extends HttpServlet { private final RESTCatalogHandler restCatalogAdapter; private final HttpMetrics httpMetrics; + private final RequestTracing tracing; - public RESTCatalogServlet(RESTCatalogHandler restCatalogAdapter) { + public RESTCatalogServlet(RESTCatalogHandler restCatalogAdapter, RequestTracing tracing) { this.restCatalogAdapter = restCatalogAdapter; this.httpMetrics = HttpMetrics.getInstance(); + this.tracing = tracing; } protected void handle(HttpServletRequest request, HttpServletResponse response) @@ -87,6 +89,24 @@ protected void handle(HttpServletRequest request, HttpServletResponse response) HTTPRequest.HTTPMethod method = HTTPRequest.HTTPMethod.valueOf(request.getMethod()); String path = request.getRequestURI().substring(1); + Session session = Session.from(request); + String clientId = tracing.resolveClientId(request, session); + String requestId = tracing.resolveRequestId(request); + tracing.begin(response, clientId, requestId); + try { + handleTraced(method, path, session, request, response); + } finally { + tracing.end(); + } + } + + private void handleTraced( + HTTPRequest.HTTPMethod method, + String path, + Session session, + HttpServletRequest request, + HttpServletResponse response) + throws IOException { Pair> routeContext = Route.from(method, path); if (routeContext == null) { // Track unknown route requests @@ -110,7 +130,6 @@ protected void handle(HttpServletRequest request, HttpServletResponse response) // Track request with metrics try (var timer = httpMetrics.startRequest(method.name(), route.name())) { - Session session = Session.from(request); String userToLog = ""; if (session != null) { userToLog = "@" + session.uid() + " "; diff --git a/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RequestTracing.java b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RequestTracing.java new file mode 100644 index 00000000..26d6f8eb --- /dev/null +++ b/ice-rest-catalog/src/main/java/com/altinity/ice/rest/catalog/internal/rest/RequestTracing.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.rest.catalog.internal.rest; + +import com.altinity.ice.internal.strings.Strings; +import com.altinity.ice.rest.catalog.internal.auth.Session; +import com.altinity.ice.rest.catalog.internal.config.TracingConfig; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import java.util.zip.CRC32; +import org.slf4j.MDC; + +/** + * Resolves and propagates per-client / per-request correlation identifiers. + * + *

{@code clientId} groups every call from a single Iceberg client instance; {@code requestId} is + * unique per HTTP request. Both are placed in the SLF4J {@link MDC} (rendered by the logback + * pattern) and echoed back as response headers. + * + *

Client id resolution degrades gracefully: + * + *

    + *
  • If the request carries the client-id header (Iceberg Java / PyIceberg echo it once the + * server advertises it via the {@code /v1/config} response), that value is used. + *
  • Otherwise a stable fingerprint derived from the auth token, remote address and User-Agent + * is used. This covers clients (e.g. ClickHouse) that do not echo config headers. + *
+ */ +public final class RequestTracing { + + public static final String MDC_CLIENT_ID = "clientId"; + public static final String MDC_REQUEST_ID = "requestId"; + + private final boolean enabled; + private final String clientIdHeader; + private final String requestIdHeader; + + public RequestTracing(TracingConfig config) { + TracingConfig c = config != null ? config : TracingConfig.defaults(); + this.enabled = c.enabledOrDefault(); + this.clientIdHeader = c.clientIdHeaderOrDefault(); + this.requestIdHeader = c.requestIdHeaderOrDefault(); + } + + public boolean enabled() { + return enabled; + } + + public String clientIdHeader() { + return clientIdHeader; + } + + /** A freshly generated client id, suitable for advertising via the config response. */ + public static String newClientId() { + return UUID.randomUUID().toString(); + } + + /** Client id from the request header, falling back to a stable fingerprint. */ + public String resolveClientId(HttpServletRequest request, Session session) { + String fromHeader = request.getHeader(clientIdHeader); + if (!Strings.isNullOrEmpty(fromHeader)) { + return fromHeader; + } + return fingerprint(request, session); + } + + /** Request id from the request header, falling back to a freshly generated UUID. */ + public String resolveRequestId(HttpServletRequest request) { + String fromHeader = request.getHeader(requestIdHeader); + if (!Strings.isNullOrEmpty(fromHeader)) { + return fromHeader; + } + return UUID.randomUUID().toString(); + } + + public void begin(HttpServletResponse response, String clientId, String requestId) { + MDC.put(MDC_CLIENT_ID, clientId); + MDC.put(MDC_REQUEST_ID, requestId); + if (response != null && !response.isCommitted()) { + response.setHeader(clientIdHeader, clientId); + response.setHeader(requestIdHeader, requestId); + } + } + + public void end() { + MDC.remove(MDC_CLIENT_ID); + MDC.remove(MDC_REQUEST_ID); + } + + private static String fingerprint(HttpServletRequest request, Session session) { + String uid = session != null ? session.uid() : "anonymous"; + String remote = request.getRemoteAddr(); + String userAgent = request.getHeader("User-Agent"); + String raw = + (uid != null ? uid : "") + + "|" + + (remote != null ? remote : "") + + "|" + + (userAgent != null ? userAgent : ""); + CRC32 crc = new CRC32(); + crc.update(raw.getBytes(StandardCharsets.UTF_8)); + return "fp-" + Long.toHexString(crc.getValue()); + } +} diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerSparkClientIdIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerSparkClientIdIT.java new file mode 100644 index 00000000..881a18e4 --- /dev/null +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerSparkClientIdIT.java @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package com.altinity.ice.rest.catalog; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.URI; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.MountableFile; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; + +/** + * Docker-based integration test that verifies request/client-id grouping. + * + *

Runs Apache Spark (spark-iceberg image) as an Iceberg REST client against the ice-rest-catalog + * container and asserts that every catalog call made by a single Spark session shares one + * server-assigned client id (the {@code X-Ice-Client-Id} UUID advertised via {@code /v1/config} and + * echoed back by the Iceberg Java client). Grouping is checked from the catalog container logs, + * where the logback pattern renders {@code %X{clientId}} on each request line. + * + *

Excluded from the default Failsafe run (see {@code ice-rest-catalog/pom.xml}); run explicitly + * with {@code ./mvnw -pl ice-rest-catalog verify -Dit.test=DockerSparkClientIdIT}. Requires Docker + * and a locally-built, tracing-enabled catalog image. This test depends on the request/client-id + * tracing feature, which is not present in published images, so the image must be built from + * current source first via {@code .bin/local-docker-build-ice-rest-catalog} (or {@code docker build + * --build-arg BASE_IMAGE_TAG=debug -t altinity/ice-rest-catalog:debug-with-ice-local -f + * ice-rest-catalog/Dockerfile.debug-with-ice .}). Override the image with {@code + * -Ddocker.image=...}. + */ +public class DockerSparkClientIdIT { + + private static final Logger logger = LoggerFactory.getLogger(DockerSparkClientIdIT.class); + + private static final String SPARK_IMAGE = "tabulario/spark-iceberg:3.5.5_1.8.1"; + + // Strips ANSI escape codes (present only if the catalog runs with a color-capable terminal). + private static final Pattern ANSI = Pattern.compile("\u001B\\[[;\\d]*m"); + + // Matches request log lines emitted by RESTCatalogServlet: + // "... RESTCatalogServlet > @token:anonymous v1/". + // Anchoring on the logger name avoids false matches when MDC ids are absent. + private static final Pattern REQUEST_LINE = + Pattern.compile( + "RESTCatalogServlet > (\\S+) (\\S+) @token:anonymous (GET|POST|HEAD|DELETE) (v1/\\S+)"); + + private static final Pattern UUID_PATTERN = + Pattern.compile( + "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}"); + + private Network network; + private GenericContainer minio; + private GenericContainer catalog; + private GenericContainer spark; + + @BeforeClass + @SuppressWarnings("resource") + public void setUp() throws Exception { + String dockerImage = + System.getProperty("docker.image", "altinity/ice-rest-catalog:debug-with-ice-local"); + logger.info("Using catalog Docker image: {}", dockerImage); + + network = Network.newNetwork(); + + // Start MinIO + minio = + new GenericContainer<>("minio/minio:latest") + .withNetwork(network) + .withNetworkAliases("minio") + .withExposedPorts(9000) + .withEnv("MINIO_ACCESS_KEY", "minioadmin") + .withEnv("MINIO_SECRET_KEY", "minioadmin") + .withCommand("server", "/data") + .waitingFor(Wait.forHttp("/minio/health/live").forPort(9000)); + minio.start(); + + // Create test bucket via MinIO's host-mapped port + String minioHostEndpoint = "http://" + minio.getHost() + ":" + minio.getMappedPort(9000); + try (var s3Client = + S3Client.builder() + .endpointOverride(URI.create(minioHostEndpoint)) + .region(Region.US_EAST_1) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create("minioadmin", "minioadmin"))) + .forcePathStyle(true) + .build()) { + s3Client.createBucket(CreateBucketRequest.builder().bucket("test-bucket").build()); + logger.info("Created test-bucket in MinIO"); + } + + // Catalog config (SQLite metastore, MinIO warehouse, anonymous read-write, tracing defaults on) + URL configResource = getClass().getClassLoader().getResource("docker-catalog-config.yaml"); + if (configResource == null) { + throw new IllegalStateException("docker-catalog-config.yaml not found on classpath"); + } + String catalogConfig = Files.readString(Paths.get(configResource.toURI())); + + // Start the ice-rest-catalog container + catalog = + new GenericContainer<>(dockerImage) + .withNetwork(network) + .withNetworkAliases("catalog") + .withExposedPorts(5000) + .withEnv("ICE_REST_CATALOG_CONFIG", "") + .withEnv("ICE_REST_CATALOG_CONFIG_YAML", catalogConfig) + .waitingFor(Wait.forHttp("/v1/config").forPort(5000).forStatusCode(200)); + + try { + catalog.start(); + } catch (Exception e) { + if (catalog != null) { + logger.error("Catalog container logs: {}", catalog.getLogs()); + } + throw e; + } + + // Spark client, configured to talk to the co-located catalog + MinIO over the Docker network. + Path sparkDefaults = Files.createTempFile("spark-defaults-", ".conf"); + Files.writeString(sparkDefaults, sparkDefaultsConf(), StandardCharsets.UTF_8); + + spark = + new GenericContainer<>(SPARK_IMAGE) + .withNetwork(network) + .withNetworkAliases("spark") + .withExposedPorts(8888) + .withCopyFileToContainer( + MountableFile.forHostPath(sparkDefaults), "/opt/spark/conf/spark-defaults.conf") + .waitingFor(Wait.forListeningPort().withStartupTimeout(Duration.ofMinutes(5))); + + try { + spark.start(); + } catch (Exception e) { + if (spark != null) { + logger.error("Spark container logs: {}", spark.getLogs()); + } + throw e; + } + + logger.info( + "Catalog started at {}:{}, Spark started ({})", + catalog.getHost(), + catalog.getMappedPort(5000), + spark.getContainerId()); + } + + @AfterClass + public void tearDown() { + if (spark != null) { + spark.close(); + } + if (catalog != null) { + catalog.close(); + } + if (minio != null) { + minio.close(); + } + if (network != null) { + network.close(); + } + } + + @Test + public void sparkRequestsShareOneClientId() throws Exception { + // A single spark-sql invocation == one SparkSession == one Iceberg RESTCatalog client, which + // performs one /v1/config handshake followed by several catalog operations. + String sql = + "CREATE NAMESPACE IF NOT EXISTS spark_ns; " + + "CREATE TABLE spark_ns.t (id BIGINT) USING iceberg; " + + "INSERT INTO spark_ns.t VALUES (1),(2),(3); " + + "SELECT COUNT(*) FROM spark_ns.t;"; + + Container.ExecResult result = + spark.execInContainer("bash", "-c", "/opt/spark/bin/spark-sql -e \"" + sql + "\""); + + if (result.getExitCode() != 0) { + logger.error("spark-sql stdout:\n{}", result.getStdout()); + logger.error("spark-sql stderr:\n{}", result.getStderr()); + logger.error("catalog logs:\n{}", catalog.getLogs()); + } + assertThat(result.getExitCode()) + .as("spark-sql should succeed; stderr: %s", result.getStderr()) + .isEqualTo(0); + + String logs = ANSI.matcher(catalog.getLogs()).replaceAll(""); + + Set operationClientIds = new LinkedHashSet<>(); + int operationRequests = 0; + String configClientId = null; + Matcher m = REQUEST_LINE.matcher(logs); + while (m.find()) { + String clientId = m.group(1); + String path = m.group(4); + if (path.startsWith("v1/config")) { + configClientId = clientId; + } else { + operationClientIds.add(clientId); + operationRequests++; + } + } + + logger.info( + "Parsed {} operation requests; config clientId={}, operation clientIds={}", + operationRequests, + configClientId, + operationClientIds); + + assertThat(operationRequests) + .as("expected multiple catalog operations from the Spark session") + .isGreaterThanOrEqualTo(2); + + assertThat(operationClientIds) + .as("all operations from one Spark client must share a single client id") + .hasSize(1); + + String sharedClientId = operationClientIds.iterator().next(); + assertThat(sharedClientId) + .as("shared client id must be the server-advertised UUID echoed by Spark") + .matches(UUID_PATTERN); + } + + private static String sparkDefaultsConf() { + return String.join( + "\n", + "spark.sql.extensions org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", + "spark.sql.catalog.demo org.apache.iceberg.spark.SparkCatalog", + "spark.sql.catalog.demo.type rest", + "spark.sql.catalog.demo.uri http://catalog:5000", + "spark.sql.catalog.demo.io-impl org.apache.iceberg.aws.s3.S3FileIO", + "spark.sql.catalog.demo.warehouse s3://test-bucket/warehouse", + "spark.sql.catalog.demo.s3.endpoint http://minio:9000", + "spark.sql.catalog.demo.s3.path-style-access true", + "spark.sql.catalog.demo.s3.access-key minioadmin", + "spark.sql.catalog.demo.s3.secret-key minioadmin", + "spark.sql.catalog.demo.client.region us-east-1", + "spark.sql.catalog.demo.s3.ssl-enabled false", + "spark.sql.defaultCatalog demo", + "spark.sql.catalogImplementation in-memory") + + "\n"; + } +} diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java index eb47c919..59c4cb61 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java @@ -106,7 +106,8 @@ public void setUp() throws Exception { null, // commitRetry null, // commitLock null, // loadTableProperties - null // icebergProperties + null, // icebergProperties + null // tracing ); // Create backend catalog from config diff --git a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogAdapterCreateIT.java b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogAdapterCreateIT.java index ebde55a2..7bad5564 100644 --- a/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogAdapterCreateIT.java +++ b/ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/internal/rest/RESTCatalogAdapterCreateIT.java @@ -82,6 +82,7 @@ public void beforeClass() throws IOException { null, null, null, + null, null); catalog = CatalogUtil.buildIcebergCatalog("it", config.toIcebergConfig(), null); adapter = new RESTCatalogAdapter(catalog); diff --git a/ice/src/main/resources/logback.xml b/ice/src/main/resources/logback.xml index 27fc8463..a4e62776 100644 --- a/ice/src/main/resources/logback.xml +++ b/ice/src/main/resources/logback.xml @@ -33,7 +33,7 @@ - %gray(%d{yyyy-MM-dd HH:mm:ss} [%.11thread/%processId]) %highlight(%-4level) %gray(%logger{27} >) %X{msgContext}%msg%n%replace(%ex{full, + %gray(%d{yyyy-MM-dd HH:mm:ss} [%.11thread/%processId]) %highlight(%-4level) %gray(%logger{27} >) %gray(%X{clientId} %X{requestId}) %X{msgContext}%msg%n%replace(%ex{full, org.eclipse.jetty, jakarta.servlet, software.amazon.awssdk.core.internal,