feat(gax): implement operation-level client request spans (T3)

java-compute/google-cloud-compute/src/test/java/com/google/cloud/compute/v1/integration/ITComputeGoldenSignals.java

@@ -318,17 +318,18 @@ private void fetchAndValidateTrace(String traceId, boolean expectError) throws E
         continue;
       }
 
-      // Assert RPC span name pattern {method} {url template}
-      assertThat(span.getName())
-          .isEqualTo("GET compute/v1/projects/{project=*}/zones/{zone=*}/instances");
+      boolean isOperationSpan = span.getName().equals("google.cloud.compute.v1.Instances.List");
+      if (!isOperationSpan) {
+        // Assert RPC attempt span name pattern {method} {url template}
+        assertThat(span.getName())
+            .isEqualTo("GET compute/v1/projects/{project=*}/zones/{zone=*}/instances");
+      }
 
       // Compute uses HTTP/REST, so we check for rpc.system.name and other HTTP attributes
       assertThat(span.getLabelsMap().get(ObservabilityAttributes.RPC_SYSTEM_NAME_ATTRIBUTE))
           .isEqualTo("http");
       assertThat(span.getLabelsMap().get(ObservabilityAttributes.URL_DOMAIN_ATTRIBUTE))
           .isEqualTo("compute.googleapis.com");
-      assertThat(span.getLabelsMap().get(ObservabilityAttributes.HTTP_METHOD_ATTRIBUTE))
-          .isEqualTo("GET");
       assertThat(span.getLabelsMap().get(ObservabilityAttributes.GCP_CLIENT_SERVICE_ATTRIBUTE))
           .isEqualTo("compute");
       assertThat(span.getLabelsMap().get(ObservabilityAttributes.REPO_ATTRIBUTE))
@@ -345,18 +346,25 @@ private void fetchAndValidateTrace(String traceId, boolean expectError) throws E
       assertThat(span.getLabelsMap().get(ObservabilityAttributes.DESTINATION_RESOURCE_ID_ATTRIBUTE))
           .startsWith(expectedDestinationResource);
 
-      String expectedUrl;
-      if (expectError) {
-        expectedUrl =
-            "https://compute.googleapis.com:443/compute/v1/projects/invalid-project-id/zones/us-central1-a/instances";
+      if (isOperationSpan) {
+        assertThat(span.getLabelsMap().get(ObservabilityAttributes.GRPC_RPC_METHOD_ATTRIBUTE))
+            .isEqualTo("google.cloud.compute.v1.Instances/List");
       } else {
-        expectedUrl =
-            "https://compute.googleapis.com:443/compute/v1/projects/"
-                + DEFAULT_PROJECT
-                + "/zones/us-central1-a/instances";
+        assertThat(span.getLabelsMap().get(ObservabilityAttributes.HTTP_METHOD_ATTRIBUTE))
+            .isEqualTo("GET");
+        String expectedUrl;
+        if (expectError) {
+          expectedUrl =
+              "https://compute.googleapis.com:443/compute/v1/projects/invalid-project-id/zones/us-central1-a/instances";
+        } else {
+          expectedUrl =
+              "https://compute.googleapis.com:443/compute/v1/projects/"
+                  + DEFAULT_PROJECT
+                  + "/zones/us-central1-a/instances";
+        }
+        assertThat(span.getLabelsMap().get(ObservabilityAttributes.HTTP_URL_FULL_ATTRIBUTE))
+            .isEqualTo(expectedUrl);
       }
-      assertThat(span.getLabelsMap().get(ObservabilityAttributes.HTTP_URL_FULL_ATTRIBUTE))
-          .isEqualTo(expectedUrl);
 
       if (expectError) {
         assertThat(span.getLabelsMap().get(ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE))

java-showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITOtelTracing.java

@@ -78,7 +78,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
 import java.util.List;
+import org.awaitility.Awaitility;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -149,6 +151,19 @@ void tearDown() {
     GlobalOpenTelemetry.resetForTest();
   }
 
+  /**
+   * Waits for the expected number of spans to be exported and returns the finished spans.
+   *
+   * @param expectedCount the expected number of exported spans
+   * @return the list of finished spans
+   */
+  private List<SpanData> waitForSpans(int expectedCount) {
+    Awaitility.await()
+        .atMost(Duration.ofSeconds(5))
+        .until(() -> spanExporter.getFinishedSpanItems().size() == expectedCount);
+    return spanExporter.getFinishedSpanItems();
+  }
+
   @Test
   void testTracing_successfulEcho_grpc() throws Exception {
     OpenTelemetryTracingFactory tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk);
@@ -160,7 +175,7 @@ void testTracing_successfulEcho_grpc() throws Exception {
 
       client.echo(EchoRequest.newBuilder().setContent("tracing-test").build());
 
-      List<SpanData> spans = spanExporter.getFinishedSpanItems();
+      List<SpanData> spans = waitForSpans(2);
       assertThat(spans).isNotEmpty();
 
       SpanData attemptSpan =
@@ -229,7 +244,7 @@ void testTracing_successfulEcho_httpjson() throws Exception {
 
       client.echo(EchoRequest.newBuilder().setContent("tracing-test").build());
 
-      List<SpanData> spans = spanExporter.getFinishedSpanItems();
+      List<SpanData> spans = waitForSpans(2);
       assertThat(spans).isNotEmpty();
 
       SpanData attemptSpan =
@@ -327,7 +342,7 @@ void testTracing_successfulIdentityGetUser_grpc() throws Exception {
         // generated.
       }
 
-      List<SpanData> spans = spanExporter.getFinishedSpanItems();
+      List<SpanData> spans = waitForSpans(2);
       assertThat(spans).isNotEmpty();
 
       SpanData attemptSpan =
@@ -356,7 +371,7 @@ void testTracing_successfulIdentityGetUser_httpjson() throws Exception {
         // generated.
       }
 
-      List<SpanData> spans = spanExporter.getFinishedSpanItems();
+      List<SpanData> spans = waitForSpans(2);
       assertThat(spans).isNotEmpty();
 
       SpanData attemptSpan =
@@ -413,8 +428,8 @@ void testTracing_retry_grpc() throws Exception {
 
     assertThrows(UnavailableException.class, () -> grpcClient.echo(echoRequest));
 
-    List<SpanData> spans = spanExporter.getFinishedSpanItems();
-    assertThat(spans).hasSize(attempts); // Expect exactly one span for the successful retry
+    List<SpanData> spans = waitForSpans(attempts + 1);
+    assertThat(spans).hasSize(attempts + 1); // attempts CLIENT spans + 1 INTERNAL operation span
 
     // This single span represents the successful retry, which has resend_count=1
     // The first attempt has no resend_count. The subsequent retries will have a
@@ -491,8 +506,8 @@ void testTracing_retry_httpjson() throws Exception {
 
     assertThrows(InvalidArgumentException.class, () -> httpClient.echo(echoRequest));
 
-    List<SpanData> spans = spanExporter.getFinishedSpanItems();
-    assertThat(spans).hasSize(attempts); // Expect exactly one span for the successful retry
+    List<SpanData> spans = waitForSpans(attempts + 1);
+    assertThat(spans).hasSize(attempts + 1); // attempts CLIENT spans + 1 INTERNAL operation span
 
     // This single span represents the successful retry, which has resend_count=1
     // The first attempt has no resend_count. The subsequent retries will have a
@@ -520,6 +535,12 @@ void testTracing_retry_httpjson() throws Exception {
   }
 
   private SpanData getErrorSpan() {
+    Awaitility.await()
+        .atMost(Duration.ofSeconds(5))
+        .until(
+            () ->
+                spanExporter.getFinishedSpanItems().stream()
+                    .anyMatch(span -> !span.getParentSpanContext().isValid()));
     List<SpanData> spans = spanExporter.getFinishedSpanItems();
     assertThat(spans).isNotEmpty();
 
@@ -707,26 +728,30 @@ void testTracing_statusCodes_grpc() throws Exception {
           com.google.api.gax.rpc.InvalidArgumentException.class,
           () -> grpcClient.echo(errorRequest));
 
-      List<SpanData> spans = spanExporter.getFinishedSpanItems();
-      assertThat(spans).hasSize(2);
-
-      SpanData grpcSuccessSpan = spans.get(0);
-      assertThat(
-              grpcSuccessSpan
-                  .getAttributes()
-                  .get(
-                      AttributeKey.stringKey(
-                          ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE)))
-          .isEqualTo("OK");
+      List<SpanData> spans = waitForSpans(4);
+      assertThat(spans).hasSize(4);
+
+      for (int i = 0; i < 2; i++) {
+        assertThat(
+                spans
+                    .get(i)
+                    .getAttributes()
+                    .get(
+                        AttributeKey.stringKey(
+                            ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE)))
+            .isEqualTo("OK");
+      }
 
-      SpanData grpcErrorSpan = spans.get(1);
-      assertThat(
-              grpcErrorSpan
-                  .getAttributes()
-                  .get(
-                      AttributeKey.stringKey(
-                          ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE)))
-          .isEqualTo("INVALID_ARGUMENT");
+      for (int i = 2; i < 4; i++) {
+        assertThat(
+                spans
+                    .get(i)
+                    .getAttributes()
+                    .get(
+                        AttributeKey.stringKey(
+                            ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE)))
+            .isEqualTo("INVALID_ARGUMENT");
+      }
     }
   }
 
@@ -748,24 +773,30 @@ void testTracing_statusCodes_httpjson() throws Exception {
           com.google.api.gax.rpc.InvalidArgumentException.class,
           () -> httpClient.echo(errorRequest));
 
-      List<SpanData> spans = spanExporter.getFinishedSpanItems();
-      assertThat(spans).hasSize(2);
-
-      SpanData httpSuccessSpan = spans.get(0);
-      assertThat(
-              httpSuccessSpan
-                  .getAttributes()
-                  .get(
-                      AttributeKey.longKey(ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE)))
-          .isEqualTo(200L);
+      List<SpanData> spans = waitForSpans(4);
+      assertThat(spans).hasSize(4);
+
+      for (int i = 0; i < 2; i++) {
+        assertThat(
+                spans
+                    .get(i)
+                    .getAttributes()
+                    .get(
+                        AttributeKey.longKey(
+                            ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE)))
+            .isEqualTo(200L);
+      }
 
-      SpanData httpErrorSpan = spans.get(1);
-      assertThat(
-              httpErrorSpan
-                  .getAttributes()
-                  .get(
-                      AttributeKey.longKey(ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE)))
-          .isEqualTo((long) StatusCode.Code.INVALID_ARGUMENT.getHttpStatusCode());
+      for (int i = 2; i < 4; i++) {
+        assertThat(
+                spans
+                    .get(i)
+                    .getAttributes()
+                    .get(
+                        AttributeKey.longKey(
+                            ObservabilityAttributes.HTTP_RESPONSE_STATUS_ATTRIBUTE)))
+            .isEqualTo((long) StatusCode.Code.INVALID_ARGUMENT.getHttpStatusCode());
+      }
     }
   }
 

sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ApiTracerContext.java

@@ -268,6 +268,18 @@ Map<String, Object> getMetricsAttributes() {
     return attributes;
   }
 
+  /**
+   * @return a map of attributes to be included in operation-level spans
+   */
+  Map<String, Object> getOperationAttributes() {
+    Map<String, Object> attributes = getMetricsAttributes();
+    if (!Strings.isNullOrEmpty(destinationResourceId())) {
+      attributes.put(
+          ObservabilityAttributes.DESTINATION_RESOURCE_ID_ATTRIBUTE, destinationResourceId());
+    }
+    return attributes;
+  }
+
   /**
    * Merges this context with another context. The values in the other context take precedence.
    *