diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/build.gradle b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/build.gradle index 10c111c9aed..b5713c764d0 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/build.gradle +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/build.gradle @@ -71,4 +71,3 @@ tasks.named("iastLatestDepTest3", Test) { javaLauncher = getJavaLauncherFor(17) jvmArgs = ['--add-opens', 'java.base/java.util=ALL-UNNAMED'] } - diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/gradle.lockfile b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/gradle.lockfile deleted file mode 100644 index 1f6f2ea2ea0..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/gradle.lockfile +++ /dev/null @@ -1,235 +0,0 @@ -# This is a Gradle generated file for dependency locking. -# Manual edits can break the build and are not advised. -# This file is expected to be part of source control. -# To regenerate this file, run: ./gradlew :dd-java-agent:instrumentation:kafka:kafka-clients-0.11:dependencies --write-locks -at.yawk.lz4:lz4-java:1.10.1=iastLatestDepTest3RuntimeClasspath -cafe.cryptography:curve25519-elisabeth:0.1.0=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -cafe.cryptography:ed25519-elisabeth:0.1.0=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -ch.qos.logback:logback-classic:1.2.13=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -ch.qos.logback:logback-core:1.2.13=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.101tec:zkclient:0.10=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.blogspot.mydailyjava:weak-lock-free:0.17=buildTimeInstrumentationPlugin,compileClasspath,iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq.okhttp3:okhttp:3.12.15=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq.okio:okio:1.17.6=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:dd-instrument-java:0.0.4=buildTimeInstrumentationPlugin,compileClasspath,iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:dd-javac-plugin-client:0.2.2=buildTimeInstrumentationPlugin,compileClasspath,iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:java-dogstatsd-client:4.4.5=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.datadoghq:sketches-java:0.8.3=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-annotations:2.11.4=latestDepTestCompileClasspath,testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-annotations:2.13.3=latestDepTestRuntimeClasspath -com.fasterxml.jackson.core:jackson-annotations:2.15.4=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -com.fasterxml.jackson.core:jackson-core:2.11.4=latestDepTestCompileClasspath,testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-core:2.13.3=latestDepTestRuntimeClasspath -com.fasterxml.jackson.core:jackson-core:2.15.4=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -com.fasterxml.jackson.core:jackson-databind:2.11.4=latestDepTestCompileClasspath,testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-databind:2.13.3=latestDepTestRuntimeClasspath -com.fasterxml.jackson.core:jackson-databind:2.15.4=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.13.3=latestDepTestRuntimeClasspath -com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.3=latestDepTestRuntimeClasspath -com.fasterxml.jackson.module:jackson-module-scala_2.13:2.13.3=latestDepTestRuntimeClasspath -com.fasterxml.jackson:jackson-bom:2.13.3=latestDepTestRuntimeClasspath -com.fasterxml.jackson:jackson-bom:2.15.4=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -com.github.javaparser:javaparser-core:3.25.6=codenarc -com.github.jnr:jffi:1.3.15=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-a64asm:1.0.0=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-constants:0.10.4=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-enxio:0.32.20=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-ffi:2.2.19=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-posix:3.1.22=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-unixsocket:0.38.25=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-x86asm:1.0.2=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.luben:zstd-jni:1.5.2-1=latestDepTestRuntimeClasspath -com.github.luben:zstd-jni:1.5.6-4=iastLatestDepTest3RuntimeClasspath -com.github.spotbugs:spotbugs-annotations:4.10.3=compileClasspath,spotbugs -com.github.spotbugs:spotbugs:4.10.3=spotbugs -com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs -com.google.auto.service:auto-service-annotations:1.1.1=annotationProcessor,compileClasspath,iastLatestDepTest3AnnotationProcessor,iastLatestDepTest3CompileClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,testAnnotationProcessor,testCompileClasspath -com.google.auto.service:auto-service:1.1.1=annotationProcessor,iastLatestDepTest3AnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.auto:auto-common:1.2.1=annotationProcessor,iastLatestDepTest3AnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,compileClasspath,iastLatestDepTest3AnnotationProcessor,iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,spotbugs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath -com.google.code.gson:gson:2.14.0=spotbugs -com.google.errorprone:error_prone_annotations:2.18.0=annotationProcessor,iastLatestDepTest3AnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.errorprone:error_prone_annotations:2.47.0=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.errorprone:error_prone_annotations:2.48.0=spotbugs -com.google.guava:failureaccess:1.0.1=annotationProcessor,iastLatestDepTest3AnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.guava:failureaccess:1.0.3=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.guava:guava:32.0.1-jre=annotationProcessor,iastLatestDepTest3AnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.guava:guava:33.6.0-jre=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,iastLatestDepTest3AnnotationProcessor,iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath -com.google.j2objc:j2objc-annotations:2.8=annotationProcessor,iastLatestDepTest3AnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.j2objc:j2objc-annotations:3.1=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.re2j:re2j:1.8=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.squareup.moshi:moshi:1.11.0=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.squareup.okhttp3:logging-interceptor:3.12.12=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.squareup.okhttp3:okhttp:3.12.12=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.squareup.okio:okio:1.17.5=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.thoughtworks.paranamer:paranamer:2.8=latestDepTestRuntimeClasspath -com.thoughtworks.qdox:qdox:1.12.1=codenarc -com.typesafe.scala-logging:scala-logging_2.13:3.9.4=latestDepTestRuntimeClasspath -com.yammer.metrics:metrics-core:2.2.0=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -commons-cli:commons-cli:1.4=latestDepTestRuntimeClasspath -commons-fileupload:commons-fileupload:1.5=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -commons-io:commons-io:2.11.0=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -commons-io:commons-io:2.21.0=spotbugs -commons-logging:commons-logging:1.2=testCompileClasspath,testRuntimeClasspath -de.thetaphi:forbiddenapis:3.10=compileClasspath,iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.dropwizard.metrics:metrics-core:4.1.12.1=latestDepTestRuntimeClasspath -io.leangen.geantyref:geantyref:1.3.16=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -io.micrometer:micrometer-commons:1.15.12=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -io.micrometer:micrometer-observation:1.15.12=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -io.netty:netty-buffer:4.1.63.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-codec:4.1.63.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-common:4.1.63.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-handler:4.1.63.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-resolver:4.1.63.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-transport-native-epoll:4.1.63.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-transport-native-unix-common:4.1.63.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-transport:4.1.63.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.projectreactor.kafka:reactor-kafka:1.0.0.RELEASE=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.projectreactor:reactor-core:3.1.0.RELEASE=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.sqreen:libsqreen:17.5.0=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -javax.activation:activation:1.1=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -javax.servlet:javax.servlet-api:3.1.0=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -javax.xml.bind:jaxb-api:2.2.3=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -javax.xml.stream:stax-api:1.0-2=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -jaxen:jaxen:2.0.6=spotbugs -junit:junit:4.12=iastLatestDepTest3CompileClasspath,testCompileClasspath -junit:junit:4.13.2=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -log4j:log4j:1.2.16=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,testCompileClasspath,testRuntimeClasspath -net.bytebuddy:byte-buddy-agent:1.18.10=buildTimeInstrumentationPlugin,compileClasspath,iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -net.bytebuddy:byte-buddy:1.18.10=buildTimeInstrumentationPlugin,compileClasspath,iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -net.java.dev.jna:jna-platform:5.8.0=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -net.java.dev.jna:jna:5.8.0=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -net.jpountz.lz4:lz4:1.3.0=compileClasspath,testCompileClasspath,testRuntimeClasspath -net.sf.jopt-simple:jopt-simple:5.0.3=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,testCompileClasspath,testRuntimeClasspath -net.sf.jopt-simple:jopt-simple:5.0.4=latestDepTestRuntimeClasspath -net.sf.saxon:Saxon-HE:12.10=spotbugs -net.sourceforge.argparse4j:argparse4j:0.7.0=latestDepTestRuntimeClasspath -org.apache.ant:ant-antlr:1.10.14=codenarc -org.apache.ant:ant-junit:1.10.14=codenarc -org.apache.bcel:bcel:6.12.0=spotbugs -org.apache.commons:commons-lang3:3.20.0=spotbugs -org.apache.commons:commons-text:1.15.0=spotbugs -org.apache.kafka:kafka-clients:0.11.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka-clients:3.2.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-clients:3.9.2=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -org.apache.kafka:kafka-metadata:3.2.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-raft:3.2.3=latestDepTestRuntimeClasspath -org.apache.kafka:kafka-server-common:3.2.3=latestDepTestRuntimeClasspath -org.apache.kafka:kafka-storage-api:3.2.3=latestDepTestRuntimeClasspath -org.apache.kafka:kafka-storage:3.2.3=latestDepTestRuntimeClasspath -org.apache.kafka:kafka-streams-test-utils:3.2.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-streams:3.2.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka_2.11:0.11.0.0=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka_2.13:3.2.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.logging.log4j:log4j-api:2.26.1=spotbugs -org.apache.logging.log4j:log4j-core:2.26.1=spotbugs -org.apache.yetus:audience-annotations:0.5.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.zookeeper:zookeeper-jute:3.6.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.zookeeper:zookeeper:3.4.10=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.apache.zookeeper:zookeeper:3.6.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apiguardian:apiguardian-api:1.1.2=iastLatestDepTest3CompileClasspath,latestDepTestCompileClasspath,testCompileClasspath -org.assertj:assertj-core:2.9.1=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.bitbucket.b_c:jose4j:0.7.9=latestDepTestRuntimeClasspath -org.checkerframework:checker-qual:3.33.0=annotationProcessor,iastLatestDepTest3AnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -org.codehaus.groovy:groovy-ant:3.0.23=codenarc -org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc -org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc -org.codehaus.groovy:groovy-json:3.0.23=codenarc -org.codehaus.groovy:groovy-json:3.0.25=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.codehaus.groovy:groovy-templates:3.0.23=codenarc -org.codehaus.groovy:groovy-xml:3.0.23=codenarc -org.codehaus.groovy:groovy:3.0.23=codenarc -org.codehaus.groovy:groovy:3.0.25=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.codenarc:CodeNarc:3.7.0=codenarc -org.dom4j:dom4j:2.2.0=spotbugs -org.gmetrics:GMetrics:2.1.0=codenarc -org.hamcrest:hamcrest-core:1.3=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.hamcrest:hamcrest:3.0=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.jctools:jctools-core-jdk11:4.0.6=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.jctools:jctools-core:4.0.6=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.jspecify:jspecify:1.0.0=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-api:5.14.1=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-engine:5.14.1=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-params:5.14.1=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter:5.14.1=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-commons:1.14.1=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-engine:1.14.1=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-launcher:1.14.1=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-runner:1.14.1=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-suite-api:1.14.1=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-suite-commons:1.14.1=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit:junit-bom:5.14.1=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit:junit-bom:6.1.2=spotbugs -org.lz4:lz4-java:1.8.0=latestDepTestRuntimeClasspath -org.mockito:mockito-core:2.19.0=iastLatestDepTest3CompileClasspath,latestDepTestCompileClasspath,testCompileClasspath -org.mockito:mockito-core:4.4.0=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.objenesis:objenesis:3.3=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.opentest4j:opentest4j:1.3.0=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.ow2.asm:asm-analysis:9.10.1=spotbugs -org.ow2.asm:asm-analysis:9.7.1=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.ow2.asm:asm-commons:9.10.1=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,spotbugs,testRuntimeClasspath -org.ow2.asm:asm-tree:9.10.1=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,spotbugs,testRuntimeClasspath -org.ow2.asm:asm-util:9.10.1=spotbugs -org.ow2.asm:asm-util:9.7.1=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.ow2.asm:asm:9.10.1=buildTimeInstrumentationPlugin,compileClasspath,iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath -org.reactivestreams:reactive-streams:1.0.1=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.rocksdb:rocksdbjni:6.29.4.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.scala-lang.modules:scala-collection-compat_2.13:2.6.0=latestDepTestRuntimeClasspath -org.scala-lang.modules:scala-java8-compat_2.13:1.0.2=latestDepTestRuntimeClasspath -org.scala-lang.modules:scala-parser-combinators_2.11:1.0.4=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.scala-lang:scala-library:2.11.11=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.scala-lang:scala-library:2.13.8=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.scala-lang:scala-reflect:2.13.8=latestDepTestRuntimeClasspath -org.slf4j:jcl-over-slf4j:1.7.30=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:jul-to-slf4j:1.7.30=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:log4j-over-slf4j:1.7.30=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:1.7.30=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath -org.slf4j:slf4j-api:1.7.32=iastLatestDepTest3CompileClasspath,latestDepTestCompileClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:1.7.36=iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath -org.slf4j:slf4j-api:2.0.17=spotbugsSlf4j -org.slf4j:slf4j-api:2.0.18=spotbugs -org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j -org.snakeyaml:snakeyaml-engine:2.9=buildTimeInstrumentationPlugin,iastLatestDepTest3RuntimeClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testRuntimeClasspath -org.spockframework:spock-bom:2.4-groovy-3.0=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.spockframework:spock-core:2.4-groovy-3.0=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.springframework.kafka:spring-kafka-test:1.3.3.RELEASE=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.springframework.kafka:spring-kafka-test:2.9.13=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework.kafka:spring-kafka:1.3.3.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework.kafka:spring-kafka:2.9.13=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework.kafka:spring-kafka:3.3.16=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -org.springframework.retry:spring-retry:1.2.2.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework.retry:spring-retry:1.3.4=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework.retry:spring-retry:2.0.13=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -org.springframework:spring-aop:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-aop:5.3.29=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-aop:6.2.19=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -org.springframework:spring-beans:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-beans:5.3.29=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-beans:6.2.19=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -org.springframework:spring-context:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-context:5.3.29=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-context:6.2.19=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -org.springframework:spring-core:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-core:5.3.29=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-core:6.2.19=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -org.springframework:spring-expression:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-expression:5.3.29=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-expression:6.2.19=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -org.springframework:spring-jcl:5.3.29=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-jcl:6.2.19=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -org.springframework:spring-messaging:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-messaging:5.3.29=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-messaging:6.2.19=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -org.springframework:spring-test:4.3.14.RELEASE=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.springframework:spring-test:5.3.29=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-tx:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-tx:5.3.29=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-tx:6.2.19=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath -org.tabletest:tabletest-junit:1.2.2=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.tabletest:tabletest-parser:1.2.1=iastLatestDepTest3CompileClasspath,iastLatestDepTest3RuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.xerial.snappy:snappy-java:1.1.10.5=iastLatestDepTest3RuntimeClasspath -org.xerial.snappy:snappy-java:1.1.2.6=compileClasspath,testCompileClasspath,testRuntimeClasspath -org.xerial.snappy:snappy-java:1.1.8.4=latestDepTestRuntimeClasspath -org.xmlresolver:xmlresolver:5.3.3=spotbugs -empty=spotbugsPlugins diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/iastLatestDepTest3/groovy/iast/KafkaIastDeserializerTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/iastLatestDepTest3/groovy/iast/KafkaIastDeserializerTest.groovy deleted file mode 100644 index 3b2c0c41f22..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/iastLatestDepTest3/groovy/iast/KafkaIastDeserializerTest.groovy +++ /dev/null @@ -1,196 +0,0 @@ -package iast - -import com.datadog.iast.propagation.PropagationModuleImpl -import com.datadog.iast.test.IastRequestTestRunner -import datadog.trace.api.iast.InstrumentationBridge -import datadog.trace.api.iast.SourceTypes -import org.apache.kafka.common.header.internals.RecordHeaders -import org.apache.kafka.common.serialization.ByteArrayDeserializer -import org.apache.kafka.common.serialization.ByteBufferDeserializer -import org.apache.kafka.common.serialization.Deserializer -import org.apache.kafka.common.serialization.StringDeserializer -import org.springframework.kafka.support.serializer.JsonDeserializer - -import java.nio.ByteBuffer - -import static org.hamcrest.CoreMatchers.instanceOf -import static org.hamcrest.core.IsEqual.equalTo - -class KafkaIastDeserializerTest extends IastRequestTestRunner { - - private static final int BUFF_OFFSET = 10 - - void 'test string deserializer: #test'() { - given: - final origin = test.origin - final propagationModule = new PropagationModuleImpl() - InstrumentationBridge.registerIastModule(propagationModule) - - and: - final payload = "Hello World!".bytes - final deserializer = new StringDeserializer() - - when: - runUnderIastTrace { - deserializer.configure([:], origin == SourceTypes.KAFKA_MESSAGE_KEY) - test.method.deserialize(deserializer, "test", payload) - } - - then: - final to = finReqTaintedObjects - to.hasTaintedObject { - value('Hello World!') - range(0, 12, source(origin)) - } - - where: - test << testSuite() - } - - void 'test byte array deserializer: #test'() { - given: - final origin = test.origin - final propagationModule = new PropagationModuleImpl() - InstrumentationBridge.registerIastModule(propagationModule) - - and: - final payload = "Hello World!".bytes - final deserializer = new ByteArrayDeserializer() - - when: - runUnderIastTrace { - deserializer.configure([:], origin == SourceTypes.KAFKA_MESSAGE_KEY) - test.method.deserialize(deserializer, "test", payload) - } - - then: - final to = finReqTaintedObjects - to.hasTaintedObject { - value(equalTo(payload)) - range(0, Integer.MAX_VALUE, source(origin)) - } - - where: - test << testSuite() - } - - void 'test byte buffer deserializer: #test'() { - given: - final origin = test.origin - final propagationModule = new PropagationModuleImpl() - InstrumentationBridge.registerIastModule(propagationModule) - - and: - final payload = "Hello World!".bytes - final deserializer = new ByteBufferDeserializer() - - when: - runUnderIastTrace { - deserializer.configure([:], origin == SourceTypes.KAFKA_MESSAGE_KEY) - test.method.deserialize(deserializer, "test", payload) - } - - then: - final to = finReqTaintedObjects - to.hasTaintedObject { - value(instanceOf(ByteBuffer)) - range(0, Integer.MAX_VALUE, source(origin)) - } - - where: - test << testSuite() - } - - void 'test json deserialization: #test'() { - given: - final origin = test.origin - final propagationModule = new PropagationModuleImpl() - InstrumentationBridge.registerIastModule(propagationModule) - - and: - final json = '{ "name": "Mr Bean" }' - final payload = json.bytes - final deserializer = new JsonDeserializer(TestBean) - - when: - runUnderIastTrace { - deserializer.configure([:], origin == SourceTypes.KAFKA_MESSAGE_KEY) - test.method.deserialize(deserializer, 'test', payload) - } - - then: - final to = finReqTaintedObjects - to.hasTaintedObject { - value(instanceOf(TestBean)) - range(0, Integer.MAX_VALUE, source(origin as byte)) - } - to.hasTaintedObject { - value('Mr Bean') - range(0, 7, source(origin as byte, 'name', 'Mr Bean')) - } - - where: - test << testSuite() - } - - private static List testSuite() { - return [SourceTypes.KAFKA_MESSAGE_KEY, SourceTypes.KAFKA_MESSAGE_VALUE].collectMany { origin -> - return [ - new Suite(origin: origin, method: Method.DEFAULT), - new Suite(origin: origin, method: Method.WITH_HEADERS), - new Suite(origin: origin, method: Method.WITH_BYTE_BUFFER), - new Suite(origin: origin, method: Method.WITH_BYTE_BUFFER_OFFSET) - ] - } - } - - enum Method { - DEFAULT{ - @Override - T deserialize(Deserializer deserializer, String topic, byte[] payload) { - return deserializer.deserialize(topic, payload) - } - }, - WITH_HEADERS{ - @Override - T deserialize(Deserializer deserializer, String topic, byte[] payload) { - return deserializer.deserialize(topic, new RecordHeaders(), payload) - } - }, - WITH_BYTE_BUFFER{ - @SuppressWarnings('GroovyAssignabilityCheck') - @Override - T deserialize(Deserializer deserializer, String topic, byte[] payload) { - ByteBuffer buffer = ByteBuffer.allocateDirect(payload.length) - buffer.put(payload) - buffer.position(0) - return deserializer.deserialize(topic, new RecordHeaders(), buffer) - } - }, - WITH_BYTE_BUFFER_OFFSET{ - @SuppressWarnings('GroovyAssignabilityCheck') - @Override - T deserialize(Deserializer deserializer, String topic, byte[] payload) { - final byte[] buffer = new byte[payload.length + BUFF_OFFSET] - System.arraycopy(payload, 0, buffer, BUFF_OFFSET, payload.length) - return deserializer.deserialize(topic, new RecordHeaders(), ByteBuffer.wrap(buffer, BUFF_OFFSET, payload.length)) - } - } - - abstract T deserialize(Deserializer deserializer, String topic, byte[] payload) - } - - static class Suite { - byte origin - Method method - - @Override - String toString() { - return "${method.name()}: ${SourceTypes.toString(origin)}" - } - } - - static class TestBean { - String name - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/latestDepTest/groovy/KafkaClientTestBase.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/latestDepTest/groovy/KafkaClientTestBase.groovy deleted file mode 100644 index 1d4ead5f6f7..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/latestDepTest/groovy/KafkaClientTestBase.groovy +++ /dev/null @@ -1,449 +0,0 @@ -import datadog.trace.agent.test.asserts.TraceAssert -import datadog.trace.agent.test.naming.VersionedNamingTestBase -import datadog.trace.api.Config -import datadog.trace.api.DDTags -import datadog.trace.api.datastreams.DataStreamsTags -import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags -import datadog.trace.bootstrap.instrumentation.api.Tags -import datadog.trace.common.writer.ListWriter -import datadog.trace.core.DDSpan -import datadog.trace.core.datastreams.StatsGroup -import org.apache.kafka.clients.consumer.ConsumerConfig -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.apache.kafka.clients.producer.KafkaProducer -import org.apache.kafka.clients.producer.ProducerConfig -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.serialization.StringSerializer -import org.springframework.kafka.core.DefaultKafkaConsumerFactory -import org.springframework.kafka.listener.KafkaMessageListenerContainer -import org.springframework.kafka.listener.MessageListener -import org.springframework.kafka.test.EmbeddedKafkaBroker -import org.springframework.kafka.test.utils.ContainerTestUtils -import org.springframework.kafka.test.utils.KafkaTestUtils -import spock.lang.Shared - -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.TimeUnit - -import static datadog.trace.agent.test.utils.TraceUtils.basicSpan -import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.isAsyncPropagationEnabled - -abstract class KafkaClientTestBase extends VersionedNamingTestBase { - static final SHARED_TOPIC = "shared.topic" - - protected EmbeddedKafkaBroker embeddedKafka - - @Override - void configurePreAgent() { - super.configurePreAgent() - - injectSysConfig("dd.kafka.e2e.duration.enabled", "true") - } - - public static final LinkedHashMap PRODUCER_PATHWAY_EDGE_TAGS - - // filter out Kafka poll, since the function is called in a loop, giving inconsistent results - @Shared - static final ListWriter.Filter DROP_KAFKA_POLL = new ListWriter.Filter() { - @Override - boolean accept(List trace) { - return !(trace.size() == 1 && - trace.get(0).getResourceName().toString().equals("kafka.poll")) - } - } - - final ListWriter.Filter dropEmptyKafkaPoll = new ListWriter.Filter() { - @Override - boolean accept(List trace) { - return !(trace.size() == 1 && - trace.get(0).getResourceName().toString().equals("kafka.poll") && - trace.get(0).getTag(InstrumentationTags.KAFKA_RECORDS_COUNT).equals(0)) - } - } - - // TraceID, start times & names changed based on the configuration, so overriding the sort to give consistent test results - private static class SortKafkaTraces implements Comparator> { - @Override - int compare(List o1, List o2) { - return rootSpanTrace(o1) - rootSpanTrace(o2) - } - - int rootSpanTrace(List trace) { - assert !trace.isEmpty() - def rootSpan = trace.get(0).localRootSpan - switch (rootSpan.operationName.toString()) { - case "parent": - return 3 - case "kafka.poll": - return 2 - default: - return 1 - } - } - } - - - static { - PRODUCER_PATHWAY_EDGE_TAGS = new LinkedHashMap<>(3) - PRODUCER_PATHWAY_EDGE_TAGS.put("direction", "out") - PRODUCER_PATHWAY_EDGE_TAGS.put("topic", SHARED_TOPIC) - PRODUCER_PATHWAY_EDGE_TAGS.put("type", "kafka") - } - - def setup() { - embeddedKafka = new EmbeddedKafkaBroker(1, true, SHARED_TOPIC) - embeddedKafka.afterPropertiesSet() - TEST_WRITER.setFilter(DROP_KAFKA_POLL) - } - - def cleanup() { - embeddedKafka?.destroy() - } - - @Override - int version() { - 0 - } - - @Override - String operation() { - return null - } - - String operationForProducer() { - "kafka.produce" - } - - String operationForConsumer() { - "kafka.consume" - } - - String serviceForTimeInQueue() { - "kafka" - } - - abstract boolean hasQueueSpan() - - abstract boolean splitByDestination() - - @Override - protected boolean isDataStreamsEnabled() { - return true - } - - def "test kafka produce and consume"() { - setup: - def producerProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - if (isDataStreamsEnabled()) { - producerProps.put(ProducerConfig.METADATA_MAX_AGE_CONFIG, 1000) - } - TEST_WRITER.setFilter(dropEmptyKafkaPoll) - KafkaProducer producer = new KafkaProducer<>(producerProps, new StringSerializer(), new StringSerializer()) - String clusterId = null - while (clusterId == null || clusterId.isEmpty()) { - Thread.sleep(1500) - clusterId = producer.metadata.fetch().clusterResource().clusterId() - } - - // set up the Kafka consumer properties - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - - // create a Kafka consumer factory - def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - - // set the topic that needs to be consumed - def containerProperties = containerProperties() - - // create a Kafka MessageListenerContainer - def container = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties) - - // create a thread safe queue to store the received message - def records = new LinkedBlockingQueue>() - - // setup a Kafka message listener - container.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records.add(record) - } - }) - - // start the container and underlying message listener - container.start() - - // wait until the container has the required number of assigned partitions - ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()) - - when: - String greeting = "Hello Spring Kafka Sender!" - runUnderTrace("parent") { - producer.send(new ProducerRecord(SHARED_TOPIC, greeting)) { meta, ex -> - assert isAsyncPropagationEnabled() - if (ex == null) { - runUnderTrace("producer callback") {} - } else { - runUnderTrace("producer exception: " + ex) {} - } - } - blockUntilChildSpansFinished(2) - } - if (isDataStreamsEnabled()) { - TEST_DATA_STREAMS_WRITER.waitForGroups(2) - // wait for produce offset 0, and commit offset 1 on 1 partition (Commit offset of 0 is not reported on partition 0 & 1) - TEST_DATA_STREAMS_WRITER.waitForBacklogs(2) - } - - then: - // check that the message was received - def received = records.poll(5, TimeUnit.SECONDS) - received.value() == greeting - received.key() == null - - int nTraces = isDataStreamsEnabled() ? 3 : 2 - int produceTraceIdx = nTraces - 1 - TEST_WRITER.waitForTraces(nTraces) - def traces = (Arrays.asList(TEST_WRITER.toArray()) as List>) - Collections.sort(traces, new SortKafkaTraces()) - assertTraces(nTraces, new SortKafkaTraces()) { - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, span(1)) - queueSpan(it, trace(produceTraceIdx)[2]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(produceTraceIdx)[2]) - } - } - if (isDataStreamsEnabled()) { - trace(1, { - pollSpan(it) - }) - } - trace(3) { - basicSpan(it, "parent") - basicSpan(it, "producer callback", span(0)) - producerSpan(it, producerProps, span(0), false) - } - } - def headers = received.headers() - headers.iterator().hasNext() - new String(headers.headers("x-datadog-trace-id").iterator().next().value()) == "${traces[produceTraceIdx][2].traceId}" - new String(headers.headers("x-datadog-parent-id").iterator().next().value()) == "${traces[produceTraceIdx][2].spanId}" - - if (isDataStreamsEnabled()) { - StatsGroup first = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == 0 } - verifyAll(first) { - tags.hasAllTags("direction:out", "kafka_cluster_id:$clusterId", "topic:$SHARED_TOPIC".toString(), "type:kafka") - } - - StatsGroup second = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == first.hash } - verifyAll(second) { - tags.hasAllTags( - "direction:in", - "group:sender", - "kafka_cluster_id:$clusterId", - "topic:$SHARED_TOPIC".toString(), - "type:kafka" - ) - } - def sorted = new ArrayList(TEST_DATA_STREAMS_WRITER.backlogs).sort({it.type+it.partition}) - verifyAll(sorted) { - size() == 2 - get(0).hasAllTags("consumer_group:sender", - "kafka_cluster_id:$clusterId", - "partition:" + received.partition(), - "topic:" + SHARED_TOPIC, - "type:kafka_commit") - get(1).hasAllTags("kafka_cluster_id:$clusterId", - "partition:" + received.partition(), - "topic:" + SHARED_TOPIC, - "type:kafka_produce") - } - } - - cleanup: - producer.close() - container?.stop() - } - - def containerProperties() { - try { - // Different class names for test and latestDepTest. - return Class.forName("org.springframework.kafka.listener.config.ContainerProperties").newInstance(SHARED_TOPIC) - } catch (ClassNotFoundException | NoClassDefFoundError e) { - return Class.forName("org.springframework.kafka.listener.ContainerProperties").newInstance(SHARED_TOPIC) - } - } - - def producerSpan( - TraceAssert trace, - Map config, - DDSpan parentSpan = null, - boolean partitioned = true, - boolean tombstone = false - ) { - trace.span { - serviceName service() - operationName operationForProducer() - resourceName "Produce Topic $SHARED_TOPIC" - spanType "queue" - errored false - measured true - if (parentSpan) { - childOf parentSpan - } else { - parent() - } - final boolean isV0 = version() == 0 - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_PRODUCER - "$InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS" config.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG) - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$SHARED_TOPIC" - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" { it >= 0 } - "$InstrumentationTags.KAFKA_CLUSTER_ID" { String } - if (tombstone) { - "$InstrumentationTags.TOMBSTONE" true - } - if ({ isDataStreamsEnabled() }) { - "$DDTags.PATHWAY_HASH" { String } - } - peerServiceFrom(InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS) - if (isV0) { - // in v0 the service name is always set to DD_SERVICE while it should just be unset as v1 - // this is a buggy behaviour that could not be easily fixed. - serviceNameSource "java-kafka" - } - defaultTags() - } - } - } - - def queueSpan( - TraceAssert trace, - DDSpan parentSpan = null - ) { - trace.span { - serviceName splitByDestination() ? "$SHARED_TOPIC" : serviceForTimeInQueue() - operationName "kafka.deliver" - resourceName "$SHARED_TOPIC" - spanType "queue" - errored false - measured true - if (parentSpan) { - childOf parentSpan - } else { - parent() - } - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_BROKER - defaultTags(true) - } - } - } - - def consumerSpan( - TraceAssert trace, - Map config, - DDSpan parentSpan = null, - Range offset = 0..0, - boolean tombstone = false, - boolean distributedRootSpan = !hasQueueSpan() - ) { - trace.span { - serviceName service() - operationName operationForConsumer() - resourceName "Consume Topic $SHARED_TOPIC" - spanType "queue" - errored false - measured true - if (parentSpan) { - childOf parentSpan - } else { - parent() - } - final boolean isV0 = version() == 0 - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_CONSUMER - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" { offset.containsWithinBounds(it as int) } - "$InstrumentationTags.CONSUMER_GROUP" "sender" - "$InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS" config.get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG) - "$InstrumentationTags.KAFKA_CLUSTER_ID" { String } - "$InstrumentationTags.RECORD_QUEUE_TIME_MS" { it >= 0 } - "$InstrumentationTags.RECORD_END_TO_END_DURATION_MS" { it >= 0 } - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$SHARED_TOPIC" - if (tombstone) { - "$InstrumentationTags.TOMBSTONE" true - } - if ({ isDataStreamsEnabled() }) { - "$DDTags.PATHWAY_HASH" { String } - } - if (isV0) { - // in v0 the service name is always set to DD_SERVICE while it should just be unset as v1 - // this is a buggy behaviour that could not be easily fixed. - serviceNameSource "java-kafka" - } - defaultTags(distributedRootSpan) - } - } - } - - def pollSpan( - TraceAssert trace, - int recordCount = 1, - DDSpan parentSpan = null, - Range offset = 0..0, - boolean tombstone = false, - boolean distributedRootSpan = !hasQueueSpan() - ) { - trace.span { - serviceName Config.get().getServiceName() - operationName "kafka.poll" - resourceName "kafka.poll" - errored false - measured false - tags { - "$InstrumentationTags.KAFKA_RECORDS_COUNT" recordCount - defaultTags(true) - } - } - } -} - -class KafkaClientLatestTest extends KafkaClientTestBase { - @Override - int version() { - 1 - } - - @Override - String operationForProducer() { - "kafka.send" - } - - @Override - String operationForConsumer() { - return "kafka.process" - } - - @Override - boolean hasQueueSpan() { - return false - } - - @Override - boolean splitByDestination() { - return false - } - - @Override - String service() { - return Config.get().getServiceName() - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/ConsumerCoordinatorInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/ConsumerCoordinatorInstrumentation.java deleted file mode 100644 index 57de8c42ff3..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/ConsumerCoordinatorInstrumentation.java +++ /dev/null @@ -1,176 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.not; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.api.datastreams.DataStreamsTags; -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import datadog.trace.instrumentation.kafka_common.KafkaConfigHelper; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import java.util.HashMap; -import java.util.Map; -import net.bytebuddy.asm.Advice; -import net.bytebuddy.matcher.ElementMatcher; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; -import org.apache.kafka.clients.consumer.internals.RequestFuture; -import org.apache.kafka.common.TopicPartition; - -@AutoService(InstrumenterModule.class) -public final class ConsumerCoordinatorInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { - - public ConsumerCoordinatorInstrumentation() { - super("kafka", "kafka-0.11"); - } - - @Override - public String muzzleDirective() { - return "before-3.8"; - } - - @Override - public ElementMatcher.Junction classLoaderMatcher() { - return not(hasClassNamed("org.apache.kafka.clients.MetadataRecoveryStrategy")); // < 3.8 - } - - @Override - public Map contextStore() { - Map contextStores = new HashMap<>(); - contextStores.put( - "org.apache.kafka.clients.Metadata", - "datadog.trace.instrumentation.kafka_common.MetadataState"); - contextStores.put( - "org.apache.kafka.clients.consumer.internals.ConsumerCoordinator", - KafkaConsumerInfo.class.getName()); - return contextStores; - } - - @Override - public String instrumentedType() { - return "org.apache.kafka.clients.consumer.internals.ConsumerCoordinator"; - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".KafkaConsumerInfo", - "datadog.trace.instrumentation.kafka_common.KafkaConfigHelper", - "datadog.trace.instrumentation.kafka_common.PendingConfig", - "datadog.trace.instrumentation.kafka_common.MetadataState", - }; - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isMethod().and(named("sendOffsetCommitRequest")).and(takesArguments(1)), - ConsumerCoordinatorInstrumentation.class.getName() + "$CommitOffsetAdvice"); - transformer.applyAdvice( - isMethod().and(named("onJoinComplete")).and(takesArguments(4)), - ConsumerCoordinatorInstrumentation.class.getName() + "$JoinGroupAdvice"); - } - - public static class CommitOffsetAdvice { - @Advice.OnMethodExit(suppress = Throwable.class) - public static void trackCommitOffset( - @Advice.This ConsumerCoordinator coordinator, - @Advice.Return RequestFuture requestFuture, - @Advice.Argument(0) final Map offsets) { - if (requestFuture == null || requestFuture.failed()) { - return; - } - if (offsets == null) { - return; - } - KafkaConsumerInfo kafkaConsumerInfo = - InstrumentationContext.get(ConsumerCoordinator.class, KafkaConsumerInfo.class) - .get(coordinator); - - if (kafkaConsumerInfo == null) { - return; - } - - String consumerGroup = kafkaConsumerInfo.getConsumerGroup(); - Metadata consumerMetadata = kafkaConsumerInfo.getClientMetadata(); - String clusterId = null; - if (consumerMetadata != null) { - MetadataState metadataState = - InstrumentationContext.get(Metadata.class, MetadataState.class).get(consumerMetadata); - clusterId = metadataState != null ? metadataState.clusterId : null; - } - - for (Map.Entry entry : offsets.entrySet()) { - if (consumerGroup == null) { - consumerGroup = ""; - } - if (entry.getKey() == null || entry.getValue() == null) { - continue; - } - - DataStreamsTags tags = - DataStreamsTags.createWithPartition( - "kafka_commit", - entry.getKey().topic(), - String.valueOf(entry.getKey().partition()), - clusterId, - consumerGroup); - AgentTracer.get().getDataStreamsMonitoring().trackBacklog(tags, entry.getValue().offset()); - } - } - - public static void muzzleCheck(ConsumerRecord record) { - // KafkaConsumerInstrumentation only applies for kafka versions with headers - // Make an explicit call so ConsumerCoordinatorInstrumentation does the same - record.headers(); - } - } - - public static class JoinGroupAdvice { - @Advice.OnMethodExit(suppress = Throwable.class) - public static void trackJoinGroup( - @Advice.This ConsumerCoordinator coordinator, - @Advice.Argument(0) final int generationId, - @Advice.Argument(1) final String memberId, - @Advice.Argument(2) final String memberProtocol) { - if (memberId == null || memberId.isEmpty()) { - return; - } - KafkaConsumerInfo kafkaConsumerInfo = - InstrumentationContext.get(ConsumerCoordinator.class, KafkaConsumerInfo.class) - .get(coordinator); - if (kafkaConsumerInfo == null) { - return; - } - if (!kafkaConsumerInfo.hasMembershipChanged(memberId, generationId)) { - return; - } - - String consumerGroup = kafkaConsumerInfo.getConsumerGroup(); - Metadata consumerMetadata = kafkaConsumerInfo.getClientMetadata(); - String clusterId = null; - if (consumerMetadata != null) { - MetadataState metadataState = - InstrumentationContext.get(Metadata.class, MetadataState.class).get(consumerMetadata); - clusterId = metadataState != null ? metadataState.clusterId : null; - } - if (KafkaConfigHelper.reportConsumerGroupMember( - clusterId, consumerGroup, memberId, generationId, memberProtocol)) { - kafkaConsumerInfo.setLastReportedMembership(memberId, generationId); - } - } - - public static void muzzleCheck(ConsumerRecord record) { - record.headers(); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfo.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfo.java index 2cd25004a2c..495847befe6 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfo.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfo.java @@ -8,8 +8,6 @@ public class KafkaConsumerInfo { private final String consumerGroup; private final Metadata clientMetadata; private final String bootstrapServers; - private volatile String lastReportedMemberId; - private volatile int lastReportedGenerationId = Integer.MIN_VALUE; public KafkaConsumerInfo(String consumerGroup, Metadata clientMetadata, String bootstrapServers) { this.consumerGroup = consumerGroup; @@ -38,16 +36,6 @@ public String getBootstrapServers() { return bootstrapServers; } - public boolean hasMembershipChanged(String memberId, int generationId) { - return !Objects.equals(memberId, lastReportedMemberId) - || generationId != lastReportedGenerationId; - } - - public void setLastReportedMembership(String memberId, int generationId) { - this.lastReportedMemberId = memberId; - this.lastReportedGenerationId = generationId; - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDecorator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDecorator.java index 53c579a4dee..3bba4cd506b 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDecorator.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDecorator.java @@ -4,6 +4,7 @@ import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.KAFKA_CLUSTER_ID; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.MESSAGING_DESTINATION_NAME; +import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.MESSAGING_SYSTEM; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.OFFSET; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.PARTITION; import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.RECORD_QUEUE_TIME_MS; @@ -123,6 +124,7 @@ public void onConsume( if (record != null) { final String topic = record.topic() == null ? "kafka" : record.topic(); span.setResourceName(CONSUMER_RESOURCE_NAME_CACHE.computeIfAbsent(topic, CONSUMER_PREFIX)); + span.setTag(MESSAGING_SYSTEM, KAFKA); span.setTag(PARTITION, record.partition()); span.setTag(OFFSET, record.offset()); span.setTag(MESSAGING_DESTINATION_NAME, topic); @@ -161,6 +163,7 @@ public void onProduce( final ProducerConfig producerConfig, final String clusterId) { if (record != null) { + span.setTag(MESSAGING_SYSTEM, KAFKA); if (record.partition() != null) { span.setTag(PARTITION, record.partition()); } diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDeserializerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDeserializerInstrumentation.java deleted file mode 100644 index 98fbb90d925..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaDeserializerInstrumentation.java +++ /dev/null @@ -1,179 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.hasInterface; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static java.util.Collections.singletonMap; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.agent.tooling.muzzle.Reference; -import datadog.trace.api.iast.IastContext; -import datadog.trace.api.iast.Source; -import datadog.trace.api.iast.SourceTypes; -import datadog.trace.bootstrap.ContextStore; -import datadog.trace.bootstrap.InstrumentationContext; -import java.nio.ByteBuffer; -import java.util.Map; -import net.bytebuddy.asm.Advice; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.matcher.ElementMatcher; -import org.apache.kafka.common.serialization.Deserializer; - -@AutoService(InstrumenterModule.class) -public class KafkaDeserializerInstrumentation extends InstrumenterModule.Iast - implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { - - private static final String DESERIALIZER_CLASS = - "org.apache.kafka.common.serialization.Deserializer"; - - /** Ensure same compatibility as the tracer */ - static final Reference[] MUZZLE_CHECK = { - new Reference.Builder("org.apache.kafka.clients.consumer.ConsumerRecord") - .withMethod( - new String[0], - Reference.EXPECTS_PUBLIC | Reference.EXPECTS_NON_STATIC, - "headers", - "Lorg/apache/kafka/common/header/Headers;") - .build() - }; - - public KafkaDeserializerInstrumentation() { - super("kafka"); - } - - @Override - public String muzzleDirective() { - return "since-0.11"; - } - - @Override - public String hierarchyMarkerType() { - return DESERIALIZER_CLASS; - } - - @Override - public ElementMatcher hierarchyMatcher() { - return hasInterface(named(hierarchyMarkerType())); - } - - @Override - public Map contextStore() { - return singletonMap(DESERIALIZER_CLASS, Boolean.class.getName()); - } - - @Override - public String[] helperClassNames() { - return new String[] {packageName + ".KafkaIastHelper"}; - } - - @Override - public Reference[] additionalMuzzleReferences() { - return MUZZLE_CHECK; - } - - @Override - public void methodAdvice(final MethodTransformer transformer) { - final String baseName = KafkaDeserializerInstrumentation.class.getName(); - transformer.applyAdvice( - named("configure").and(takesArguments(Map.class, boolean.class)), - baseName + "$ConfigureAdvice"); - transformer.applyAdvice( - named("deserialize").and(takesArguments(2)).and(takesArgument(1, byte[].class)), - baseName + "$Deserialize2Advice"); - transformer.applyAdvice( - named("deserialize").and(takesArguments(3)).and(takesArgument(2, byte[].class)), - baseName + "$Deserialize3Advice"); - transformer.applyAdvice( - named("deserialize").and(takesArguments(3)).and(takesArgument(2, ByteBuffer.class)), - baseName + "$DeserializeByteBufferAdvice"); - } - - @SuppressWarnings("rawtypes") - public static class ConfigureAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void configure( - @Advice.This final Deserializer deserializer, @Advice.Argument(1) final boolean isKey) { - final ContextStore store = - InstrumentationContext.get(Deserializer.class, Boolean.class); - KafkaIastHelper.configure(store, deserializer, isKey); - } - } - - @SuppressWarnings("rawtypes") - public static class Deserialize2Advice { - - @Source(SourceTypes.KAFKA_MESSAGE) - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void deserialize( - @Advice.This final Deserializer deserializer, - @Advice.Argument(1) byte[] data, - @Advice.Local("iastCtx") IastContext ctx) { - final ContextStore store = - InstrumentationContext.get(Deserializer.class, Boolean.class); - ctx = KafkaIastHelper.beforeDeserialize(store, deserializer, data); - } - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void afterDeserialize( - @Advice.This final Deserializer deserializer, - @Advice.Return Object result, - @Advice.Local("iastCtx") IastContext ctx) { - final ContextStore store = - InstrumentationContext.get(Deserializer.class, Boolean.class); - KafkaIastHelper.afterDeserialize(ctx, store, deserializer, result); - } - } - - @SuppressWarnings({"rawtypes", "JavaExistingMethodCanBeUsed"}) - public static class Deserialize3Advice { - - @Source(SourceTypes.KAFKA_MESSAGE) - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void deserialize( - @Advice.This final Deserializer deserializer, - @Advice.Argument(2) byte[] data, - @Advice.Local("iastCtx") IastContext ctx) { - final ContextStore store = - InstrumentationContext.get(Deserializer.class, Boolean.class); - ctx = KafkaIastHelper.beforeDeserialize(store, deserializer, data); - } - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void afterDeserialize( - @Advice.This final Deserializer deserializer, - @Advice.Return Object result, - @Advice.Local("iastCtx") IastContext ctx) { - final ContextStore store = - InstrumentationContext.get(Deserializer.class, Boolean.class); - KafkaIastHelper.afterDeserialize(ctx, store, deserializer, result); - } - } - - @SuppressWarnings({"rawtypes", "JavaExistingMethodCanBeUsed"}) - public static class DeserializeByteBufferAdvice { - - @Source(SourceTypes.KAFKA_MESSAGE) - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void deserialize( - @Advice.This final Deserializer deserializer, - @Advice.Argument(2) ByteBuffer data, - @Advice.Local("iastCtx") IastContext ctx) { - final ContextStore store = - InstrumentationContext.get(Deserializer.class, Boolean.class); - ctx = KafkaIastHelper.beforeDeserialize(store, deserializer, data); - } - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void afterDeserialize( - @Advice.This final Deserializer deserializer, - @Advice.Return final Object result, - @Advice.Local("iastCtx") IastContext ctx) { - final ContextStore store = - InstrumentationContext.get(Deserializer.class, Boolean.class); - KafkaIastHelper.afterDeserialize(ctx, store, deserializer, result); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaIastHelper.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaIastHelper.java deleted file mode 100644 index 4063904a4d6..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaIastHelper.java +++ /dev/null @@ -1,111 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients; - -import static datadog.trace.api.iast.SourceTypes.KAFKA_MESSAGE_KEY; -import static datadog.trace.api.iast.SourceTypes.KAFKA_MESSAGE_VALUE; - -import datadog.trace.api.iast.IastContext; -import datadog.trace.api.iast.InstrumentationBridge; -import datadog.trace.api.iast.propagation.PropagationModule; -import datadog.trace.bootstrap.CallDepthThreadLocalMap; -import datadog.trace.bootstrap.ContextStore; -import java.nio.ByteBuffer; -import org.apache.kafka.common.serialization.Deserializer; - -@SuppressWarnings("rawtypes") -public class KafkaIastHelper { - - public static void configure( - final ContextStore store, - final Deserializer deserializer, - final boolean isKey) { - if (store != null) { - store.put(deserializer, isKey); - } - } - - public static IastContext beforeDeserialize( - final ContextStore store, - final Deserializer deserializer, - final Object data) { - CallDepthThreadLocalMap.incrementCallDepth(Deserializer.class); - if (data == null) { - return null; - } - final PropagationModule module = InstrumentationBridge.PROPAGATION; - if (module == null) { - return null; - } - final IastContext ctx = IastContext.Provider.get(); - if (ctx == null) { - return null; - } - if (module.isTainted(ctx, data)) { - return ctx; // prevent double tainting on reentrant calls - } - final byte source = getSource(store, deserializer); - if (data instanceof String) { - module.taintString(ctx, (String) data, source); - } else { - module.taintObject(ctx, data, source); - } - return ctx; - } - - public static IastContext beforeDeserialize( - final ContextStore store, - final Deserializer deserializer, - final ByteBuffer data) { - CallDepthThreadLocalMap.incrementCallDepth(Deserializer.class); - if (data == null || data.remaining() == 0) { - return null; - } - final PropagationModule module = InstrumentationBridge.PROPAGATION; - if (module == null) { - return null; - } - final IastContext ctx = IastContext.Provider.get(); - if (ctx == null) { - return null; - } - if (module.isTainted(ctx, data)) { - return ctx; // prevent double tainting on reentrant calls - } - final byte source = getSource(store, deserializer); - int start = data.position(); - if (data.hasArray()) { - start += data.arrayOffset(); - } - module.taintObjectRange(ctx, data, source, start, data.remaining()); - return ctx; - } - - public static void afterDeserialize( - final IastContext ctx, - final ContextStore store, - final Deserializer deserializer, - final Object result) { - // final exit of the method - if (CallDepthThreadLocalMap.decrementCallDepth(Deserializer.class) != 0) { - return; - } - final PropagationModule module = InstrumentationBridge.PROPAGATION; - if (module == null) { - return; - } - final byte source = getSource(store, deserializer); - if (result instanceof String) { - module.taintString(ctx, (String) result, source); - } else { - module.taintObject(ctx, result, source); - } - } - - private static byte getSource( - final ContextStore store, final Deserializer deserializer) { - if (store == null) { - return KAFKA_MESSAGE_VALUE; - } - final Boolean key = store.get(deserializer); - return key != null && key ? KAFKA_MESSAGE_KEY : KAFKA_MESSAGE_VALUE; - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java index 227d8872648..5c97194808a 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java @@ -206,7 +206,8 @@ public static void onEnter( TextMapInjectAdapterInterface setter = NoopTextMapInjectAdapter.NOOP_SETTER; // Do not inject headers for batch versions below 2 // This is how similar check is being done in Kafka client itself: - // https://github.com/apache/kafka/blob/05fcfde8f69b0349216553f711fdfc3f0259c601/clients/src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java#L411-L412 + // https://github.com/apache/kafka/blob/05fcfde8f69b0349216553f711fdfc3f0259c601/clients + // /src/main/java/org/apache/kafka/common/record/MemoryRecordsBuilder.java#L411-L412 // Also, do not inject headers if specified by JVM option or environment variable // This can help in mixed client environments where clients < 0.11 that do not support // headers attempt to read messages that were produced by clients > 0.11 and the magic diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/MetadataInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/MetadataInstrumentation.java deleted file mode 100644 index d6acfe30369..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/MetadataInstrumentation.java +++ /dev/null @@ -1,126 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static java.util.Collections.singletonMap; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.not; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.instrumentation.kafka_common.KafkaConfigHelper; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import java.util.Map; -import net.bytebuddy.asm.Advice; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.matcher.ElementMatcher; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.Cluster; -import org.apache.kafka.common.requests.MetadataResponse; - -@AutoService(InstrumenterModule.class) -public class MetadataInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { - - public MetadataInstrumentation() { - super("kafka", "kafka-0.11"); - } - - @Override - public String muzzleDirective() { - return "before-3.8"; - } - - @Override - public ElementMatcher.Junction classLoaderMatcher() { - return not(hasClassNamed("org.apache.kafka.clients.MetadataRecoveryStrategy")); // < 3.8 - } - - @Override - public String hierarchyMarkerType() { - return "org.apache.kafka.clients.Metadata"; - } - - @Override - public ElementMatcher hierarchyMatcher() { - return extendsClass(named(hierarchyMarkerType())); - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".KafkaDecorator", - "datadog.trace.instrumentation.kafka_common.KafkaConfigHelper", - "datadog.trace.instrumentation.kafka_common.PendingConfig", - "datadog.trace.instrumentation.kafka_common.MetadataState", - }; - } - - @Override - public Map contextStore() { - return singletonMap( - "org.apache.kafka.clients.Metadata", - "datadog.trace.instrumentation.kafka_common.MetadataState"); - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isMethod() - .and(named("update")) - .and(takesArgument(0, named("org.apache.kafka.common.Cluster"))), - MetadataInstrumentation.class.getName() + "$MetadataUpdateBefore22Advice"); - transformer.applyAdvice( - isMethod() - .and(named("update")) - .and(takesArgument(1, named("org.apache.kafka.common.requests.MetadataResponse"))), - MetadataInstrumentation.class.getName() + "$MetadataUpdate22AndAfterAdvice"); - } - - public static class MetadataUpdateBefore22Advice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter( - @Advice.This final Metadata metadata, @Advice.Argument(0) final Cluster newCluster) { - if (newCluster != null && !newCluster.isBootstrapConfigured()) { - String clusterId = newCluster.clusterResource().clusterId(); - MetadataState state = - InstrumentationContext.get(Metadata.class, MetadataState.class) - .putIfAbsent(metadata, MetadataState::new); - state.clusterId = clusterId; - KafkaConfigHelper.reportPendingConfig(state, clusterId); - } - } - - public static void muzzleCheck(ConsumerRecord record) { - // KafkaConsumerInstrumentation only applies for kafka versions with headers - // Make an explicit call so MetadataInstrumentation does the same - record.headers(); - } - } - - public static class MetadataUpdate22AndAfterAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter( - @Advice.This final Metadata metadata, @Advice.Argument(1) final MetadataResponse response) { - if (response != null) { - String clusterId = response.clusterId(); - MetadataState state = - InstrumentationContext.get(Metadata.class, MetadataState.class) - .putIfAbsent(metadata, MetadataState::new); - state.clusterId = clusterId; - KafkaConfigHelper.reportPendingConfig(state, clusterId); - } - } - - public static void muzzleCheck(ConsumerRecord record) { - // KafkaConsumerInstrumentation only applies for kafka versions with headers - // Make an explicit call so MetadataInstrumentation does the same - record.headers(); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy index 98460601e5c..35f5ab5088b 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy @@ -39,7 +39,6 @@ class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecificatio @Override boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly return false } @@ -57,7 +56,6 @@ class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecificatio @Override void configurePreAgent() { super.configurePreAgent() - injectSysConfig("dd.kafka.e2e.duration.enabled", "true") } @@ -69,52 +67,42 @@ class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecificatio def producerFactory = new DefaultKafkaProducerFactory(senderProps) def kafkaTemplate = new KafkaTemplate(producerFactory) - // set up the Kafka consumer properties def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - - // create a Kafka consumer factory def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - // set the topic that needs to be consumed def containerProperties1 = containerProperties(SHARED_TOPIC[0]) as ContainerProperties def containerProperties2 = containerProperties(SHARED_TOPIC[1]) as ContainerProperties def containerProperties3 = containerProperties(SHARED_TOPIC[2]) as ContainerProperties def containerProperties4 = containerProperties(SHARED_TOPIC[3]) as ContainerProperties - // create a Kafka MessageListenerContainer def container1 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties1) def container2 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties2) def container3 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties3) def container4 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties4) - // create a thread safe queue to store the received message def records1 = new LinkedBlockingQueue>() def records2 = new LinkedBlockingQueue>() def records3 = new LinkedBlockingQueue>() def records4 = new LinkedBlockingQueue>() - // setup a Kafka message listener container1.setupMessageListener(new MessageListener() { @Override void onMessage(ConsumerRecord record) { records1.add(record) } }) - container2.setupMessageListener(new MessageListener() { @Override void onMessage(ConsumerRecord record) { records2.add(record) } }) - container3.setupMessageListener(new MessageListener() { @Override void onMessage(ConsumerRecord record) { records3.add(record) } }) - container4.setupMessageListener(new MessageListener() { @Override void onMessage(ConsumerRecord record) { @@ -122,13 +110,11 @@ class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecificatio } }) - // start the container and underlying message listener container1.start() container2.start() container3.start() container4.start() - // wait until the container has the required number of assigned partitions ContainerTestUtils.waitForAssignment(container1, embeddedKafka.getPartitionsPerTopic()) ContainerTestUtils.waitForAssignment(container2, embeddedKafka.getPartitionsPerTopic()) ContainerTestUtils.waitForAssignment(container3, embeddedKafka.getPartitionsPerTopic()) @@ -140,7 +126,6 @@ class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecificatio } then: - // check that the message was received def received1 = records1.poll(5, TimeUnit.SECONDS) def received2 = records2.poll(5, TimeUnit.SECONDS) def received3 = records3.poll(5, TimeUnit.SECONDS) @@ -159,7 +144,7 @@ class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecificatio container4?.stop() where: - [value, expected1, expected2, expected3, expected4]<< dataTable() + [value, expected1, expected2, expected3, expected4] << dataTable() } def "test consumer with topic filters"() { @@ -170,52 +155,42 @@ class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecificatio def producerFactory = new DefaultKafkaProducerFactory(senderProps) def kafkaTemplate = new KafkaTemplate(producerFactory) - // set up the Kafka consumer properties def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - - // create a Kafka consumer factory def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - // set the topic that needs to be consumed def containerProperties1 = containerProperties(SHARED_TOPIC[0]) as ContainerProperties def containerProperties2 = containerProperties(SHARED_TOPIC[1]) as ContainerProperties def containerProperties3 = containerProperties(SHARED_TOPIC[2]) as ContainerProperties def containerProperties4 = containerProperties(SHARED_TOPIC[3]) as ContainerProperties - // create a Kafka MessageListenerContainer def container1 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties1) def container2 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties2) def container3 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties3) def container4 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties4) - // create a thread safe queue to store the received message def records1 = new LinkedBlockingQueue() def records2 = new LinkedBlockingQueue() def records3 = new LinkedBlockingQueue() def records4 = new LinkedBlockingQueue() - // setup a Kafka message listener container1.setupMessageListener(new MessageListener() { @Override void onMessage(ConsumerRecord record) { records1.add(activeSpan()) } }) - container2.setupMessageListener(new MessageListener() { @Override void onMessage(ConsumerRecord record) { records2.add(activeSpan()) } }) - container3.setupMessageListener(new MessageListener() { @Override void onMessage(ConsumerRecord record) { records3.add(activeSpan()) } }) - container4.setupMessageListener(new MessageListener() { @Override void onMessage(ConsumerRecord record) { @@ -223,13 +198,11 @@ class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecificatio } }) - // start the container and underlying message listener container1.start() container2.start() container3.start() container4.start() - // wait until the container has the required number of assigned partitions ContainerTestUtils.waitForAssignment(container1, embeddedKafka.getPartitionsPerTopic()) ContainerTestUtils.waitForAssignment(container2, embeddedKafka.getPartitionsPerTopic()) ContainerTestUtils.waitForAssignment(container3, embeddedKafka.getPartitionsPerTopic()) @@ -254,7 +227,6 @@ class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecificatio span.finish() then: - // check that the message was received def received1 = records1.poll(5, TimeUnit.SECONDS) def received2 = records2.poll(5, TimeUnit.SECONDS) def received3 = records3.poll(5, TimeUnit.SECONDS) @@ -288,9 +260,8 @@ class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecificatio container3?.stop() container4?.stop() - where: - [value, expected1, expected2, expected3, expected4]<< dataTable() + [value, expected1, expected2, expected3, expected4] << dataTable() } def containerProperties(String topic) { diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy index 6d311a1b8d5..4bdc12227a2 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy @@ -1,15 +1,7 @@ -import datadog.trace.api.datastreams.DataStreamsTags -import datadog.trace.api.datastreams.DataStreamsTransactionExtractor -import datadog.trace.api.config.TraceInstrumentationConfig -import datadog.trace.api.config.TracerConfig -import datadog.trace.instrumentation.kafka_common.ClusterIdHolder - import static datadog.trace.agent.test.utils.TraceUtils.basicSpan import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.isAsyncPropagationEnabled -import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.agent.test.asserts.TraceAssert import datadog.trace.agent.test.naming.VersionedNamingTestBase import datadog.trace.api.Config @@ -18,25 +10,16 @@ import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.common.writer.ListWriter import datadog.trace.core.DDSpan -import datadog.trace.core.datastreams.StatsGroup import org.apache.kafka.clients.consumer.ConsumerConfig import org.apache.kafka.clients.consumer.ConsumerRecord import org.apache.kafka.clients.consumer.KafkaConsumer import org.apache.kafka.clients.producer.KafkaProducer -import org.apache.kafka.clients.producer.Producer import org.apache.kafka.clients.producer.ProducerConfig import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.clients.producer.RecordMetadata -import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.header.internals.RecordHeader -import org.apache.kafka.common.header.internals.RecordHeaders import org.apache.kafka.common.serialization.StringSerializer - -import java.nio.charset.StandardCharsets import org.springframework.kafka.core.DefaultKafkaConsumerFactory import org.springframework.kafka.core.DefaultKafkaProducerFactory import org.springframework.kafka.core.KafkaTemplate -import org.springframework.kafka.listener.BatchMessageListener import org.springframework.kafka.listener.KafkaMessageListenerContainer import org.springframework.kafka.listener.MessageListener import org.springframework.kafka.test.rule.KafkaEmbedded @@ -44,8 +27,6 @@ import org.springframework.kafka.test.utils.ContainerTestUtils import org.springframework.kafka.test.utils.KafkaTestUtils import spock.lang.Shared -import java.util.concurrent.ExecutionException -import java.util.concurrent.Future import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.TimeUnit @@ -54,16 +35,8 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { KafkaEmbedded embeddedKafka - @Override - void configurePreAgent() { - super.configurePreAgent() - - injectSysConfig("dd.kafka.e2e.duration.enabled", "true") - } - - public static final LinkedHashMap PRODUCER_PATHWAY_EDGE_TAGS - - // filter out Kafka poll, since the function is called in a loop, giving inconsistent results + // Filter out Kafka poll spans that appear due to the consumer's polling loop, + // which gives inconsistent results across test runs. @Shared static final ListWriter.Filter DROP_KAFKA_POLL = new ListWriter.Filter() { @Override @@ -73,16 +46,6 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { } } - final ListWriter.Filter dropEmptyKafkaPoll = new ListWriter.Filter() { - @Override - boolean accept(List trace) { - return !(trace.size() == 1 && - trace.get(0).getResourceName().toString().equals("kafka.poll") && - trace.get(0).getTag(InstrumentationTags.KAFKA_RECORDS_COUNT).equals(0)) - } - } - - // TraceID, start times & names changed based on the configuration, so overriding the sort to give consistent test results private static class SortKafkaTraces implements Comparator> { @Override int compare(List o1, List o2) { @@ -103,40 +66,6 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { } } - private static class SortBatchKafkaTraces implements Comparator> { - @Override - int compare(List o1, List o2) { - return Long.compare(batchSortKey(o1), batchSortKey(o2)) - } - } - - private static long batchSortKey(List trace) { - assert !trace.isEmpty() - if (trace.get(0).localRootSpan.operationName.toString() == "parent") { - return Long.MIN_VALUE - } - def deliverSpan = trace.find { it.operationName.toString() == "kafka.deliver" } - return deliverSpan ? deliverSpan.parentId : trace.get(0).parentId - } - - private static List producerSpans(List> traces) { - def producerTrace = traces.find { trace -> - !trace.isEmpty() && trace.get(0).localRootSpan.operationName.toString() == "parent" - } - assert producerTrace != null - return producerTrace - .findAll { it.getTag(Tags.SPAN_KIND) == Tags.SPAN_KIND_PRODUCER } - .sort { it.spanId } - } - - - static { - PRODUCER_PATHWAY_EDGE_TAGS = new LinkedHashMap<>(3) - PRODUCER_PATHWAY_EDGE_TAGS.put("direction", "out") - PRODUCER_PATHWAY_EDGE_TAGS.put("topic", SHARED_TOPIC) - PRODUCER_PATHWAY_EDGE_TAGS.put("type", "kafka") - } - def setup() { embeddedKafka = new KafkaEmbedded(1, true, SHARED_TOPIC) embeddedKafka.before() @@ -184,52 +113,28 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { if (isDataStreamsEnabled()) { senderProps.put(ProducerConfig.METADATA_MAX_AGE_CONFIG, 1000) } - TEST_WRITER.setFilter(dropEmptyKafkaPoll) KafkaProducer producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) - String clusterId = "" - if (isDataStreamsEnabled()) { - producer.flush() - clusterId = producer.metadata.cluster.clusterResource().clusterId() - while (clusterId == null || clusterId.isEmpty()) { - Thread.sleep(1500) - clusterId = producer.metadata.cluster.clusterResource().clusterId() - } - } - // set up the Kafka consumer properties + // set up Kafka consumer def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - - // create a Kafka consumer factory def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - - // set the topic that needs to be consumed def containerProperties = containerProperties() - - // create a Kafka MessageListenerContainer def container = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties) - - // create a thread safe queue to store the received message def records = new LinkedBlockingQueue>() - // setup a Kafka message listener container.setupMessageListener(new MessageListener() { @Override void onMessage(ConsumerRecord record) { records.add(record) } }) - - // start the container and underlying message listener container.start() - - // wait until the container has the required number of assigned partitions ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()) when: String greeting = "Hello Spring Kafka Sender!" runUnderTrace("parent") { producer.send(new ProducerRecord(SHARED_TOPIC, greeting)) { meta, ex -> - assert isAsyncPropagationEnabled() if (ex == null) { runUnderTrace("producer callback") {} } else { @@ -238,94 +143,33 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { } blockUntilChildSpansFinished(2) } - if (isDataStreamsEnabled()) { - TEST_DATA_STREAMS_WRITER.waitForGroups(2) - // wait for produce offset 0, commit offset 0 on partition 0 and 1, and commit offset 1 on 1 partition. - TEST_DATA_STREAMS_WRITER.waitForBacklogs(3) - } then: - // check that the message was received def received = records.poll(5, TimeUnit.SECONDS) received.value() == greeting received.key() == null - // verify ClusterIdHolder was properly cleaned up after produce and consume - ClusterIdHolder.get() == null - - int nTraces = isDataStreamsEnabled() ? 3 : 2 - int produceTraceIdx = nTraces - 1 - TEST_WRITER.waitForTraces(nTraces) - def traces = (Arrays.asList(TEST_WRITER.toArray()) as List>) - Collections.sort(traces, new SortKafkaTraces()) - assertTraces(nTraces, new SortKafkaTraces()) { + assertTraces(2, new SortKafkaTraces()) { if (hasQueueSpan()) { trace(2) { consumerSpan(it, consumerProperties, span(1)) - queueSpan(it, trace(produceTraceIdx)[2]) + queueSpan(it, trace(1)[2]) } } else { trace(1) { - consumerSpan(it, consumerProperties, trace(produceTraceIdx)[2]) + consumerSpan(it, consumerProperties, trace(1)[2]) } } - if (isDataStreamsEnabled()) { - trace(1, { - pollSpan(it) - }) - } trace(3) { basicSpan(it, "parent") basicSpan(it, "producer callback", span(0)) producerSpan(it, senderProps, span(0), false) } } + + // Verify trace context was propagated via message headers def headers = received.headers() headers.iterator().hasNext() - new String(headers.headers("x-datadog-trace-id").iterator().next().value()) == "${traces[produceTraceIdx][2].traceId}" - new String(headers.headers("x-datadog-parent-id").iterator().next().value()) == "${traces[produceTraceIdx][2].spanId}" - - if (isDataStreamsEnabled()) { - StatsGroup first = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == 0 } - verifyAll(first) { - tags.hasAllTags("direction:out", "kafka_cluster_id:$clusterId", "topic:$SHARED_TOPIC".toString(), "type:kafka") - } - - StatsGroup second = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == first.hash } - verifyAll(second) { - tags.hasAllTags( - "direction:in", - "group:sender", - "kafka_cluster_id:$clusterId", - "topic:$SHARED_TOPIC".toString(), - "type:kafka" - ) - } - def items = new ArrayList(TEST_DATA_STREAMS_WRITER.backlogs).sort { it.type + it.partition} - verifyAll(items) { - size() == 3 - get(0).hasAllTags( - "consumer_group:sender", - "kafka_cluster_id:$clusterId", - "partition:0", - "topic:" + SHARED_TOPIC, - "type:kafka_commit" - ) - get(1).hasAllTags( - "consumer_group:sender", - "kafka_cluster_id:$clusterId", - "partition:1", - "topic:" + SHARED_TOPIC, - "type:kafka_commit" - ) - get(2).hasAllTags( - "kafka_cluster_id:$clusterId", - "partition:" + received.partition(), - "topic:" + SHARED_TOPIC, - "type:kafka_produce" - ) - } - } cleanup: producer.close() @@ -335,17 +179,17 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { def "test producing message too large"() { setup: def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - // set a low max request size, so that we can crash it senderProps.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, 10) - Producer producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) + def producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) when: String greeting = "Hello Spring Kafka" - Future future = producer.send(new ProducerRecord(SHARED_TOPIC, greeting)) { meta, ex -> - } + def future = producer.send(new ProducerRecord(SHARED_TOPIC, greeting)) { meta, ex -> } future.get() + then: - thrown ExecutionException + thrown java.util.concurrent.ExecutionException + cleanup: producer.close() } @@ -358,38 +202,20 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { } def producerFactory = new DefaultKafkaProducerFactory(senderProps) def kafkaTemplate = new KafkaTemplate(producerFactory) - String clusterId = null - if (isDataStreamsEnabled()) { - clusterId = waitForKafkaMetadataUpdate(kafkaTemplate) - } - // set up the Kafka consumer properties def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - - // create a Kafka consumer factory def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - - // set the topic that needs to be consumed def containerProperties = containerProperties() - - // create a Kafka MessageListenerContainer def container = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties) - - // create a thread safe queue to store the received message def records = new LinkedBlockingQueue>() - // setup a Kafka message listener container.setupMessageListener(new MessageListener() { @Override void onMessage(ConsumerRecord record) { records.add(record) } }) - - // start the container and underlying message listener container.start() - - // wait until the container has the required number of assigned partitions ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()) when: @@ -402,14 +228,8 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { }) blockUntilChildSpansFinished(2) } - if (isDataStreamsEnabled()) { - TEST_DATA_STREAMS_WRITER.waitForGroups(2) - // wait for produce offset 0, commit offset 0 on partition 0 and 1, and commit offset 1 on 1 partition. - TEST_DATA_STREAMS_WRITER.waitForBacklogs(3) - } then: - // check that the message was received def received = records.poll(5, TimeUnit.SECONDS) received.value() == greeting received.key() == null @@ -432,61 +252,6 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { } } - // sort a snapshot so the producer trace is deterministically first, regardless of write order - def sortedTraces = new ArrayList<>(TEST_WRITER) - sortedTraces.sort(SORT_TRACES_BY_ID) - def headers = received.headers() - headers.iterator().hasNext() - new String(headers.headers("x-datadog-trace-id").iterator().next().value()) == "${sortedTraces[0][2].traceId}" - new String(headers.headers("x-datadog-parent-id").iterator().next().value()) == "${sortedTraces[0][2].spanId}" - - if (isDataStreamsEnabled()) { - StatsGroup first = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == 0 } - verifyAll(first) { - tags.hasAllTags( - "direction:out", - "kafka_cluster_id:$clusterId".toString(), - "topic:$SHARED_TOPIC".toString(), - "type:kafka" - ) - } - - StatsGroup second = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == first.hash } - verifyAll(second) { - tags.hasAllTags( - "direction:in", - "group:sender", - "kafka_cluster_id:$clusterId".toString(), - "topic:$SHARED_TOPIC".toString(), - "type:kafka" - ) - } - def items = new ArrayList(TEST_DATA_STREAMS_WRITER.backlogs).sort {it.type + it.partition} - verifyAll(items) { - size() == 3 - get(0).hasAllTags( - "consumer_group:sender", - "kafka_cluster_id:$clusterId".toString(), - "partition:0", - "topic:" + SHARED_TOPIC, - "type:kafka_commit" - ) - get(1).hasAllTags( - "consumer_group:sender", - "kafka_cluster_id:$clusterId".toString(), - "partition:1", - "topic:" + SHARED_TOPIC, - "type:kafka_commit" - ) - get(2).hasAllTags( - "kafka_cluster_id:$clusterId".toString(), - "partition:" + received.partition(), - "topic:" + SHARED_TOPIC, - "type:kafka_produce" - ) - } - } - cleanup: producerFactory.stop() container?.stop() @@ -498,43 +263,27 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { def producerFactory = new DefaultKafkaProducerFactory(senderProps) def kafkaTemplate = new KafkaTemplate(producerFactory) - // set up the Kafka consumer properties def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - - // create a Kafka consumer factory def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - - // set the topic that needs to be consumed def containerProperties = containerProperties() - - // create a Kafka MessageListenerContainer def container = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties) - - // create a thread safe queue to store the received message def records = new LinkedBlockingQueue>() - // setup a Kafka message listener container.setupMessageListener(new MessageListener() { @Override void onMessage(ConsumerRecord record) { records.add(record) } }) - - // start the container and underlying message listener container.start() - - // wait until the container has the required number of assigned partitions ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()) when: kafkaTemplate.send(SHARED_TOPIC, null) then: - // check that the message was received def received = records.poll(5, TimeUnit.SECONDS) received.value() == null - received.key() == null assertTraces(2, SORT_TRACES_BY_ID) { trace(1) { @@ -542,7 +291,7 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { } if (hasQueueSpan()) { trace(2) { - consumerSpan(it, consumerProperties, trace(1)[1], 0..0, true) + consumerSpan(it, consumerProperties, span(1), 0..0, true) queueSpan(it, trace(0)[0]) } } else { @@ -557,638 +306,53 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { container?.stop() } - def "test records(TopicPartition) kafka consume"() { - setup: - // set up the Kafka consumer properties - def kafkaPartition = 0 - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer(senderProps) - - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - - when: - def greeting = "Hello from MockConsumer!" - producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, greeting)) - - then: - TEST_WRITER.waitForTraces(1) - def pollResult = KafkaTestUtils.getRecords(consumer) - - def recs = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() - - def first = null - if (recs.hasNext()) { - first = recs.next() - } - - then: - recs.hasNext() == false - first.value() == greeting - first.key() == null - - assertTraces(2, SORT_TRACES_BY_ID) { - trace(1) { - producerSpan(it, senderProps) - } - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, trace(1)[1]) - queueSpan(it, trace(0)[0]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[0]) - } - } - } - - cleanup: - consumer.close() - producer.close() - } - - def "test records(TopicPartition).subList kafka consume"() { - setup: - // set up the Kafka consumer properties - def kafkaPartition = 0 - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer(senderProps) - - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - - when: - def greeting = "Hello from MockConsumer!" - producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, greeting)) - - then: - TEST_WRITER.waitForTraces(1) - def pollResult = KafkaTestUtils.getRecords(consumer) - - def records = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)) - def recs = records.subList(0, records.size()).iterator() - - def first = null - if (recs.hasNext()) { - first = recs.next() - } - - then: - recs.hasNext() == false - first.value() == greeting - first.key() == null - - assertTraces(2, SORT_TRACES_BY_ID) { - trace(1) { - producerSpan(it, senderProps) - } - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, trace(1)[1]) - queueSpan(it, trace(0)[0]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[0]) - } - } - } - - cleanup: - consumer.close() - producer.close() - } - - def "test records(TopicPartition).forEach kafka consume"() { - setup: - // set up the Kafka consumer properties - def kafkaPartition = 0 - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer(senderProps) - - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - - when: - def greeting = "Hello from MockConsumer!" - producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, greeting)) - - then: - TEST_WRITER.waitForTraces(1) - def pollResult = KafkaTestUtils.getRecords(consumer) - - def records = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)) - - def last = null - records.forEach { - last = it - assert activeSpan() != null - } - - then: - records.size() == 1 - last.value() == greeting - last.key() == null - - assertTraces(2, SORT_TRACES_BY_ID) { - trace(1) { - producerSpan(it, senderProps) - } - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, trace(1)[1]) - queueSpan(it, trace(0)[0]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[0]) - } - } - } - - cleanup: - consumer.close() - producer.close() - } - - def "test iteration backwards over ConsumerRecords"() { + def "test kafka consume with existing headers does not overwrite"() { setup: - def kafkaPartition = 0 - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer(senderProps) - - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - - when: - def greetings = ["msg 1", "msg 2", "msg 3"] - greetings.each { - producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, it)) - } - - then: - TEST_WRITER.waitForTraces(3) - def pollRecords = KafkaTestUtils.getRecords(consumer) - - def listIter = - pollRecords.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).listIterator() - - then: - def receivedSet = greetings.toSet() - while (listIter.hasNext()) { - listIter.next() - } - while (listIter.hasPrevious()) { - def record = listIter.previous() - receivedSet.remove(record.value()) - assert record.key() == null - } - receivedSet.isEmpty() - - assertTraces(9, SORT_TRACES_BY_ID) { - - // producing traces - trace(1) { - producerSpan(it, senderProps) - } - trace(1) { - producerSpan(it, senderProps) - } - trace(1) { - producerSpan(it, senderProps) - } - - // iterating to the end of ListIterator: - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, trace(3)[1], 0..0) - queueSpan(it, trace(0)[0]) - } - trace(2) { - consumerSpan(it, consumerProperties, trace(4)[1], 1..1) - queueSpan(it, trace(1)[0]) - } - trace(2) { - consumerSpan(it, consumerProperties, trace(5)[1], 2..2) - queueSpan(it, trace(2)[0]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[0], 0..0) - } - trace(1) { - consumerSpan(it, consumerProperties, trace(1)[0], 1..1) - } - trace(1) { - consumerSpan(it, consumerProperties, trace(2)[0], 2..2) - } - } - - // backwards iteration over ListIterator to the beginning - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, trace(6)[1], 2..2) - queueSpan(it, trace(2)[0]) - } - trace(2) { - consumerSpan(it, consumerProperties, trace(7)[1], 1..1) - queueSpan(it, trace(1)[0]) - } - trace(2) { - consumerSpan(it, consumerProperties, trace(8)[1], 0..0) - queueSpan(it, trace(0)[0]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(2)[0], 2..2) - } - trace(1) { - consumerSpan(it, consumerProperties, trace(1)[0], 1..1) - } - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[0], 0..0) - } - } - } - - cleanup: - consumer.close() - producer.close() - } - - def "test spring kafka template produce and batch consume"() { - setup: - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - if (isDataStreamsEnabled()) { - senderProps.put(ProducerConfig.METADATA_MAX_AGE_CONFIG, 1000) - } - def producerFactory = new DefaultKafkaProducerFactory(senderProps) - def kafkaTemplate = new KafkaTemplate(producerFactory) - String clusterId = null - if (isDataStreamsEnabled()) { - clusterId = waitForKafkaMetadataUpdate(kafkaTemplate) - } - - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - def containerProperties = containerProperties() - - - def container = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties) - def records = new LinkedBlockingQueue>() - container.setupMessageListener(new BatchMessageListener() { - @Override - void onMessage(List> consumerRecords) { - consumerRecords.each { - records.add(it) - } - } - }) - container.start() - ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()) - - when: - List greetings = ["msg 1", "msg 2", "msg 3"] - runUnderTrace("parent") { - for (g in greetings) { - kafkaTemplate.send(SHARED_TOPIC, g).addCallback({ - runUnderTrace("producer callback") {} - }, { - ex -> - runUnderTrace("producer exception: " + ex) {} - }) - } - blockUntilChildSpansFinished(2 * greetings.size()) - } - if (isDataStreamsEnabled()) { - TEST_DATA_STREAMS_WRITER.waitForGroups(2) - // wait for produce offset 0, commit offset 0 on partition 0 and 1, and commit offset 1 on 1 partition. - TEST_DATA_STREAMS_WRITER.waitForBacklogs(4) - } - - then: - def receivedSet = greetings.toSet() - def receivedRecords = [] - greetings.eachWithIndex { - g, i -> - def received = records.poll(5, TimeUnit.SECONDS) - receivedSet.remove(received.value()) //maybe received out of order in case several partitions - assert received.key() == null - - def headers = received.headers() - assert headers.iterator().hasNext() - receivedRecords.add(received) - } - assert receivedSet.isEmpty() - - TEST_WRITER.waitForTraces(4) - def traces = Arrays.asList(TEST_WRITER.toArray()) as List> - def produceSpans = producerSpans(traces) - def spanIdToRecord = receivedRecords.collectEntries { - record -> - def header = record.headers().headers("x-datadog-parent-id").iterator() - assert header.hasNext() - [(Long.parseLong(new String(header.next().value(), StandardCharsets.UTF_8))): record] - } - - // Batch listener delivery order can vary; match each consumer trace to its producer via the propagated parent ID. - assertTraces(4, new SortBatchKafkaTraces()) { - trace(7) { - basicSpan(it, "parent") - basicSpan(it, "producer callback", span(0)) - producerSpan(it, senderProps, span(0), false) - basicSpan(it, "producer callback", span(0)) - producerSpan(it, senderProps, span(0), false) - basicSpan(it, "producer callback", span(0)) - producerSpan(it, senderProps, span(0), false) - } - - if (hasQueueSpan()) { - [0, 1, 2].each { - i -> - def expectedOffset = spanIdToRecord[produceSpans[i].spanId].offset() - trace(2) { - consumerSpan(it, consumerProperties, span(1), expectedOffset..expectedOffset) - queueSpan(it, produceSpans[i]) - } - } - } else { - [0, 1, 2].each { - i -> - def expectedOffset = spanIdToRecord[produceSpans[i].spanId].offset() - trace(1) { - consumerSpan(it, consumerProperties, produceSpans[i], expectedOffset..expectedOffset) - } - } - } - } - - if (isDataStreamsEnabled()) { - StatsGroup first = TEST_DATA_STREAMS_WRITER.groups.find { - it.parentHash == 0 - } - verifyAll(first) { - tags.hasAllTags("direction:out", "kafka_cluster_id:$clusterId", "topic:$SHARED_TOPIC".toString(), "type:kafka") - } - - StatsGroup second = TEST_DATA_STREAMS_WRITER.groups.find { - it.parentHash == first.hash - } - verifyAll(second) { - tags.hasAllTags( - "direction:in", - "group:sender", - "kafka_cluster_id:$clusterId", - "topic:$SHARED_TOPIC".toString(), - "type:kafka" - ) - } - } - - cleanup: - producerFactory.stop() - container?.stop() - } - - def "test kafka client header propagation manual config"() { - setup: - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producerFactory = new DefaultKafkaProducerFactory(senderProps) - def kafkaTemplate = new KafkaTemplate(producerFactory) + KafkaProducer producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) - // set up the Kafka consumer properties def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - - // create a Kafka consumer factory def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - - // set the topic that needs to be consumed def containerProperties = containerProperties() - - // create a Kafka MessageListenerContainer def container = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties) - - // create a thread safe queue to store the received message def records = new LinkedBlockingQueue>() - // setup a Kafka message listener container.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records.add(record) - if (isDataStreamsEnabled()) { - // even if header propagation is disabled, we want data streams to work. - TEST_DATA_STREAMS_WRITER.waitForGroups(2) + @Override + void onMessage(ConsumerRecord record) { + records.add(record) } - } - }) - - // start the container and underlying message listener + }) container.start() - - // wait until the container has the required number of assigned partitions ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()) when: - String message = "Testing without headers" - injectSysConfig("kafka.client.propagation.enabled", value) - kafkaTemplate.send(SHARED_TOPIC, message) + def producerRecord = new ProducerRecord(SHARED_TOPIC, "test-with-headers") + producerRecord.headers().add("x-datadog-trace-id", "1".getBytes()) + producerRecord.headers().add("x-datadog-parent-id", "1".getBytes()) + runUnderTrace("parent") { + producer.send(producerRecord) { meta, ex -> } + blockUntilChildSpansFinished(1) + } then: - // check that the message was received def received = records.poll(5, TimeUnit.SECONDS) - - received.headers().iterator().hasNext() == expected + received.value() == "test-with-headers" + // The instrumentation should have replaced the existing headers with new trace context + def traceIdHeader = received.headers().headers("x-datadog-trace-id").iterator().next() + new String(traceIdHeader.value()) != "1" cleanup: - producerFactory.stop() + producer.close() container?.stop() - - where: - value | expected - "false" | false - "true" | true - } - - def "test producer extracts and uses existing trace context from record headers"() { - setup: - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer<>(senderProps) - - def existingTraceId = 1234567890123456L - def existingSpanId = 9876543210987654L - def headers = new RecordHeaders() - headers.add(new RecordHeader("x-datadog-trace-id", - String.valueOf(existingTraceId).getBytes(StandardCharsets.UTF_8))) - headers.add(new RecordHeader("x-datadog-parent-id", - String.valueOf(existingSpanId).getBytes(StandardCharsets.UTF_8))) - - when: - def record = new ProducerRecord(SHARED_TOPIC, 0, null, "test-context-extraction", headers) - producer.send(record).get() - - then: - TEST_WRITER.waitForTraces(1) - def producedSpan = TEST_WRITER[0][0] - // Verify the span used the extracted context as parent - producedSpan.traceId.toLong() == existingTraceId - producedSpan.parentId == existingSpanId - // Verify a new span was created (not reusing the extracted span ID) - producedSpan.spanId != existingSpanId - - cleanup: - producer?.close() - } - - def "test producer DSM transaction tracking extracts transaction id from headers"() { - setup: - if (!isDataStreamsEnabled()) { - return - } - - injectEnvConfig("DD_DATA_STREAMS_ENABLED", "true") - - // Configure a DSM transaction extractor for KAFKA_PRODUCE_HEADERS - def extractorsByTypeField = TEST_DATA_STREAMS_MONITORING.getClass().getDeclaredField("extractorsByType") - extractorsByTypeField.setAccessible(true) - def oldExtractorsByType = extractorsByTypeField.get(TEST_DATA_STREAMS_MONITORING) - - def extractor = new DataStreamsTransactionExtractor() { - String getName() { - return "kafka-produce-test" - } - DataStreamsTransactionExtractor.Type getType() { - return DataStreamsTransactionExtractor.Type.KAFKA_PRODUCE_HEADERS - } - String getValue() { - return "x-transaction-id" - } - } - def extractorsByType = new EnumMap<>(DataStreamsTransactionExtractor.Type) - extractorsByType.put(DataStreamsTransactionExtractor.Type.KAFKA_PRODUCE_HEADERS, [extractor]) - extractorsByTypeField.set(TEST_DATA_STREAMS_MONITORING, extractorsByType) - - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) - - def headers = new RecordHeaders() - headers.add(new RecordHeader("x-transaction-id", "txn-123".getBytes(StandardCharsets.UTF_8))) - - when: - def record = new ProducerRecord(SHARED_TOPIC, 0, null, "test-dsm-transaction", headers) - producer.send(record).get() - - then: - TEST_WRITER.waitForTraces(1) - def producedSpan = TEST_WRITER[0][0] - producedSpan.getTag(Tags.DSM_TRANSACTION_ID) == "txn-123" - producedSpan.getTag(Tags.DSM_TRANSACTION_CHECKPOINT) == "kafka-produce-test" - - cleanup: - extractorsByTypeField?.set(TEST_DATA_STREAMS_MONITORING, oldExtractorsByType) - producer?.close() - } - - def "test consumer DSM transaction tracking extracts transaction id from headers"() { - setup: - if (!isDataStreamsEnabled()) { - return - } - - injectEnvConfig("DD_DATA_STREAMS_ENABLED", "true") - - // Configure a DSM transaction extractor for KAFKA_CONSUME_HEADERS - def extractorsByTypeField = TEST_DATA_STREAMS_MONITORING.getClass().getDeclaredField("extractorsByType") - extractorsByTypeField.setAccessible(true) - def oldExtractorsByType = extractorsByTypeField.get(TEST_DATA_STREAMS_MONITORING) - - def extractor = new DataStreamsTransactionExtractor() { - String getName() { - return "kafka-consume-test" - } - DataStreamsTransactionExtractor.Type getType() { - return DataStreamsTransactionExtractor.Type.KAFKA_CONSUME_HEADERS - } - String getValue() { - return "x-transaction-id" - } - } - def extractorsByType = new EnumMap<>(DataStreamsTransactionExtractor.Type) - extractorsByType.put(DataStreamsTransactionExtractor.Type.KAFKA_CONSUME_HEADERS, [extractor]) - extractorsByTypeField.set(TEST_DATA_STREAMS_MONITORING, extractorsByType) - - def kafkaPartition = 0 - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer<>(senderProps, new StringSerializer(), new StringSerializer()) - - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - - def headers = new RecordHeaders() - headers.add(new RecordHeader("x-transaction-id", "txn-456".getBytes(StandardCharsets.UTF_8))) - - when: - def record = new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, "test-dsm-consume-transaction", headers) - producer.send(record).get() - - then: - TEST_WRITER.waitForTraces(1) - def pollResult = KafkaTestUtils.getRecords(consumer) - def recs = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() - recs.hasNext() - recs.next().value() == "test-dsm-consume-transaction" - !recs.hasNext() - - // The consume span is created by TracingIterator when iterating over records - // Find the consumer span with the DSM transaction tags - TEST_WRITER.waitForTraces(2) - def allTraces = TEST_WRITER.toArray() as List> - def consumerSpan = allTraces.collectMany { - it - }.find { - it.getTag(Tags.DSM_TRANSACTION_ID) == "txn-456" - } - consumerSpan != null - consumerSpan.getTag(Tags.DSM_TRANSACTION_ID) == "txn-456" - consumerSpan.getTag(Tags.DSM_TRANSACTION_CHECKPOINT) == "kafka-consume-test" - - cleanup: - extractorsByTypeField?.set(TEST_DATA_STREAMS_MONITORING, oldExtractorsByType) - consumer?.close() - producer?.close() - } - - def containerProperties() { - try { - // Different class names for test and latestDepTest. - return Class.forName("org.springframework.kafka.listener.config.ContainerProperties").newInstance(SHARED_TOPIC) - } catch (ClassNotFoundException | NoClassDefFoundError e) { - return Class.forName("org.springframework.kafka.listener.ContainerProperties").newInstance(SHARED_TOPIC) - } } def producerSpan( - TraceAssert trace, - Map config, - DDSpan parentSpan = null, - boolean partitioned = true, - boolean tombstone = false + TraceAssert trace, + Map config, + DDSpan parentSpan = null, + boolean partitioned = true, + boolean tombstone = false ) { trace.span { serviceName service() @@ -1202,10 +366,10 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { } else { parent() } - final boolean isV0 = version() == 0 tags { "$Tags.COMPONENT" "java-kafka" "$Tags.SPAN_KIND" Tags.SPAN_KIND_PRODUCER + "$InstrumentationTags.MESSAGING_SYSTEM" "kafka" "$InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS" config.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG) "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$SHARED_TOPIC" "$InstrumentationTags.PARTITION" { it >= 0 } @@ -1218,19 +382,14 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { "$DDTags.PATHWAY_HASH" { String } } peerServiceFrom(InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS) - if (isV0) { - // in v0 the service name is always set to DD_SERVICE while it should just be unset as v1 - // this is a buggy behaviour that could not be easily fixed. - serviceNameSource "java-kafka" - } defaultTags() } } } def queueSpan( - TraceAssert trace, - DDSpan parentSpan = null + TraceAssert trace, + DDSpan parentSpan = null ) { trace.span { serviceName splitByDestination() ? "$SHARED_TOPIC" : serviceForTimeInQueue() @@ -1253,12 +412,12 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { } def consumerSpan( - TraceAssert trace, - Map config, - DDSpan parentSpan = null, - Range offset = 0..0, - boolean tombstone = false, - boolean distributedRootSpan = !hasQueueSpan() + TraceAssert trace, + Map config, + DDSpan parentSpan = null, + Range offset = 0..0, + boolean tombstone = false, + boolean distributedRootSpan = !hasQueueSpan() ) { trace.span { serviceName service() @@ -1272,10 +431,10 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { } else { parent() } - final boolean isV0 = version() == 0 tags { "$Tags.COMPONENT" "java-kafka" "$Tags.SPAN_KIND" Tags.SPAN_KIND_CONSUMER + "$InstrumentationTags.MESSAGING_SYSTEM" "kafka" "$InstrumentationTags.PARTITION" { it >= 0 } "$InstrumentationTags.OFFSET" { offset.containsWithinBounds(it as int) } "$InstrumentationTags.CONSUMER_GROUP" "sender" @@ -1290,23 +449,15 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { if ({ isDataStreamsEnabled() }) { "$DDTags.PATHWAY_HASH" { String } } - if (isV0) { - // in v0 the service name is always set to DD_SERVICE while it should just be unset as v1 - // this is a buggy behaviour that could not be easily fixed. - serviceNameSource "java-kafka" - } defaultTags(distributedRootSpan) } } } def pollSpan( - TraceAssert trace, - int recordCount = 1, - DDSpan parentSpan = null, - Range offset = 0..0, - boolean tombstone = false, - boolean distributedRootSpan = !hasQueueSpan() + TraceAssert trace, + int recordCount = 1, + DDSpan parentSpan = null ) { trace.span { serviceName Config.get().getServiceName() @@ -1320,20 +471,6 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { } } } - - def waitForKafkaMetadataUpdate(KafkaTemplate kafkaTemplate) { - kafkaTemplate.flush() - Producer wrappedProducer = kafkaTemplate.getTheProducer() - assert (wrappedProducer instanceof DefaultKafkaProducerFactory.CloseSafeProducer) - Producer producer = wrappedProducer.delegate - assert (producer instanceof KafkaProducer) - String clusterId = producer.metadata.cluster.clusterResource().clusterId() - while (clusterId == null || clusterId.isEmpty()) { - Thread.sleep(1500) - clusterId = producer.metadata.cluster.clusterResource().clusterId() - } - return clusterId - } } abstract class KafkaClientForkedTest extends KafkaClientTestBase { @@ -1497,50 +634,3 @@ class KafkaClientDataStreamsDisabledForkedTest extends KafkaClientTestBase { return false } } - -class KafkaClientContextSwapForkedTest extends KafkaClientV0ForkedTest { - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig(TraceInstrumentationConfig.LEGACY_CONTEXT_MANAGER_ENABLED, "false") - } -} - -class KafkaClientBadBase64HeaderForkedTest extends InstrumentationSpecification { - KafkaEmbedded embeddedKafka - - def setup() { - embeddedKafka = new KafkaEmbedded(1, true, KafkaClientTestBase.SHARED_TOPIC) - embeddedKafka.before() - } - - def cleanup() { - embeddedKafka?.after() - } - - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig(TraceInstrumentationConfig.KAFKA_CLIENT_BASE64_DECODING_ENABLED, "true") - injectSysConfig(TracerConfig.HEADER_TAGS, "x-custom-header:my.custom.tag") - } - - def "producer span is created when message carries non-Base64 headers and base64 decoding is enabled"() { - setup: - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer(senderProps, new StringSerializer(), new StringSerializer()) - - when: - def headers = new RecordHeaders([ - new RecordHeader("x-custom-header", "not-valid-base64!@#".getBytes(StandardCharsets.UTF_8)), - new RecordHeader("x-another-header", "also-not-base64!!".getBytes(StandardCharsets.UTF_8)) - ]) - producer.send(new ProducerRecord<>(KafkaClientTestBase.SHARED_TOPIC, 0, null, "hello", headers)).get() - - then: - TEST_WRITER.waitForTraces(1) - !TEST_WRITER.isEmpty() - - cleanup: - producer?.close() - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaReactorForkedTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaReactorForkedTest.groovy deleted file mode 100644 index 49ec8900712..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaReactorForkedTest.groovy +++ /dev/null @@ -1,239 +0,0 @@ -import static datadog.trace.agent.test.utils.TraceUtils.basicSpan -import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace - -import datadog.trace.agent.test.InstrumentationSpecification -import datadog.trace.agent.test.asserts.TraceAssert -import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags -import datadog.trace.bootstrap.instrumentation.api.Tags -import datadog.trace.core.DDSpan -import org.apache.kafka.clients.consumer.ConsumerConfig -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.apache.kafka.clients.producer.ProducerConfig -import org.apache.kafka.clients.producer.ProducerRecord -import org.springframework.kafka.test.rule.KafkaEmbedded -import org.springframework.kafka.test.utils.KafkaTestUtils -import reactor.core.publisher.Flux -import reactor.core.publisher.Mono -import reactor.core.scheduler.Schedulers -import reactor.kafka.receiver.KafkaReceiver -import reactor.kafka.receiver.ReceiverOptions -import reactor.kafka.sender.KafkaSender -import reactor.kafka.sender.SenderOptions -import reactor.kafka.sender.SenderRecord - -import java.util.concurrent.CountDownLatch -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.TimeUnit - -class KafkaReactorForkedTest extends InstrumentationSpecification { - // create 4 partitions for more parallelism - KafkaEmbedded embeddedKafka - - @Override - boolean useStrictTraceWrites() { - false - } - - def setup() { - embeddedKafka = new KafkaEmbedded(1, true, 4, KafkaClientTestBase.SHARED_TOPIC) - embeddedKafka.before() - TEST_WRITER.setFilter(KafkaClientTestBase.DROP_KAFKA_POLL) - } - - def cleanup() { - embeddedKafka?.after() - } - - def "test reactive produce and consume"() { - setup: - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - - def kafkaSender = KafkaSender.create(SenderOptions.create(senderProps)) - // set up the Kafka consumer properties - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - def subscriptionReady = new CountDownLatch(embeddedKafka.getPartitionsPerTopic()) - - final KafkaReceiver kafkaReceiver = KafkaReceiver.create(ReceiverOptions. create(consumerProperties) - .subscription([KafkaClientTestBase.SHARED_TOPIC]) - .addAssignListener { - it.each { - subscriptionReady.countDown() - } - }) - - // create a thread safe queue to store the received message - def records = new LinkedBlockingQueue>() - kafkaReceiver.receive() - // publish on another thread to be sure we're propagating that receive span correctly - .publishOn(Schedulers.parallel()) - .flatMap { - receiverRecord -> { - records.add(receiverRecord) - receiverRecord.receiverOffset().commit() - } - }.subscribe() - - - // wait until the container has the required number of assigned partitions - subscriptionReady.await() - - when: - String greeting = "Hello Reactor Kafka Sender!" - runUnderTrace("parent") { - kafkaSender.send(Mono.just(SenderRecord.create(new ProducerRecord<>(KafkaClientTestBase.SHARED_TOPIC, greeting), null))) - .doOnError { - ex -> runUnderTrace("producer exception: " + ex) {} - } - .doOnNext { - runUnderTrace("producer callback") {} - } - .blockFirst() - blockUntilChildSpansFinished(2) - } - then: - // check that the message was received - def received = records.poll(5, TimeUnit.SECONDS) - received.value() == greeting - received.key() == null - - - assertTraces(2, SORT_TRACES_BY_START) { - trace(3) { - basicSpan(it, "parent") - basicSpan(it, "producer callback", span(0)) - producerSpan(it, senderProps, span(0)) - } - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[2]) - } - } - } - - def "test reactive 100 msg produce and consume have only one parent"() { - setup: - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - if (isDataStreamsEnabled()) { - senderProps.put(ProducerConfig.METADATA_MAX_AGE_CONFIG, 1000) - } - - def kafkaSender = KafkaSender.create(SenderOptions.create(senderProps)) - // set up the Kafka consumer properties - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - def subscriptionReady = new CountDownLatch(embeddedKafka.getPartitionsPerTopic()) - - final KafkaReceiver kafkaReceiver = KafkaReceiver.create(ReceiverOptions. create(consumerProperties) - .subscription([KafkaClientTestBase.SHARED_TOPIC]) - .addAssignListener { - it.each { - subscriptionReady.countDown() - } - }) - - // create a thread safe queue to store the received message - kafkaReceiver.receive() - // publish on another thread to be sure we're propagating that receive span correctly - .publishOn(Schedulers.parallel()) - .flatMap { - receiverRecord -> { - receiverRecord.receiverOffset().commit() - } - } - .subscribeOn(Schedulers.parallel()) - .subscribe() - - - // wait until the container has the required number of assigned partitions - subscriptionReady.await() - - when: - String greeting = "Hello Reactor Kafka Sender!" - Flux.range(0, 100) - .flatMap { - kafkaSender.send(Mono.just(SenderRecord.create(new ProducerRecord<>(KafkaClientTestBase.SHARED_TOPIC, greeting), null))) - } - .publishOn(Schedulers.parallel()) - .subscribe() - then: - // check that the all the consume (100) and the send (100) are reported - TEST_WRITER.waitForTraces(200) - Map> traces = TEST_WRITER.inject([:]) { - map, entry -> - def key = entry.get(0).getTraceId().toString() - map[key] = (map[key] ?: []) + entry - return map - } - traces.values().each { - assert it.size() == 2 - int produceIndex = 0 - int consumeIndex = 1 - if ("kafka.produce".contentEquals(it.get(1).getOperationName().toString())) { - produceIndex = 1 - consumeIndex = 0 - } - //assert that the consumer has the producer as parent and that the producer is root - assert it.get(consumeIndex).getParentId() == it.get(produceIndex).getSpanId() - assert it.get(produceIndex).getParentId() == 0 - } - } - - def producerSpan( - TraceAssert trace, - Map config, - DDSpan parentSpan = null) { - trace.span { - serviceName "kafka" - operationName "kafka.produce" - resourceName "Produce Topic $KafkaClientTestBase.SHARED_TOPIC" - spanType "queue" - errored false - measured true - if (parentSpan) { - childOf parentSpan - } else { - parent() - } - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_PRODUCER - "$InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS" config.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG) - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$KafkaClientTestBase.SHARED_TOPIC" - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" { it >= 0 } - "$InstrumentationTags.KAFKA_CLUSTER_ID" { String } - peerServiceFrom(InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS) - defaultTags() - } - } - } - - def consumerSpan( - TraceAssert trace, - Map config, - DDSpan parentSpan = null) { - trace.span { - serviceName "kafka" - operationName "kafka.consume" - resourceName "Consume Topic $KafkaClientTestBase.SHARED_TOPIC" - spanType "queue" - errored false - measured true - if (parentSpan) { - childOf parentSpan - } else { - parent() - } - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_CONSUMER - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" { Integer } - "$InstrumentationTags.CONSUMER_GROUP" "sender" - "$InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS" config.get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG) - "$InstrumentationTags.KAFKA_CLUSTER_ID" { String } - "$InstrumentationTags.RECORD_QUEUE_TIME_MS" { it >= 0 } - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$KafkaClientTestBase.SHARED_TOPIC" - defaultTags(true) - } - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/iast/KafkaIastDeserializerForkedTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/iast/KafkaIastDeserializerForkedTest.groovy deleted file mode 100644 index d8dfeabce50..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/iast/KafkaIastDeserializerForkedTest.groovy +++ /dev/null @@ -1,139 +0,0 @@ -package iast - -import com.datadog.iast.propagation.PropagationModuleImpl -import com.datadog.iast.test.IastRequestTestRunner -import datadog.trace.api.iast.InstrumentationBridge -import datadog.trace.api.iast.SourceTypes -import org.apache.kafka.common.serialization.ByteArrayDeserializer -import org.apache.kafka.common.serialization.ByteBufferDeserializer -import org.apache.kafka.common.serialization.StringDeserializer -import org.springframework.kafka.support.serializer.JsonDeserializer - -import java.nio.ByteBuffer - -import static org.hamcrest.CoreMatchers.equalTo -import static org.hamcrest.CoreMatchers.instanceOf - -class KafkaIastDeserializerForkedTest extends IastRequestTestRunner { - - void 'test string deserializer'() { - given: - final propagationModule = new PropagationModuleImpl() - InstrumentationBridge.registerIastModule(propagationModule) - - and: - final payload = "Hello World!".bytes - final deserializer = new StringDeserializer() - - when: - runUnderIastTrace { - deserializer.configure([:], origin == SourceTypes.KAFKA_MESSAGE_KEY) - deserializer.deserialize("test", payload) - } - - then: - final to = finReqTaintedObjects - to.hasTaintedObject { - value('Hello World!') - range(0, 12, source(origin as byte)) - } - - where: - origin | _ - SourceTypes.KAFKA_MESSAGE_KEY | _ - SourceTypes.KAFKA_MESSAGE_VALUE | _ - } - - void 'test byte array deserializer'() { - given: - final propagationModule = new PropagationModuleImpl() - InstrumentationBridge.registerIastModule(propagationModule) - - and: - final payload = "Hello World!".bytes - final deserializer = new ByteArrayDeserializer() - - when: - runUnderIastTrace { - deserializer.configure([:], origin == SourceTypes.KAFKA_MESSAGE_KEY) - deserializer.deserialize("test", payload) - } - - then: - final to = finReqTaintedObjects - to.hasTaintedObject { - value(equalTo(payload)) - range(0, Integer.MAX_VALUE, source(origin as byte)) - } - - where: - origin | _ - SourceTypes.KAFKA_MESSAGE_KEY | _ - SourceTypes.KAFKA_MESSAGE_VALUE | _ - } - - void 'test byte buffer deserializer'() { - given: - final propagationModule = new PropagationModuleImpl() - InstrumentationBridge.registerIastModule(propagationModule) - - and: - final payload = "Hello World!".bytes - final deserializer = new ByteBufferDeserializer() - - when: - runUnderIastTrace { - deserializer.configure([:], origin == SourceTypes.KAFKA_MESSAGE_KEY) - deserializer.deserialize("test", payload) - } - - then: - final to = finReqTaintedObjects - to.hasTaintedObject { - value(instanceOf(ByteBuffer)) - range(0, Integer.MAX_VALUE, source(origin as byte)) - } - - where: - origin | _ - SourceTypes.KAFKA_MESSAGE_KEY | _ - SourceTypes.KAFKA_MESSAGE_VALUE | _ - } - - void 'test json deserialization'() { - given: - final propagationModule = new PropagationModuleImpl() - InstrumentationBridge.registerIastModule(propagationModule) - - and: - final json = '{ "name": "Mr Bean" }' - final payload = json.bytes - final deserializer = new JsonDeserializer(TestBean) - - when: - runUnderIastTrace { - deserializer.configure([:], origin == SourceTypes.KAFKA_MESSAGE_KEY) - deserializer.deserialize('test', payload) - } - - then: - final to = finReqTaintedObjects - to.hasTaintedObject { - value(instanceOf(TestBean)) - range(0, Integer.MAX_VALUE, source(origin as byte)) - } - to.hasTaintedObject { - value('Mr Bean') - range(0, 7, source(origin as byte, 'name', 'Mr Bean')) - } - - where: - origin | _ - SourceTypes.KAFKA_MESSAGE_KEY | _ - SourceTypes.KAFKA_MESSAGE_VALUE | _ - } - - static class TestBean { - String name - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/build.gradle b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/build.gradle deleted file mode 100644 index 4a32748dbb9..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/build.gradle +++ /dev/null @@ -1,68 +0,0 @@ -plugins { - id 'dd-trace-java.module.instrumentation' -} - -muzzle { - extraRepository("confluent-releases", "https://packages.confluent.io/maven/") - - pass { - group = "org.apache.kafka" - module = "kafka-clients" - versions = "[3.8.0,5),[7.8.0,)" - assertInverse = true - javaVersion = "17" - } -} - -tracerJava { - addSourceSetFor(JavaVersion.VERSION_17) -} - -addTestSuiteForDir('latestDepTest', 'test') -addTestSuiteExtendingForDir('latestDepForkedTest', 'latestDepTest', 'test') - -["compileMain_java17Java", "compileTestJava", "compileLatestDepTestJava"].each { - tasks.named(it, JavaCompile) { - configureCompiler(it, 17, JavaVersion.VERSION_1_8) - } -} - -tasks.withType(JavaCompile).configureEach { - configureCompiler(it, 17, JavaVersion.VERSION_1_8) -} - -tasks.withType(GroovyCompile).configureEach { - configureCompiler(it, 17) -} - -dependencies { - implementation project(':dd-java-agent:instrumentation:kafka:kafka-common') - implementation project(':dd-java-agent:instrumentation:datadog:dynamic-instrumentation:span-origin') - - main_java17CompileOnly group: 'org.apache.kafka', name: 'kafka-clients', version: '3.8.0' - main_java17Implementation project(':dd-java-agent:instrumentation:kafka:kafka-common') - - testImplementation group: 'org.springframework.kafka', name: 'spring-kafka', version: '3.3.4', { - exclude group: 'org.apache.kafka' - } - testImplementation group: 'org.springframework.kafka', name: 'spring-kafka-test', version: '3.3.4', { - exclude group: 'org.apache.kafka' - } - - testImplementation 'org.apache.kafka:kafka-server-common:3.8.0:test' - - testImplementation 'org.apache.kafka:kafka-clients:3.8.0' - testImplementation 'org.apache.kafka:kafka-clients:3.8.0:test' - - testImplementation 'org.apache.kafka:kafka_2.13:3.8.0' - testImplementation 'org.apache.kafka:kafka_2.13:3.8.0:test' - - testImplementation group: 'javax.xml.bind', name: 'jaxb-api', version: '2.2.3' - testImplementation group: 'org.mockito', name: 'mockito-core', version: '2.19.0' - testRuntimeOnly project(':dd-java-agent:instrumentation:spring:spring-scheduling-3.1') - - latestDepTestImplementation group: 'org.springframework.kafka', name: 'spring-kafka', version: '3.+' - latestDepTestImplementation group: 'org.springframework.kafka', name: 'spring-kafka-test', version: '3.+' - latestDepTestImplementation group: 'io.dropwizard.metrics', name: 'metrics-core', version: '+' - latestDepTestImplementation libs.guava -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/gradle.lockfile b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/gradle.lockfile deleted file mode 100644 index e34e0dcd377..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/gradle.lockfile +++ /dev/null @@ -1,244 +0,0 @@ -# This is a Gradle generated file for dependency locking. -# Manual edits can break the build and are not advised. -# This file is expected to be part of source control. -# To regenerate this file, run: ./gradlew :dd-java-agent:instrumentation:kafka:kafka-clients-3.8:dependencies --write-locks -cafe.cryptography:curve25519-elisabeth:0.1.0=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -cafe.cryptography:ed25519-elisabeth:0.1.0=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -ch.qos.logback:logback-classic:1.2.13=testCompileClasspath,testRuntimeClasspath -ch.qos.logback:logback-classic:1.3.15=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -ch.qos.logback:logback-core:1.2.13=testCompileClasspath,testRuntimeClasspath -ch.qos.logback:logback-core:1.3.15=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -com.blogspot.mydailyjava:weak-lock-free:0.17=buildTimeInstrumentationPlugin,compileClasspath,latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,main_java17CompileClasspath,main_java17RuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq.okhttp3:okhttp:3.12.15=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq.okio:okio:1.17.6=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:dd-instrument-java:0.0.4=buildTimeInstrumentationPlugin,compileClasspath,latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,main_java17CompileClasspath,main_java17RuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:dd-javac-plugin-client:0.2.2=buildTimeInstrumentationPlugin,compileClasspath,latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,main_java17CompileClasspath,main_java17RuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:java-dogstatsd-client:4.4.5=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.datadoghq:sketches-java:0.8.3=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-annotations:2.16.2=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-core:2.16.2=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-databind:2.16.2=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.16.2=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.16.2=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.fasterxml.jackson.module:jackson-module-scala_2.13:2.16.2=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.fasterxml.jackson:jackson-bom:2.16.2=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.ben-manes.caffeine:caffeine:2.9.3=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.javaparser:javaparser-core:3.25.6=codenarc -com.github.jnr:jffi:1.3.15=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-a64asm:1.0.0=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-constants:0.10.4=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-enxio:0.32.20=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-ffi:2.2.19=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-posix:3.1.22=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-unixsocket:0.38.25=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-x86asm:1.0.2=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.luben:zstd-jni:1.5.6-3=testRuntimeClasspath -com.github.luben:zstd-jni:1.5.6-4=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath -com.github.spotbugs:spotbugs-annotations:4.10.3=compileClasspath,spotbugs -com.github.spotbugs:spotbugs:4.10.3=spotbugs -com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs -com.google.auto.service:auto-service-annotations:1.1.1=annotationProcessor,compileClasspath,latestDepForkedTestAnnotationProcessor,latestDepForkedTestCompileClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,testAnnotationProcessor,testCompileClasspath -com.google.auto.service:auto-service:1.1.1=annotationProcessor,latestDepForkedTestAnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.auto:auto-common:1.2.1=annotationProcessor,latestDepForkedTestAnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,compileClasspath,latestDepForkedTestAnnotationProcessor,latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,spotbugs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath -com.google.code.gson:gson:2.14.0=spotbugs -com.google.errorprone:error_prone_annotations:2.18.0=annotationProcessor,latestDepForkedTestAnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.errorprone:error_prone_annotations:2.47.0=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.errorprone:error_prone_annotations:2.48.0=spotbugs -com.google.guava:failureaccess:1.0.1=annotationProcessor,latestDepForkedTestAnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.guava:failureaccess:1.0.3=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.guava:guava:32.0.1-jre=annotationProcessor,latestDepForkedTestAnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.guava:guava:33.6.0-jre=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,latestDepForkedTestAnnotationProcessor,latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath -com.google.j2objc:j2objc-annotations:2.8=annotationProcessor,latestDepForkedTestAnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.j2objc:j2objc-annotations:3.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.re2j:re2j:1.8=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.squareup.moshi:moshi:1.11.0=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.squareup.okhttp3:logging-interceptor:3.12.12=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.squareup.okhttp3:okhttp:3.12.12=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.squareup.okio:okio:1.17.5=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.thoughtworks.paranamer:paranamer:2.8=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.thoughtworks.qdox:qdox:1.12.1=codenarc -com.typesafe.scala-logging:scala-logging_2.13:3.9.4=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -com.yammer.metrics:metrics-core:2.2.0=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -commons-beanutils:commons-beanutils:1.9.4=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -commons-cli:commons-cli:1.4=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -commons-collections:commons-collections:3.2.2=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -commons-digester:commons-digester:2.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -commons-fileupload:commons-fileupload:1.5=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -commons-io:commons-io:2.11.0=testCompileClasspath,testRuntimeClasspath -commons-io:commons-io:2.17.0=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -commons-io:commons-io:2.21.0=spotbugs -commons-logging:commons-logging:1.2=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -commons-validator:commons-validator:1.7=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -de.thetaphi:forbiddenapis:3.10=compileClasspath,latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.dropwizard.metrics:metrics-core:4.1.12.1=testRuntimeClasspath -io.dropwizard.metrics:metrics-core:4.2.39=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.leangen.geantyref:geantyref:1.3.16=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -io.micrometer:micrometer-commons:1.14.5=testCompileClasspath,testRuntimeClasspath -io.micrometer:micrometer-commons:1.15.12=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.micrometer:micrometer-observation:1.14.5=testCompileClasspath,testRuntimeClasspath -io.micrometer:micrometer-observation:1.15.12=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-buffer:4.1.105.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-buffer:4.1.130.Final=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-codec:4.1.105.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec:4.1.130.Final=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-common:4.1.105.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-common:4.1.130.Final=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-handler:4.1.105.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-handler:4.1.130.Final=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-resolver:4.1.105.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-resolver:4.1.130.Final=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-transport-classes-epoll:4.1.105.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport-classes-epoll:4.1.130.Final=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-transport-native-epoll:4.1.105.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport-native-epoll:4.1.130.Final=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-transport-native-unix-common:4.1.105.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport-native-unix-common:4.1.130.Final=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-transport:4.1.105.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport:4.1.130.Final=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.sqreen:libsqreen:17.5.0=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -javax.activation:activation:1.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -javax.servlet:javax.servlet-api:3.1.0=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -javax.xml.bind:jaxb-api:2.2.3=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -javax.xml.stream:stax-api:1.0-2=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -jaxen:jaxen:2.0.6=spotbugs -junit:junit:4.13.2=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -net.bytebuddy:byte-buddy-agent:1.18.10=buildTimeInstrumentationPlugin,compileClasspath,latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,main_java17CompileClasspath,main_java17RuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -net.bytebuddy:byte-buddy:1.18.10=buildTimeInstrumentationPlugin,compileClasspath,latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,main_java17CompileClasspath,main_java17RuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -net.java.dev.jna:jna-platform:5.8.0=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -net.java.dev.jna:jna:5.8.0=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -net.sf.jopt-simple:jopt-simple:5.0.4=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -net.sf.saxon:Saxon-HE:12.10=spotbugs -net.sourceforge.argparse4j:argparse4j:0.7.0=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.apache.ant:ant-antlr:1.10.14=codenarc -org.apache.ant:ant-junit:1.10.14=codenarc -org.apache.bcel:bcel:6.12.0=spotbugs -org.apache.commons:commons-lang3:3.20.0=spotbugs -org.apache.commons:commons-text:1.15.0=spotbugs -org.apache.kafka:kafka-clients:3.8.0=main_java17CompileClasspath,testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka-clients:3.8.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-group-coordinator-api:3.8.0=testRuntimeClasspath -org.apache.kafka:kafka-group-coordinator-api:3.8.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-group-coordinator:3.8.0=testRuntimeClasspath -org.apache.kafka:kafka-group-coordinator:3.8.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-metadata:3.8.0=testRuntimeClasspath -org.apache.kafka:kafka-metadata:3.8.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-raft:3.8.0=testRuntimeClasspath -org.apache.kafka:kafka-raft:3.8.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-server-common:3.8.0=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka-server-common:3.8.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-server:3.8.0=testRuntimeClasspath -org.apache.kafka:kafka-server:3.8.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-storage-api:3.8.0=testRuntimeClasspath -org.apache.kafka:kafka-storage-api:3.8.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-storage:3.8.0=testRuntimeClasspath -org.apache.kafka:kafka-storage:3.8.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-streams-test-utils:3.8.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-streams:3.8.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-tools-api:3.8.0=testRuntimeClasspath -org.apache.kafka:kafka-tools-api:3.8.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-transaction-coordinator:3.8.0=testRuntimeClasspath -org.apache.kafka:kafka-transaction-coordinator:3.8.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka_2.13:3.8.0=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka_2.13:3.8.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.logging.log4j:log4j-api:2.26.1=spotbugs -org.apache.logging.log4j:log4j-core:2.26.1=spotbugs -org.apache.yetus:audience-annotations:0.12.0=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.apache.zookeeper:zookeeper-jute:3.8.4=testCompileClasspath,testRuntimeClasspath -org.apache.zookeeper:zookeeper-jute:3.8.6=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.zookeeper:zookeeper:3.8.4=testCompileClasspath,testRuntimeClasspath -org.apache.zookeeper:zookeeper:3.8.6=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apiguardian:apiguardian-api:1.1.2=latestDepForkedTestCompileClasspath,latestDepTestCompileClasspath,testCompileClasspath -org.bitbucket.b_c:jose4j:0.9.4=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.checkerframework:checker-qual:3.19.0=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.checkerframework:checker-qual:3.33.0=annotationProcessor,latestDepForkedTestAnnotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -org.codehaus.groovy:groovy-ant:3.0.23=codenarc -org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc -org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc -org.codehaus.groovy:groovy-json:3.0.23=codenarc -org.codehaus.groovy:groovy-json:3.0.25=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.codehaus.groovy:groovy-templates:3.0.23=codenarc -org.codehaus.groovy:groovy-xml:3.0.23=codenarc -org.codehaus.groovy:groovy:3.0.23=codenarc -org.codehaus.groovy:groovy:3.0.25=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.codenarc:CodeNarc:3.7.0=codenarc -org.dom4j:dom4j:2.2.0=spotbugs -org.gmetrics:GMetrics:2.1.0=codenarc -org.hamcrest:hamcrest-core:1.3=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.hamcrest:hamcrest:3.0=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.jctools:jctools-core-jdk11:4.0.6=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.jctools:jctools-core:4.0.6=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.jspecify:jspecify:1.0.0=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-api:5.14.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-engine:5.14.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-params:5.14.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter:5.14.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-commons:1.14.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-engine:1.14.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-launcher:1.14.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-runner:1.14.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-suite-api:1.14.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-suite-commons:1.14.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit:junit-bom:5.14.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit:junit-bom:6.1.2=spotbugs -org.lz4:lz4-java:1.8.0=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.mockito:mockito-core:2.19.0=latestDepForkedTestCompileClasspath,latestDepTestCompileClasspath,testCompileClasspath -org.mockito:mockito-core:4.4.0=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.objenesis:objenesis:3.3=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.opentest4j:opentest4j:1.3.0=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.ow2.asm:asm-analysis:9.10.1=spotbugs -org.ow2.asm:asm-analysis:9.7.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.ow2.asm:asm-commons:9.10.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,spotbugs,testRuntimeClasspath -org.ow2.asm:asm-tree:9.10.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,spotbugs,testRuntimeClasspath -org.ow2.asm:asm-util:9.10.1=spotbugs -org.ow2.asm:asm-util:9.7.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.ow2.asm:asm:9.10.1=buildTimeInstrumentationPlugin,compileClasspath,latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,main_java17CompileClasspath,main_java17RuntimeClasspath,muzzleTooling,runtimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath -org.pcollections:pcollections:4.0.1=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.rocksdb:rocksdbjni:7.9.2=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.scala-lang.modules:scala-collection-compat_2.13:2.10.0=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.scala-lang.modules:scala-java8-compat_2.13:1.0.2=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.scala-lang:scala-library:2.13.14=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.scala-lang:scala-reflect:2.13.14=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.slf4j:jcl-over-slf4j:1.7.30=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:jul-to-slf4j:1.7.30=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:log4j-over-slf4j:1.7.30=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:1.7.30=buildTimeInstrumentationPlugin,compileClasspath,main_java17CompileClasspath,main_java17RuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath -org.slf4j:slf4j-api:1.7.32=testCompileClasspath -org.slf4j:slf4j-api:1.7.36=testRuntimeClasspath -org.slf4j:slf4j-api:2.0.13=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.slf4j:slf4j-api:2.0.17=spotbugsSlf4j -org.slf4j:slf4j-api:2.0.18=spotbugs -org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j -org.snakeyaml:snakeyaml-engine:2.9=buildTimeInstrumentationPlugin,latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,main_java17RuntimeClasspath,muzzleTooling,runtimeClasspath,testRuntimeClasspath -org.spockframework:spock-bom:2.4-groovy-3.0=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.spockframework:spock-core:2.4-groovy-3.0=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.springframework.kafka:spring-kafka-test:3.3.16=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework.kafka:spring-kafka-test:3.3.4=testCompileClasspath,testRuntimeClasspath -org.springframework.kafka:spring-kafka:3.3.16=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework.kafka:spring-kafka:3.3.4=testCompileClasspath,testRuntimeClasspath -org.springframework.retry:spring-retry:2.0.11=testCompileClasspath,testRuntimeClasspath -org.springframework.retry:spring-retry:2.0.13=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-aop:6.2.19=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-aop:6.2.4=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-beans:6.2.19=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-beans:6.2.4=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-context:6.2.19=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-context:6.2.4=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-core:6.2.19=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-core:6.2.4=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-expression:6.2.19=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-expression:6.2.4=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-jcl:6.2.19=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-jcl:6.2.4=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-messaging:6.2.19=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-messaging:6.2.4=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-test:6.2.19=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-test:6.2.4=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-tx:6.2.19=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-tx:6.2.4=testCompileClasspath,testRuntimeClasspath -org.tabletest:tabletest-junit:1.2.2=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.tabletest:tabletest-parser:1.2.1=latestDepForkedTestCompileClasspath,latestDepForkedTestRuntimeClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.xerial.snappy:snappy-java:1.1.10.5=latestDepForkedTestRuntimeClasspath,latestDepTestRuntimeClasspath,testRuntimeClasspath -org.xmlresolver:xmlresolver:5.3.3=spotbugs -empty=main_java17AnnotationProcessor,spotbugsPlugins diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorInstrumentation.java deleted file mode 100644 index f2d99348473..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorInstrumentation.java +++ /dev/null @@ -1,64 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import java.util.HashMap; -import java.util.Map; -import net.bytebuddy.matcher.ElementMatcher; - -@AutoService(InstrumenterModule.class) -public final class ConsumerCoordinatorInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { - - public ConsumerCoordinatorInstrumentation() { - super("kafka", "kafka-3.8"); - } - - @Override - public ElementMatcher.Junction classLoaderMatcher() { - return hasClassNamed("org.apache.kafka.clients.MetadataRecoveryStrategy"); // since 3.8 - } - - @Override - public Map contextStore() { - Map contextStores = new HashMap<>(2); - contextStores.put( - "org.apache.kafka.clients.Metadata", - "datadog.trace.instrumentation.kafka_common.MetadataState"); - contextStores.put( - "org.apache.kafka.clients.consumer.internals.ConsumerCoordinator", - KafkaConsumerInfo.class.getName()); - return contextStores; - } - - @Override - public String instrumentedType() { - return "org.apache.kafka.clients.consumer.internals.ConsumerCoordinator"; - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".KafkaConsumerInfo", - "datadog.trace.instrumentation.kafka_common.KafkaConfigHelper", - "datadog.trace.instrumentation.kafka_common.PendingConfig", - "datadog.trace.instrumentation.kafka_common.MetadataState", - }; - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isMethod().and(named("sendOffsetCommitRequest")).and(takesArguments(1)), - packageName + ".ConsumerCoordinatorAdvice"); - transformer.applyAdvice( - isMethod().and(named("onJoinComplete")).and(takesArguments(4)), - packageName + ".JoinGroupAdvice"); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java deleted file mode 100644 index c94c49369ed..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java +++ /dev/null @@ -1,104 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.declaresField; -import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.implementsInterface; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isConstructor; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.isPublic; -import static net.bytebuddy.matcher.ElementMatchers.returns; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import java.util.HashMap; -import java.util.Map; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.matcher.ElementMatcher; - -/** - * This instrumentation saves additional information from the KafkaConsumer, such as consumer group - * and cluster ID, in the context store for later use. - */ -@AutoService(InstrumenterModule.class) -public final class KafkaConsumerInfoInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForTypeHierarchy, - Instrumenter.HasMethodAdvice, - Instrumenter.WithTypeStructure { - - public KafkaConsumerInfoInstrumentation() { - super("kafka", "kafka-3.8"); - } - - @Override - public ElementMatcher.Junction classLoaderMatcher() { - return hasClassNamed("org.apache.kafka.clients.MetadataRecoveryStrategy"); // since 3.8 - } - - @Override - public Map contextStore() { - Map contextStores = new HashMap<>(4); - contextStores.put( - "org.apache.kafka.clients.Metadata", - "datadog.trace.instrumentation.kafka_common.MetadataState"); - contextStores.put( - "org.apache.kafka.clients.consumer.ConsumerRecords", KafkaConsumerInfo.class.getName()); - // new- here we are storing the callbackinvoker and consumerdelegate in the context store - // as opposed to the old consumercoordinator and kafkaconsumer - contextStores.put( - "org.apache.kafka.clients.consumer.internals.OffsetCommitCallbackInvoker", - KafkaConsumerInfo.class.getName()); - contextStores.put( - "org.apache.kafka.clients.consumer.internals.ConsumerDelegate", - KafkaConsumerInfo.class.getName()); - return contextStores; - } - - @Override - public String hierarchyMarkerType() { - return "org.apache.kafka.clients.consumer.internals.ConsumerDelegate"; - } - - @Override - public ElementMatcher hierarchyMatcher() { - return implementsInterface(named(hierarchyMarkerType())); - } - - @Override - public ElementMatcher structureMatcher() { - return declaresField(named("offsetCommitCallbackInvoker")); - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".KafkaDecorator", - packageName + ".KafkaConsumerInfo", - packageName + ".KafkaConsumerInstrumentationHelper", - "datadog.trace.instrumentation.kafka_common.ClusterIdHolder", - "datadog.trace.instrumentation.kafka_common.KafkaConfigHelper", - "datadog.trace.instrumentation.kafka_common.PendingConfig", - "datadog.trace.instrumentation.kafka_common.MetadataState", - }; - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isConstructor() - .and(takesArgument(0, named("org.apache.kafka.clients.consumer.ConsumerConfig"))) - .and(takesArgument(1, named("org.apache.kafka.common.serialization.Deserializer"))) - .and(takesArgument(2, named("org.apache.kafka.common.serialization.Deserializer"))), - packageName + ".ConstructorAdvice"); - transformer.applyAdvice( - isMethod() - .and(isPublic()) - .and(named("poll")) - .and(takesArguments(1)) - .and(returns(named("org.apache.kafka.clients.consumer.ConsumerRecords"))), - packageName + ".RecordsAdvice"); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentation.java deleted file mode 100644 index 70f3f6dbc92..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentation.java +++ /dev/null @@ -1,96 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.isPublic; -import static net.bytebuddy.matcher.ElementMatchers.returns; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import net.bytebuddy.matcher.ElementMatcher; - -@AutoService(InstrumenterModule.class) -public final class KafkaConsumerInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { - - public KafkaConsumerInstrumentation() { - super("kafka", "kafka-3.8"); - } - - @Override - public ElementMatcher.Junction classLoaderMatcher() { - return hasClassNamed("org.apache.kafka.clients.MetadataRecoveryStrategy"); // since 3.8 - } - - @Override - public Map contextStore() { - Map contextStores = new HashMap<>(2); - contextStores.put( - "org.apache.kafka.clients.Metadata", - "datadog.trace.instrumentation.kafka_common.MetadataState"); - contextStores.put( - "org.apache.kafka.clients.consumer.ConsumerRecords", - "datadog.trace.instrumentation.kafka_clients38.KafkaConsumerInfo"); - return Collections.unmodifiableMap(contextStores); - } - - @Override - public String instrumentedType() { - return "org.apache.kafka.clients.consumer.ConsumerRecords"; - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".TextMapInjectAdapterInterface", - packageName + ".KafkaConsumerInfo", - packageName + ".KafkaConsumerInstrumentationHelper", - packageName + ".KafkaDecorator", - packageName + ".TextMapExtractAdapter", - packageName + ".TracingIterableDelegator", - packageName + ".TracingIterable", - packageName + ".TracingIterator", - packageName + ".TracingList", - packageName + ".TracingListIterator", - packageName + ".TextMapInjectAdapter", - "datadog.trace.instrumentation.kafka_common.Utils", - "datadog.trace.instrumentation.kafka_common.StreamingContext", - "datadog.trace.instrumentation.kafka_common.PendingConfig", - "datadog.trace.instrumentation.kafka_common.MetadataState", - }; - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isMethod() - .and(isPublic()) - .and(named("records")) - .and(takesArgument(0, String.class)) - .and(returns(Iterable.class)), - packageName + ".IterableAdvice"); - transformer.applyAdvice( - isMethod() - .and(isPublic()) - .and(named("records")) - .and(takesArgument(0, named("org.apache.kafka.common.TopicPartition"))) - .and(returns(List.class)), - packageName + ".ListAdvice"); - transformer.applyAdvice( - isMethod() - .and(isPublic()) - .and(named("iterator")) - .and(takesArguments(0)) - .and(returns(Iterator.class)), - packageName + ".IteratorAdvice"); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java deleted file mode 100644 index 0680c757e37..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java +++ /dev/null @@ -1,88 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static java.util.Collections.singletonMap; -import static net.bytebuddy.matcher.ElementMatchers.isConstructor; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.isPrivate; -import static net.bytebuddy.matcher.ElementMatchers.isPublic; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import java.util.Map; -import net.bytebuddy.matcher.ElementMatcher; - -@AutoService(InstrumenterModule.class) -public final class KafkaProducerInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { - - public KafkaProducerInstrumentation() { - super("kafka", "kafka-3.8"); - } - - @Override - public ElementMatcher.Junction classLoaderMatcher() { - return hasClassNamed("org.apache.kafka.clients.MetadataRecoveryStrategy"); // since 3.8 - } - - @Override - public String instrumentedType() { - return "org.apache.kafka.clients.producer.KafkaProducer"; - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".KafkaDecorator", - packageName + ".TextMapInjectAdapterInterface", - packageName + ".TextMapInjectAdapter", - packageName + ".TextMapExtractAdapter", - packageName + ".NoopTextMapInjectAdapter", - packageName + ".KafkaProducerCallback", - "datadog.trace.instrumentation.kafka_common.StreamingContext", - "datadog.trace.instrumentation.kafka_common.ClusterIdHolder", - "datadog.trace.instrumentation.kafka_common.Utils", - "datadog.trace.instrumentation.kafka_common.KafkaConfigHelper", - "datadog.trace.instrumentation.kafka_common.PendingConfig", - "datadog.trace.instrumentation.kafka_common.MetadataState", - packageName + ".ProducerContextPropagationAdvice", - }; - } - - @Override - public Map contextStore() { - return singletonMap( - "org.apache.kafka.clients.Metadata", - "datadog.trace.instrumentation.kafka_common.MetadataState"); - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isConstructor() - .and(takesArgument(0, named("org.apache.kafka.clients.producer.ProducerConfig"))) - .and(takesArgument(1, named("org.apache.kafka.common.serialization.Serializer"))) - .and(takesArgument(2, named("org.apache.kafka.common.serialization.Serializer"))), - packageName + ".ProducerConstructorAdvice"); - - transformer.applyAdvices( - isMethod() - .and(isPublic()) - .and(named("send")) - .and(takesArgument(0, named("org.apache.kafka.clients.producer.ProducerRecord"))) - .and(takesArgument(1, named("org.apache.kafka.clients.producer.Callback"))), - packageName + ".ProducerAdvice", - packageName + ".ProducerContextPropagationAdvice"); - - transformer.applyAdvice( - isMethod() - .and(isPrivate()) - .and(takesArgument(0, int.class)) - .and(named("ensureValidRecordSize")), // intercepting this call allows us to see the - // estimated message size - packageName + ".PayloadSizeAdvice"); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java deleted file mode 100644 index dd36ff1d934..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java +++ /dev/null @@ -1,104 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.declaresField; -import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.implementsInterface; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isConstructor; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.isPublic; -import static net.bytebuddy.matcher.ElementMatchers.returns; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import java.util.HashMap; -import java.util.Map; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.matcher.ElementMatcher; - -/** - * This instrumentation saves additional information from the KafkaConsumer, such as consumer group - * and cluster ID, in the context store for later use. - */ -@AutoService(InstrumenterModule.class) -public final class LegacyKafkaConsumerInfoInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForTypeHierarchy, - Instrumenter.HasMethodAdvice, - Instrumenter.WithTypeStructure { - - public LegacyKafkaConsumerInfoInstrumentation() { - super("kafka", "kafka-3.8"); - } - - @Override - public ElementMatcher.Junction classLoaderMatcher() { - return hasClassNamed("org.apache.kafka.clients.MetadataRecoveryStrategy"); // since 3.8 - } - - @Override - public Map contextStore() { - Map contextStores = new HashMap<>(4); - contextStores.put( - "org.apache.kafka.clients.Metadata", - "datadog.trace.instrumentation.kafka_common.MetadataState"); - contextStores.put( - "org.apache.kafka.clients.consumer.ConsumerRecords", KafkaConsumerInfo.class.getName()); - contextStores.put( - "org.apache.kafka.clients.consumer.internals.ConsumerCoordinator", - KafkaConsumerInfo.class.getName()); - // new - we are storing the ConsumerDelegate instead of the KafkaConsumer - contextStores.put( - "org.apache.kafka.clients.consumer.internals.ConsumerDelegate", - KafkaConsumerInfo.class.getName()); - return contextStores; - } - - @Override - public String hierarchyMarkerType() { - return "org.apache.kafka.clients.consumer.internals.ConsumerDelegate"; - } - - // new - we are instrumenting the ConsumerDelegate class instead of the KafkaConsumer class - @Override - public ElementMatcher hierarchyMatcher() { - return implementsInterface(named(hierarchyMarkerType())); - } - - @Override - public ElementMatcher structureMatcher() { - return declaresField(named("coordinator")); - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".KafkaDecorator", - packageName + ".KafkaConsumerInfo", - packageName + ".KafkaConsumerInstrumentationHelper", - "datadog.trace.instrumentation.kafka_common.ClusterIdHolder", - "datadog.trace.instrumentation.kafka_common.KafkaConfigHelper", - "datadog.trace.instrumentation.kafka_common.PendingConfig", - "datadog.trace.instrumentation.kafka_common.MetadataState", - }; - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isConstructor() - .and(takesArgument(0, named("org.apache.kafka.clients.consumer.ConsumerConfig"))) - .and(takesArgument(1, named("org.apache.kafka.common.serialization.Deserializer"))) - .and(takesArgument(2, named("org.apache.kafka.common.serialization.Deserializer"))), - packageName + ".LegacyConstructorAdvice"); - transformer.applyAdvice( - isMethod() - .and(isPublic()) - .and(named("poll")) - .and(takesArguments(1)) - .and(returns(named("org.apache.kafka.clients.consumer.ConsumerRecords"))), - packageName + ".RecordsAdvice"); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java deleted file mode 100644 index da17835a1a2..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MessageListenerInstrumentation.java +++ /dev/null @@ -1,49 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.implementsInterface; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.api.InstrumenterConfig; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.matcher.ElementMatcher; - -@AutoService(InstrumenterModule.class) -public class MessageListenerInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { - - public MessageListenerInstrumentation() { - super("kafka", "kafka-3.8"); - } - - @Override - public boolean isEnabled() { - return InstrumenterConfig.get().isCodeOriginEnabled() && super.isEnabled(); - } - - @Override - public ElementMatcher.Junction classLoaderMatcher() { - return hasClassNamed("org.apache.kafka.clients.MetadataRecoveryStrategy"); // since 3.8 - } - - @Override - public String hierarchyMarkerType() { - return "org.springframework.kafka.listener.MessageListener"; - } - - @Override - public ElementMatcher hierarchyMatcher() { - return implementsInterface(named(hierarchyMarkerType())); - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isMethod().and(named("onMessage")), - datadog.trace.instrumentation.codeorigin.EntrySpanOriginAdvice.class.getName()); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MetadataInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MetadataInstrumentation.java deleted file mode 100644 index 3907ad0c18c..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/MetadataInstrumentation.java +++ /dev/null @@ -1,70 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static java.util.Collections.singletonMap; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import java.util.Map; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.matcher.ElementMatcher; - -@AutoService(InstrumenterModule.class) -public class MetadataInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { - - public MetadataInstrumentation() { - super("kafka", "kafka-3.8"); - } - - @Override - public ElementMatcher.Junction classLoaderMatcher() { - return hasClassNamed("org.apache.kafka.clients.MetadataRecoveryStrategy"); // since 3.8 - } - - @Override - public String hierarchyMarkerType() { - return "org.apache.kafka.clients.Metadata"; - } - - @Override - public ElementMatcher hierarchyMatcher() { - return extendsClass(named(hierarchyMarkerType())); - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".KafkaDecorator", - "datadog.trace.instrumentation.kafka_common.KafkaConfigHelper", - "datadog.trace.instrumentation.kafka_common.PendingConfig", - "datadog.trace.instrumentation.kafka_common.MetadataState", - }; - } - - @Override - public Map contextStore() { - return singletonMap( - "org.apache.kafka.clients.Metadata", - "datadog.trace.instrumentation.kafka_common.MetadataState"); - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isMethod() - .and(named("update")) - .and(takesArgument(0, named("org.apache.kafka.common.Cluster"))), - packageName + ".MetadataUpdateBefore22Advice"); - transformer.applyAdvice( - isMethod() - .and(named("update")) - .and(takesArgument(1, named("org.apache.kafka.common.requests.MetadataResponse"))), - packageName + ".MetadataUpdate22AndAfterAdvice"); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerInstrumentation.java deleted file mode 100644 index e64beeb1ebd..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerInstrumentation.java +++ /dev/null @@ -1,35 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; - -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import net.bytebuddy.matcher.ElementMatcher; - -// new - this instrumentation is completely new. -// the purpose of this class is to provide us with information on consumer group and cluster ID -public class OffsetCommitCallbackInvokerInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { - public OffsetCommitCallbackInvokerInstrumentation() { - super("kafka", "kafka-3.8"); - } - - @Override - public ElementMatcher.Junction classLoaderMatcher() { - return hasClassNamed("org.apache.kafka.clients.MetadataRecoveryStrategy"); // since 3.8 - } - - @Override - public String instrumentedType() { - return "org.apache.kafka.clients.consumer.internals.OffsetCommitCallbackInvoker"; - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isMethod().and(named("enqueueUserCallbackInvocation")), - packageName + ".OffsetCommitCallbackInvokerAdvice"); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ConstructorAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ConstructorAdvice.java deleted file mode 100644 index f704eecade4..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ConstructorAdvice.java +++ /dev/null @@ -1,73 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.trace.api.Config; -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.instrumentation.kafka_common.KafkaConfigHelper; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import java.util.List; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerGroupMetadata; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.internals.ConsumerDelegate; -import org.apache.kafka.clients.consumer.internals.OffsetCommitCallbackInvoker; - -public class ConstructorAdvice { - // new - capturing OffsetCommitCallbackInvoker instead of the old ConsumerCoordinator - @Advice.OnMethodExit(suppress = Throwable.class) - public static void captureGroup( - @Advice.This ConsumerDelegate consumer, - @Advice.Argument(0) ConsumerConfig consumerConfig, - @Advice.FieldValue("offsetCommitCallbackInvoker") - OffsetCommitCallbackInvoker offsetCommitCallbackInvoker, - @Advice.FieldValue("metadata") Metadata metadata) { - ConsumerGroupMetadata groupMetadata = consumer.groupMetadata(); - String consumerGroup = consumerConfig.getString(ConsumerConfig.GROUP_ID_CONFIG); - String normalizedConsumerGroup = - consumerGroup != null && !consumerGroup.isEmpty() ? consumerGroup : null; - if (normalizedConsumerGroup == null) { - if (groupMetadata != null) { - normalizedConsumerGroup = groupMetadata.groupId(); - } - } - List bootstrapServersList = - consumerConfig.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG); - String bootstrapServers = null; - if (bootstrapServersList != null && !bootstrapServersList.isEmpty()) { - bootstrapServers = String.join(",", bootstrapServersList); - } - KafkaConsumerInfo kafkaConsumerInfo; - if (Config.get().isDataStreamsEnabled()) { - kafkaConsumerInfo = - new KafkaConsumerInfo(normalizedConsumerGroup, metadata, bootstrapServers); - } else { - kafkaConsumerInfo = new KafkaConsumerInfo(normalizedConsumerGroup, bootstrapServers); - } - // new - searching context for ConsumerDelegate and OffsetCommitCallbackInvoker instead of - // ConsumerCoordinator and KafkaConsumer - if (kafkaConsumerInfo.getConsumerGroup().isPresent() - || kafkaConsumerInfo.getmetadata().isPresent()) { - InstrumentationContext.get(ConsumerDelegate.class, KafkaConsumerInfo.class) - .put(consumer, kafkaConsumerInfo); - } - if (offsetCommitCallbackInvoker != null) { - InstrumentationContext.get(OffsetCommitCallbackInvoker.class, KafkaConsumerInfo.class) - .put(offsetCommitCallbackInvoker, kafkaConsumerInfo); - } - - if (Config.get().isDataStreamsEnabled()) { - MetadataState state = - InstrumentationContext.get(Metadata.class, MetadataState.class) - .putIfAbsent(metadata, MetadataState::new); - KafkaConfigHelper.storePendingConsumerConfig( - state, normalizedConsumerGroup, KafkaConfigHelper.extractConsumerConfig(consumerConfig)); - } - } - - public static void muzzleCheck(ConsumerRecord record) { - // KafkaConsumerInstrumentation only applies for kafka versions with headers - // Make an explicit call so KafkaConsumerGroupInstrumentation does the same - record.headers(); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorAdvice.java deleted file mode 100644 index 075590d2206..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ConsumerCoordinatorAdvice.java +++ /dev/null @@ -1,68 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.trace.api.datastreams.DataStreamsTags; -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import java.util.Map; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; -import org.apache.kafka.clients.consumer.internals.RequestFuture; -import org.apache.kafka.common.TopicPartition; - -public class ConsumerCoordinatorAdvice { - @Advice.OnMethodExit(suppress = Throwable.class) - public static void trackCommitOffset( - @Advice.This ConsumerCoordinator coordinator, - @Advice.Return RequestFuture requestFuture, - @Advice.Argument(0) final Map offsets) { - if (requestFuture == null || requestFuture.failed()) { - return; - } - if (offsets == null) { - return; - } - KafkaConsumerInfo kafkaConsumerInfo = - InstrumentationContext.get(ConsumerCoordinator.class, KafkaConsumerInfo.class) - .get(coordinator); - - if (kafkaConsumerInfo == null) { - return; - } - - String consumerGroup = kafkaConsumerInfo.getConsumerGroup().orElse(null); - Metadata consumerMetadata = kafkaConsumerInfo.getmetadata().orElse(null); - String clusterId = null; - if (consumerMetadata != null) { - MetadataState metadataState = - InstrumentationContext.get(Metadata.class, MetadataState.class).get(consumerMetadata); - clusterId = metadataState != null ? metadataState.clusterId : null; - } - - for (Map.Entry entry : offsets.entrySet()) { - if (consumerGroup == null) { - consumerGroup = ""; - } - if (entry.getKey() == null || entry.getValue() == null) { - continue; - } - DataStreamsTags tags = - DataStreamsTags.createWithPartition( - "kafka_commit", - entry.getKey().topic(), - String.valueOf(entry.getKey().partition()), - clusterId, - consumerGroup); - AgentTracer.get().getDataStreamsMonitoring().trackBacklog(tags, entry.getValue().offset()); - } - } - - public static void muzzleCheck(ConsumerRecord record) { - // KafkaConsumerInstrumentation only applies for kafka versions with headers - // Make an explicit call so ConsumerCoordinatorInstrumentation does the same - record.headers(); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/DDOffsetCommitCallback.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/DDOffsetCommitCallback.java deleted file mode 100644 index c1eeb0141d0..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/DDOffsetCommitCallback.java +++ /dev/null @@ -1,55 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.trace.api.datastreams.DataStreamsTags; -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import java.util.Map; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.OffsetAndMetadata; -import org.apache.kafka.clients.consumer.OffsetCommitCallback; -import org.apache.kafka.common.TopicPartition; - -public class DDOffsetCommitCallback implements OffsetCommitCallback { - OffsetCommitCallback callback; - KafkaConsumerInfo kafkaConsumerInfo; - - public DDOffsetCommitCallback( - OffsetCommitCallback callback, KafkaConsumerInfo kafkaConsumerInfo) { - this.callback = callback; - this.kafkaConsumerInfo = kafkaConsumerInfo; - } - - @Override - public void onComplete(Map map, Exception e) { - if (callback != null) { - callback.onComplete(map, e); - } - for (Map.Entry entry : map.entrySet()) { - if (entry.getKey() == null || entry.getValue() == null) { - continue; - } - String consumerGroup = null; - String clusterId = null; - - if (kafkaConsumerInfo != null) { - consumerGroup = kafkaConsumerInfo.getConsumerGroup().orElse(null); - Metadata consumerMetadata = kafkaConsumerInfo.getmetadata().orElse(null); - if (consumerMetadata != null) { - MetadataState metadataState = - InstrumentationContext.get(Metadata.class, MetadataState.class).get(consumerMetadata); - clusterId = metadataState != null ? metadataState.clusterId : null; - } - } - - DataStreamsTags tags = - DataStreamsTags.createWithPartition( - "kafka_commit", - entry.getKey().topic(), - String.valueOf(entry.getKey().partition()), - clusterId, - consumerGroup); - AgentTracer.get().getDataStreamsMonitoring().trackBacklog(tags, entry.getValue().offset()); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/IterableAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/IterableAdvice.java deleted file mode 100644 index c6cc52a3d9e..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/IterableAdvice.java +++ /dev/null @@ -1,35 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; - -public class IterableAdvice { - - @Advice.OnMethodExit(suppress = Throwable.class) - public static void wrap( - @Advice.Return(readOnly = false) Iterable> iterable, - @Advice.This ConsumerRecords records) { - if (iterable != null) { - KafkaConsumerInfo kafkaConsumerInfo = - InstrumentationContext.get(ConsumerRecords.class, KafkaConsumerInfo.class).get(records); - String group = KafkaConsumerInstrumentationHelper.extractGroup(kafkaConsumerInfo); - String clusterId = - KafkaConsumerInstrumentationHelper.extractClusterId( - kafkaConsumerInfo, InstrumentationContext.get(Metadata.class, MetadataState.class)); - String bootstrapServers = - KafkaConsumerInstrumentationHelper.extractBootstrapServers(kafkaConsumerInfo); - iterable = - new TracingIterable( - iterable, - KafkaDecorator.KAFKA_CONSUME, - KafkaDecorator.CONSUMER_DECORATE, - group, - clusterId, - bootstrapServers); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/IteratorAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/IteratorAdvice.java deleted file mode 100644 index 39859c20e19..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/IteratorAdvice.java +++ /dev/null @@ -1,36 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import java.util.Iterator; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; - -public class IteratorAdvice { - - @Advice.OnMethodExit(suppress = Throwable.class) - public static void wrap( - @Advice.Return(readOnly = false) Iterator> iterator, - @Advice.This ConsumerRecords records) { - if (iterator != null) { - KafkaConsumerInfo kafkaConsumerInfo = - InstrumentationContext.get(ConsumerRecords.class, KafkaConsumerInfo.class).get(records); - String group = KafkaConsumerInstrumentationHelper.extractGroup(kafkaConsumerInfo); - String clusterId = - KafkaConsumerInstrumentationHelper.extractClusterId( - kafkaConsumerInfo, InstrumentationContext.get(Metadata.class, MetadataState.class)); - String bootstrapServers = - KafkaConsumerInstrumentationHelper.extractBootstrapServers(kafkaConsumerInfo); - iterator = - new TracingIterator( - iterator, - KafkaDecorator.KAFKA_CONSUME, - KafkaDecorator.CONSUMER_DECORATE, - group, - clusterId, - bootstrapServers); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/JoinGroupAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/JoinGroupAdvice.java deleted file mode 100644 index 41a3ed170be..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/JoinGroupAdvice.java +++ /dev/null @@ -1,48 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.instrumentation.kafka_common.KafkaConfigHelper; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; - -public class JoinGroupAdvice { - @Advice.OnMethodExit(suppress = Throwable.class) - public static void trackJoinGroup( - @Advice.This ConsumerCoordinator coordinator, - @Advice.Argument(0) final int generationId, - @Advice.Argument(1) final String memberId, - @Advice.Argument(2) final String memberProtocol) { - if (memberId == null || memberId.isEmpty()) { - return; - } - KafkaConsumerInfo kafkaConsumerInfo = - InstrumentationContext.get(ConsumerCoordinator.class, KafkaConsumerInfo.class) - .get(coordinator); - if (kafkaConsumerInfo == null) { - return; - } - if (!kafkaConsumerInfo.hasMembershipChanged(memberId, generationId)) { - return; - } - - String consumerGroup = kafkaConsumerInfo.getConsumerGroup().orElse(null); - Metadata consumerMetadata = kafkaConsumerInfo.getmetadata().orElse(null); - String clusterId = null; - if (consumerMetadata != null) { - MetadataState metadataState = - InstrumentationContext.get(Metadata.class, MetadataState.class).get(consumerMetadata); - clusterId = metadataState != null ? metadataState.clusterId : null; - } - if (KafkaConfigHelper.reportConsumerGroupMember( - clusterId, consumerGroup, memberId, generationId, memberProtocol)) { - kafkaConsumerInfo.setLastReportedMembership(memberId, generationId); - } - } - - public static void muzzleCheck(ConsumerRecord record) { - record.headers(); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfo.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfo.java deleted file mode 100644 index 8b6f78cd672..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfo.java +++ /dev/null @@ -1,62 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import java.util.Objects; -import java.util.Optional; -import org.apache.kafka.clients.Metadata; - -public class KafkaConsumerInfo { - private final String consumerGroup; - private final Metadata metadata; - private final String bootstrapServers; - private volatile String lastReportedMemberId; - private volatile int lastReportedGenerationId = Integer.MIN_VALUE; - - public KafkaConsumerInfo(String consumerGroup, Metadata metadata, String bootstrapServers) { - this.consumerGroup = consumerGroup; - this.metadata = metadata; - this.bootstrapServers = bootstrapServers; - } - - public KafkaConsumerInfo(String consumerGroup, String bootstrapServers) { - this.consumerGroup = consumerGroup; - this.metadata = null; - this.bootstrapServers = bootstrapServers; - } - - public Optional getConsumerGroup() { - return Optional.ofNullable(consumerGroup); - } - - public Optional getmetadata() { - return Optional.ofNullable(metadata); - } - - public Optional getBootstrapServers() { - return Optional.ofNullable(bootstrapServers); - } - - public boolean hasMembershipChanged(String memberId, int generationId) { - return !Objects.equals(memberId, lastReportedMemberId) - || generationId != lastReportedGenerationId; - } - - public void setLastReportedMembership(String memberId, int generationId) { - this.lastReportedMemberId = memberId; - this.lastReportedGenerationId = generationId; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - KafkaConsumerInfo consumerInfo = (KafkaConsumerInfo) o; - return Objects.equals(consumerGroup, consumerInfo.consumerGroup) - && Objects.equals(metadata, consumerInfo.metadata); - } - - @Override - public int hashCode() { - return 31 * (null == consumerGroup ? 0 : consumerGroup.hashCode()) - + (null == metadata ? 0 : metadata.hashCode()); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentationHelper.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentationHelper.java deleted file mode 100644 index 57e0baa7edd..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentationHelper.java +++ /dev/null @@ -1,32 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.trace.bootstrap.ContextStore; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import java.util.Optional; -import org.apache.kafka.clients.Metadata; - -public class KafkaConsumerInstrumentationHelper { - public static String extractGroup(KafkaConsumerInfo kafkaConsumerInfo) { - if (kafkaConsumerInfo != null) { - return kafkaConsumerInfo.getConsumerGroup().orElse(null); - } - return null; - } - - public static String extractClusterId( - KafkaConsumerInfo kafkaConsumerInfo, - ContextStore metadataContextStore) { - if (kafkaConsumerInfo != null) { - Optional metadata = kafkaConsumerInfo.getmetadata(); - if (metadata.isPresent()) { - MetadataState state = metadataContextStore.get(metadata.get()); - return state != null ? state.clusterId : null; - } - } - return null; - } - - public static String extractBootstrapServers(KafkaConsumerInfo kafkaConsumerInfo) { - return kafkaConsumerInfo == null ? null : kafkaConsumerInfo.getBootstrapServers().orElse(null); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaDecorator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaDecorator.java deleted file mode 100644 index d2d6f53b8a9..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaDecorator.java +++ /dev/null @@ -1,181 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.CONSUMER_GROUP; -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS; -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.KAFKA_CLUSTER_ID; -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.MESSAGING_DESTINATION_NAME; -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.OFFSET; -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.PARTITION; -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.RECORD_QUEUE_TIME_MS; -import static datadog.trace.bootstrap.instrumentation.api.ServiceNameSources.MESSAGE_BROKER_SPLIT_BY_DESTINATION; -import static java.util.concurrent.TimeUnit.NANOSECONDS; - -import datadog.trace.api.Config; -import datadog.trace.api.Functions; -import datadog.trace.api.cache.DDCache; -import datadog.trace.api.cache.DDCaches; -import datadog.trace.api.naming.SpanNaming; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; -import datadog.trace.bootstrap.instrumentation.api.Tags; -import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; -import datadog.trace.bootstrap.instrumentation.decorator.MessagingClientDecorator; -import java.util.function.Function; -import java.util.function.Supplier; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.record.TimestampType; - -public class KafkaDecorator extends MessagingClientDecorator { - private static final String KAFKA = "kafka"; - public static final CharSequence JAVA_KAFKA = UTF8BytesString.create("java-kafka"); - public static final CharSequence KAFKA_CONSUME = - UTF8BytesString.create( - SpanNaming.instance().namingSchema().messaging().inboundOperation(KAFKA)); - - public static final CharSequence KAFKA_POLL = UTF8BytesString.create("kafka.poll"); - public static final CharSequence KAFKA_PRODUCE = - UTF8BytesString.create( - SpanNaming.instance().namingSchema().messaging().outboundOperation(KAFKA)); - public static final CharSequence KAFKA_DELIVER = UTF8BytesString.create("kafka.deliver"); - public static final boolean KAFKA_LEGACY_TRACING = Config.get().isKafkaLegacyTracingEnabled(); - public static final boolean TIME_IN_QUEUE_ENABLED = - Config.get().isTimeInQueueEnabled(!KAFKA_LEGACY_TRACING, KAFKA); - public static final String KAFKA_PRODUCED_KEY = "x_datadog_kafka_produced"; - private final String spanKind; - private final CharSequence spanType; - private final Supplier serviceNameSupplier; - - private static final DDCache PRODUCER_RESOURCE_NAME_CACHE = - DDCaches.newFixedSizeCache(32); - private static final Functions.Prefix PRODUCER_PREFIX = new Functions.Prefix("Produce Topic "); - private static final DDCache CONSUMER_RESOURCE_NAME_CACHE = - DDCaches.newFixedSizeCache(32); - private static final DDCache PRODUCER_BOOSTRAP_SERVERS_CACHE = - DDCaches.newFixedSizeWeakKeyCache(16); - private static final Function BOOTSTRAP_SERVERS_JOINER = - pc -> String.join(",", pc.getList(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG)); - private static final Functions.Prefix CONSUMER_PREFIX = new Functions.Prefix("Consume Topic "); - - public static final KafkaDecorator PRODUCER_DECORATE = - new KafkaDecorator( - Tags.SPAN_KIND_PRODUCER, - InternalSpanTypes.MESSAGE_PRODUCER, - SpanNaming.instance() - .namingSchema() - .messaging() - .outboundService(KAFKA, KAFKA_LEGACY_TRACING)); - - public static final KafkaDecorator CONSUMER_DECORATE = - new KafkaDecorator( - Tags.SPAN_KIND_CONSUMER, - InternalSpanTypes.MESSAGE_CONSUMER, - SpanNaming.instance() - .namingSchema() - .messaging() - .inboundService(KAFKA, KAFKA_LEGACY_TRACING)); - - public static final KafkaDecorator BROKER_DECORATE = - new KafkaDecorator( - Tags.SPAN_KIND_BROKER, - InternalSpanTypes.MESSAGE_BROKER, - SpanNaming.instance().namingSchema().messaging().timeInQueueService(KAFKA)); - - protected KafkaDecorator( - String spanKind, CharSequence spanType, Supplier serviceNameSupplier) { - this.spanKind = spanKind; - this.spanType = spanType; - this.serviceNameSupplier = serviceNameSupplier; - } - - @Override - protected CharSequence spanType() { - return spanType; - } - - @Override - protected String[] instrumentationNames() { - return new String[] {"kafka"}; - } - - @Override - protected String service() { - return serviceNameSupplier.get(); - } - - @Override - protected CharSequence component() { - return JAVA_KAFKA; - } - - @Override - protected String spanKind() { - return spanKind; - } - - public void onConsume( - final AgentSpan span, - final ConsumerRecord record, - String consumerGroup, - String clusterId, - String bootstrapServers) { - if (record != null) { - final String topic = record.topic() == null ? "kafka" : record.topic(); - span.setResourceName(CONSUMER_RESOURCE_NAME_CACHE.computeIfAbsent(topic, CONSUMER_PREFIX)); - span.setTag(PARTITION, record.partition()); - span.setTag(OFFSET, record.offset()); - span.setTag(MESSAGING_DESTINATION_NAME, topic); - if (consumerGroup != null) { - span.setTag(CONSUMER_GROUP, consumerGroup); - } - if (clusterId != null) { - span.setTag(KAFKA_CLUSTER_ID, clusterId); - } - if (bootstrapServers != null) { - span.setTag(KAFKA_BOOTSTRAP_SERVERS, bootstrapServers); - } - // TODO - do we really need both? This mechanism already adds a lot of... baggage. - // check to not record a duration if the message was sent from an old Kafka client - if (record.timestampType() != TimestampType.NO_TIMESTAMP_TYPE) { - long consumeTime = NANOSECONDS.toMillis(span.getStartTime()); - final long produceTime = record.timestamp(); - span.setTag(RECORD_QUEUE_TIME_MS, Math.max(0L, consumeTime - produceTime)); - } - } - } - - public void onTimeInQueue(final AgentSpan span, final ConsumerRecord record) { - if (record != null) { - String topic = record.topic() == null ? "kafka" : record.topic(); - span.setResourceName(topic); - if (Config.get().isMessageBrokerSplitByDestination()) { - span.setServiceName(topic, MESSAGE_BROKER_SPLIT_BY_DESTINATION); - } - } - } - - public void onProduce( - final AgentSpan span, - final ProducerRecord record, - final ProducerConfig producerConfig, - final String clusterId) { - if (record != null) { - if (record.partition() != null) { - span.setTag(PARTITION, record.partition()); - } - if (producerConfig != null) { - span.setTag( - KAFKA_BOOTSTRAP_SERVERS, - PRODUCER_BOOSTRAP_SERVERS_CACHE.computeIfAbsent( - producerConfig, BOOTSTRAP_SERVERS_JOINER)); - } - if (clusterId != null) { - span.setTag(KAFKA_CLUSTER_ID, clusterId); - } - final String topic = record.topic() == null ? "kafka" : record.topic(); - span.setResourceName(PRODUCER_RESOURCE_NAME_CACHE.computeIfAbsent(topic, PRODUCER_PREFIX)); - span.setTag(MESSAGING_DESTINATION_NAME, topic); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaProducerCallback.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaProducerCallback.java deleted file mode 100644 index dd7bb52989c..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/KafkaProducerCallback.java +++ /dev/null @@ -1,62 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.OFFSET; -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.PARTITION; -import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.PRODUCER_DECORATE; - -import datadog.context.ContextScope; -import datadog.trace.api.datastreams.DataStreamsTags; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; - -public class KafkaProducerCallback implements Callback { - private final Callback callback; - private final AgentSpan parent; - private final AgentSpan span; - private final String clusterId; - - public KafkaProducerCallback( - final Callback callback, - final AgentSpan parent, - final AgentSpan span, - final String clusterId) { - this.callback = callback; - this.parent = parent; - this.span = span; - this.clusterId = clusterId; - } - - @Override - public void onCompletion(final RecordMetadata metadata, final Exception exception) { - if (metadata != null) { - span.setTag(PARTITION, metadata.partition()); - span.setTag(OFFSET, metadata.offset()); - } - PRODUCER_DECORATE.onError(span, exception); - PRODUCER_DECORATE.beforeFinish(span); - span.finish(); - if (callback != null) { - if (parent != null) { - try (final ContextScope scope = activateSpan(parent)) { - callback.onCompletion(metadata, exception); - } - } else { - callback.onCompletion(metadata, exception); - } - } - if (metadata == null) { - return; - } - DataStreamsTags tags = - DataStreamsTags.createWithPartition( - "kafka_produce", - metadata.topic(), - String.valueOf(metadata.partition()), - clusterId, - null); - AgentTracer.get().getDataStreamsMonitoring().trackBacklog(tags, metadata.offset()); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/LegacyConstructorAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/LegacyConstructorAdvice.java deleted file mode 100644 index 6557c1b6821..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/LegacyConstructorAdvice.java +++ /dev/null @@ -1,66 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.trace.api.Config; -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.instrumentation.kafka_common.KafkaConfigHelper; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import java.util.List; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerGroupMetadata; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.internals.ConsumerCoordinator; -import org.apache.kafka.clients.consumer.internals.ConsumerDelegate; - -public class LegacyConstructorAdvice { - // new - capture the ConsumerDelegate instead of KafkaConsumer - @Advice.OnMethodExit(suppress = Throwable.class) - public static void captureGroup( - @Advice.This ConsumerDelegate consumer, - @Advice.Argument(0) ConsumerConfig consumerConfig, - @Advice.FieldValue("coordinator") ConsumerCoordinator coordinator, - @Advice.FieldValue("metadata") Metadata metadata) { - ConsumerGroupMetadata groupMetadata = consumer.groupMetadata(); - String consumerGroup = consumerConfig.getString(ConsumerConfig.GROUP_ID_CONFIG); - String normalizedConsumerGroup = - consumerGroup != null && !consumerGroup.isEmpty() ? consumerGroup : null; - if (normalizedConsumerGroup == null) { - if (groupMetadata != null) { - normalizedConsumerGroup = groupMetadata.groupId(); - } - } - List bootstrapServersList = - consumerConfig.getList(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG); - String bootstrapServers = null; - if (bootstrapServersList != null && !bootstrapServersList.isEmpty()) { - bootstrapServers = String.join(",", bootstrapServersList); - } - KafkaConsumerInfo kafkaConsumerInfo; - kafkaConsumerInfo = new KafkaConsumerInfo(normalizedConsumerGroup, metadata, bootstrapServers); - // new - search for the ConsumerDelegate instead of KafkaConsumer - if (kafkaConsumerInfo.getConsumerGroup().isPresent() - || kafkaConsumerInfo.getmetadata().isPresent()) { - InstrumentationContext.get(ConsumerDelegate.class, KafkaConsumerInfo.class) - .put(consumer, kafkaConsumerInfo); - if (coordinator != null) { - InstrumentationContext.get(ConsumerCoordinator.class, KafkaConsumerInfo.class) - .put(coordinator, kafkaConsumerInfo); - } - } - - if (Config.get().isDataStreamsEnabled()) { - MetadataState state = - InstrumentationContext.get(Metadata.class, MetadataState.class) - .putIfAbsent(metadata, MetadataState::new); - KafkaConfigHelper.storePendingConsumerConfig( - state, normalizedConsumerGroup, KafkaConfigHelper.extractConsumerConfig(consumerConfig)); - } - } - - public static void muzzleCheck(ConsumerRecord record) { - // KafkaConsumerInstrumentation only applies for kafka versions with headers - // Make an explicit call so KafkaConsumerGroupInstrumentation does the same - record.headers(); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ListAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ListAdvice.java deleted file mode 100644 index 1c65b8a2ccd..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ListAdvice.java +++ /dev/null @@ -1,34 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.CONSUMER_DECORATE; -import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.KAFKA_CONSUME; - -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import java.util.List; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; - -public class ListAdvice { - - @Advice.OnMethodExit(suppress = Throwable.class) - public static void wrap( - @Advice.Return(readOnly = false) List> iterable, - @Advice.This ConsumerRecords records) { - if (iterable != null) { - KafkaConsumerInfo kafkaConsumerInfo = - InstrumentationContext.get(ConsumerRecords.class, KafkaConsumerInfo.class).get(records); - String group = KafkaConsumerInstrumentationHelper.extractGroup(kafkaConsumerInfo); - String clusterId = - KafkaConsumerInstrumentationHelper.extractClusterId( - kafkaConsumerInfo, InstrumentationContext.get(Metadata.class, MetadataState.class)); - String bootstrapServers = - KafkaConsumerInstrumentationHelper.extractBootstrapServers(kafkaConsumerInfo); - iterable = - new TracingList( - iterable, KAFKA_CONSUME, CONSUMER_DECORATE, group, clusterId, bootstrapServers); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/MetadataUpdate22AndAfterAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/MetadataUpdate22AndAfterAdvice.java deleted file mode 100644 index 307440b06fb..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/MetadataUpdate22AndAfterAdvice.java +++ /dev/null @@ -1,30 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.instrumentation.kafka_common.KafkaConfigHelper; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.requests.MetadataResponse; - -public class MetadataUpdate22AndAfterAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter( - @Advice.This final Metadata metadata, @Advice.Argument(1) final MetadataResponse response) { - if (response != null) { - String clusterId = response.clusterId(); - MetadataState state = - InstrumentationContext.get(Metadata.class, MetadataState.class) - .putIfAbsent(metadata, MetadataState::new); - state.clusterId = clusterId; - KafkaConfigHelper.reportPendingConfig(state, clusterId); - } - } - - public static void muzzleCheck(ConsumerRecord record) { - // KafkaConsumerInstrumentation only applies for kafka versions with headers - // Make an explicit call so MetadataInstrumentation does the same - record.headers(); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/MetadataUpdateBefore22Advice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/MetadataUpdateBefore22Advice.java deleted file mode 100644 index a1abe7b1959..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/MetadataUpdateBefore22Advice.java +++ /dev/null @@ -1,30 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.instrumentation.kafka_common.KafkaConfigHelper; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.Cluster; - -public class MetadataUpdateBefore22Advice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter( - @Advice.This final Metadata metadata, @Advice.Argument(0) final Cluster newCluster) { - if (newCluster != null && !newCluster.isBootstrapConfigured()) { - String clusterId = newCluster.clusterResource().clusterId(); - MetadataState state = - InstrumentationContext.get(Metadata.class, MetadataState.class) - .putIfAbsent(metadata, MetadataState::new); - state.clusterId = clusterId; - KafkaConfigHelper.reportPendingConfig(state, clusterId); - } - } - - public static void muzzleCheck(ConsumerRecord record) { - // KafkaConsumerInstrumentation only applies for kafka versions with headers - // Make an explicit call so MetadataInstrumentation does the same - record.headers(); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/NoopTextMapInjectAdapter.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/NoopTextMapInjectAdapter.java deleted file mode 100644 index 653d382184c..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/NoopTextMapInjectAdapter.java +++ /dev/null @@ -1,13 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import org.apache.kafka.common.header.Headers; - -public class NoopTextMapInjectAdapter implements TextMapInjectAdapterInterface { - - public static final NoopTextMapInjectAdapter NOOP_SETTER = new NoopTextMapInjectAdapter(); - - @Override - public void set(final Headers headers, final String key, final String value) {} - - public void injectTimeInQueue(Headers headers) {} -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerAdvice.java deleted file mode 100644 index fc58fff319d..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/OffsetCommitCallbackInvokerAdvice.java +++ /dev/null @@ -1,18 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.trace.bootstrap.InstrumentationContext; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.consumer.OffsetCommitCallback; -import org.apache.kafka.clients.consumer.internals.OffsetCommitCallbackInvoker; - -public class OffsetCommitCallbackInvokerAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnqueue( - @Advice.Argument(value = 0, readOnly = false) OffsetCommitCallback callback, - @Advice.This OffsetCommitCallbackInvoker callbackInvoker) { - KafkaConsumerInfo kafkaConsumerInfo = - InstrumentationContext.get(OffsetCommitCallbackInvoker.class, KafkaConsumerInfo.class) - .get(callbackInvoker); - callback = new DDOffsetCommitCallback(callback, kafkaConsumerInfo); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/PayloadSizeAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/PayloadSizeAdvice.java deleted file mode 100644 index 3b4effb3979..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/PayloadSizeAdvice.java +++ /dev/null @@ -1,36 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; - -import datadog.trace.api.datastreams.StatsPoint; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import net.bytebuddy.asm.Advice; - -public class PayloadSizeAdvice { - - /** - * Instrumentation for the method KafkaProducer.ensureValidRecordSize that is called as part of - * sending a kafka payload. This gives us access to an estimate of the payload size "for free", - * that we send as a metric. - */ - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter(@Advice.Argument(value = 0) int estimatedPayloadSize) { - StatsPoint saved = activeSpan().spanContext().getPathwayContext().getSavedStats(); - if (saved != null) { - // create new stats including the payload size - StatsPoint updated = - new StatsPoint( - saved.getTags(), - saved.getHash(), - saved.getParentHash(), - saved.getAggregationHash(), - saved.getTimestampNanos(), - saved.getPathwayLatencyNano(), - saved.getEdgeLatencyNano(), - estimatedPayloadSize, - saved.getServiceNameOverride()); - // then send the point - AgentTracer.get().getDataStreamsMonitoring().add(updated); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java deleted file mode 100644 index 01905ee65e4..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java +++ /dev/null @@ -1,81 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.bootstrap.instrumentation.api.AgentPropagation.extractContextAndGetSpanContext; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; -import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.JAVA_KAFKA; -import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.KAFKA_PRODUCE; -import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.PRODUCER_DECORATE; - -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.bootstrap.instrumentation.api.AgentScope; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; -import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags; -import datadog.trace.instrumentation.kafka_common.ClusterIdHolder; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.internals.Sender; - -public class ProducerAdvice { - - @Advice.OnMethodEnter(suppress = Throwable.class) - public static AgentScope onEnter( - @Advice.FieldValue("producerConfig") ProducerConfig producerConfig, - @Advice.FieldValue("sender") Sender sender, - @Advice.FieldValue("metadata") Metadata metadata, - @Advice.Argument(value = 0, readOnly = false) ProducerRecord record, - @Advice.Argument(value = 1, readOnly = false) Callback callback) { - MetadataState metadataState = - InstrumentationContext.get(Metadata.class, MetadataState.class).get(metadata); - String clusterId = metadataState != null ? metadataState.clusterId : null; - - // Set cluster ID for Schema Registry instrumentation - if (clusterId != null) { - ClusterIdHolder.set(clusterId); - } - - // Try to extract existing trace context from record headers - final AgentSpanContext extractedContext = - extractContextAndGetSpanContext(record.headers(), TextMapExtractAdapter.GETTER); - - final AgentSpan localActiveSpan = activeSpan(); - - final AgentSpan span; - final AgentSpan callbackParentSpan; - - if (extractedContext != null) { - span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE, extractedContext); - callbackParentSpan = span; - } else { - span = startSpan(JAVA_KAFKA.toString(), KAFKA_PRODUCE); - callbackParentSpan = localActiveSpan; - } - PRODUCER_DECORATE.afterStart(span); - PRODUCER_DECORATE.onProduce(span, record, producerConfig, clusterId); - - callback = new KafkaProducerCallback(callback, callbackParentSpan, span, clusterId); - - if (record.value() == null) { - span.setTag(InstrumentationTags.TOMBSTONE, true); - } - - return activateSpan(span); - } - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void stopSpan( - @Advice.Enter final AgentScope scope, @Advice.Thrown final Throwable throwable) { - // Clear cluster ID from Schema Registry instrumentation - ClusterIdHolder.clear(); - - PRODUCER_DECORATE.onError(scope, throwable); - PRODUCER_DECORATE.beforeFinish(scope); - scope.close(); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerConstructorAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerConstructorAdvice.java deleted file mode 100644 index a97caca83b2..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerConstructorAdvice.java +++ /dev/null @@ -1,25 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.trace.api.Config; -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.instrumentation.kafka_common.KafkaConfigHelper; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.producer.ProducerConfig; - -public class ProducerConstructorAdvice { - - @Advice.OnMethodExit(suppress = Throwable.class) - public static void captureConfiguration( - @Advice.FieldValue("metadata") Metadata metadata, - @Advice.Argument(0) ProducerConfig producerConfig) { - MetadataState state = - InstrumentationContext.get(Metadata.class, MetadataState.class) - .putIfAbsent(metadata, MetadataState::new); - if (Config.get().isDataStreamsEnabled()) { - KafkaConfigHelper.storePendingProducerConfig( - state, KafkaConfigHelper.extractProducerConfig(producerConfig)); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerContextPropagationAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerContextPropagationAdvice.java deleted file mode 100644 index e3094e44e0d..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerContextPropagationAdvice.java +++ /dev/null @@ -1,79 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.context.propagation.Propagators.defaultPropagator; -import static datadog.trace.agent.tooling.InstrumenterModule.TargetSystem.CONTEXT_TRACKING; -import static datadog.trace.api.datastreams.DataStreamsContext.fromTagsWithoutCheckpoint; -import static datadog.trace.api.datastreams.DataStreamsTags.Direction.OUTBOUND; -import static datadog.trace.api.datastreams.DataStreamsTags.create; -import static datadog.trace.bootstrap.instrumentation.api.AgentPropagation.DSM_CONCERN; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; -import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.TIME_IN_QUEUE_ENABLED; -import static datadog.trace.instrumentation.kafka_common.StreamingContext.STREAMING_CONTEXT; - -import datadog.context.propagation.Propagator; -import datadog.context.propagation.Propagators; -import datadog.trace.agent.tooling.annotation.AppliesOn; -import datadog.trace.api.Config; -import datadog.trace.api.datastreams.DataStreamsContext; -import datadog.trace.api.datastreams.DataStreamsTags; -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.producer.ProducerRecord; - -@AppliesOn(CONTEXT_TRACKING) -public class ProducerContextPropagationAdvice { - - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter( - @Advice.FieldValue("metadata") Metadata metadata, - @Advice.Argument(value = 0, readOnly = false) ProducerRecord record) { - AgentSpan span = activeSpan(); - if (span == null) return; - MetadataState metadataState = - InstrumentationContext.get(Metadata.class, MetadataState.class).get(metadata); - String clusterId = metadataState != null ? metadataState.clusterId : null; - TextMapInjectAdapterInterface setter = NoopTextMapInjectAdapter.NOOP_SETTER; - // Please note that the minimum magic for kafka 3.8+ is 2 so there is no need to check this - if (Config.get().isKafkaClientPropagationEnabled() - && !Config.get().isKafkaClientPropagationDisabledForTopic(record.topic())) { - setter = TextMapInjectAdapter.SETTER; - } - DataStreamsTags tags = create("kafka", OUTBOUND, record.topic(), null, clusterId); - try { - defaultPropagator().inject(span, record.headers(), setter); - if (STREAMING_CONTEXT.isDisabledForTopic(record.topic()) - || STREAMING_CONTEXT.isSinkTopic(record.topic())) { - // inject the context in the headers, but delay sending the stats until we know the - // message size. - // The stats are saved in the pathway context and sent in PayloadSizeAdvice. - Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); - DataStreamsContext dsmContext = fromTagsWithoutCheckpoint(tags); - dsmPropagator.inject(span.with(dsmContext), record.headers(), setter); - } - } catch (final IllegalStateException e) { - // headers must be read-only from reused record. try again with new one. - record = - new ProducerRecord<>( - record.topic(), - record.partition(), - record.timestamp(), - record.key(), - record.value(), - record.headers()); - - defaultPropagator().inject(span, record.headers(), setter); - if (STREAMING_CONTEXT.isDisabledForTopic(record.topic()) - || STREAMING_CONTEXT.isSinkTopic(record.topic())) { - Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); - DataStreamsContext dsmContext = fromTagsWithoutCheckpoint(tags); - dsmPropagator.inject(span.with(dsmContext), record.headers(), setter); - } - } - if (TIME_IN_QUEUE_ENABLED) { - setter.injectTimeInQueue(record.headers()); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java deleted file mode 100644 index b8f3dff049a..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java +++ /dev/null @@ -1,81 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.traceConfig; -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.KAFKA_RECORDS_COUNT; -import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.JAVA_KAFKA; -import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.KAFKA_POLL; - -import datadog.trace.api.Config; -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.bootstrap.instrumentation.api.AgentScope; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.instrumentation.kafka_common.ClusterIdHolder; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.Metadata; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.internals.ConsumerDelegate; -import org.apache.kafka.common.errors.WakeupException; - -/** - * this method transfers the consumer group from the KafkaConsumer class key to the ConsumerRecords - * key. This is necessary because in the poll method, we don't have access to the KafkaConsumer - * class. - */ -public class RecordsAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static AgentScope onEnter(@Advice.This ConsumerDelegate consumer) { - // Set cluster ID in ClusterIdHolder for Schema Registry instrumentation - KafkaConsumerInfo kafkaConsumerInfo = - InstrumentationContext.get(ConsumerDelegate.class, KafkaConsumerInfo.class).get(consumer); - if (kafkaConsumerInfo != null && Config.get().isDataStreamsEnabled()) { - String clusterId = - KafkaConsumerInstrumentationHelper.extractClusterId( - kafkaConsumerInfo, InstrumentationContext.get(Metadata.class, MetadataState.class)); - if (clusterId != null) { - ClusterIdHolder.set(clusterId); - } - } - - if (traceConfig().isDataStreamsEnabled()) { - final AgentSpan span = startSpan(JAVA_KAFKA.toString(), KAFKA_POLL); - return activateSpan(span); - } - return null; - } - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void captureGroup( - @Advice.Enter final AgentScope scope, - @Advice.This ConsumerDelegate consumer, - @Advice.Return ConsumerRecords records, - @Advice.Thrown Throwable throwable) { - int recordsCount = 0; - if (records != null) { - // new - we are getting the KafkaConsumerInfo from the ConsumerDelegate instead of - // KafkaConsumer - KafkaConsumerInfo kafkaConsumerInfo = - InstrumentationContext.get(ConsumerDelegate.class, KafkaConsumerInfo.class).get(consumer); - if (kafkaConsumerInfo != null) { - InstrumentationContext.get(ConsumerRecords.class, KafkaConsumerInfo.class) - .put(records, kafkaConsumerInfo); - } - recordsCount = records.count(); - } - // Clear cluster ID from Schema Registry instrumentation - ClusterIdHolder.clear(); - - if (scope == null) { - return; - } - AgentSpan span = scope.span(); - span.setTag(KAFKA_RECORDS_COUNT, recordsCount); - if (!(throwable instanceof WakeupException)) { - span.addThrowable(throwable); - } - scope.close(); - span.finish(); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TextMapExtractAdapter.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TextMapExtractAdapter.java deleted file mode 100644 index 30b35216a4b..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TextMapExtractAdapter.java +++ /dev/null @@ -1,70 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.api.Functions.BASE64_DECODE; -import static datadog.trace.api.Functions.UTF8_BYTES_TO_STRING; -import static datadog.trace.api.telemetry.LogCollector.EXCLUDE_TELEMETRY; - -import datadog.trace.api.Config; -import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; -import datadog.trace.bootstrap.instrumentation.api.AgentPropagation.ContextVisitor; -import java.nio.ByteBuffer; -import java.util.Base64; -import java.util.function.Function; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.Headers; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TextMapExtractAdapter implements ContextVisitor { - private static final Logger log = LoggerFactory.getLogger(TextMapExtractAdapter.class); - - public static final TextMapExtractAdapter GETTER = - new TextMapExtractAdapter(Config.get().isKafkaClientBase64DecodingEnabled()); - - private final Function headerValueTransformer; - private final Base64.Decoder decoder; - - public TextMapExtractAdapter(boolean decodeBase64Headers) { - if (decodeBase64Headers) { - this.headerValueTransformer = BASE64_DECODE; - this.decoder = Base64.getDecoder(); - } else { - this.headerValueTransformer = UTF8_BYTES_TO_STRING; - this.decoder = null; - } - } - - @Override - public void forEachKey(Headers carrier, AgentPropagation.KeyClassifier classifier) { - for (Header header : carrier) { - String key = header.key(); - byte[] value = header.value(); - if (null == value) { - continue; - } - String decoded = headerValueTransformer.apply(value); - if (decoded == null) { - log.debug(EXCLUDE_TELEMETRY, "Failed to Base64-decode Kafka header '{}', skipping", key); - continue; - } - if (!classifier.accept(key, decoded)) { - return; - } - } - } - - public long extractTimeInQueueStart(Headers carrier) { - Header header = carrier.lastHeader(KafkaDecorator.KAFKA_PRODUCED_KEY); - if (null != header) { - try { - ByteBuffer buf = ByteBuffer.allocate(8); - buf.put(decoder != null ? decoder.decode(header.value()) : header.value()); - buf.flip(); - return buf.getLong(); - } catch (Exception e) { - log.debug(EXCLUDE_TELEMETRY, "Unable to get kafka produced time", e); - } - } - return 0; - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TextMapInjectAdapter.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TextMapInjectAdapter.java deleted file mode 100644 index 2b58ff6bae4..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TextMapInjectAdapter.java +++ /dev/null @@ -1,21 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.nio.ByteBuffer; -import org.apache.kafka.common.header.Headers; - -public class TextMapInjectAdapter implements TextMapInjectAdapterInterface { - public static final TextMapInjectAdapter SETTER = new TextMapInjectAdapter(); - - @Override - public void set(final Headers headers, final String key, final String value) { - headers.remove(key).add(key, value.getBytes(UTF_8)); - } - - public void injectTimeInQueue(Headers headers) { - ByteBuffer buf = ByteBuffer.allocate(8); - buf.putLong(System.currentTimeMillis()); - headers.add(KafkaDecorator.KAFKA_PRODUCED_KEY, buf.array()); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TextMapInjectAdapterInterface.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TextMapInjectAdapterInterface.java deleted file mode 100644 index 869613c656b..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TextMapInjectAdapterInterface.java +++ /dev/null @@ -1,8 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import datadog.context.propagation.CarrierSetter; -import org.apache.kafka.common.header.Headers; - -public interface TextMapInjectAdapterInterface extends CarrierSetter { - void injectTimeInQueue(Headers headers); -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterable.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterable.java deleted file mode 100644 index 59b53b1c363..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterable.java +++ /dev/null @@ -1,40 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import java.util.Iterator; -import org.apache.kafka.clients.consumer.ConsumerRecord; - -public class TracingIterable implements Iterable>, TracingIterableDelegator { - private final Iterable> delegate; - private final CharSequence operationName; - private final KafkaDecorator decorator; - private final String group; - private final String clusterId; - private final String bootstrapServers; - - public TracingIterable( - final Iterable> delegate, - final CharSequence operationName, - final KafkaDecorator decorator, - String group, - String clusterId, - String bootstrapServers) { - this.delegate = delegate; - this.operationName = operationName; - this.decorator = decorator; - this.group = group; - this.clusterId = clusterId; - this.bootstrapServers = bootstrapServers; - } - - @Override - public Iterator> iterator() { - // every iteration will add spans. Not only the very first one - return new TracingIterator( - delegate.iterator(), operationName, decorator, group, clusterId, bootstrapServers); - } - - @Override - public Iterable> getDelegate() { - return delegate; - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterableDelegator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterableDelegator.java deleted file mode 100644 index 8b9f94a3e4f..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterableDelegator.java +++ /dev/null @@ -1,8 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import org.apache.kafka.clients.consumer.ConsumerRecord; - -public interface TracingIterableDelegator { - // Used by the streams instrumentation to unwrap (disable) the iteration advice. - Iterable> getDelegate(); -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterator.java deleted file mode 100644 index 7beba848473..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingIterator.java +++ /dev/null @@ -1,175 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.api.datastreams.DataStreamsContext.create; -import static datadog.trace.api.datastreams.DataStreamsTags.Direction.INBOUND; -import static datadog.trace.api.datastreams.DataStreamsTags.create; -import static datadog.trace.bootstrap.instrumentation.api.AgentPropagation.DSM_CONCERN; -import static datadog.trace.bootstrap.instrumentation.api.AgentPropagation.extractContextAndGetSpanContext; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateNext; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.closePrevious; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.traceConfig; -import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.JAVA_KAFKA; -import static datadog.trace.instrumentation.kafka_clients38.TextMapExtractAdapter.GETTER; -import static datadog.trace.instrumentation.kafka_clients38.TextMapInjectAdapter.SETTER; -import static java.util.concurrent.TimeUnit.MILLISECONDS; - -import datadog.context.Context; -import datadog.context.propagation.Propagator; -import datadog.context.propagation.Propagators; -import datadog.trace.api.Config; -import datadog.trace.api.InstrumenterConfig; -import datadog.trace.api.datastreams.DataStreamsContext; -import datadog.trace.api.datastreams.DataStreamsTags; -import datadog.trace.api.datastreams.DataStreamsTransactionExtractor; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags; -import datadog.trace.instrumentation.kafka_common.StreamingContext; -import datadog.trace.instrumentation.kafka_common.Utils; -import java.util.Iterator; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TracingIterator implements Iterator> { - - private static final Logger log = LoggerFactory.getLogger(TracingIterator.class); - - private final Iterator> delegateIterator; - private final CharSequence operationName; - private final KafkaDecorator decorator; - private final String group; - private final String clusterId; - private final String bootstrapServers; - - public TracingIterator( - final Iterator> delegateIterator, - final CharSequence operationName, - final KafkaDecorator decorator, - String group, - String clusterId, - String bootstrapServers) { - this.delegateIterator = delegateIterator; - this.operationName = operationName; - this.decorator = decorator; - this.group = group; - this.clusterId = clusterId; - this.bootstrapServers = bootstrapServers; - } - - @Override - public boolean hasNext() { - boolean moreRecords = delegateIterator.hasNext(); - if (!moreRecords) { - // no more records, use this as a signal to close the last iteration scope - if (InstrumenterConfig.get().isLegacyContextManagerEnabled()) { - closePrevious(true); - } else { - final AgentSpan previousSpan = AgentSpan.fromContext(Context.root().swap()); - if (previousSpan != null) { - previousSpan.finishWithEndToEnd(); - } - } - } - return moreRecords; - } - - @Override - public ConsumerRecord next() { - final ConsumerRecord next = delegateIterator.next(); - startNewRecordSpan(next); - return next; - } - - protected void startNewRecordSpan(ConsumerRecord val) { - try { - if (InstrumenterConfig.get().isLegacyContextManagerEnabled()) { - closePrevious(true); - } else if (val == null) { // previous message span was the last - final AgentSpan previousSpan = AgentSpan.fromContext(Context.root().swap()); - if (previousSpan != null) { - previousSpan.finishWithEndToEnd(); - } - } - AgentSpan span, queueSpan = null; - if (val != null) { - if (!Config.get().isKafkaClientPropagationDisabledForTopic(val.topic())) { - final AgentSpanContext spanContext = - extractContextAndGetSpanContext(val.headers(), GETTER); - long timeInQueueStart = GETTER.extractTimeInQueueStart(val.headers()); - if (timeInQueueStart == 0 || !KafkaDecorator.TIME_IN_QUEUE_ENABLED) { - span = startSpan(JAVA_KAFKA.toString(), operationName, spanContext); - } else { - queueSpan = - startSpan( - JAVA_KAFKA.toString(), - KafkaDecorator.KAFKA_DELIVER, - spanContext, - MILLISECONDS.toMicros(timeInQueueStart)); - KafkaDecorator.BROKER_DECORATE.afterStart(queueSpan); - KafkaDecorator.BROKER_DECORATE.onTimeInQueue(queueSpan, val); - span = startSpan(JAVA_KAFKA.toString(), operationName, queueSpan.spanContext()); - KafkaDecorator.BROKER_DECORATE.beforeFinish(queueSpan); - // The queueSpan will be finished after inner span has been activated to ensure that - // spans are written out together by TraceStructureWriter when running in strict mode - } - - DataStreamsTags tags = create("kafka", INBOUND, val.topic(), group, clusterId); - final long payloadSize = - traceConfig().isDataStreamsEnabled() ? Utils.computePayloadSizeBytes(val) : 0; - if (StreamingContext.STREAMING_CONTEXT.isDisabledForTopic(val.topic())) { - AgentTracer.get() - .getDataStreamsMonitoring() - .setCheckpoint(span, create(tags, val.timestamp(), payloadSize)); - } else { - // when we're in a streaming context we want to consume only from source topics - if (StreamingContext.STREAMING_CONTEXT.isSourceTopic(val.topic())) { - // We have to inject the context to headers here, - // since the data received from the source may leave the topology on - // some other instance of the application, breaking the context propagation - // for DSM users - Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); - DataStreamsContext dsmContext = create(tags, val.timestamp(), payloadSize); - dsmPropagator.inject(span.with(dsmContext), val.headers(), SETTER); - } - } - } else { - span = startSpan(JAVA_KAFKA.toString(), operationName, null); - } - if (val.value() == null) { - span.setTag(InstrumentationTags.TOMBSTONE, true); - } - decorator.afterStart(span); - decorator.onConsume(span, val, group, clusterId, bootstrapServers); - if (InstrumenterConfig.get().isLegacyContextManagerEnabled()) { - activateNext(span); - } else { - final AgentSpan previousSpan = AgentSpan.fromContext(span.swap()); - if (previousSpan != null) { - previousSpan.finishWithEndToEnd(); - } - } - if (null != queueSpan) { - queueSpan.finish(); - } - - AgentTracer.get() - .getDataStreamsMonitoring() - .trackTransaction( - span, - DataStreamsTransactionExtractor.Type.KAFKA_CONSUME_HEADERS, - val.headers(), - Utils.DSM_TRANSACTION_SOURCE_READER); - } - } catch (final Exception e) { - log.debug("Error starting new record span", e); - } - } - - @Override - public void remove() { - delegateIterator.remove(); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingList.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingList.java deleted file mode 100644 index 24cd318d175..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingList.java +++ /dev/null @@ -1,161 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.ListIterator; -import org.apache.kafka.clients.consumer.ConsumerRecord; - -public class TracingList implements List>, TracingIterableDelegator { - - private final List> delegate; - private final CharSequence operationName; - private final KafkaDecorator decorator; - private final String group; - private final String clusterId; - private final String bootstrapServers; - - public TracingList( - final List> delegate, - final CharSequence operationName, - final KafkaDecorator decorator, - String group, - String clusterId, - String bootstrapServers) { - this.operationName = operationName; - this.decorator = decorator; - this.delegate = delegate; - this.group = group; - this.clusterId = clusterId; - this.bootstrapServers = bootstrapServers; - } - - @Override - public int size() { - return delegate.size(); - } - - @Override - public boolean isEmpty() { - return delegate.isEmpty(); - } - - @Override - public boolean contains(final Object o) { - return delegate.contains(o); - } - - @Override - public Iterator> iterator() { - return listIterator(0); - } - - @Override - public Object[] toArray() { - return delegate.toArray(); - } - - @Override - public T[] toArray(final T[] a) { - return delegate.toArray(a); - } - - @Override - public boolean add(final ConsumerRecord consumerRecord) { - return delegate.add(consumerRecord); - } - - @Override - public boolean remove(final Object o) { - return delegate.remove(o); - } - - @Override - public boolean containsAll(final Collection c) { - return delegate.containsAll(c); - } - - @Override - public boolean addAll(final Collection> c) { - return delegate.addAll(c); - } - - @Override - public boolean addAll(final int index, final Collection> c) { - return delegate.addAll(index, c); - } - - @Override - public boolean removeAll(final Collection c) { - return delegate.removeAll(c); - } - - @Override - public boolean retainAll(final Collection c) { - return delegate.retainAll(c); - } - - @Override - public void clear() { - delegate.clear(); - } - - @Override - public ConsumerRecord get(final int index) { - // TODO: should this be instrumented as well? - return delegate.get(index); - } - - @Override - public ConsumerRecord set(final int index, final ConsumerRecord element) { - return delegate.set(index, element); - } - - @Override - public void add(final int index, final ConsumerRecord element) { - delegate.add(index, element); - } - - @Override - public ConsumerRecord remove(final int index) { - return delegate.remove(index); - } - - @Override - public int indexOf(final Object o) { - return delegate.indexOf(o); - } - - @Override - public int lastIndexOf(final Object o) { - return delegate.lastIndexOf(o); - } - - @Override - public ListIterator> listIterator() { - return listIterator(0); - } - - @Override - public ListIterator> listIterator(final int index) { - // every iteration will add spans. Not only the very first one - return new TracingListIterator( - delegate.listIterator(index), operationName, decorator, group, clusterId, bootstrapServers); - } - - @Override - public List> subList(final int fromIndex, final int toIndex) { - return new TracingList( - delegate.subList(fromIndex, toIndex), - operationName, - decorator, - group, - clusterId, - bootstrapServers); - } - - @Override - public List> getDelegate() { - return delegate; - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingListIterator.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingListIterator.java deleted file mode 100644 index 3b17006d562..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/TracingListIterator.java +++ /dev/null @@ -1,75 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.closePrevious; - -import datadog.context.Context; -import datadog.trace.api.InstrumenterConfig; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import java.util.ListIterator; -import org.apache.kafka.clients.consumer.ConsumerRecord; - -public class TracingListIterator extends TracingIterator - implements ListIterator> { - - private final ListIterator> delegateIterator; - - public TracingListIterator( - ListIterator> delegateIterator, - CharSequence operationName, - KafkaDecorator decorator, - String group, - String clusterId, - String bootstrapServers) { - super(delegateIterator, operationName, decorator, group, clusterId, bootstrapServers); - this.delegateIterator = delegateIterator; - } - - @Override - public boolean hasPrevious() { - boolean moreRecords = delegateIterator.hasPrevious(); - if (!moreRecords) { - // no more records, use this as a signal to close the last iteration scope - if (InstrumenterConfig.get().isLegacyContextManagerEnabled()) { - closePrevious(true); - } else { - final AgentSpan previousSpan = AgentSpan.fromContext(Context.root().swap()); - if (previousSpan != null) { - previousSpan.finishWithEndToEnd(); - } - } - } - return moreRecords; - } - - @Override - public ConsumerRecord previous() { - final ConsumerRecord prev = delegateIterator.previous(); - startNewRecordSpan(prev); - return prev; - } - - @Override - public int nextIndex() { - return delegateIterator.nextIndex(); - } - - @Override - public int previousIndex() { - return delegateIterator.previousIndex(); - } - - /* - * org.apache.kafka.clients.consumer.ConsumerRecords::records(TopicPartition) always returns - * UnmodifiableList. Modifiable operations will lead to exception - */ - - @Override - public void set(ConsumerRecord consumerRecord) { - delegateIterator.set(consumerRecord); - } - - @Override - public void add(ConsumerRecord consumerRecord) { - delegateIterator.add(consumerRecord); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/DDOffsetCommitCallbackNullGroupForkedTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/DDOffsetCommitCallbackNullGroupForkedTest.groovy deleted file mode 100644 index dac578f1137..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/DDOffsetCommitCallbackNullGroupForkedTest.groovy +++ /dev/null @@ -1,135 +0,0 @@ -import datadog.trace.agent.test.naming.VersionedNamingTestBase -import datadog.trace.instrumentation.kafka_clients38.DDOffsetCommitCallback -import datadog.trace.instrumentation.kafka_clients38.KafkaConsumerInfo -import org.apache.kafka.clients.consumer.OffsetAndMetadata -import org.apache.kafka.clients.consumer.OffsetCommitCallback -import org.apache.kafka.common.TopicPartition - -/** - * Unit test to verify that DSM backlog tracking works even when consumer group is null. - * - * This test reproduces a bug where calling Optional.get() on an empty Optional - * (when consumer group is null) throws NoSuchElementException, causing the entire - * backlog tracking to fail silently. - * - * Before the fix: - * - KafkaConsumerInfo.getConsumerGroup() returns Optional.empty() when consumer group is null - * - DDOffsetCommitCallback calls .get() on this empty Optional - * - NoSuchElementException is thrown - * - No backlog metrics are emitted - * - * After the fix (using .orElse(null) instead of .get()): - * - Backlog metrics are emitted even with null consumer group - */ -class DDOffsetCommitCallbackNullGroupForkedTest extends VersionedNamingTestBase { - - @Override - int version() { - return 0 - } - - @Override - String operation() { - return null - } - - @Override - String service() { - return "test-service" - } - - @Override - protected boolean isDataStreamsEnabled() { - return true - } - - def "test DDOffsetCommitCallback handles null consumer group without exception"() { - setup: - // Create KafkaConsumerInfo with NULL consumer group - this is the key to reproducing the bug - def kafkaConsumerInfo = new KafkaConsumerInfo(null, null, "localhost:9092") - - // Verify the consumer group is indeed empty - assert !kafkaConsumerInfo.getConsumerGroup().isPresent() : "Consumer group should be empty for this test" - - def callbackInvoked = false - def innerCallback = new OffsetCommitCallback() { - @Override - void onComplete(Map offsets, Exception exception) { - callbackInvoked = true - } - } - - def ddCallback = new DDOffsetCommitCallback(innerCallback, kafkaConsumerInfo) - - def offsets = [ - (new TopicPartition("test-topic", 0)): new OffsetAndMetadata(100L) - ] - - when: - // This should NOT throw an exception - // Before the fix: NoSuchElementException would be thrown here - // After the fix: completes successfully with null consumer group - ddCallback.onComplete(offsets, null) - - then: - // Inner callback should have been invoked - callbackInvoked - - // No exception was thrown (implicit - test would fail otherwise) - noExceptionThrown() - } - - def "test DDOffsetCommitCallback handles null KafkaConsumerInfo"() { - setup: - def callbackInvoked = false - def innerCallback = new OffsetCommitCallback() { - @Override - void onComplete(Map offsets, Exception exception) { - callbackInvoked = true - } - } - - // KafkaConsumerInfo is completely null - def ddCallback = new DDOffsetCommitCallback(innerCallback, null) - - def offsets = [ - (new TopicPartition("test-topic", 0)): new OffsetAndMetadata(100L) - ] - - when: - ddCallback.onComplete(offsets, null) - - then: - callbackInvoked - noExceptionThrown() - } - - def "test DDOffsetCommitCallback with valid consumer group still works"() { - setup: - def kafkaConsumerInfo = new KafkaConsumerInfo("test-group", null, "localhost:9092") - - assert kafkaConsumerInfo.getConsumerGroup().isPresent() - assert kafkaConsumerInfo.getConsumerGroup().get() == "test-group" - - def callbackInvoked = false - def innerCallback = new OffsetCommitCallback() { - @Override - void onComplete(Map offsets, Exception exception) { - callbackInvoked = true - } - } - - def ddCallback = new DDOffsetCommitCallback(innerCallback, kafkaConsumerInfo) - - def offsets = [ - (new TopicPartition("test-topic", 0)): new OffsetAndMetadata(100L) - ] - - when: - ddCallback.onComplete(offsets, null) - - then: - callbackInvoked - noExceptionThrown() - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy deleted file mode 100644 index 0dec7bbe6bc..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientCustomPropagationConfigTest.groovy +++ /dev/null @@ -1,297 +0,0 @@ -import datadog.trace.agent.test.InstrumentationSpecification -import datadog.trace.api.config.TraceInstrumentationConfig -import datadog.trace.bootstrap.instrumentation.api.AgentSpan -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.header.Headers -import org.apache.kafka.common.header.internals.RecordHeaders -import org.springframework.kafka.core.DefaultKafkaConsumerFactory -import org.springframework.kafka.core.DefaultKafkaProducerFactory -import org.springframework.kafka.core.KafkaTemplate -import org.springframework.kafka.listener.ContainerProperties -import org.springframework.kafka.listener.KafkaMessageListenerContainer -import org.springframework.kafka.listener.MessageListener -import org.springframework.kafka.test.EmbeddedKafkaBroker -import org.springframework.kafka.test.EmbeddedKafkaKraftBroker -import org.springframework.kafka.test.utils.ContainerTestUtils -import org.springframework.kafka.test.utils.KafkaTestUtils -import spock.lang.Shared - -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.TimeUnit - -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan -import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.KAFKA_PRODUCE - -class KafkaClientCustomPropagationConfigTest extends InstrumentationSpecification { - static final SHARED_TOPIC = ["topic1", "topic2", "topic3", "topic4"] - static final MESSAGE = "Testing without headers for certain topics" - - @Shared - EmbeddedKafkaBroker embeddedKafka - - def setupSpec() { - embeddedKafka = new EmbeddedKafkaKraftBroker(1, 2, *SHARED_TOPIC) - embeddedKafka.afterPropertiesSet() - } - - def cleanupSpec() { - embeddedKafka.destroy() - } - - static final dataTable() { - [ - ["topic1,topic2,topic3,topic4", false, false, false, false], - ["topic1,topic2", false, false, true, true], - ["topic1", false, true, true, true], - ["", true, true, true, true], - ["randomTopic", true, true, true, true] - ] - } - - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - - - @Override - void configurePreAgent() { - super.configurePreAgent() - - injectSysConfig("dd.kafka.e2e.duration.enabled", "true") - } - - def "test kafka client header propagation with topic filters"() { - setup: - injectSysConfig(TraceInstrumentationConfig.KAFKA_CLIENT_PROPAGATION_DISABLED_TOPICS, value as String) - - def senderProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - def producerFactory = new DefaultKafkaProducerFactory(senderProps) - def kafkaTemplate = new KafkaTemplate(producerFactory) - - // set up the Kafka consumer properties - def consumerProperties = KafkaTestUtils.consumerProps( embeddedKafka.getBrokersAsString(),"sender", "false") - - // create a Kafka consumer factory - def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - - // set the topic that needs to be consumed - def containerProperties1 = new ContainerProperties(SHARED_TOPIC[0]) - def containerProperties2 = new ContainerProperties(SHARED_TOPIC[1]) - def containerProperties3 = new ContainerProperties(SHARED_TOPIC[2]) - def containerProperties4 = new ContainerProperties(SHARED_TOPIC[3]) - - // create a Kafka MessageListenerContainer - def container1 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties1) - def container2 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties2) - def container3 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties3) - def container4 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties4) - - // create a thread safe queue to store the received message - def records1 = new LinkedBlockingQueue>() - def records2 = new LinkedBlockingQueue>() - def records3 = new LinkedBlockingQueue>() - def records4 = new LinkedBlockingQueue>() - - // setup a Kafka message listener - container1.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records1.add(record) - } - }) - - container2.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records2.add(record) - } - }) - - container3.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records3.add(record) - } - }) - - container4.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records4.add(record) - } - }) - - // start the container and underlying message listener - container1.start() - container2.start() - container3.start() - container4.start() - - // wait until the container has the required number of assigned partitions - ContainerTestUtils.waitForAssignment(container1, container1.assignedPartitions.size()) - ContainerTestUtils.waitForAssignment(container2, container2.assignedPartitions.size()) - ContainerTestUtils.waitForAssignment(container3, container3.assignedPartitions.size()) - ContainerTestUtils.waitForAssignment(container4, container4.assignedPartitions.size()) - - when: - for (String topic : SHARED_TOPIC) { - kafkaTemplate.send(topic, MESSAGE) - } - - then: - // check that the message was received - def received1 = records1.poll(5, TimeUnit.SECONDS) - def received2 = records2.poll(5, TimeUnit.SECONDS) - def received3 = records3.poll(5, TimeUnit.SECONDS) - def received4 = records4.poll(5, TimeUnit.SECONDS) - - received1.headers().iterator().hasNext() == expected1 - received2.headers().iterator().hasNext() == expected2 - received3.headers().iterator().hasNext() == expected3 - received4.headers().iterator().hasNext() == expected4 - - cleanup: - producerFactory.stop() - container1?.stop() - container2?.stop() - container3?.stop() - container4?.stop() - - where: - [value, expected1, expected2, expected3, expected4]<< dataTable() - } - - def "test consumer with topic filters"() { - setup: - injectSysConfig(TraceInstrumentationConfig.KAFKA_CLIENT_PROPAGATION_DISABLED_TOPICS, value as String) - def senderProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - def producerFactory = new DefaultKafkaProducerFactory(senderProps) - def kafkaTemplate = new KafkaTemplate(producerFactory) - - // set up the Kafka consumer properties - def consumerProperties = KafkaTestUtils.consumerProps( embeddedKafka.getBrokersAsString(),"sender", "false") - - // create a Kafka consumer factory - def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - - // set the topic that needs to be consumed - def containerProperties1 = new ContainerProperties(SHARED_TOPIC[0]) - def containerProperties2 = new ContainerProperties(SHARED_TOPIC[1]) - def containerProperties3 = new ContainerProperties(SHARED_TOPIC[2]) - def containerProperties4 = new ContainerProperties(SHARED_TOPIC[3]) - - // create a Kafka MessageListenerContainer - def container1 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties1) - def container2 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties2) - def container3 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties3) - def container4 = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties4) - - // create a thread safe queue to store the received message - def records1 = new LinkedBlockingQueue() - def records2 = new LinkedBlockingQueue() - def records3 = new LinkedBlockingQueue() - def records4 = new LinkedBlockingQueue() - - // setup a Kafka message listener - container1.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records1.add(activeSpan()) - } - }) - - container2.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records2.add(activeSpan()) - } - }) - - container3.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records3.add(activeSpan()) - } - }) - - container4.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records4.add(activeSpan()) - } - }) - - // start the container and underlying message listener - container1.start() - container2.start() - container3.start() - container4.start() - - // wait until the container has the required number of assigned partitions - ContainerTestUtils.waitForAssignment(container1, container1.assignedPartitions.size()) - ContainerTestUtils.waitForAssignment(container2, container2.assignedPartitions.size()) - ContainerTestUtils.waitForAssignment(container3, container3.assignedPartitions.size()) - ContainerTestUtils.waitForAssignment(container4, container4.assignedPartitions.size()) - - when: - Headers header = new RecordHeaders() - - AgentSpan span = startSpan("kafka", KAFKA_PRODUCE) - activateSpan(span).withCloseable { - for (String topic : SHARED_TOPIC) { - ProducerRecord record = new ProducerRecord<>( - topic, - 0, - null, - MESSAGE, - header - ) - kafkaTemplate.send(record as ProducerRecord) - } - } - span.finish() - - then: - // check that the message was received - def received1 = records1.poll(5, TimeUnit.SECONDS) - def received2 = records2.poll(5, TimeUnit.SECONDS) - def received3 = records3.poll(5, TimeUnit.SECONDS) - def received4 = records4.poll(5, TimeUnit.SECONDS) - - if (expected1) { - assert received1.getTraceId() == span.getTraceId() - } else { - assert received1.getTraceId() != span.getTraceId() - } - if (expected2) { - assert received2.getTraceId() == span.getTraceId() - } else { - assert received2.getTraceId() != span.getTraceId() - } - if (expected3) { - assert received3.getTraceId() == span.getTraceId() - } else { - assert received3.getTraceId() != span.getTraceId() - } - if (expected4) { - assert received4.getTraceId() == span.getTraceId() - } else { - assert received4.getTraceId() != span.getTraceId() - } - - cleanup: - producerFactory.stop() - container1?.stop() - container2?.stop() - container3?.stop() - container4?.stop() - - where: - [value, expected1, expected2, expected3, expected4]<< dataTable() - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientTestBase.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientTestBase.groovy deleted file mode 100644 index 579532664bd..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientTestBase.groovy +++ /dev/null @@ -1,1250 +0,0 @@ -import datadog.trace.agent.test.InstrumentationSpecification -import datadog.trace.agent.test.asserts.TraceAssert -import datadog.trace.agent.test.naming.VersionedNamingTestBase -import datadog.trace.api.Config -import datadog.trace.api.config.TraceInstrumentationConfig -import datadog.trace.api.config.TracerConfig -import datadog.trace.api.DDTags -import datadog.trace.api.datastreams.DataStreamsTags -import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags -import datadog.trace.bootstrap.instrumentation.api.Tags -import datadog.trace.common.writer.ListWriter -import datadog.trace.core.DDSpan -import datadog.trace.core.datastreams.StatsGroup -import datadog.trace.instrumentation.kafka_common.ClusterIdHolder -import org.apache.kafka.clients.consumer.ConsumerConfig -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.apache.kafka.clients.consumer.KafkaConsumer -import org.apache.kafka.clients.producer.KafkaProducer -import org.apache.kafka.clients.producer.Producer -import org.apache.kafka.clients.producer.ProducerConfig -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.clients.producer.RecordMetadata -import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.header.internals.RecordHeader -import org.apache.kafka.common.header.internals.RecordHeaders -import org.apache.kafka.common.serialization.StringSerializer - -import java.nio.charset.StandardCharsets -import org.springframework.kafka.core.DefaultKafkaConsumerFactory -import org.springframework.kafka.core.DefaultKafkaProducerFactory -import org.springframework.kafka.core.KafkaTemplate -import org.springframework.kafka.listener.ContainerProperties -import org.springframework.kafka.listener.KafkaMessageListenerContainer -import org.springframework.kafka.listener.MessageListener -import org.springframework.kafka.test.EmbeddedKafkaBroker -import org.springframework.kafka.test.EmbeddedKafkaKraftBroker -import org.springframework.kafka.test.utils.ContainerTestUtils -import org.springframework.kafka.test.utils.KafkaTestUtils - -import java.util.concurrent.ExecutionException -import java.util.concurrent.Future -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.TimeUnit - -import static datadog.trace.agent.test.asserts.TagsAssert.codeOriginTags -import static datadog.trace.agent.test.utils.TraceUtils.basicSpan -import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.isAsyncPropagationEnabled - -abstract class KafkaClientTestBase extends VersionedNamingTestBase { - static final SHARED_TOPIC = "shared.topic" - - EmbeddedKafkaBroker embeddedKafka - - def setup() { - embeddedKafka = new EmbeddedKafkaKraftBroker(1, 2, SHARED_TOPIC) - embeddedKafka.afterPropertiesSet() - - TEST_WRITER.setFilter(dropKafkaPoll) - } - - def cleanup() { - embeddedKafka.destroy() - } - - @Override - boolean useStrictTraceWrites() { - // TODO fix this by making sure that spans get closed properly - return false - } - - @Override - void configurePreAgent() { - super.configurePreAgent() - codeOriginSetup() - injectSysConfig("dd.kafka.e2e.duration.enabled", "true") - } - - public static final LinkedHashMap PRODUCER_PATHWAY_EDGE_TAGS - - // filter out Kafka poll, since the function is called in a loop, giving inconsistent results - final ListWriter.Filter dropKafkaPoll = new ListWriter.Filter() { - @Override - boolean accept(List trace) { - return !(trace.size() == 1 && - trace.get(0).getResourceName().toString().equals("kafka.poll")) - } - } - - final ListWriter.Filter dropEmptyKafkaPoll = new ListWriter.Filter() { - @Override - boolean accept(List trace) { - return !(trace.size() == 1 && - trace.get(0).getResourceName().toString().equals("kafka.poll") && - trace.get(0).getTag(InstrumentationTags.KAFKA_RECORDS_COUNT).equals(0)) - } - } - - // TraceID, start times & names changed based on the configuration, so overriding the sort to give consistent test results - private static class SortKafkaTraces implements Comparator> { - @Override - int compare(List o1, List o2) { - return rootSpanTrace(o1) - rootSpanTrace(o2) - } - - int rootSpanTrace(List trace) { - assert !trace.isEmpty() - def rootSpan = trace.get(0).localRootSpan - switch (rootSpan.operationName.toString()) { - case "parent": - return 3 - case "kafka.poll": - return 2 - default: - return 1 - } - } - } - - static { - PRODUCER_PATHWAY_EDGE_TAGS = new LinkedHashMap<>(3) - PRODUCER_PATHWAY_EDGE_TAGS.put("direction", "out") - PRODUCER_PATHWAY_EDGE_TAGS.put("topic", SHARED_TOPIC) - PRODUCER_PATHWAY_EDGE_TAGS.put("type", "kafka") - } - - @Override - int version() { - 0 - } - - @Override - String operation() { - return null - } - - String operationForProducer() { - "kafka.produce" - } - - String operationForConsumer() { - "kafka.consume" - } - - String serviceForTimeInQueue() { - "kafka" - } - - abstract boolean hasQueueSpan() - - abstract boolean splitByDestination() - - @Override - protected boolean isDataStreamsEnabled() { - return true - } - - def "test kafka produce and consume"() { - setup: - def producerProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - if (isDataStreamsEnabled()) { - producerProps.put(ProducerConfig.METADATA_MAX_AGE_CONFIG, 1000) - } - // set a low max request size, so that we can crash it - TEST_WRITER.setFilter(dropEmptyKafkaPoll) - KafkaProducer producer = new KafkaProducer<>(producerProps, new StringSerializer(), new StringSerializer()) - String clusterId = null - while (clusterId == null || clusterId.isEmpty()) { - Thread.sleep(1500) - clusterId = producer.metadata.fetch().clusterResource().clusterId() - } - - // set up the Kafka consumer properties - def consumerProperties = KafkaTestUtils.consumerProps("sender", "false", embeddedKafka) - - // create a Kafka consumer factory - def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - - // set the topic that needs to be consumed - def containerProperties = new ContainerProperties(SHARED_TOPIC) - - // create a Kafka MessageListenerContainer - def container = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties) - - // create a thread safe queue to store the received message - def records = new LinkedBlockingQueue>() - - // setup a Kafka message listener - container.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records.add(record) - } - }) - - // start the container and underlying message listener - container.start() - - // wait until the container has the required number of assigned partitions - ContainerTestUtils.waitForAssignment(container, embeddedKafka.getPartitionsPerTopic()) - when: - String greeting = "Hello Spring Kafka Sender!" - runUnderTrace("parent") { - producer.send(new ProducerRecord(SHARED_TOPIC, greeting)) { meta, ex -> - assert isAsyncPropagationEnabled() - if (ex == null) { - runUnderTrace("producer callback") {} - } else { - runUnderTrace("producer exception: " + ex) {} - } - } - blockUntilChildSpansFinished(2) - } - if (isDataStreamsEnabled()) { - TEST_DATA_STREAMS_WRITER.waitForGroups(2) - // wait for produce offset 0, commit offset 0 on partition 0 and 1, and commit offset 1 on 1 partition - //TODO - TEST_DATA_STREAMS_WRITER.waitForBacklogs(2) - } - - then: - // // check that the message was received - def received = records.poll(10, TimeUnit.SECONDS) - received.value() == greeting - received.key() == null - - // verify ClusterIdHolder was properly cleaned up after produce and consume - ClusterIdHolder.get() == null - int nTraces = isDataStreamsEnabled() ? 3 : 2 - int produceTraceIdx = nTraces - 1 - TEST_WRITER.waitForTraces(nTraces) - def traces = new ArrayList<>(TEST_WRITER) - traces.sort(new SortKafkaTraces()) - codeOriginTags(TEST_WRITER) - assertTraces(nTraces, new SortKafkaTraces()) { - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, span(1)) - queueSpan(it, trace(produceTraceIdx)[2]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(produceTraceIdx)[2]) - } - } - if (isDataStreamsEnabled()) { - trace(1, { pollSpan(it) }) - } - trace(3) { - basicSpan(it, "parent") - basicSpan(it, "producer callback", span(0)) - producerSpan(it, producerProps, span(0), false) - } - } - def headers = received.headers() - headers.iterator().hasNext() - new String(headers.headers("x-datadog-trace-id").iterator().next().value()) == "${traces[produceTraceIdx][2].traceId}" - new String(headers.headers("x-datadog-parent-id").iterator().next().value()) == "${traces[produceTraceIdx][2].spanId}" - if (isDataStreamsEnabled()) { - StatsGroup first = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == 0 } - verifyAll(first) { - tags.hasAllTags("direction:out", "kafka_cluster_id:$clusterId", "topic:$SHARED_TOPIC".toString(), "type:kafka") - } - StatsGroup second = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == first.hash } - verifyAll(second) { - tags.hasAllTags( - "direction:in", - "group:sender", - "kafka_cluster_id:$clusterId", - "topic:$SHARED_TOPIC".toString(), - "type:kafka" - ) - } - - def sorted = new ArrayList(TEST_DATA_STREAMS_WRITER.backlogs).sort { it.type } - verifyAll(sorted) { - size() == 2 - get(0).hasAllTags( - "consumer_group:sender", - "kafka_cluster_id:$clusterId", - "partition:" + received.partition(), - "topic:$SHARED_TOPIC", - "type:kafka_commit" - ) - get(1).hasAllTags( - "kafka_cluster_id:$clusterId", - "partition:" + received.partition(), - "topic:" + SHARED_TOPIC, - "type:kafka_produce" - ) - } - } - - cleanup: - producer.close() - container?.stop() - } - - def "test producing message too large"() { - setup: - def producerProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - producerProps.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, 10) - - KafkaProducer producer = new KafkaProducer<>(producerProps, new StringSerializer(), new StringSerializer()) - - when: - String greeting = "Hello Spring Kafka" - Future future = producer.send(new ProducerRecord(SHARED_TOPIC, greeting)) { meta, ex -> - } - future.get() - then: - thrown ExecutionException - cleanup: - producer.close() - } - - def "test spring kafka template produce and consume"() { - setup: - - def senderProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - if (isDataStreamsEnabled()) { - senderProps.put(ProducerConfig.METADATA_MAX_AGE_CONFIG, 1000) - } - def producerFactory = new DefaultKafkaProducerFactory(senderProps) - def kafkaTemplate = new KafkaTemplate(producerFactory) - String clusterId = null - if (isDataStreamsEnabled()) { - clusterId = waitForKafkaMetadataUpdate(kafkaTemplate) - } - - // set up the Kafka consumer properties - def consumerProperties = KafkaTestUtils.consumerProps(embeddedKafka.getBrokersAsString(), "sender", "false") - - // create a Kafka consumer factory - def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - - // set the topic that needs to be consumed - def containerProperties = new ContainerProperties(SHARED_TOPIC) - - // create a Kafka MessageListenerContainer - def container = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties) - - // create a thread safe queue to store the received message - def records = new LinkedBlockingQueue>() - - // setup a Kafka message listener - container.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records.add(record) - } - }) - - // start the container and underlying message listener - container.start() - - // wait until the container has the required number of assigned partitions - ContainerTestUtils.waitForAssignment(container, container.assignedPartitions.size()) - - when: - String greeting = "Hello Spring Kafka Sender!" - runUnderTrace("parent") { - kafkaTemplate.send(SHARED_TOPIC, greeting).whenComplete { meta, ex -> - assert isAsyncPropagationEnabled() - if (ex == null) { - runUnderTrace("producer callback") {} - } else { - runUnderTrace("producer exception: " + ex) {} - } - } - blockUntilChildSpansFinished(2) - } - if (isDataStreamsEnabled()) { - TEST_DATA_STREAMS_WRITER.waitForGroups(2) - // wait for produce offset 0, commit offset 0 on partition 0 and 1, and commit offset 1 on 1 partition. - //TODO - TEST_DATA_STREAMS_WRITER.waitForBacklogs(2) - } - - then: - // check that the message was received - def received = records.poll(5, TimeUnit.SECONDS) - received.value() == greeting - received.key() == null - - assertTraces(2, SORT_TRACES_BY_ID) { - trace(3) { - basicSpan(it, "parent") - basicSpan(it, "producer callback", span(0)) - producerSpan(it, senderProps, span(0), false) - } - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, trace(1)[1]) - queueSpan(it, trace(0)[2]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[2]) - } - } - } - - // sort a snapshot so the producer trace is deterministically first, regardless of write order - def sortedTraces = new ArrayList<>(TEST_WRITER) - sortedTraces.sort(SORT_TRACES_BY_ID) - def headers = received.headers() - headers.iterator().hasNext() - new String(headers.headers("x-datadog-trace-id").iterator().next().value()) == "${sortedTraces[0][2].traceId}" - new String(headers.headers("x-datadog-parent-id").iterator().next().value()) == "${sortedTraces[0][2].spanId}" - - if (isDataStreamsEnabled()) { - StatsGroup first = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == 0 } - verifyAll(first) { - tags.hasAllTags( - "direction:out", - "kafka_cluster_id:$clusterId".toString(), - "topic:$SHARED_TOPIC".toString(), - "type:kafka" - ) - } - - StatsGroup second = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == first.hash } - verifyAll(second) { - tags.hasAllTags( - "direction:in", - "group:sender", - "kafka_cluster_id:$clusterId".toString(), - "topic:$SHARED_TOPIC".toString(), - "type:kafka" - ) - } - def items = new ArrayList(TEST_DATA_STREAMS_WRITER.backlogs).sort { it.type } - verifyAll(items) { - size() == 2 - get(0).hasAllTags( - "consumer_group:sender", - "kafka_cluster_id:$clusterId".toString(), - "partition:" + received.partition(), - "topic:" + SHARED_TOPIC, - "type:kafka_commit" - ) - get(1).hasAllTags( - "kafka_cluster_id:$clusterId".toString(), - "partition:" + received.partition(), - "topic:" + SHARED_TOPIC, - "type:kafka_produce" - ) - } - } - - cleanup: - producerFactory.stop() - container?.stop() - } - - def "test pass through tombstone"() { - setup: - - def senderProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - def producerFactory = new DefaultKafkaProducerFactory(senderProps) - def kafkaTemplate = new KafkaTemplate(producerFactory) - - // set up the Kafka consumer properties - def consumerProperties = KafkaTestUtils.consumerProps(embeddedKafka.getBrokersAsString(), "sender", "false") - - // create a Kafka consumer factory - def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - - // set the topic that needs to be consumed - def containerProperties = new ContainerProperties(SHARED_TOPIC) - - // create a Kafka MessageListenerContainer - def container = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties) - - // create a thread safe queue to store the received message - def records = new LinkedBlockingQueue>() - - // setup a Kafka message listener - container.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records.add(record) - } - }) - - // start the container and underlying message listener - container.start() - - // wait until the container has the required number of assigned partitions - ContainerTestUtils.waitForAssignment(container, container.assignedPartitions.size()) - - when: - kafkaTemplate.send(SHARED_TOPIC, null) - - then: - // check that the message was received - def received = records.poll(5, TimeUnit.SECONDS) - received.value() == null - received.key() == null - - assertTraces(2, SORT_TRACES_BY_ID) { - trace(1) { - producerSpan(it, senderProps, null, false, true) - } - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, trace(1)[1], 0..0, true) - queueSpan(it, trace(0)[0]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[0], 0..0, true) - } - } - } - - cleanup: - producerFactory.stop() - container?.stop() - } - - def "test records(TopicPartition) kafka consume"() { - setup: - // set up the Kafka consumer properties - def kafkaPartition = 0 - def consumerProperties = KafkaTestUtils.consumerProps(embeddedKafka.getBrokersAsString(), "sender", "false") - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - - def senderProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer(senderProps) - - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - - when: - def greeting = "Hello from MockConsumer!" - producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, greeting)) - - then: - TEST_WRITER.waitForTraces(1) - def pollResult = KafkaTestUtils.getRecords(consumer) - - def recs = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).iterator() - - def first = null - if (recs.hasNext()) { - first = recs.next() - } - - then: - !recs.hasNext() - first.value() == greeting - first.key() == null - - assertTraces(2, SORT_TRACES_BY_ID) { - trace(1) { - producerSpan(it, senderProps) - } - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, trace(1)[1]) - queueSpan(it, trace(0)[0]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[0]) - } - } - } - - cleanup: - consumer.close() - producer.close() - } - - def "test records(TopicPartition).subList kafka consume"() { - setup: - - def senderProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - def consumerProperties = KafkaTestUtils.consumerProps(embeddedKafka.getBrokersAsString(), "sender", "false") - - // set up the Kafka consumer properties - def kafkaPartition = 0 - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - - def producer = new KafkaProducer(senderProps) - - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - - when: - def greeting = "Hello from MockConsumer!" - producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, greeting)) - - then: - TEST_WRITER.waitForTraces(1) - def pollResult = KafkaTestUtils.getRecords(consumer) - - def records = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)) - def recs = records.subList(0, records.size()).iterator() - - def first = null - if (recs.hasNext()) { - first = recs.next() - } - - then: - !recs.hasNext() - first.value() == greeting - first.key() == null - - assertTraces(2, SORT_TRACES_BY_ID) { - trace(1) { - producerSpan(it, senderProps) - } - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, trace(1)[1]) - queueSpan(it, trace(0)[0]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[0]) - } - } - } - - cleanup: - consumer.close() - producer.close() - } - - def "test records(TopicPartition).forEach kafka consume"() { - setup: - - def senderProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - def consumerProperties = KafkaTestUtils.consumerProps(embeddedKafka.getBrokersAsString(), "sender", "false") - - // set up the Kafka consumer properties - def kafkaPartition = 0 - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - - def producer = new KafkaProducer(senderProps) - - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - - when: - def greeting = "Hello from MockConsumer!" - producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, greeting)) - - then: - TEST_WRITER.waitForTraces(1) - def pollResult = KafkaTestUtils.getRecords(consumer) - - def records = pollResult.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)) - - def last = null - records.forEach { - last = it - assert activeSpan() != null - } - - then: - records.size() == 1 - last.value() == greeting - last.key() == null - - assertTraces(2, SORT_TRACES_BY_ID) { - trace(1) { - producerSpan(it, senderProps) - } - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, trace(1)[1]) - queueSpan(it, trace(0)[0]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[0]) - } - } - } - - cleanup: - consumer.close() - producer.close() - } - - def "test iteration backwards over ConsumerRecords"() { - setup: - - def senderProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - def consumerProperties = KafkaTestUtils.consumerProps(embeddedKafka.getBrokersAsString(), "sender", "false") - - def kafkaPartition = 0 - consumerProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - def consumer = new KafkaConsumer(consumerProperties) - - def producer = new KafkaProducer(senderProps) - - consumer.assign(Arrays.asList(new TopicPartition(SHARED_TOPIC, kafkaPartition))) - - when: - def greetings = ["msg 1", "msg 2", "msg 3"] - greetings.each { - producer.send(new ProducerRecord(SHARED_TOPIC, kafkaPartition, null, it)) - } - - then: - TEST_WRITER.waitForTraces(3) - def pollRecords = KafkaTestUtils.getRecords(consumer) - - def listIter = - pollRecords.records(new TopicPartition(SHARED_TOPIC, kafkaPartition)).listIterator() - - then: - def receivedSet = greetings.toSet() - while (listIter.hasNext()) { - listIter.next() - } - while (listIter.hasPrevious()) { - def record = listIter.previous() - receivedSet.remove(record.value()) - assert record.key() == null - } - receivedSet.isEmpty() - - assertTraces(9, SORT_TRACES_BY_ID) { - - // producing traces - trace(1) { - producerSpan(it, senderProps) - } - trace(1) { - producerSpan(it, senderProps) - } - trace(1) { - producerSpan(it, senderProps) - } - - // iterating to the end of ListIterator: - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, trace(3)[1], 0..0) - queueSpan(it, trace(0)[0]) - } - trace(2) { - consumerSpan(it, consumerProperties, trace(4)[1], 1..1) - queueSpan(it, trace(1)[0]) - } - trace(2) { - consumerSpan(it, consumerProperties, trace(5)[1], 2..2) - queueSpan(it, trace(2)[0]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[0], 0..0) - } - trace(1) { - consumerSpan(it, consumerProperties, trace(1)[0], 1..1) - } - trace(1) { - consumerSpan(it, consumerProperties, trace(2)[0], 2..2) - } - } - - // backwards iteration over ListIterator to the beginning - if (hasQueueSpan()) { - trace(2) { - consumerSpan(it, consumerProperties, trace(6)[1], 2..2) - queueSpan(it, trace(2)[0]) - } - trace(2) { - consumerSpan(it, consumerProperties, trace(7)[1], 1..1) - queueSpan(it, trace(1)[0]) - } - trace(2) { - consumerSpan(it, consumerProperties, trace(8)[1], 0..0) - queueSpan(it, trace(0)[0]) - } - } else { - trace(1) { - consumerSpan(it, consumerProperties, trace(2)[0], 2..2) - } - trace(1) { - consumerSpan(it, consumerProperties, trace(1)[0], 1..1) - } - trace(1) { - consumerSpan(it, consumerProperties, trace(0)[0], 0..0) - } - } - } - - cleanup: - consumer.close() - producer.close() - } - - def "test kafka client header propagation manual config"() { - setup: - - def senderProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - def consumerProperties = KafkaTestUtils.consumerProps(embeddedKafka.getBrokersAsString(), "sender", "false") - - def producerFactory = new DefaultKafkaProducerFactory(senderProps) - def kafkaTemplate = new KafkaTemplate(producerFactory) - - // create a Kafka consumer factory - def consumerFactory = new DefaultKafkaConsumerFactory(consumerProperties) - - // set the topic that needs to be consumed - def containerProperties = new ContainerProperties(SHARED_TOPIC) - - // create a Kafka MessageListenerContainer - def container = new KafkaMessageListenerContainer<>(consumerFactory, containerProperties) - - // create a thread safe queue to store the received message - def records = new LinkedBlockingQueue>() - - // setup a Kafka message listener - container.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - records.add(record) - if (isDataStreamsEnabled()) { - // even if header propagation is disabled, we want data streams to work. - TEST_DATA_STREAMS_WRITER.waitForGroups(2) - } - } - }) - - // start the container and underlying message listener - container.start() - - // wait until the container has the required number of assigned partitions - ContainerTestUtils.waitForAssignment(container, container.assignedPartitions.size()) - - when: - String message = "Testing without headers" - injectSysConfig("kafka.client.propagation.enabled", value) - kafkaTemplate.send(SHARED_TOPIC, message) - - then: - // check that the message was received - def received = records.poll(5, TimeUnit.SECONDS) - - received.headers().iterator().hasNext() == expected - - cleanup: - producerFactory.stop() - container?.stop() - - where: - value | expected - "false" | false - "true" | true - } - - def "test producer extracts and uses existing trace context from record headers"() { - setup: - def senderProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer<>(senderProps) - - def existingTraceId = 1234567890123456L - def existingSpanId = 9876543210987654L - def headers = new RecordHeaders() - headers.add(new RecordHeader("x-datadog-trace-id", - String.valueOf(existingTraceId).getBytes(StandardCharsets.UTF_8))) - headers.add(new RecordHeader("x-datadog-parent-id", - String.valueOf(existingSpanId).getBytes(StandardCharsets.UTF_8))) - - when: - def record = new ProducerRecord(SHARED_TOPIC, 0, null, "test-context-extraction", headers) - producer.send(record).get() - - then: - TEST_WRITER.waitForTraces(1) - def producedSpan = TEST_WRITER[0][0] - // Verify the span used the extracted context as parent - producedSpan.traceId.toLong() == existingTraceId - producedSpan.parentId == existingSpanId - // Verify a new span was created (not reusing the extracted span ID) - producedSpan.spanId != existingSpanId - - cleanup: - producer?.close() - } - - def producerSpan( - TraceAssert trace, - Map config, - DDSpan parentSpan = null, - boolean partitioned = true, - boolean tombstone = false - ) { - trace.span { - serviceName service() - operationName operationForProducer() - resourceName "Produce Topic $SHARED_TOPIC" - spanType "queue" - errored false - measured true - if (parentSpan) { - childOf parentSpan - } else { - parent() - } - final boolean isV0 = version() == 0 - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_PRODUCER - "$InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS" config.get(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG) - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$SHARED_TOPIC" - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" { it >= 0 } - "$InstrumentationTags.KAFKA_CLUSTER_ID" { String } - if (tombstone) { - "$InstrumentationTags.TOMBSTONE" true - } - if ({ isDataStreamsEnabled() }) { - "$DDTags.PATHWAY_HASH" { String } - } - peerServiceFrom(InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS) - if (isV0) { - // in v0 the service name is always set to DD_SERVICE while it should just be unset as v1 - // this is a buggy behaviour that could not be easily fixed. - serviceNameSource "java-kafka" - } - defaultTags() - } - } - } - - def queueSpan( - TraceAssert trace, - DDSpan parentSpan = null - ) { - trace.span { - serviceName splitByDestination() ? "$SHARED_TOPIC" : serviceForTimeInQueue() - operationName "kafka.deliver" - resourceName "$SHARED_TOPIC" - spanType "queue" - errored false - measured true - if (parentSpan) { - childOf parentSpan - } else { - parent() - } - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_BROKER - defaultTags(true) - } - } - } - - def consumerSpan( - TraceAssert trace, - Map config, - DDSpan parentSpan = null, - Range offset = 0..0, - boolean tombstone = false, - boolean distributedRootSpan = !hasQueueSpan() - ) { - trace.span { - serviceName service() - operationName operationForConsumer() - resourceName "Consume Topic $SHARED_TOPIC" - spanType "queue" - errored false - measured true - if (parentSpan) { - childOf parentSpan - } else { - parent() - } - final boolean isV0 = version() == 0 - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_CONSUMER - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" { offset.containsWithinBounds(it as int) } - "$InstrumentationTags.CONSUMER_GROUP" "sender" - "$InstrumentationTags.KAFKA_BOOTSTRAP_SERVERS" config.get(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG) - "$InstrumentationTags.KAFKA_CLUSTER_ID" { String } - "$InstrumentationTags.RECORD_QUEUE_TIME_MS" { it >= 0 } - "$InstrumentationTags.RECORD_END_TO_END_DURATION_MS" { it >= 0 } - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$SHARED_TOPIC" - if (tombstone) { - "$InstrumentationTags.TOMBSTONE" true - } - if ({ isDataStreamsEnabled() }) { - "$DDTags.PATHWAY_HASH" { String } - } - if (isV0) { - serviceNameSource "java-kafka" - } - defaultTags(distributedRootSpan) - } - } - } - - def pollSpan( - TraceAssert trace, - int recordCount = 1, - DDSpan parentSpan = null, - Range offset = 0..0, - boolean tombstone = false, - boolean distributedRootSpan = !hasQueueSpan() - ) { - trace.span { - serviceName Config.get().getServiceName() - operationName "kafka.poll" - resourceName "kafka.poll" - errored false - measured false - tags { - "$InstrumentationTags.KAFKA_RECORDS_COUNT" recordCount - defaultTags(true) - } - } - } - - def waitForKafkaMetadataUpdate(KafkaTemplate kafkaTemplate) { - kafkaTemplate.flush() - Producer wrappedProducer = kafkaTemplate.getTheProducer() - assert (wrappedProducer instanceof DefaultKafkaProducerFactory.CloseSafeProducer) - Producer producer = wrappedProducer.delegate - assert (producer instanceof KafkaProducer) - String clusterId = producer.metadata.fetch().clusterResource().clusterId() - while (clusterId == null || clusterId.isEmpty()) { - Thread.sleep(1500) - clusterId = producer.metadata.fetch().clusterResource().clusterId() - } - return clusterId - } -} - -abstract class KafkaClientForkedTest extends KafkaClientTestBase { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("dd.kafka.legacy.tracing.enabled", "false") - injectSysConfig("dd.service", "KafkaClientTest") - } - - @Override - boolean hasQueueSpan() { - return true - } - - @Override - boolean splitByDestination() { - return false - } -} - -class KafkaClientV0ForkedTest extends KafkaClientForkedTest { - @Override - String service() { - return "KafkaClientTest" - } -} - -class KafkaClientV1ForkedTest extends KafkaClientForkedTest { - @Override - int version() { - 1 - } - - @Override - String service() { - return "KafkaClientTest" - } - - @Override - String operationForProducer() { - "kafka.send" - } - - @Override - String operationForConsumer() { - return "kafka.process" - } - - @Override - String serviceForTimeInQueue() { - "kafka-queue" - } - - @Override - boolean hasQueueSpan() { - false - } -} - -class KafkaClientSplitByDestinationForkedTest extends KafkaClientTestBase { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("dd.service", "KafkaClientTest") - injectSysConfig("dd.kafka.legacy.tracing.enabled", "false") - injectSysConfig("dd.message.broker.split-by-destination", "true") - } - - @Override - String service() { - return "KafkaClientTest" - } - - @Override - boolean hasQueueSpan() { - return true - } - - @Override - boolean splitByDestination() { - return true - } -} - -abstract class KafkaClientLegacyTracingForkedTest extends KafkaClientTestBase { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("dd.service", "KafkaClientLegacyTest") - injectSysConfig("dd.kafka.legacy.tracing.enabled", "true") - } - - @Override - String service() { - return "kafka" - } - - @Override - boolean hasQueueSpan() { - return false - } - - @Override - boolean splitByDestination() { - return false - } -} - -class KafkaClientLegacyTracingV0ForkedTest extends KafkaClientLegacyTracingForkedTest { -} - -class KafkaClientLegacyTracingV1ForkedTest extends KafkaClientLegacyTracingForkedTest { - - @Override - int version() { - 1 - } - - @Override - String operationForProducer() { - "kafka.send" - } - - @Override - String operationForConsumer() { - return "kafka.process" - } - - @Override - String service() { - return Config.get().getServiceName() - } -} - -class KafkaClientDataStreamsDisabledForkedTest extends KafkaClientTestBase { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("dd.service", "KafkaClientDataStreamsDisabledForkedTest") - injectSysConfig("dd.kafka.legacy.tracing.enabled", "true") - } - - @Override - String service() { - return "kafka" - } - - @Override - boolean hasQueueSpan() { - return false - } - - @Override - boolean splitByDestination() { - return false - } - - @Override - boolean isDataStreamsEnabled() { - return false - } -} - -class KafkaClientContextSwapForkedTest extends KafkaClientV0ForkedTest { - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig(TraceInstrumentationConfig.LEGACY_CONTEXT_MANAGER_ENABLED, "false") - } -} - -class KafkaClientBadBase64HeaderForkedTest extends InstrumentationSpecification { - EmbeddedKafkaBroker embeddedKafka - - def setup() { - embeddedKafka = new EmbeddedKafkaKraftBroker(1, 2, KafkaClientTestBase.SHARED_TOPIC) - embeddedKafka.afterPropertiesSet() - } - - def cleanup() { - embeddedKafka.destroy() - } - - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig(TraceInstrumentationConfig.KAFKA_CLIENT_BASE64_DECODING_ENABLED, "true") - injectSysConfig(TracerConfig.HEADER_TAGS, "x-custom-header:my.custom.tag") - } - - def "producer span is created when message carries non-Base64 headers and base64 decoding is enabled"() { - setup: - def producerProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - def producer = new KafkaProducer(producerProps, new StringSerializer(), new StringSerializer()) - - when: - def headers = new RecordHeaders([ - new RecordHeader("x-custom-header", "not-valid-base64!@#".getBytes(StandardCharsets.UTF_8)), - new RecordHeader("x-another-header", "also-not-base64!!".getBytes(StandardCharsets.UTF_8)) - ]) - producer.send(new ProducerRecord<>(KafkaClientTestBase.SHARED_TOPIC, 0, null, "hello", headers)).get() - - then: - TEST_WRITER.waitForTraces(1) - !TEST_WRITER.isEmpty() - - cleanup: - producer?.close() - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/TextMapExtractAdapterTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/TextMapExtractAdapterTest.groovy deleted file mode 100644 index e0d155f588e..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/TextMapExtractAdapterTest.groovy +++ /dev/null @@ -1,57 +0,0 @@ -import com.google.common.io.BaseEncoding -import datadog.trace.agent.test.InstrumentationSpecification -import datadog.trace.bootstrap.instrumentation.api.AgentPropagation -import datadog.trace.instrumentation.kafka_clients38.TextMapExtractAdapter -import java.nio.charset.StandardCharsets -import org.apache.kafka.common.header.Headers -import org.apache.kafka.common.header.internals.RecordHeader -import org.apache.kafka.common.header.internals.RecordHeaders - -class TextMapExtractAdapterTest extends InstrumentationSpecification { - - def "check can decode base64 mangled headers"() { - given: - def base64 = BaseEncoding.base64().encode("foo".getBytes(StandardCharsets.UTF_8)) - def expectedValue = base64Decode ? "foo" : base64 - Headers headers = new RecordHeaders(new RecordHeader("key", base64.getBytes(StandardCharsets.UTF_8))) - TextMapExtractAdapter adapter = new TextMapExtractAdapter(base64Decode) - when: - String extracted = null - adapter.forEachKey(headers, new AgentPropagation.KeyClassifier() { - @Override - boolean accept(String key, String value) { - extracted = value - return false - } - }) - - then: - extracted == expectedValue - - where: - base64Decode << [true, false] - } - - def "invalid base64 header is skipped and subsequent valid headers are still processed"() { - given: - def validBase64 = BaseEncoding.base64().encode("bar".getBytes(StandardCharsets.UTF_8)) - Headers headers = new RecordHeaders([ - new RecordHeader("bad-key", "not-valid-base64!@#".getBytes(StandardCharsets.UTF_8)), - new RecordHeader("good-key", validBase64.getBytes(StandardCharsets.UTF_8)) - ]) - TextMapExtractAdapter adapter = new TextMapExtractAdapter(true) - when: - Map extracted = [:] - adapter.forEachKey(headers, new AgentPropagation.KeyClassifier() { - @Override - boolean accept(String key, String value) { - extracted[key] = value - return true - } - }) - then: - noExceptionThrown() - !extracted.containsKey("bad-key") - extracted["good-key"] == "bar" - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentationHelperTest.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentationHelperTest.java deleted file mode 100644 index dee19831317..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInstrumentationHelperTest.java +++ /dev/null @@ -1,94 +0,0 @@ -package datadog.trace.instrumentation.kafka_clients38; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import datadog.trace.bootstrap.ContextStore; -import datadog.trace.instrumentation.kafka_common.MetadataState; -import org.apache.kafka.clients.Metadata; -import org.junit.jupiter.api.Test; - -class KafkaConsumerInstrumentationHelperTest { - - @SuppressWarnings("unchecked") - private final ContextStore metadataContextStore = - mock(ContextStore.class); - - @Test - void extractGroupReturnsNullForNullKafkaConsumerInfo() { - assertNull(KafkaConsumerInstrumentationHelper.extractGroup(null)); - } - - @Test - void extractGroupReturnsNullWhenConsumerGroupIsNull() { - KafkaConsumerInfo kafkaConsumerInfo = new KafkaConsumerInfo(null, null, "localhost:9092"); - assertNull(KafkaConsumerInstrumentationHelper.extractGroup(kafkaConsumerInfo)); - } - - @Test - void extractGroupReturnsConsumerGroupWhenPresent() { - KafkaConsumerInfo kafkaConsumerInfo = - new KafkaConsumerInfo("test-group", null, "localhost:9092"); - assertEquals("test-group", KafkaConsumerInstrumentationHelper.extractGroup(kafkaConsumerInfo)); - } - - @Test - void extractBootstrapServersReturnsNullForNullKafkaConsumerInfo() { - assertNull(KafkaConsumerInstrumentationHelper.extractBootstrapServers(null)); - } - - @Test - void extractBootstrapServersReturnsNullWhenBootstrapServersIsNull() { - KafkaConsumerInfo kafkaConsumerInfo = new KafkaConsumerInfo("test-group", null, null); - assertNull(KafkaConsumerInstrumentationHelper.extractBootstrapServers(kafkaConsumerInfo)); - } - - @Test - void extractBootstrapServersReturnsValueWhenPresent() { - KafkaConsumerInfo kafkaConsumerInfo = - new KafkaConsumerInfo("test-group", null, "localhost:9092"); - assertEquals( - "localhost:9092", - KafkaConsumerInstrumentationHelper.extractBootstrapServers(kafkaConsumerInfo)); - } - - @Test - void extractClusterIdReturnsNullForNullKafkaConsumerInfo() { - assertNull(KafkaConsumerInstrumentationHelper.extractClusterId(null, metadataContextStore)); - } - - @Test - void extractClusterIdReturnsNullWhenMetadataIsNull() { - KafkaConsumerInfo kafkaConsumerInfo = new KafkaConsumerInfo("test-group", "localhost:9092"); - assertNull( - KafkaConsumerInstrumentationHelper.extractClusterId( - kafkaConsumerInfo, metadataContextStore)); - } - - @Test - void extractClusterIdReturnsNullWhenNoStateForMetadata() { - Metadata metadata = mock(Metadata.class); - KafkaConsumerInfo kafkaConsumerInfo = - new KafkaConsumerInfo("test-group", metadata, "localhost:9092"); - when(metadataContextStore.get(metadata)).thenReturn(null); - assertNull( - KafkaConsumerInstrumentationHelper.extractClusterId( - kafkaConsumerInfo, metadataContextStore)); - } - - @Test - void extractClusterIdReturnsClusterIdWhenStatePresent() { - Metadata metadata = mock(Metadata.class); - KafkaConsumerInfo kafkaConsumerInfo = - new KafkaConsumerInfo("test-group", metadata, "localhost:9092"); - MetadataState state = new MetadataState(); - state.clusterId = "cluster-1"; - when(metadataContextStore.get(metadata)).thenReturn(state); - assertEquals( - "cluster-1", - KafkaConsumerInstrumentationHelper.extractClusterId( - kafkaConsumerInfo, metadataContextStore)); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-common/gradle.lockfile b/dd-java-agent/instrumentation/kafka/kafka-common/gradle.lockfile deleted file mode 100644 index 8085cdad5b6..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-common/gradle.lockfile +++ /dev/null @@ -1,128 +0,0 @@ -# This is a Gradle generated file for dependency locking. -# Manual edits can break the build and are not advised. -# This file is expected to be part of source control. -# To regenerate this file, run: ./gradlew :dd-java-agent:instrumentation:kafka:kafka-common:dependencies --write-locks -cafe.cryptography:curve25519-elisabeth:0.1.0=testRuntimeClasspath -cafe.cryptography:ed25519-elisabeth:0.1.0=testRuntimeClasspath -ch.qos.logback:logback-classic:1.2.13=testCompileClasspath,testRuntimeClasspath -ch.qos.logback:logback-core:1.2.13=testCompileClasspath,testRuntimeClasspath -com.blogspot.mydailyjava:weak-lock-free:0.17=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq.okhttp3:okhttp:3.12.15=testCompileClasspath,testRuntimeClasspath -com.datadoghq.okio:okio:1.17.6=testCompileClasspath,testRuntimeClasspath -com.datadoghq:dd-instrument-java:0.0.4=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:dd-javac-plugin-client:0.2.2=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:java-dogstatsd-client:4.4.5=testRuntimeClasspath -com.datadoghq:sketches-java:0.8.3=testRuntimeClasspath -com.github.javaparser:javaparser-core:3.25.6=codenarc -com.github.jnr:jffi:1.3.15=testRuntimeClasspath -com.github.jnr:jnr-a64asm:1.0.0=testRuntimeClasspath -com.github.jnr:jnr-constants:0.10.4=testRuntimeClasspath -com.github.jnr:jnr-enxio:0.32.20=testRuntimeClasspath -com.github.jnr:jnr-ffi:2.2.19=testRuntimeClasspath -com.github.jnr:jnr-posix:3.1.22=testRuntimeClasspath -com.github.jnr:jnr-unixsocket:0.38.25=testRuntimeClasspath -com.github.jnr:jnr-x86asm:1.0.2=testRuntimeClasspath -com.github.spotbugs:spotbugs-annotations:4.10.3=compileClasspath,spotbugs -com.github.spotbugs:spotbugs:4.10.3=spotbugs -com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs -com.google.auto.service:auto-service-annotations:1.1.1=annotationProcessor,compileClasspath,testAnnotationProcessor,testCompileClasspath -com.google.auto.service:auto-service:1.1.1=annotationProcessor,testAnnotationProcessor -com.google.auto:auto-common:1.2.1=annotationProcessor,testAnnotationProcessor -com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,compileClasspath,spotbugs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath -com.google.code.gson:gson:2.14.0=spotbugs -com.google.errorprone:error_prone_annotations:2.18.0=annotationProcessor,testAnnotationProcessor -com.google.errorprone:error_prone_annotations:2.47.0=testCompileClasspath,testRuntimeClasspath -com.google.errorprone:error_prone_annotations:2.48.0=spotbugs -com.google.guava:failureaccess:1.0.1=annotationProcessor,testAnnotationProcessor -com.google.guava:failureaccess:1.0.3=testCompileClasspath,testRuntimeClasspath -com.google.guava:guava:32.0.1-jre=annotationProcessor,testAnnotationProcessor -com.google.guava:guava:33.6.0-jre=testCompileClasspath,testRuntimeClasspath -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath -com.google.j2objc:j2objc-annotations:2.8=annotationProcessor,testAnnotationProcessor -com.google.j2objc:j2objc-annotations:3.1=testCompileClasspath,testRuntimeClasspath -com.google.re2j:re2j:1.8=testRuntimeClasspath -com.squareup.moshi:moshi:1.11.0=testCompileClasspath,testRuntimeClasspath -com.squareup.okhttp3:logging-interceptor:3.12.12=testCompileClasspath,testRuntimeClasspath -com.squareup.okhttp3:okhttp:3.12.12=testCompileClasspath,testRuntimeClasspath -com.squareup.okio:okio:1.17.5=testCompileClasspath,testRuntimeClasspath -com.thoughtworks.qdox:qdox:1.12.1=codenarc -commons-fileupload:commons-fileupload:1.5=testCompileClasspath,testRuntimeClasspath -commons-io:commons-io:2.11.0=testCompileClasspath,testRuntimeClasspath -commons-io:commons-io:2.21.0=spotbugs -de.thetaphi:forbiddenapis:3.10=compileClasspath,testCompileClasspath,testRuntimeClasspath -io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath -io.sqreen:libsqreen:17.5.0=testRuntimeClasspath -javax.servlet:javax.servlet-api:3.1.0=testCompileClasspath,testRuntimeClasspath -jaxen:jaxen:2.0.6=spotbugs -junit:junit:4.13.2=testRuntimeClasspath -net.bytebuddy:byte-buddy-agent:1.18.10=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -net.bytebuddy:byte-buddy:1.18.10=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -net.java.dev.jna:jna-platform:5.8.0=testRuntimeClasspath -net.java.dev.jna:jna:5.8.0=testRuntimeClasspath -net.jpountz.lz4:lz4:1.3.0=compileClasspath,testCompileClasspath,testRuntimeClasspath -net.sf.saxon:Saxon-HE:12.10=spotbugs -org.apache.ant:ant-antlr:1.10.14=codenarc -org.apache.ant:ant-junit:1.10.14=codenarc -org.apache.bcel:bcel:6.12.0=spotbugs -org.apache.commons:commons-lang3:3.20.0=spotbugs -org.apache.commons:commons-text:1.15.0=spotbugs -org.apache.kafka:kafka-clients:0.11.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath -org.apache.logging.log4j:log4j-api:2.26.1=spotbugs -org.apache.logging.log4j:log4j-core:2.26.1=spotbugs -org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath -org.checkerframework:checker-qual:3.33.0=annotationProcessor,testAnnotationProcessor -org.codehaus.groovy:groovy-ant:3.0.23=codenarc -org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc -org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc -org.codehaus.groovy:groovy-json:3.0.23=codenarc -org.codehaus.groovy:groovy-json:3.0.25=testCompileClasspath,testRuntimeClasspath -org.codehaus.groovy:groovy-templates:3.0.23=codenarc -org.codehaus.groovy:groovy-xml:3.0.23=codenarc -org.codehaus.groovy:groovy:3.0.23=codenarc -org.codehaus.groovy:groovy:3.0.25=testCompileClasspath,testRuntimeClasspath -org.codenarc:CodeNarc:3.7.0=codenarc -org.dom4j:dom4j:2.2.0=spotbugs -org.gmetrics:GMetrics:2.1.0=codenarc -org.hamcrest:hamcrest-core:1.3=testRuntimeClasspath -org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath -org.jctools:jctools-core-jdk11:4.0.6=testRuntimeClasspath -org.jctools:jctools-core:4.0.6=testRuntimeClasspath -org.jspecify:jspecify:1.0.0=testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-api:5.14.1=testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-engine:5.14.1=testRuntimeClasspath -org.junit.jupiter:junit-jupiter-params:5.14.1=testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter:5.14.1=testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-commons:1.14.1=testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-engine:1.14.1=testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-launcher:1.14.1=testRuntimeClasspath -org.junit.platform:junit-platform-runner:1.14.1=testRuntimeClasspath -org.junit.platform:junit-platform-suite-api:1.14.1=testRuntimeClasspath -org.junit.platform:junit-platform-suite-commons:1.14.1=testRuntimeClasspath -org.junit:junit-bom:5.14.1=testCompileClasspath,testRuntimeClasspath -org.junit:junit-bom:6.1.2=spotbugs -org.mockito:mockito-core:4.4.0=testRuntimeClasspath -org.objenesis:objenesis:3.3=testCompileClasspath,testRuntimeClasspath -org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath -org.ow2.asm:asm-analysis:9.10.1=spotbugs -org.ow2.asm:asm-analysis:9.7.1=testRuntimeClasspath -org.ow2.asm:asm-commons:9.10.1=spotbugs,testRuntimeClasspath -org.ow2.asm:asm-tree:9.10.1=spotbugs,testRuntimeClasspath -org.ow2.asm:asm-util:9.10.1=spotbugs -org.ow2.asm:asm-util:9.7.1=testRuntimeClasspath -org.ow2.asm:asm:9.10.1=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath -org.slf4j:jcl-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath -org.slf4j:jul-to-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath -org.slf4j:log4j-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:1.7.30=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath -org.slf4j:slf4j-api:1.7.32=testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:2.0.17=spotbugsSlf4j -org.slf4j:slf4j-api:2.0.18=spotbugs -org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j -org.snakeyaml:snakeyaml-engine:2.9=buildTimeInstrumentationPlugin,muzzleTooling,runtimeClasspath,testRuntimeClasspath -org.spockframework:spock-bom:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath -org.spockframework:spock-core:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath -org.tabletest:tabletest-junit:1.2.2=testCompileClasspath,testRuntimeClasspath -org.tabletest:tabletest-parser:1.2.1=testCompileClasspath,testRuntimeClasspath -org.xerial.snappy:snappy-java:1.1.2.6=compileClasspath,testCompileClasspath,testRuntimeClasspath -org.xmlresolver:xmlresolver:5.3.3=spotbugs -empty=spotbugsPlugins diff --git a/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/KafkaConfigHelper.java b/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/KafkaConfigHelper.java index 2ffbf11f46f..b9c5089f70a 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/KafkaConfigHelper.java +++ b/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/KafkaConfigHelper.java @@ -105,28 +105,6 @@ public static void storePendingConsumerConfig( log.debug("Stored pending consumer config (cluster ID not yet known)"); } - public static boolean reportConsumerGroupMember( - String clusterId, - String consumerGroup, - String memberId, - int generationId, - String memberProtocol) { - if (memberId == null || memberId.isEmpty() || clusterId == null || clusterId.isEmpty()) { - return false; - } - if (Config.get().isDataStreamsEnabled()) { - AgentTracer.get() - .getDataStreamsMonitoring() - .reportKafkaConsumerGroupMember( - clusterId, - consumerGroup != null ? consumerGroup : "", - memberId, - generationId, - memberProtocol != null ? memberProtocol : ""); - } - return true; - } - /** Called from metadata update advice when the cluster ID becomes available. */ public static void reportPendingConfig(MetadataState state, String clusterId) { PendingConfig pending = state.takePendingConfig(); diff --git a/dd-java-agent/instrumentation/kafka/kafka-common/src/test/groovy/KafkaConfigHelperTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-common/src/test/groovy/KafkaConfigHelperTest.groovy deleted file mode 100644 index c45598c3afa..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-common/src/test/groovy/KafkaConfigHelperTest.groovy +++ /dev/null @@ -1,84 +0,0 @@ -import datadog.trace.instrumentation.kafka_common.KafkaConfigHelper -import spock.lang.Specification - -class KafkaConfigHelperTest extends Specification { - - def "extractConsumerConfigFromMap includes allowed keys with values and masks others"() { - given: - def rawConfig = [ - "bootstrap.servers" : "localhost:9092", - "group.id" : "my-group", - "ssl.keystore.password": "secret123", - "ssl.key.password" : "secret456", - "ssl.truststore.password": "secret789", - "sasl.jaas.config" : "org.apache.kafka.common.security.plain.PlainLoginModule required;", - "auto.offset.reset" : "earliest", - "some.unknown.key" : "unknown-value" - ] as Map - - when: - def result = KafkaConfigHelper.extractConsumerConfigFromMap(rawConfig) - - then: - result["bootstrap.servers"] == "localhost:9092" - result["group.id"] == "my-group" - result["auto.offset.reset"] == "earliest" - result["ssl.keystore.password"] == KafkaConfigHelper.MASKED_VALUE - result["ssl.key.password"] == KafkaConfigHelper.MASKED_VALUE - result["ssl.truststore.password"] == KafkaConfigHelper.MASKED_VALUE - result["sasl.jaas.config"] == KafkaConfigHelper.MASKED_VALUE - result["some.unknown.key"] == KafkaConfigHelper.MASKED_VALUE - } - - def "non-allowed keys are masked, not dropped"() { - given: - def rawConfig = [:] as Map - ["ssl.keystore.password", "ssl.key.password", "sasl.jaas.config", "custom.key"].each { key -> - rawConfig[key] = "secret-value" - } - rawConfig["bootstrap.servers"] = "localhost:9092" - - when: - def result = KafkaConfigHelper.extractConsumerConfigFromMap(rawConfig) - - then: - result.size() == 5 - result["bootstrap.servers"] == "localhost:9092" - result["ssl.keystore.password"] == KafkaConfigHelper.MASKED_VALUE - result["ssl.key.password"] == KafkaConfigHelper.MASKED_VALUE - result["sasl.jaas.config"] == KafkaConfigHelper.MASKED_VALUE - result["custom.key"] == KafkaConfigHelper.MASKED_VALUE - } - - def "all allowed keys are captured with their actual values"() { - given: - def rawConfig = [:] as Map - KafkaConfigHelper.ALLOWED_KEYS.each { key -> - rawConfig[key] = "test-value" - } - - when: - def result = KafkaConfigHelper.extractConsumerConfigFromMap(rawConfig) - - then: - result.size() == KafkaConfigHelper.ALLOWED_KEYS.size() - KafkaConfigHelper.ALLOWED_KEYS.each { key -> - assert result[key] == "test-value" - } - } - - def "null values are converted to empty string for allowed keys"() { - given: - def rawConfig = [ - "bootstrap.servers": "localhost:9092", - "client.id" : null - ] as Map - - when: - def result = KafkaConfigHelper.extractConsumerConfigFromMap(rawConfig) - - then: - result["bootstrap.servers"] == "localhost:9092" - result["client.id"] == "" - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/build.gradle b/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/build.gradle deleted file mode 100644 index 5d1a056c2f6..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/build.gradle +++ /dev/null @@ -1,45 +0,0 @@ -plugins { - id 'dd-trace-java.module.instrumentation' -} - -muzzle { - extraRepository("confluent-releases", "https://packages.confluent.io/maven/") - - pass { - group = "org.apache.kafka" - module = "connect-runtime" - versions = "[0.11.0.0,)" - javaVersion = "17" - // Some Confluent 7.x POMs reference unpublished Jetty artifacts that Muzzle does not need. - // See: https://github.com/confluentinc/kafka-connect-storage-common/issues/468 - excludeDependency "org.eclipse.jetty:*" - excludeDependency "io.confluent.cloud:*" - excludeDependency "io.confluent.observability:*" - excludeDependency "io.confluent.secure.compute:*" - assertInverse = true - } -} - -dependencies { - compileOnly group: 'org.apache.kafka', name: 'connect-runtime', version: '0.11.0.0' - - testImplementation project(':dd-java-agent:agent-iast:iast-test-fixtures') - testRuntimeOnly project(':dd-java-agent:instrumentation:datadog:asm:iast-instrumenter') - testRuntimeOnly project(':dd-java-agent:instrumentation:java:java-lang:java-lang-1.8') - testRuntimeOnly project(':dd-java-agent:instrumentation:java:java-io-1.8') - testRuntimeOnly project(':dd-java-agent:instrumentation:jackson-core:jackson-core-common') - testRuntimeOnly project(':dd-java-agent:instrumentation:jackson-core:jackson-core-2.8') - testImplementation libs.jackson.databind - testImplementation group: 'org.mockito', name: 'mockito-core', version: '2.19.0' - testImplementation group: 'javax.xml.bind', name: 'jaxb-api', version: '2.2.3' - testImplementation 'org.apache.kafka:connect-api:2.7.0' // Fixed version - testImplementation 'org.apache.kafka:connect-runtime:2.7.0' - testImplementation 'org.apache.kafka:connect-file:2.7.0' // For FileStreamSourceConnector - testImplementation 'org.apache.kafka:kafka-clients:2.7.0' - // Spring Kafka Test library - testImplementation 'org.springframework.kafka:spring-kafka-test:2.7.9' // Version compatible with Kafka 2.7.x - testRuntimeOnly project(':dd-java-agent:instrumentation:kafka:kafka-clients-0.11') - testRuntimeOnly project(':dd-java-agent:instrumentation:kafka:kafka-clients-3.8') - testRuntimeOnly project(':dd-java-agent:instrumentation:kafka:kafka-streams-0.11') - testRuntimeOnly project(':dd-java-agent:instrumentation:kafka:kafka-streams-1.0') -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/gradle.lockfile b/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/gradle.lockfile deleted file mode 100644 index f4224fe48c8..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/gradle.lockfile +++ /dev/null @@ -1,264 +0,0 @@ -# This is a Gradle generated file for dependency locking. -# Manual edits can break the build and are not advised. -# This file is expected to be part of source control. -# To regenerate this file, run: ./gradlew :dd-java-agent:instrumentation:kafka:kafka-connect-0.11:dependencies --write-locks -cafe.cryptography:curve25519-elisabeth:0.1.0=testRuntimeClasspath -cafe.cryptography:ed25519-elisabeth:0.1.0=testRuntimeClasspath -ch.qos.logback:logback-classic:1.2.13=testCompileClasspath,testRuntimeClasspath -ch.qos.logback:logback-core:1.2.13=testCompileClasspath,testRuntimeClasspath -com.blogspot.mydailyjava:weak-lock-free:0.17=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq.okhttp3:okhttp:3.12.15=testCompileClasspath,testRuntimeClasspath -com.datadoghq.okio:okio:1.17.6=testCompileClasspath,testRuntimeClasspath -com.datadoghq:dd-instrument-java:0.0.4=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:dd-javac-plugin-client:0.2.2=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:java-dogstatsd-client:4.4.5=testRuntimeClasspath -com.datadoghq:sketches-java:0.8.3=testRuntimeClasspath -com.fasterxml.jackson.core:jackson-annotations:2.20=testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-annotations:2.8.5=compileClasspath -com.fasterxml.jackson.core:jackson-core:2.20.0=testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-core:2.8.5=compileClasspath -com.fasterxml.jackson.core:jackson-databind:2.20.0=testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-databind:2.8.5=compileClasspath -com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.20.0=testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.20.0=testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.20.0=testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.8.5=compileClasspath -com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.20.0=testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.8.5=compileClasspath -com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.20.0=testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.8.5=compileClasspath -com.fasterxml.jackson.module:jackson-module-scala_2.13:2.20.0=testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson:jackson-bom:2.20.0=testCompileClasspath,testRuntimeClasspath -com.github.javaparser:javaparser-core:3.25.6=codenarc -com.github.jnr:jffi:1.3.15=testRuntimeClasspath -com.github.jnr:jnr-a64asm:1.0.0=testRuntimeClasspath -com.github.jnr:jnr-constants:0.10.4=testRuntimeClasspath -com.github.jnr:jnr-enxio:0.32.20=testRuntimeClasspath -com.github.jnr:jnr-ffi:2.2.19=testRuntimeClasspath -com.github.jnr:jnr-posix:3.1.22=testRuntimeClasspath -com.github.jnr:jnr-unixsocket:0.38.25=testRuntimeClasspath -com.github.jnr:jnr-x86asm:1.0.2=testRuntimeClasspath -com.github.luben:zstd-jni:1.4.5-6=testCompileClasspath,testRuntimeClasspath -com.github.spotbugs:spotbugs-annotations:4.10.3=compileClasspath,spotbugs -com.github.spotbugs:spotbugs:4.10.3=spotbugs -com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs -com.google.auto.service:auto-service-annotations:1.1.1=annotationProcessor,compileClasspath,testAnnotationProcessor,testCompileClasspath -com.google.auto.service:auto-service:1.1.1=annotationProcessor,testAnnotationProcessor -com.google.auto:auto-common:1.2.1=annotationProcessor,testAnnotationProcessor -com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,compileClasspath,spotbugs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath -com.google.code.gson:gson:2.14.0=spotbugs -com.google.errorprone:error_prone_annotations:2.18.0=annotationProcessor,testAnnotationProcessor -com.google.errorprone:error_prone_annotations:2.47.0=testCompileClasspath,testRuntimeClasspath -com.google.errorprone:error_prone_annotations:2.48.0=spotbugs -com.google.guava:failureaccess:1.0.1=annotationProcessor,testAnnotationProcessor -com.google.guava:failureaccess:1.0.3=testCompileClasspath,testRuntimeClasspath -com.google.guava:guava:20.0=compileClasspath -com.google.guava:guava:32.0.1-jre=annotationProcessor,testAnnotationProcessor -com.google.guava:guava:33.6.0-jre=testCompileClasspath,testRuntimeClasspath -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath -com.google.j2objc:j2objc-annotations:2.8=annotationProcessor,testAnnotationProcessor -com.google.j2objc:j2objc-annotations:3.1=testCompileClasspath,testRuntimeClasspath -com.google.re2j:re2j:1.8=testRuntimeClasspath -com.squareup.moshi:moshi:1.11.0=testCompileClasspath,testRuntimeClasspath -com.squareup.okhttp3:logging-interceptor:3.12.12=testCompileClasspath,testRuntimeClasspath -com.squareup.okhttp3:okhttp:3.12.12=testCompileClasspath,testRuntimeClasspath -com.squareup.okio:okio:1.17.5=testCompileClasspath,testRuntimeClasspath -com.thoughtworks.paranamer:paranamer:2.8.3=testCompileClasspath,testRuntimeClasspath -com.thoughtworks.qdox:qdox:1.12.1=codenarc -com.typesafe.scala-logging:scala-logging_2.13:3.9.2=testCompileClasspath,testRuntimeClasspath -com.yammer.metrics:metrics-core:2.2.0=testCompileClasspath,testRuntimeClasspath -commons-cli:commons-cli:1.4=testCompileClasspath,testRuntimeClasspath -commons-fileupload:commons-fileupload:1.5=testCompileClasspath,testRuntimeClasspath -commons-io:commons-io:2.11.0=testCompileClasspath,testRuntimeClasspath -commons-io:commons-io:2.21.0=spotbugs -de.thetaphi:forbiddenapis:3.10=compileClasspath,testCompileClasspath,testRuntimeClasspath -io.leangen.geantyref:geantyref:1.3.16=testRuntimeClasspath -io.netty:netty-buffer:4.1.50.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec:4.1.50.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-common:4.1.50.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-handler:4.1.50.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-resolver:4.1.50.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport-native-epoll:4.1.50.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport-native-unix-common:4.1.50.Final=testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport:4.1.50.Final=testCompileClasspath,testRuntimeClasspath -io.sqreen:libsqreen:17.5.0=testRuntimeClasspath -jakarta.activation:jakarta.activation-api:1.2.2=testCompileClasspath,testRuntimeClasspath -jakarta.annotation:jakarta.annotation-api:1.3.5=testCompileClasspath,testRuntimeClasspath -jakarta.validation:jakarta.validation-api:2.0.2=testCompileClasspath,testRuntimeClasspath -jakarta.ws.rs:jakarta.ws.rs-api:2.1.6=testCompileClasspath,testRuntimeClasspath -javax.activation:activation:1.1.1=testCompileClasspath,testRuntimeClasspath -javax.annotation:javax.annotation-api:1.2=compileClasspath -javax.inject:javax.inject:1=compileClasspath -javax.servlet:javax.servlet-api:3.1.0=compileClasspath,testCompileClasspath,testRuntimeClasspath -javax.validation:validation-api:1.1.0.Final=compileClasspath -javax.ws.rs:javax.ws.rs-api:2.0.1=compileClasspath -javax.ws.rs:javax.ws.rs-api:2.1.1=testCompileClasspath,testRuntimeClasspath -javax.xml.bind:jaxb-api:2.3.0=testCompileClasspath,testRuntimeClasspath -jaxen:jaxen:2.0.6=spotbugs -junit:junit:4.13.2=testRuntimeClasspath -log4j:log4j:1.2.17=compileClasspath,testCompileClasspath,testRuntimeClasspath -net.bytebuddy:byte-buddy-agent:1.18.10=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -net.bytebuddy:byte-buddy:1.18.10=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -net.java.dev.jna:jna-platform:5.8.0=testRuntimeClasspath -net.java.dev.jna:jna:5.8.0=testRuntimeClasspath -net.jpountz.lz4:lz4:1.3.0=compileClasspath -net.sf.jopt-simple:jopt-simple:5.0.4=testCompileClasspath,testRuntimeClasspath -net.sf.saxon:Saxon-HE:12.10=spotbugs -net.sourceforge.argparse4j:argparse4j:0.7.0=compileClasspath,testCompileClasspath,testRuntimeClasspath -org.apache.ant:ant-antlr:1.10.14=codenarc -org.apache.ant:ant-junit:1.10.14=codenarc -org.apache.bcel:bcel:6.12.0=spotbugs -org.apache.commons:commons-lang3:3.20.0=spotbugs -org.apache.commons:commons-lang3:3.5=compileClasspath -org.apache.commons:commons-lang3:3.8.1=testCompileClasspath,testRuntimeClasspath -org.apache.commons:commons-text:1.15.0=spotbugs -org.apache.kafka:connect-api:0.11.0.0=compileClasspath -org.apache.kafka:connect-api:2.7.2=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:connect-file:2.7.0=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:connect-json:2.7.2=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:connect-runtime:0.11.0.0=compileClasspath -org.apache.kafka:connect-runtime:2.7.0=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:connect-transforms:0.11.0.0=compileClasspath -org.apache.kafka:connect-transforms:2.7.0=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka-clients:0.11.0.0=compileClasspath -org.apache.kafka:kafka-clients:2.7.2=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka-log4j-appender:0.11.0.0=compileClasspath -org.apache.kafka:kafka-log4j-appender:2.7.0=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka-raft:2.7.2=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka-streams-test-utils:2.7.2=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka-streams:2.7.2=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka-tools:0.11.0.0=compileClasspath -org.apache.kafka:kafka-tools:2.7.0=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka_2.13:2.7.2=testCompileClasspath,testRuntimeClasspath -org.apache.logging.log4j:log4j-api:2.26.1=spotbugs -org.apache.logging.log4j:log4j-core:2.26.1=spotbugs -org.apache.maven:maven-artifact:3.5.0=compileClasspath -org.apache.maven:maven-artifact:3.6.3=testCompileClasspath,testRuntimeClasspath -org.apache.yetus:audience-annotations:0.5.0=testCompileClasspath,testRuntimeClasspath -org.apache.zookeeper:zookeeper-jute:3.5.9=testCompileClasspath,testRuntimeClasspath -org.apache.zookeeper:zookeeper:3.5.9=testCompileClasspath,testRuntimeClasspath -org.apiguardian:apiguardian-api:1.1.2=testCompileClasspath -org.checkerframework:checker-qual:3.33.0=annotationProcessor,testAnnotationProcessor -org.codehaus.groovy:groovy-ant:3.0.23=codenarc -org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc -org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc -org.codehaus.groovy:groovy-json:3.0.23=codenarc -org.codehaus.groovy:groovy-json:3.0.25=testCompileClasspath,testRuntimeClasspath -org.codehaus.groovy:groovy-templates:3.0.23=codenarc -org.codehaus.groovy:groovy-xml:3.0.23=codenarc -org.codehaus.groovy:groovy:3.0.23=codenarc -org.codehaus.groovy:groovy:3.0.25=testCompileClasspath,testRuntimeClasspath -org.codehaus.plexus:plexus-utils:3.0.24=compileClasspath -org.codehaus.plexus:plexus-utils:3.2.1=testCompileClasspath,testRuntimeClasspath -org.codenarc:CodeNarc:3.7.0=codenarc -org.dom4j:dom4j:2.2.0=spotbugs -org.eclipse.jetty:jetty-client:9.4.33.v20201020=testCompileClasspath,testRuntimeClasspath -org.eclipse.jetty:jetty-continuation:9.2.15.v20160210=compileClasspath -org.eclipse.jetty:jetty-continuation:9.4.33.v20201020=testCompileClasspath,testRuntimeClasspath -org.eclipse.jetty:jetty-http:9.2.15.v20160210=compileClasspath -org.eclipse.jetty:jetty-http:9.4.33.v20201020=testCompileClasspath,testRuntimeClasspath -org.eclipse.jetty:jetty-io:9.2.15.v20160210=compileClasspath -org.eclipse.jetty:jetty-io:9.4.33.v20201020=testCompileClasspath,testRuntimeClasspath -org.eclipse.jetty:jetty-security:9.2.15.v20160210=compileClasspath -org.eclipse.jetty:jetty-security:9.4.33.v20201020=testCompileClasspath,testRuntimeClasspath -org.eclipse.jetty:jetty-server:9.2.15.v20160210=compileClasspath -org.eclipse.jetty:jetty-server:9.4.33.v20201020=testCompileClasspath,testRuntimeClasspath -org.eclipse.jetty:jetty-servlet:9.2.15.v20160210=compileClasspath -org.eclipse.jetty:jetty-servlet:9.4.33.v20201020=testCompileClasspath,testRuntimeClasspath -org.eclipse.jetty:jetty-servlets:9.2.15.v20160210=compileClasspath -org.eclipse.jetty:jetty-servlets:9.4.33.v20201020=testCompileClasspath,testRuntimeClasspath -org.eclipse.jetty:jetty-util:9.2.15.v20160210=compileClasspath -org.eclipse.jetty:jetty-util:9.4.33.v20201020=testCompileClasspath,testRuntimeClasspath -org.glassfish.hk2.external:aopalliance-repackaged:2.5.0-b05=compileClasspath -org.glassfish.hk2.external:aopalliance-repackaged:2.6.1=testCompileClasspath,testRuntimeClasspath -org.glassfish.hk2.external:jakarta.inject:2.6.1=testCompileClasspath,testRuntimeClasspath -org.glassfish.hk2.external:javax.inject:2.5.0-b05=compileClasspath -org.glassfish.hk2:hk2-api:2.5.0-b05=compileClasspath -org.glassfish.hk2:hk2-api:2.6.1=testCompileClasspath,testRuntimeClasspath -org.glassfish.hk2:hk2-locator:2.5.0-b05=compileClasspath -org.glassfish.hk2:hk2-locator:2.6.1=testCompileClasspath,testRuntimeClasspath -org.glassfish.hk2:hk2-utils:2.5.0-b05=compileClasspath -org.glassfish.hk2:hk2-utils:2.6.1=testCompileClasspath,testRuntimeClasspath -org.glassfish.hk2:osgi-resource-locator:1.0.1=compileClasspath -org.glassfish.hk2:osgi-resource-locator:1.0.3=testCompileClasspath,testRuntimeClasspath -org.glassfish.jersey.bundles.repackaged:jersey-guava:2.24=compileClasspath -org.glassfish.jersey.containers:jersey-container-servlet-core:2.24=compileClasspath -org.glassfish.jersey.containers:jersey-container-servlet-core:2.31=testCompileClasspath,testRuntimeClasspath -org.glassfish.jersey.containers:jersey-container-servlet:2.24=compileClasspath -org.glassfish.jersey.containers:jersey-container-servlet:2.31=testCompileClasspath,testRuntimeClasspath -org.glassfish.jersey.core:jersey-client:2.24=compileClasspath -org.glassfish.jersey.core:jersey-client:2.31=testCompileClasspath,testRuntimeClasspath -org.glassfish.jersey.core:jersey-common:2.24=compileClasspath -org.glassfish.jersey.core:jersey-common:2.31=testCompileClasspath,testRuntimeClasspath -org.glassfish.jersey.core:jersey-server:2.24=compileClasspath -org.glassfish.jersey.core:jersey-server:2.31=testCompileClasspath,testRuntimeClasspath -org.glassfish.jersey.inject:jersey-hk2:2.31=testCompileClasspath,testRuntimeClasspath -org.glassfish.jersey.media:jersey-media-jaxb:2.24=compileClasspath -org.glassfish.jersey.media:jersey-media-jaxb:2.31=testCompileClasspath,testRuntimeClasspath -org.gmetrics:GMetrics:2.1.0=codenarc -org.hamcrest:hamcrest-core:1.3=testRuntimeClasspath -org.hamcrest:hamcrest:3.0=testCompileClasspath,testRuntimeClasspath -org.javassist:javassist:3.21.0-GA=compileClasspath -org.javassist:javassist:3.26.0-GA=testCompileClasspath,testRuntimeClasspath -org.jctools:jctools-core-jdk11:4.0.6=testRuntimeClasspath -org.jctools:jctools-core:4.0.6=testRuntimeClasspath -org.jspecify:jspecify:1.0.0=testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-api:5.14.1=testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-engine:5.14.1=testRuntimeClasspath -org.junit.jupiter:junit-jupiter-params:5.14.1=testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter:5.14.1=testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-commons:1.14.1=testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-engine:1.14.1=testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-launcher:1.14.1=testRuntimeClasspath -org.junit.platform:junit-platform-runner:1.14.1=testRuntimeClasspath -org.junit.platform:junit-platform-suite-api:1.14.1=testRuntimeClasspath -org.junit.platform:junit-platform-suite-commons:1.14.1=testRuntimeClasspath -org.junit:junit-bom:5.14.1=testCompileClasspath,testRuntimeClasspath -org.junit:junit-bom:6.1.2=spotbugs -org.lz4:lz4-java:1.7.1=testCompileClasspath,testRuntimeClasspath -org.mockito:mockito-core:2.19.0=testCompileClasspath -org.mockito:mockito-core:4.4.0=testRuntimeClasspath -org.objenesis:objenesis:3.3=testCompileClasspath,testRuntimeClasspath -org.opentest4j:opentest4j:1.3.0=testCompileClasspath,testRuntimeClasspath -org.ow2.asm:asm-analysis:9.10.1=spotbugs -org.ow2.asm:asm-analysis:9.7.1=testRuntimeClasspath -org.ow2.asm:asm-commons:9.10.1=spotbugs,testRuntimeClasspath -org.ow2.asm:asm-tree:9.10.1=spotbugs,testRuntimeClasspath -org.ow2.asm:asm-util:9.10.1=spotbugs -org.ow2.asm:asm-util:9.7.1=testRuntimeClasspath -org.ow2.asm:asm:9.10.1=buildTimeInstrumentationPlugin,compileClasspath,muzzleTooling,runtimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath -org.reflections:reflections:0.9.11=compileClasspath -org.reflections:reflections:0.9.12=testCompileClasspath,testRuntimeClasspath -org.rocksdb:rocksdbjni:5.18.4=testCompileClasspath,testRuntimeClasspath -org.scala-lang.modules:scala-collection-compat_2.13:2.2.0=testCompileClasspath,testRuntimeClasspath -org.scala-lang.modules:scala-java8-compat_2.13:0.9.1=testCompileClasspath,testRuntimeClasspath -org.scala-lang:scala-library:2.13.16=testCompileClasspath,testRuntimeClasspath -org.scala-lang:scala-reflect:2.13.3=testCompileClasspath,testRuntimeClasspath -org.slf4j:jcl-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath -org.slf4j:jul-to-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath -org.slf4j:log4j-over-slf4j:1.7.30=testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:1.7.30=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath -org.slf4j:slf4j-api:1.7.32=testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:2.0.17=spotbugsSlf4j -org.slf4j:slf4j-api:2.0.18=spotbugs -org.slf4j:slf4j-log4j12:1.7.25=compileClasspath -org.slf4j:slf4j-log4j12:1.7.30=testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j -org.snakeyaml:snakeyaml-engine:2.9=buildTimeInstrumentationPlugin,muzzleTooling,runtimeClasspath,testRuntimeClasspath -org.spockframework:spock-bom:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath -org.spockframework:spock-core:2.4-groovy-3.0=testCompileClasspath,testRuntimeClasspath -org.springframework.kafka:spring-kafka-test:2.7.9=testCompileClasspath,testRuntimeClasspath -org.springframework.retry:spring-retry:1.3.1=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-aop:5.3.13=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-beans:5.3.13=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-context:5.3.13=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-core:5.3.13=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-expression:5.3.13=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-jcl:5.3.13=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-test:5.3.13=testCompileClasspath,testRuntimeClasspath -org.tabletest:tabletest-junit:1.2.2=testCompileClasspath,testRuntimeClasspath -org.tabletest:tabletest-parser:1.2.1=testCompileClasspath,testRuntimeClasspath -org.xerial.snappy:snappy-java:1.1.2.6=compileClasspath -org.xerial.snappy:snappy-java:1.1.7.7=testCompileClasspath,testRuntimeClasspath -org.xmlresolver:xmlresolver:5.3.3=spotbugs -empty=spotbugsPlugins diff --git a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/main/java/datadog/trace/instrumentation/kafka_connect/ConnectWorkerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/main/java/datadog/trace/instrumentation/kafka_connect/ConnectWorkerInstrumentation.java deleted file mode 100644 index 5c56e1341d0..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/main/java/datadog/trace/instrumentation/kafka_connect/ConnectWorkerInstrumentation.java +++ /dev/null @@ -1,69 +0,0 @@ -package datadog.trace.instrumentation.kafka_connect; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers.extendsClass; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isConstructor; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import net.bytebuddy.asm.Advice; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.matcher.ElementMatcher; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.connect.runtime.TaskStatus.Listener; -import org.apache.kafka.connect.util.ConnectorTaskId; - -@AutoService(InstrumenterModule.class) -public final class ConnectWorkerInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { - - static final String TARGET_TYPE = "org.apache.kafka.connect.runtime.WorkerTask"; - - public ConnectWorkerInstrumentation() { - super("kafka", "kafka-connect"); - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".TaskListener", - }; - } - - @Override - public String hierarchyMarkerType() { - return TARGET_TYPE; - } - - @Override - public ElementMatcher hierarchyMatcher() { - return extendsClass(named(hierarchyMarkerType())); - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isConstructor() - .and(takesArgument(0, named("org.apache.kafka.connect.util.ConnectorTaskId"))) - .and(takesArgument(1, named("org.apache.kafka.connect.runtime.TaskStatus$Listener"))), - ConnectWorkerInstrumentation.class.getName() + "$ConstructorAdvice"); - } - - public static class ConstructorAdvice { - - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void wrap( - @Advice.Argument(value = 0, readOnly = true) ConnectorTaskId id, - @Advice.Argument(value = 1, readOnly = false) Listener statusListener) { - statusListener = new TaskListener(statusListener); - } - - public static void muzzleCheck(ConsumerRecord record) { - // KafkaConsumerInstrumentation only applies for kafka versions with headers - // Make an explicit call so ConsumerCoordinatorInstrumentation does the same - record.headers(); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/main/java/datadog/trace/instrumentation/kafka_connect/TaskListener.java b/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/main/java/datadog/trace/instrumentation/kafka_connect/TaskListener.java deleted file mode 100644 index c30ab4062e1..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/main/java/datadog/trace/instrumentation/kafka_connect/TaskListener.java +++ /dev/null @@ -1,52 +0,0 @@ -package datadog.trace.instrumentation.kafka_connect; - -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import org.apache.kafka.connect.runtime.TaskStatus.Listener; -import org.apache.kafka.connect.util.ConnectorTaskId; - -public class TaskListener implements Listener { - private final Listener delegate; - - public TaskListener(Listener delegate) { - this.delegate = delegate; - } - - @Override - public void onStartup(ConnectorTaskId connectorTaskId) { - AgentTracer.get().getDataStreamsMonitoring().setThreadServiceName(connectorTaskId.connector()); - delegate.onStartup(connectorTaskId); - } - - @Override - public void onPause(ConnectorTaskId connectorTaskId) { - try { - delegate.onPause(connectorTaskId); - } finally { - AgentTracer.get().getDataStreamsMonitoring().clearThreadServiceName(); - } - } - - @Override - public void onResume(ConnectorTaskId connectorTaskId) { - AgentTracer.get().getDataStreamsMonitoring().setThreadServiceName(connectorTaskId.connector()); - delegate.onResume(connectorTaskId); - } - - @Override - public void onFailure(ConnectorTaskId connectorTaskId, Throwable throwable) { - try { - delegate.onFailure(connectorTaskId, throwable); - } finally { - AgentTracer.get().getDataStreamsMonitoring().clearThreadServiceName(); - } - } - - @Override - public void onShutdown(ConnectorTaskId connectorTaskId) { - try { - delegate.onShutdown(connectorTaskId); - } finally { - AgentTracer.get().getDataStreamsMonitoring().clearThreadServiceName(); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/test/groovy/ConnectWorkerInstrumentationTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/test/groovy/ConnectWorkerInstrumentationTest.groovy deleted file mode 100644 index fe50abc776c..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-connect-0.11/src/test/groovy/ConnectWorkerInstrumentationTest.groovy +++ /dev/null @@ -1,312 +0,0 @@ -import datadog.trace.agent.test.InstrumentationSpecification -import datadog.trace.core.datastreams.StatsGroup -import org.apache.kafka.clients.admin.AdminClient -import org.apache.kafka.clients.admin.AdminClientConfig -import org.apache.kafka.clients.admin.DescribeClusterResult -import org.apache.kafka.clients.consumer.ConsumerConfig -import org.apache.kafka.clients.consumer.ConsumerRecords -import org.apache.kafka.clients.consumer.KafkaConsumer -import org.apache.kafka.clients.producer.KafkaProducer -import org.apache.kafka.clients.producer.ProducerConfig -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.utils.Time -import org.apache.kafka.connect.connector.policy.AllConnectorClientConfigOverridePolicy -import org.apache.kafka.connect.connector.policy.ConnectorClientConfigOverridePolicy -import org.apache.kafka.connect.runtime.Herder -import org.apache.kafka.connect.runtime.Worker -import org.apache.kafka.connect.runtime.WorkerConfig -import org.apache.kafka.connect.runtime.isolation.Plugins -import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo -import org.apache.kafka.connect.runtime.standalone.StandaloneConfig -import org.apache.kafka.connect.runtime.standalone.StandaloneHerder -import org.apache.kafka.connect.storage.FileOffsetBackingStore -import org.apache.kafka.connect.util.Callback -import org.springframework.kafka.test.EmbeddedKafkaBroker -import spock.lang.Shared - -import java.time.Duration -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit - -class ConnectWorkerInstrumentationTest extends InstrumentationSpecification { - @Shared - EmbeddedKafkaBroker embeddedKafka = new EmbeddedKafkaBroker(1, false, 1, 'test-topic') - - def setupSpec() { - embeddedKafka.afterPropertiesSet() // Initializes the broker - } - - def cleanupSpec() { - embeddedKafka.destroy() - } - - def "test kafka-connect instrumentation"() { - // Kafka bootstrap servers from the embedded broker - String bootstrapServers = embeddedKafka.getBrokersAsString() - - // Retrieve Kafka cluster ID - // Create an AdminClient to interact with the Kafka cluster - Properties adminProps = new Properties() - adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) - String clusterId - try (AdminClient adminClient = AdminClient.create(adminProps)) { - DescribeClusterResult describeClusterResult = adminClient.describeCluster() - clusterId = describeClusterResult.clusterId().get() // Retrieve the cluster ID - } - assert clusterId != null : "Cluster ID is null" - - // Create a temporary file with a test message - File tempFile = File.createTempFile("test-message", ".txt") - - // Worker properties - Properties workerProps = new Properties() - workerProps.put(WorkerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) - workerProps.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter") - workerProps.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter") - workerProps.put(StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, "/tmp/connect.offsets") - workerProps.put(WorkerConfig.INTERNAL_KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter") - workerProps.put(WorkerConfig.INTERNAL_VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter") - workerProps.put(WorkerConfig.PLUGIN_PATH_CONFIG, "") // Required but can be empty for built-in connectors - workerProps.put("plugin.scan.classpath", "true") - - Map workerPropsMap = workerProps.stringPropertyNames() - .collectEntries { - [(it): workerProps.getProperty(it)] - } - - // Create the Connect worker - Time time = Time.SYSTEM - Plugins plugins = new Plugins(workerPropsMap) - plugins.compareAndSwapWithDelegatingLoader() - String workerId = "worker-1" - - FileOffsetBackingStore offsetBackingStore = new FileOffsetBackingStore() - WorkerConfig workerConfig = new StandaloneConfig(workerPropsMap) - offsetBackingStore.configure(workerConfig) - ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy = new AllConnectorClientConfigOverridePolicy() - Worker worker = new Worker(workerId, time, plugins, workerConfig, offsetBackingStore, connectorClientConfigOverridePolicy) - Herder herder = new StandaloneHerder(worker, clusterId, connectorClientConfigOverridePolicy) - - // Start worker and herder - worker.start() - herder.start() - - // Connector configuration - Map connectorProps = [ - 'name' : 'file-source-connector', - 'connector.class': 'org.apache.kafka.connect.file.FileStreamSourceConnector', - 'tasks.max' : '1', - 'file' : tempFile.getAbsolutePath(), - 'topic' : 'test-topic' - ] - - // Latch to wait for connector addition - CountDownLatch connectorAddedLatch = new CountDownLatch(1) - Callback> addConnectorCallback = new Callback>() { - @Override - void onCompletion(Throwable error, Herder.Created result) { - if (error != null) { - error.printStackTrace() - } else { - println "Connector added successfully." - } - connectorAddedLatch.countDown() - } - } - - when: - // Add the connector to the herder - herder.putConnectorConfig("file-source-connector", connectorProps, false, addConnectorCallback) - - // Wait for the connector to be added - boolean connectorAdded = connectorAddedLatch.await(10, TimeUnit.SECONDS) - assert connectorAdded : "Connector was not added in time" - - tempFile.write("Hello Kafka\n") - - // Consume the message from Kafka - Properties consumerProps = new Properties() - consumerProps.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) - consumerProps.put(ConsumerConfig.GROUP_ID_CONFIG, "test-consumer-group") - consumerProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest") - consumerProps.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer") - consumerProps.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer") - - KafkaConsumer consumer = new KafkaConsumer<>(consumerProps) - consumer.subscribe(['test-topic']) - - String receivedMessage = null - for (int i = 0; i < 10; i++) { - // Try for up to 10 seconds - ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)) - if (!records.isEmpty()) { - receivedMessage = records.iterator().next().value() - break - } - } - TEST_DATA_STREAMS_WRITER.waitForGroups(2) - - then: - receivedMessage == "Hello Kafka" - - StatsGroup first = TEST_DATA_STREAMS_WRITER.groups.find { - it.parentHash == 0 - } - verifyAll(first) { - tags.hasAllTags( - "direction:out", - "topic:test-topic", - "type:kafka" - ) - } - - StatsGroup second = TEST_DATA_STREAMS_WRITER.groups.find { - it.parentHash == first.hash - } - verifyAll(second) { - tags.hasAllTags("direction:in", "group:test-consumer-group", "topic:test-topic", "type:kafka") - } - TEST_DATA_STREAMS_WRITER.getServices().contains('file-source-connector') - - - cleanup: - consumer?.close() - herder?.stop() - worker?.stop() - tempFile?.delete() - } - - def "test kafka-connect sink instrumentation"() { - String bootstrapServers = embeddedKafka.getBrokersAsString() - - Properties adminProps = new Properties() - adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) - String clusterId - try (AdminClient adminClient = AdminClient.create(adminProps)) { - DescribeClusterResult describeClusterResult = adminClient.describeCluster() - clusterId = describeClusterResult.clusterId().get() - } - assert clusterId != null : "Cluster ID is null" - - // Create a temporary file where the sink connector should write - File sinkFile = File.createTempFile("sink-messages", ".txt") - if (sinkFile.exists()) { - sinkFile.delete() - } - sinkFile.deleteOnExit() - - Properties workerProps = new Properties() - workerProps.put(WorkerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) - workerProps.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter") - workerProps.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter") - workerProps.put(StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG, "/tmp/connect.offsets") - workerProps.put(WorkerConfig.INTERNAL_KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter") - workerProps.put(WorkerConfig.INTERNAL_VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter") - workerProps.put(WorkerConfig.PLUGIN_PATH_CONFIG, "") // Required but can be empty for built-in connectors - workerProps.put("plugin.scan.classpath", "true") - - Map workerPropsMap = workerProps.stringPropertyNames() - .collectEntries { - [(it): workerProps.getProperty(it)] - } - - // Create the Connect worker - Time time = Time.SYSTEM - Plugins plugins = new Plugins(workerPropsMap) - plugins.compareAndSwapWithDelegatingLoader() - String workerId = "worker-1" - - FileOffsetBackingStore offsetBackingStore = new FileOffsetBackingStore() - WorkerConfig workerConfig = new StandaloneConfig(workerPropsMap) - offsetBackingStore.configure(workerConfig) - ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy = new AllConnectorClientConfigOverridePolicy() - Worker worker = new Worker(workerId, time, plugins, workerConfig, offsetBackingStore, connectorClientConfigOverridePolicy) - Herder herder = new StandaloneHerder(worker, clusterId, connectorClientConfigOverridePolicy) - - // Start worker and herder - worker.start() - herder.start() - - // Create the sink connector configuration - Map connectorProps = [ - 'name' : 'file-sink-connector', - 'connector.class': 'org.apache.kafka.connect.file.FileStreamSinkConnector', - 'tasks.max' : '1', - 'file' : sinkFile.getAbsolutePath(), - 'topics' : 'test-topic' - ] - - // Latch to wait for connector addition - CountDownLatch connectorAddedLatch = new CountDownLatch(1) - Callback> addConnectorCallback = new Callback>() { - @Override - void onCompletion(Throwable error, Herder.Created result) { - if (error != null) { - error.printStackTrace() - } else { - println "Sink connector added successfully." - } - connectorAddedLatch.countDown() - } - } - - when: - // Add the sink connector to the herder - herder.putConnectorConfig("file-sink-connector", connectorProps, false, addConnectorCallback) - - // Wait for the connector to be added - boolean connectorAdded = connectorAddedLatch.await(10, TimeUnit.SECONDS) - assert connectorAdded : "Sink connector was not added in time" - - // Produce a message to the topic that we expect to be written to the file - Properties producerProps = new Properties() - producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) - producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") - producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") - - KafkaProducer producer = new KafkaProducer<>(producerProps) - producer.send(new ProducerRecord<>("test-topic", "key1", "Hello Kafka Sink")) - producer.flush() - producer.close() - - for (int i = 0; i < 100; i++) { - // Try for up to 10 seconds - Thread.sleep(100) - if (sinkFile.text.contains("Hello Kafka Sink")) { - break - } - } - - String fileContents = sinkFile.text - TEST_DATA_STREAMS_WRITER.waitForGroups(2) - - then: - fileContents.contains("Hello Kafka Sink") - - StatsGroup first = TEST_DATA_STREAMS_WRITER.groups.find { - it.parentHash == 0 - } - verifyAll(first) { - tags.hasAllTags("direction:out", "topic:test-topic", "type:kafka") - } - - StatsGroup second = TEST_DATA_STREAMS_WRITER.groups.find { - it.parentHash == first.hash - } - verifyAll(second) { - tags.hasAllTags("direction:in", "group:connect-file-sink-connector", "topic:test-topic", "type:kafka") - } - TEST_DATA_STREAMS_WRITER.getServices().contains('file-sink-connector') - - - cleanup: - herder?.stop() - worker?.stop() - sinkFile?.delete() - } - - @Override - protected boolean isDataStreamsEnabled() { - return true - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/build.gradle b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/build.gradle deleted file mode 100644 index 907ccffcabe..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/build.gradle +++ /dev/null @@ -1,39 +0,0 @@ -plugins { - id 'dd-trace-java.module.instrumentation' -} - -muzzle { - extraRepository("confluent-releases", "https://packages.confluent.io/maven/") - - pass { - group = "org.apache.kafka" - module = "kafka-streams" - versions = "[0.11.0.0,3)" - // TODO shouldn't there be an assertInverse here, or do we support 3.x? - } -} - -addTestSuite('latestDepTest') - -dependencies { - compileOnly group: 'org.apache.kafka', name: 'kafka-streams', version: '0.11.0.0' - - implementation project(':dd-java-agent:instrumentation:kafka:kafka-common') - implementation project(':dd-java-agent:instrumentation:kafka:kafka-clients-0.11') - - testImplementation group: 'org.apache.kafka', name: 'kafka-clients', version: '0.11.0.0' - testImplementation group: 'org.apache.kafka', name: 'kafka-streams', version: '0.11.0.0' - testImplementation group: 'org.springframework.kafka', name: 'spring-kafka', version: '1.3.3.RELEASE' - testImplementation group: 'org.springframework.kafka', name: 'spring-kafka-test', version: '1.3.3.RELEASE' - testImplementation group: 'javax.xml.bind', name: 'jaxb-api', version: '2.2.3' - testImplementation group: 'org.mockito', name: 'mockito-core', version: '2.19.0' - - // Include latest version of kafka itself along with latest version of client libs. - // This seems to help with jar compatibility hell. - latestDepTestImplementation group: 'org.apache.kafka', name: 'kafka_2.13', version: '2.+' - latestDepTestImplementation group: 'org.apache.kafka', name: 'kafka-clients', version: '2.+' - latestDepTestImplementation group: 'org.apache.kafka', name: 'kafka-streams', version: '2.+' - // spring-kafka 2.8.x pulls in kafka-clients/streams 3.x which behaves differently - latestDepTestImplementation group: 'org.springframework.kafka', name: 'spring-kafka', version: '2.7+' - latestDepTestImplementation group: 'org.springframework.kafka', name: 'spring-kafka-test', version: '2.7+' -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/gradle.lockfile b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/gradle.lockfile deleted file mode 100644 index a4c10c56a91..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/gradle.lockfile +++ /dev/null @@ -1,211 +0,0 @@ -# This is a Gradle generated file for dependency locking. -# Manual edits can break the build and are not advised. -# This file is expected to be part of source control. -# To regenerate this file, run: ./gradlew :dd-java-agent:instrumentation:kafka:kafka-streams-0.11:dependencies --write-locks -cafe.cryptography:curve25519-elisabeth:0.1.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -cafe.cryptography:ed25519-elisabeth:0.1.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -ch.qos.logback:logback-classic:1.2.13=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -ch.qos.logback:logback-core:1.2.13=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.101tec:zkclient:0.10=testCompileClasspath,testRuntimeClasspath -com.blogspot.mydailyjava:weak-lock-free:0.17=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq.okhttp3:okhttp:3.12.15=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq.okio:okio:1.17.6=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:dd-instrument-java:0.0.4=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:dd-javac-plugin-client:0.2.2=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:java-dogstatsd-client:4.4.5=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.datadoghq:sketches-java:0.8.3=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-annotations:2.10.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -com.fasterxml.jackson.core:jackson-annotations:2.8.0=compileClasspath,testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-core:2.10.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -com.fasterxml.jackson.core:jackson-core:2.8.5=compileClasspath,testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-databind:2.10.5.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -com.fasterxml.jackson.core:jackson-databind:2.8.5=compileClasspath,testCompileClasspath,testRuntimeClasspath -com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.10.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.10.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -com.fasterxml.jackson.module:jackson-module-paranamer:2.10.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -com.fasterxml.jackson.module:jackson-module-scala_2.13:2.10.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -com.github.javaparser:javaparser-core:3.25.6=codenarc -com.github.jnr:jffi:1.3.15=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-a64asm:1.0.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-constants:0.10.4=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-enxio:0.32.20=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-ffi:2.2.19=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-posix:3.1.22=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-unixsocket:0.38.25=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-x86asm:1.0.2=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.luben:zstd-jni:1.4.9-1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -com.github.spotbugs:spotbugs-annotations:4.10.3=compileClasspath,spotbugs -com.github.spotbugs:spotbugs:4.10.3=spotbugs -com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs -com.google.auto.service:auto-service-annotations:1.1.1=annotationProcessor,compileClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,testAnnotationProcessor,testCompileClasspath -com.google.auto.service:auto-service:1.1.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.auto:auto-common:1.2.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,compileClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,spotbugs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath -com.google.code.gson:gson:2.14.0=spotbugs -com.google.errorprone:error_prone_annotations:2.18.0=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.errorprone:error_prone_annotations:2.47.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.errorprone:error_prone_annotations:2.48.0=spotbugs -com.google.guava:failureaccess:1.0.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.guava:failureaccess:1.0.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.guava:guava:32.0.1-jre=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.guava:guava:33.6.0-jre=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath -com.google.j2objc:j2objc-annotations:2.8=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.j2objc:j2objc-annotations:3.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.re2j:re2j:1.8=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.squareup.moshi:moshi:1.11.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.squareup.okhttp3:logging-interceptor:3.12.12=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.squareup.okhttp3:okhttp:3.12.12=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.squareup.okio:okio:1.17.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.thoughtworks.paranamer:paranamer:2.8=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -com.thoughtworks.qdox:qdox:1.12.1=codenarc -com.typesafe.scala-logging:scala-logging_2.13:3.9.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -com.yammer.metrics:metrics-core:2.2.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -commons-cli:commons-cli:1.4=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -commons-fileupload:commons-fileupload:1.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -commons-io:commons-io:2.11.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -commons-io:commons-io:2.21.0=spotbugs -commons-logging:commons-logging:1.2=testCompileClasspath,testRuntimeClasspath -de.thetaphi:forbiddenapis:3.10=compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.leangen.geantyref:geantyref:1.3.16=latestDepTestRuntimeClasspath,testRuntimeClasspath -io.netty:netty-buffer:4.1.50.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-codec:4.1.50.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-common:4.1.50.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-handler:4.1.50.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-resolver:4.1.50.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-transport-native-epoll:4.1.50.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-transport-native-unix-common:4.1.50.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.netty:netty-transport:4.1.50.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -io.sqreen:libsqreen:17.5.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -javax.activation:activation:1.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -javax.servlet:javax.servlet-api:3.1.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -javax.xml.bind:jaxb-api:2.2.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -javax.xml.stream:stax-api:1.0-2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -jaxen:jaxen:2.0.6=spotbugs -junit:junit:4.12=testCompileClasspath -junit:junit:4.13.2=latestDepTestRuntimeClasspath,testRuntimeClasspath -log4j:log4j:1.2.16=testCompileClasspath,testRuntimeClasspath -net.bytebuddy:byte-buddy-agent:1.18.10=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -net.bytebuddy:byte-buddy:1.18.10=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -net.java.dev.jna:jna-platform:5.8.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -net.java.dev.jna:jna:5.8.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -net.jpountz.lz4:lz4:1.3.0=compileClasspath,testCompileClasspath,testRuntimeClasspath -net.sf.jopt-simple:jopt-simple:5.0.3=testCompileClasspath,testRuntimeClasspath -net.sf.jopt-simple:jopt-simple:5.0.4=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -net.sf.saxon:Saxon-HE:12.10=spotbugs -net.sourceforge.argparse4j:argparse4j:0.7.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.ant:ant-antlr:1.10.14=codenarc -org.apache.ant:ant-junit:1.10.14=codenarc -org.apache.bcel:bcel:6.12.0=spotbugs -org.apache.commons:commons-lang3:3.20.0=spotbugs -org.apache.commons:commons-text:1.15.0=spotbugs -org.apache.kafka:connect-api:0.11.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath -org.apache.kafka:connect-api:2.8.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:connect-json:0.11.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath -org.apache.kafka:connect-json:2.8.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-clients:0.11.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka-clients:2.8.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-metadata:2.8.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-raft:2.8.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-streams-test-utils:2.7.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka-streams:0.11.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka-streams:2.8.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.kafka:kafka_2.11:0.11.0.0=testCompileClasspath,testRuntimeClasspath -org.apache.kafka:kafka_2.13:2.8.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.logging.log4j:log4j-api:2.26.1=spotbugs -org.apache.logging.log4j:log4j-core:2.26.1=spotbugs -org.apache.yetus:audience-annotations:0.5.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.zookeeper:zookeeper-jute:3.5.9=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apache.zookeeper:zookeeper:3.4.10=testCompileClasspath,testRuntimeClasspath -org.apache.zookeeper:zookeeper:3.5.9=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.apiguardian:apiguardian-api:1.1.2=latestDepTestCompileClasspath,testCompileClasspath -org.checkerframework:checker-qual:3.33.0=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -org.codehaus.groovy:groovy-ant:3.0.23=codenarc -org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc -org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc -org.codehaus.groovy:groovy-json:3.0.23=codenarc -org.codehaus.groovy:groovy-json:3.0.25=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.codehaus.groovy:groovy-templates:3.0.23=codenarc -org.codehaus.groovy:groovy-xml:3.0.23=codenarc -org.codehaus.groovy:groovy:3.0.23=codenarc -org.codehaus.groovy:groovy:3.0.25=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.codenarc:CodeNarc:3.7.0=codenarc -org.dom4j:dom4j:2.2.0=spotbugs -org.gmetrics:GMetrics:2.1.0=codenarc -org.hamcrest:hamcrest-core:1.3=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.hamcrest:hamcrest:3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.jctools:jctools-core-jdk11:4.0.6=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.jctools:jctools-core:4.0.6=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.jspecify:jspecify:1.0.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-api:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-engine:5.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-params:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-commons:1.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-engine:1.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-launcher:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-runner:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-suite-api:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-suite-commons:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit:junit-bom:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit:junit-bom:6.1.2=spotbugs -org.lz4:lz4-java:1.7.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.mockito:mockito-core:2.19.0=latestDepTestCompileClasspath,testCompileClasspath -org.mockito:mockito-core:4.4.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.objenesis:objenesis:3.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.opentest4j:opentest4j:1.3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.ow2.asm:asm-analysis:9.10.1=spotbugs -org.ow2.asm:asm-analysis:9.7.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.ow2.asm:asm-commons:9.10.1=latestDepTestRuntimeClasspath,spotbugs,testRuntimeClasspath -org.ow2.asm:asm-tree:9.10.1=latestDepTestRuntimeClasspath,spotbugs,testRuntimeClasspath -org.ow2.asm:asm-util:9.10.1=spotbugs -org.ow2.asm:asm-util:9.7.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.ow2.asm:asm:9.10.1=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath -org.rocksdb:rocksdbjni:5.0.1=compileClasspath,testCompileClasspath,testRuntimeClasspath -org.rocksdb:rocksdbjni:5.18.4=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.scala-lang.modules:scala-collection-compat_2.13:2.3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.scala-lang.modules:scala-java8-compat_2.13:0.9.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.scala-lang.modules:scala-parser-combinators_2.11:1.0.4=testCompileClasspath,testRuntimeClasspath -org.scala-lang:scala-library:2.11.11=testCompileClasspath,testRuntimeClasspath -org.scala-lang:scala-library:2.13.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.scala-lang:scala-reflect:2.13.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.slf4j:jcl-over-slf4j:1.7.30=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:jul-to-slf4j:1.7.30=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:log4j-over-slf4j:1.7.30=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:1.7.30=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath -org.slf4j:slf4j-api:1.7.32=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:2.0.17=spotbugsSlf4j -org.slf4j:slf4j-api:2.0.18=spotbugs -org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j -org.snakeyaml:snakeyaml-engine:2.9=buildTimeInstrumentationPlugin,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testRuntimeClasspath -org.spockframework:spock-bom:2.4-groovy-3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.spockframework:spock-core:2.4-groovy-3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.springframework.kafka:spring-kafka-test:1.3.3.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework.kafka:spring-kafka-test:2.7.14=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework.kafka:spring-kafka:1.3.3.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework.kafka:spring-kafka:2.7.14=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework.retry:spring-retry:1.2.2.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework.retry:spring-retry:1.3.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-aop:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-aop:5.3.20=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-beans:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-beans:5.3.20=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-context:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-context:5.3.20=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-core:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-core:5.3.20=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-expression:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-expression:5.3.20=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-jcl:5.3.20=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-messaging:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-messaging:5.3.20=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-test:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-test:5.3.20=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.springframework:spring-tx:4.3.14.RELEASE=testCompileClasspath,testRuntimeClasspath -org.springframework:spring-tx:5.3.20=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.tabletest:tabletest-junit:1.2.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.tabletest:tabletest-parser:1.2.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.xerial.snappy:snappy-java:1.1.2.6=compileClasspath,testCompileClasspath,testRuntimeClasspath -org.xerial.snappy:snappy-java:1.1.8.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath -org.xmlresolver:xmlresolver:5.3.3=spotbugs -empty=spotbugsPlugins diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/latestDepTest/groovy/KafkaStreamsTest.groovy b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/latestDepTest/groovy/KafkaStreamsTest.groovy deleted file mode 100644 index 49c04c57bae..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/latestDepTest/groovy/KafkaStreamsTest.groovy +++ /dev/null @@ -1,267 +0,0 @@ -import datadog.trace.agent.test.InstrumentationSpecification -import datadog.trace.api.DDTags -import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags -import datadog.trace.bootstrap.instrumentation.api.Tags -import datadog.trace.core.datastreams.StatsGroup -import datadog.trace.test.util.Flaky -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.apache.kafka.common.serialization.Serdes -import org.apache.kafka.streams.KafkaStreams -import org.apache.kafka.streams.StreamsBuilder -import org.apache.kafka.streams.StreamsConfig -import org.apache.kafka.streams.kstream.KStream -import org.apache.kafka.streams.kstream.Produced -import org.apache.kafka.streams.kstream.ValueMapper -import org.springframework.kafka.core.DefaultKafkaConsumerFactory -import org.springframework.kafka.core.DefaultKafkaProducerFactory -import org.springframework.kafka.core.KafkaTemplate -import org.springframework.kafka.listener.ContainerProperties -import org.springframework.kafka.listener.KafkaMessageListenerContainer -import org.springframework.kafka.listener.MessageListener -import org.springframework.kafka.test.EmbeddedKafkaBroker -import org.springframework.kafka.test.utils.ContainerTestUtils -import org.springframework.kafka.test.utils.KafkaTestUtils -import spock.lang.Shared - -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.TimeUnit - -@Flaky("https://github.com/DataDog/dd-trace-java/issues/3865") -class KafkaStreamsTest extends InstrumentationSpecification { - static final STREAM_PENDING = "test.pending" - static final STREAM_PROCESSED = "test.processed" - - @Shared - private EmbeddedKafkaBroker embeddedKafka - - def setupSpec() { - embeddedKafka = new EmbeddedKafkaBroker(1, true, 1, STREAM_PENDING, STREAM_PROCESSED) - embeddedKafka.afterPropertiesSet() - } - - def cleanupSpec() { - embeddedKafka?.destroy() - } - - @Override - protected boolean isDataStreamsEnabled() { - return true - } - - def "test kafka produce and consume with streams in-between"() { - setup: - def config = new Properties() - def producerProps = KafkaTestUtils.producerProps(embeddedKafka.getBrokersAsString()) - config.putAll(producerProps) - config.put(StreamsConfig.APPLICATION_ID_CONFIG, "test-application") - config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()) - config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()) - - // CONFIGURE CONSUMER - def consumerFactory = new DefaultKafkaConsumerFactory(KafkaTestUtils.consumerProps("sender", "false", embeddedKafka)) - - def consumerContainer = new KafkaMessageListenerContainer<>(consumerFactory, new ContainerProperties(STREAM_PROCESSED)) - - // create a thread safe queue to store the processed message - def records = new LinkedBlockingQueue>() - - // setup a Kafka message listener - consumerContainer.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - // ensure consistent ordering of traces - // this is the last processing step so we should see 2 traces here - TEST_WRITER.waitForTraces(2) - TEST_TRACER.activeSpan().setTag("testing", 123) - if (isDataStreamsEnabled()) { - TEST_DATA_STREAMS_WRITER.waitForGroups(1) - } - records.add(record) - } - }) - - // start the container and underlying message listener - consumerContainer.start() - - // wait until the container has the required number of assigned partitions - ContainerTestUtils.waitForAssignment(consumerContainer, embeddedKafka.getPartitionsPerTopic()) - - // CONFIGURE PROCESSOR - StreamsBuilder builder = new StreamsBuilder() - KStream textLines = builder.stream(STREAM_PENDING) - def values = textLines - .mapValues(new ValueMapper() { - @Override - String apply(String textLine) { - TEST_WRITER.waitForTraces(1) // ensure consistent ordering of traces - TEST_TRACER.activeSpan().setTag("asdf", "testing") - if (isDataStreamsEnabled()) { - TEST_DATA_STREAMS_WRITER.waitForGroups(1) - } - return textLine.toLowerCase() - } - }) - - def producer = Produced.with(Serdes.String(), Serdes.String()) - values.to(STREAM_PROCESSED, producer) - KafkaStreams streams = new KafkaStreams(builder.build(), config) - streams.start() - - // CONFIGURE PRODUCER - def producerFactory = new DefaultKafkaProducerFactory(producerProps) - def kafkaTemplate = new KafkaTemplate(producerFactory) - - when: - String greeting = "TESTING TESTING 123!" - kafkaTemplate.send(STREAM_PENDING, greeting) - - then: - // check that the message was received - def received = records.poll(10, TimeUnit.SECONDS) - received.value() == greeting.toLowerCase() - received.key() == null - - assertTraces(3) { - trace(1) { - // PRODUCER span 0 - span { - serviceName "kafka" - operationName "kafka.produce" - resourceName "Produce Topic $STREAM_PENDING" - spanType "queue" - errored false - measured true - parent() - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_PRODUCER - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$STREAM_PENDING" - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" { it >= 0 } - "$InstrumentationTags.KAFKA_CLUSTER_ID" { String } - if ({ isDataStreamsEnabled()}) { - "$DDTags.PATHWAY_HASH" { String } - } - defaultTagsNoPeerService() - } - } - } - def producerSpan = null - trace(2) { - sortSpansByStart() - - // STREAMING span 0 - span { - serviceName "kafka" - operationName "kafka.consume" - resourceName "Consume Topic $STREAM_PENDING" - spanType "queue" - errored false - measured true - childOf trace(0)[0] - - tags { - "$Tags.COMPONENT" "java-kafka-streams" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_CONSUMER - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" 0 - "$InstrumentationTags.PROCESSOR_NAME" "KSTREAM-SOURCE-0000000000" - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$STREAM_PENDING" - "asdf" "testing" - if ({ isDataStreamsEnabled()}) { - "$DDTags.PATHWAY_HASH" { String } - } - defaultTags(true) - } - } - - // STREAMING span 1 - span { - producerSpan = it.span - serviceName "kafka" - operationName "kafka.produce" - resourceName "Produce Topic $STREAM_PROCESSED" - spanType "queue" - errored false - measured true - childOf span(0) - - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_PRODUCER - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$STREAM_PROCESSED" - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" { it >= 0 } - "$InstrumentationTags.KAFKA_CLUSTER_ID" { String } - if ({ isDataStreamsEnabled()}) { - "$DDTags.PATHWAY_HASH" { String } - } - defaultTagsNoPeerService() - } - } - } - trace(1) { - // CONSUMER span 0 - span { - serviceName "kafka" - operationName "kafka.consume" - resourceName "Consume Topic $STREAM_PROCESSED" - spanType "queue" - errored false - measured true - childOf producerSpan - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_CONSUMER - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" 0 - "$InstrumentationTags.CONSUMER_GROUP" "sender" - "$InstrumentationTags.KAFKA_CLUSTER_ID" { String } - "$InstrumentationTags.RECORD_QUEUE_TIME_MS" { it >= 0 } - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$STREAM_PROCESSED" - "testing" 123 - if ({ isDataStreamsEnabled()}) { - "$DDTags.PATHWAY_HASH" { String } - } - defaultTags(true) - } - } - } - } - - def headers = received.headers() - headers.iterator().hasNext() - new String(headers.headers("x-datadog-trace-id").iterator().next().value()) == "${TEST_WRITER[1][0].traceId}" - new String(headers.headers("x-datadog-parent-id").iterator().next().value()) == "${TEST_WRITER[1][0].spanId}" - - if (isDataStreamsEnabled()) { - StatsGroup originProducerPoint = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == 0 } - verifyAll(originProducerPoint) { - tags.hasAllTags("direction:out", "topic:$STREAM_PENDING", "type:kafka") - } - - StatsGroup kafkaStreamsConsumerPoint = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == originProducerPoint.hash } - verifyAll(kafkaStreamsConsumerPoint) { - tags.hasAllTags("direction:in", - "group:test-application", - "topic:$STREAM_PENDING".toString(), - "type:kafka") - } - - StatsGroup kafkaStreamsProducerPoint = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == kafkaStreamsConsumerPoint.hash } - verifyAll(kafkaStreamsProducerPoint) { - tags.hasAllTags("direction:out", "topic:$STREAM_PROCESSED", "type:kafka") - } - - StatsGroup finalConsumerPoint = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == kafkaStreamsProducerPoint.hash } - verifyAll(finalConsumerPoint) { - tags.hasAllTags("direction:in", "group:sender", "topic:$STREAM_PROCESSED".toString(), "type:kafka") - } - } - - cleanup: - producerFactory?.destroy() - streams?.close() - consumerContainer?.stop() - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java deleted file mode 100644 index 81ddfd4202f..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamTaskInstrumentation.java +++ /dev/null @@ -1,425 +0,0 @@ -package datadog.trace.instrumentation.kafka_streams; - -import static datadog.context.propagation.Propagators.defaultPropagator; -import static datadog.trace.agent.tooling.InstrumenterModule.TargetSystem.CONTEXT_TRACKING; -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static datadog.trace.api.datastreams.DataStreamsContext.create; -import static datadog.trace.api.datastreams.DataStreamsTags.Direction.INBOUND; -import static datadog.trace.api.datastreams.DataStreamsTags.createWithGroup; -import static datadog.trace.bootstrap.instrumentation.api.AgentPropagation.DSM_CONCERN; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.traceConfig; -import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.rootContext; -import static datadog.trace.instrumentation.kafka_common.StreamingContext.STREAMING_CONTEXT; -import static datadog.trace.instrumentation.kafka_common.Utils.computePayloadSizeBytes; -import static datadog.trace.instrumentation.kafka_streams.KafkaStreamsDecorator.BROKER_DECORATE; -import static datadog.trace.instrumentation.kafka_streams.KafkaStreamsDecorator.CONSUMER_DECORATE; -import static datadog.trace.instrumentation.kafka_streams.KafkaStreamsDecorator.JAVA_KAFKA; -import static datadog.trace.instrumentation.kafka_streams.KafkaStreamsDecorator.KAFKA_CONSUME; -import static datadog.trace.instrumentation.kafka_streams.KafkaStreamsDecorator.KAFKA_DELIVER; -import static datadog.trace.instrumentation.kafka_streams.KafkaStreamsDecorator.TIME_IN_QUEUE_ENABLED; -import static datadog.trace.instrumentation.kafka_streams.ProcessorRecordContextSetter.PR_SETTER; -import static datadog.trace.instrumentation.kafka_streams.ProcessorRecordContextVisitor.PR_GETTER; -import static datadog.trace.instrumentation.kafka_streams.StampedRecordContextSetter.SR_SETTER; -import static datadog.trace.instrumentation.kafka_streams.StampedRecordContextVisitor.SR_GETTER; -import static java.util.Collections.singletonMap; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static net.bytebuddy.matcher.ElementMatchers.isConstructor; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.isPublic; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; - -import com.google.auto.service.AutoService; -import datadog.context.ContextScope; -import datadog.context.propagation.Propagator; -import datadog.context.propagation.Propagators; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.agent.tooling.annotation.AppliesOn; -import datadog.trace.api.Config; -import datadog.trace.api.datastreams.DataStreamsContext; -import datadog.trace.api.datastreams.DataStreamsTags; -import datadog.trace.bootstrap.InstrumentationContext; -import datadog.trace.bootstrap.instrumentation.api.AgentScope; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import datadog.trace.instrumentation.kafka_clients.TracingIterableDelegator; -import java.util.Map; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.streams.StreamsConfig; -import org.apache.kafka.streams.processor.internals.ProcessorNode; -import org.apache.kafka.streams.processor.internals.ProcessorRecordContext; -import org.apache.kafka.streams.processor.internals.StampedRecord; -import org.apache.kafka.streams.processor.internals.StreamTask; - -@AutoService(InstrumenterModule.class) -public class KafkaStreamTaskInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { - - public KafkaStreamTaskInstrumentation() { - super("kafka", "kafka-streams"); - } - - @Override - public String instrumentedType() { - return "org.apache.kafka.streams.processor.internals.StreamTask"; - } - - @Override - public String[] helperClassNames() { - return new String[] { - "datadog.trace.instrumentation.kafka_clients.TextMapInjectAdapterInterface", - "datadog.trace.instrumentation.kafka_clients.TracingIterableDelegator", - "datadog.trace.instrumentation.kafka_common.Utils", - "datadog.trace.instrumentation.kafka_common.StreamingContext", - packageName + ".KafkaStreamsDecorator", - packageName + ".ProcessorRecordContextHeadersAccess", - packageName + ".ProcessorRecordContextVisitor", - packageName + ".ProcessorRecordContextSetter", - packageName + ".StampedRecordContextVisitor", - packageName + ".StampedRecordContextSetter", - packageName + ".StreamTaskContext", - }; - } - - @Override - public Map contextStore() { - return singletonMap( - "org.apache.kafka.streams.processor.internals.StreamTask", - packageName + ".StreamTaskContext"); - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - // StreamsConfig was the 4th input argument to StreamTask's constructor in kafka versions 2.6 to - // 3.1. - // Starting from 3.2 StreamsConfig was no longer an input argument into StreamTask. - transformer.applyAdvice( - isConstructor().and(takesArgument(4, named("org.apache.kafka.streams.StreamsConfig"))), - KafkaStreamTaskInstrumentation.class.getName() + "$Constructor4Advice"); - - // StreamsConfig was the 5th input argument to StreamTask's constructor in kafka versions 1.1 to - // 2.5 - transformer.applyAdvice( - isConstructor().and(takesArgument(5, named("org.apache.kafka.streams.StreamsConfig"))), - KafkaStreamTaskInstrumentation.class.getName() + "$Constructor5Advice"); - - // StreamsConfig was the 6th input argument to StreamTask's constructor in kafka versions 0.11 - // to 1.0. - transformer.applyAdvice( - isConstructor().and(takesArgument(6, named("org.apache.kafka.streams.StreamsConfig"))), - KafkaStreamTaskInstrumentation.class.getName() + "$Constructor6Advice"); - - transformer.applyAdvice( - isMethod().and(named("addRecords")).and(takesArgument(1, named("java.lang.Iterable"))), - KafkaStreamTaskInstrumentation.class.getName() + "$UnwrapIterableAdvice"); - - // Before 2.7 - transformer.applyAdvices( - isMethod() - .and(named("updateProcessorContext")) - .and( - takesArgument( - 0, named("org.apache.kafka.streams.processor.internals.StampedRecord"))) - .and( - takesArgument( - 1, named("org.apache.kafka.streams.processor.internals.ProcessorNode"))), - KafkaStreamTaskInstrumentation.class.getName() + "$ContextPropagationAdvice", - KafkaStreamTaskInstrumentation.class.getName() + "$StartSpanAdvice"); - // After 2.7 - transformer.applyAdvices( - isMethod() - .and(named("updateProcessorContext")) - .and( - takesArgument( - 0, named("org.apache.kafka.streams.processor.internals.ProcessorNode"))) - .and( - takesArgument( - 2, - named("org.apache.kafka.streams.processor.internals.ProcessorRecordContext"))), - KafkaStreamTaskInstrumentation.class.getName() + "$ContextPropagationAdvice27", - KafkaStreamTaskInstrumentation.class.getName() + "$StartSpanAdvice27"); - - transformer.applyAdvice( - isMethod() - .and(isPublic()) - .and(named("process")) - // Method signature changed in 2.6. - .and(takesArguments(0).or(takesArguments(1).and(takesArgument(0, long.class)))), - KafkaStreamTaskInstrumentation.class.getName() + "$StopSpanAdvice"); - } - - public static class Constructor4Advice { - @Advice.OnMethodExit(suppress = Throwable.class) - public static void captureGroup( - @Advice.This StreamTask task, @Advice.Argument(4) StreamsConfig streamsConfig) { - String applicationId = streamsConfig.getString(StreamsConfig.APPLICATION_ID_CONFIG); - - if (applicationId != null && !applicationId.isEmpty()) { - StreamTaskContext context = - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).get(task); - if (context == null) { - context = new StreamTaskContext(); - } - context.setApplicationId(applicationId); - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).put(task, context); - } - } - } - - public static class Constructor5Advice { - @Advice.OnMethodExit(suppress = Throwable.class) - public static void captureGroup( - @Advice.This StreamTask task, @Advice.Argument(5) StreamsConfig streamsConfig) { - String applicationId = streamsConfig.getString(StreamsConfig.APPLICATION_ID_CONFIG); - - if (applicationId != null && !applicationId.isEmpty()) { - StreamTaskContext context = - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).get(task); - if (context == null) { - context = new StreamTaskContext(); - } - context.setApplicationId(applicationId); - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).put(task, context); - } - } - } - - public static class Constructor6Advice { - @Advice.OnMethodExit(suppress = Throwable.class) - public static void captureGroup( - @Advice.This StreamTask task, @Advice.Argument(6) StreamsConfig streamsConfig) { - String applicationId = streamsConfig.getString(StreamsConfig.APPLICATION_ID_CONFIG); - - if (applicationId != null && !applicationId.isEmpty()) { - StreamTaskContext context = - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).get(task); - if (context == null) { - context = new StreamTaskContext(); - } - context.setApplicationId(applicationId); - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).put(task, context); - } - } - } - - public static class UnwrapIterableAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter( - @Advice.Argument(value = 1, readOnly = false) Iterable> records) { - // This method adds the records to a queue, so we want to bypass the kafka instrumentation - // since the resulting spans are very short and uninteresting. - // KafkaStreamsProcessorInstrumentation will create a new span instead. - - // Expecting a TracingList because TaskManager.addRecordsToTasks calls records(partition). - if (records instanceof TracingIterableDelegator) { - records = ((TracingIterableDelegator) records).getDelegate(); - } - } - } - - /** Context propagation for updateProcessorContext before 2.7 (StampedRecord). */ - @AppliesOn(CONTEXT_TRACKING) - public static class ContextPropagationAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter( - @Advice.Argument(0) final StampedRecord record, - @Advice.Local("ctxScope") ContextScope scope) { - if (record == null || record.partition() == -1 || record.offset() == -1) { - return; - } - if (!Config.get().isKafkaClientPropagationDisabledForTopic(record.topic())) { - scope = defaultPropagator().extract(rootContext(), record, SR_GETTER).attach(); - } - } - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void onExit(@Advice.Local("ctxScope") ContextScope scope) { - if (scope != null) scope.close(); - } - } - - /** Context propagation for updateProcessorContext after 2.7 (ProcessorRecordContext). */ - @AppliesOn(CONTEXT_TRACKING) - public static class ContextPropagationAdvice27 { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter( - @Advice.Argument(2) final ProcessorRecordContext record, - @Advice.Local("ctxScope") ContextScope scope) { - if (record == null || record.partition() == -1 || record.offset() == -1) { - return; - } - if (!Config.get().isKafkaClientPropagationDisabledForTopic(record.topic())) { - scope = defaultPropagator().extract(rootContext(), record, PR_GETTER).attach(); - } - } - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void onExit(@Advice.Local("ctxScope") ContextScope scope) { - if (scope != null) scope.close(); - } - } - - /** Very similar to StartSpanAdvice27, but with a different argument type for record. */ - public static class StartSpanAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void start( - @Advice.Argument(0) final StampedRecord record, - @Advice.Argument(1) final ProcessorNode node, - @Advice.This StreamTask task) { - if (record == null || record.partition() == -1 || record.offset() == -1) { - // partition|offset == -1 -> punctuation call. - return; - } - - AgentSpan span, queueSpan = null; - StreamTaskContext streamTaskContext = - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).get(task); - long timeInQueueStart = SR_GETTER.extractTimeInQueueStart(record); - if (timeInQueueStart == 0 || !TIME_IN_QUEUE_ENABLED) { - span = startSpan(JAVA_KAFKA.toString(), KAFKA_CONSUME); - } else { - queueSpan = - startSpan( - JAVA_KAFKA.toString(), KAFKA_DELIVER, MILLISECONDS.toMicros(timeInQueueStart)); - BROKER_DECORATE.afterStart(queueSpan); - BROKER_DECORATE.onTimeInQueue(queueSpan, record); - span = startSpan(JAVA_KAFKA.toString(), KAFKA_CONSUME, queueSpan.spanContext()); - BROKER_DECORATE.beforeFinish(queueSpan); - // The queueSpan will be finished after inner span has been activated to ensure that - // spans are written out together by TraceStructureWriter when running in strict mode - } - - String applicationId = null; - if (streamTaskContext != null) { - applicationId = streamTaskContext.getApplicationId(); - } - DataStreamsTags tags = createWithGroup("kafka", INBOUND, applicationId, record.topic()); - - final long payloadSize = - traceConfig().isDataStreamsEnabled() ? computePayloadSizeBytes(record.value) : 0; - if (STREAMING_CONTEXT.isDisabledForTopic(record.topic())) { - AgentTracer.get() - .getDataStreamsMonitoring() - .setCheckpoint(span, create(tags, record.timestamp, payloadSize)); - } else { - if (STREAMING_CONTEXT.isSourceTopic(record.topic())) { - Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); - DataStreamsContext dsmContext = create(tags, record.timestamp, payloadSize); - dsmPropagator.inject(span.with(dsmContext), record, SR_SETTER); - } - } - - CONSUMER_DECORATE.afterStart(span); - CONSUMER_DECORATE.onConsume(span, record, node); - AgentScope agentScope = activateSpan(span); - if (null != queueSpan) { - queueSpan.finish(); - } - - if (streamTaskContext == null) { - streamTaskContext = new StreamTaskContext(); - } - streamTaskContext.setAgentScope(agentScope); - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class) - .put(task, streamTaskContext); - } - } - - /** Very similar to StartSpanAdvice, but with a different argument type for record. */ - public static class StartSpanAdvice27 { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void start( - @Advice.Argument(0) final ProcessorNode node, - @Advice.Argument(2) final ProcessorRecordContext record, - @Advice.This StreamTask task) { - if (record == null || record.partition() == -1 || record.offset() == -1) { - // partition|offset == -1 -> punctuation call. - return; - } - - AgentSpan span, queueSpan = null; - StreamTaskContext streamTaskContext = - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).get(task); - long timeInQueueStart = PR_GETTER.extractTimeInQueueStart(record); - if (timeInQueueStart == 0 || !TIME_IN_QUEUE_ENABLED) { - span = startSpan(JAVA_KAFKA.toString(), KAFKA_CONSUME); - } else { - queueSpan = - startSpan( - JAVA_KAFKA.toString(), KAFKA_DELIVER, MILLISECONDS.toMicros(timeInQueueStart)); - BROKER_DECORATE.afterStart(queueSpan); - BROKER_DECORATE.onTimeInQueue(queueSpan, record); - span = startSpan(JAVA_KAFKA.toString(), KAFKA_CONSUME, queueSpan.spanContext()); - BROKER_DECORATE.beforeFinish(queueSpan); - // The queueSpan will be finished after inner span has been activated to ensure that - // spans are written out together by TraceStructureWriter when running in strict mode - } - - String applicationId = null; - if (streamTaskContext != null) { - applicationId = streamTaskContext.getApplicationId(); - } - DataStreamsTags tags = createWithGroup("kafka", INBOUND, applicationId, record.topic()); - - long payloadSize = 0; - // we have to go through Object to get the RecordMetadata here because the class of `record` - // only implements it after 2.7 (and this class is only used if v >= 2.7) - if ((Object) record instanceof RecordMetadata) { // should always be true - RecordMetadata metadata = (RecordMetadata) (Object) record; - payloadSize = metadata.serializedKeySize() + metadata.serializedValueSize(); - } - - if (STREAMING_CONTEXT.isDisabledForTopic(record.topic())) { - AgentTracer.get() - .getDataStreamsMonitoring() - .setCheckpoint(span, create(tags, record.timestamp(), payloadSize)); - } else { - if (STREAMING_CONTEXT.isSourceTopic(record.topic())) { - Propagator dsmPropagator = Propagators.forConcern(DSM_CONCERN); - DataStreamsContext dsmContext = create(tags, record.timestamp(), payloadSize); - dsmPropagator.inject(span.with(dsmContext), record, PR_SETTER); - } - } - - CONSUMER_DECORATE.afterStart(span); - CONSUMER_DECORATE.onConsume(span, record, node); - AgentScope agentScope = activateSpan(span); - if (null != queueSpan) { - queueSpan.finish(); - } - - if (streamTaskContext == null) { - streamTaskContext = new StreamTaskContext(); - } - streamTaskContext.setAgentScope(agentScope); - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class) - .put(task, streamTaskContext); - } - } - - public static class StopSpanAdvice { - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void stop( - @Advice.Thrown final Throwable throwable, @Advice.This StreamTask task) { - StreamTaskContext streamTaskContext = - InstrumentationContext.get(StreamTask.class, StreamTaskContext.class).get(task); - if (streamTaskContext != null) { - AgentScope scope = streamTaskContext.getAgentScope(); - if (scope != null) { - AgentSpan span = scope.span(); - CONSUMER_DECORATE.onError(span, throwable); - CONSUMER_DECORATE.beforeFinish(span); - scope.close(); - span.finish(); - streamTaskContext.setAgentScope(null); - } - } - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsDecorator.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsDecorator.java deleted file mode 100644 index 97f52bae52c..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsDecorator.java +++ /dev/null @@ -1,135 +0,0 @@ -package datadog.trace.instrumentation.kafka_streams; - -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.OFFSET; -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.PARTITION; -import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.PROCESSOR_NAME; -import static datadog.trace.bootstrap.instrumentation.api.ServiceNameSources.MESSAGE_BROKER_SPLIT_BY_DESTINATION; - -import datadog.trace.api.Config; -import datadog.trace.api.Functions; -import datadog.trace.api.cache.DDCache; -import datadog.trace.api.cache.DDCaches; -import datadog.trace.api.naming.SpanNaming; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; -import datadog.trace.bootstrap.instrumentation.api.Tags; -import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; -import datadog.trace.bootstrap.instrumentation.decorator.MessagingClientDecorator; -import java.util.function.Supplier; -import org.apache.kafka.streams.processor.internals.ProcessorNode; -import org.apache.kafka.streams.processor.internals.ProcessorRecordContext; -import org.apache.kafka.streams.processor.internals.StampedRecord; - -public class KafkaStreamsDecorator extends MessagingClientDecorator { - private static final String KAFKA = "kafka"; - public static final CharSequence JAVA_KAFKA = UTF8BytesString.create("java-kafka-streams"); - public static final CharSequence KAFKA_CONSUME = - UTF8BytesString.create( - SpanNaming.instance().namingSchema().messaging().inboundOperation(KAFKA)); - public static final CharSequence KAFKA_DELIVER = UTF8BytesString.create("kafka.deliver"); - - public static final boolean KAFKA_LEGACY_TRACING = Config.get().isKafkaLegacyTracingEnabled(); - public static final boolean TIME_IN_QUEUE_ENABLED = - Config.get().isTimeInQueueEnabled(!KAFKA_LEGACY_TRACING, KAFKA); - public static final String KAFKA_PRODUCED_KEY = "x_datadog_kafka_produced"; - - private final String spanKind; - private final CharSequence spanType; - private final Supplier serviceNameSupplier; - - private static final DDCache RESOURCE_NAME_CACHE = - DDCaches.newFixedSizeCache(32); - private static final Functions.Prefix PREFIX = new Functions.Prefix("Consume Topic "); - - public static final KafkaStreamsDecorator CONSUMER_DECORATE = - new KafkaStreamsDecorator( - Tags.SPAN_KIND_CONSUMER, - InternalSpanTypes.MESSAGE_CONSUMER, - SpanNaming.instance() - .namingSchema() - .messaging() - .inboundService(KAFKA, KAFKA_LEGACY_TRACING)); - - public static final KafkaStreamsDecorator BROKER_DECORATE = - new KafkaStreamsDecorator( - Tags.SPAN_KIND_BROKER, - InternalSpanTypes.MESSAGE_BROKER, - SpanNaming.instance().namingSchema().messaging().timeInQueueService(KAFKA)); - - protected KafkaStreamsDecorator( - String spanKind, CharSequence spanType, Supplier serviceNameSupplier) { - this.spanKind = spanKind; - this.spanType = spanType; - this.serviceNameSupplier = serviceNameSupplier; - } - - @Override - protected String[] instrumentationNames() { - return new String[] {"kafka", "kafka-streams"}; - } - - @Override - protected String service() { - return serviceNameSupplier.get(); - } - - @Override - protected CharSequence component() { - return JAVA_KAFKA; - } - - @Override - protected String spanKind() { - return spanKind; - } - - @Override - protected CharSequence spanType() { - return spanType; - } - - public void onConsume( - final AgentSpan span, final StampedRecord record, final ProcessorNode node) { - if (record != null) { - onConsume(span, record.topic(), record.partition(), record.offset(), node); - } - } - - public void onConsume( - final AgentSpan span, final ProcessorRecordContext record, final ProcessorNode node) { - if (record != null) { - onConsume(span, record.topic(), record.partition(), record.offset(), node); - } - } - - private void onConsume( - AgentSpan span, String topic2, int partition, long offset, ProcessorNode node) { - String topic = topic2 == null ? "kafka" : topic2; - span.setResourceName(RESOURCE_NAME_CACHE.computeIfAbsent(topic, PREFIX)); - span.setTag(PARTITION, partition); - span.setTag(OFFSET, offset); - if (node != null) { - span.setTag(PROCESSOR_NAME, node.name()); - } - } - - public void onTimeInQueue(final AgentSpan span, final StampedRecord record) { - if (record != null) { - onTimeInQueue(span, record.topic()); - } - } - - public void onTimeInQueue(final AgentSpan span, final ProcessorRecordContext record) { - if (record != null) { - onTimeInQueue(span, record.topic()); - } - } - - public void onTimeInQueue(final AgentSpan span, final String topic2) { - String topic = topic2 == null ? "kafka" : topic2; - span.setResourceName(topic); - if (Config.get().isMessageBrokerSplitByDestination()) { - span.setServiceName(topic, MESSAGE_BROKER_SPLIT_BY_DESTINATION); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsSourceNodeRecordDeserializerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsSourceNodeRecordDeserializerInstrumentation.java deleted file mode 100644 index 81348490676..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/KafkaStreamsSourceNodeRecordDeserializerInstrumentation.java +++ /dev/null @@ -1,64 +0,0 @@ -package datadog.trace.instrumentation.kafka_streams; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.isPublic; -import static net.bytebuddy.matcher.ElementMatchers.returns; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.common.record.TimestampType; - -// This is necessary because SourceNodeRecordDeserializer drops the headers. :-( -@AutoService(InstrumenterModule.class) -public class KafkaStreamsSourceNodeRecordDeserializerInstrumentation - extends InstrumenterModule.Tracing - implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { - - public KafkaStreamsSourceNodeRecordDeserializerInstrumentation() { - super("kafka", "kafka-streams"); - } - - @Override - public String instrumentedType() { - return "org.apache.kafka.streams.processor.internals.SourceNodeRecordDeserializer"; - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isMethod() - .and(isPublic()) - .and(named("deserialize")) - .and(takesArgument(0, named("org.apache.kafka.clients.consumer.ConsumerRecord"))) - .and(returns(named("org.apache.kafka.clients.consumer.ConsumerRecord"))), - KafkaStreamsSourceNodeRecordDeserializerInstrumentation.class.getName() - + "$SaveHeadersAdvice"); - } - - public static class SaveHeadersAdvice { - - @Advice.OnMethodExit(suppress = Throwable.class) - public static void saveHeaders( - @Advice.Argument(0) final ConsumerRecord incoming, - @Advice.Return(readOnly = false) ConsumerRecord result) { - result = - new ConsumerRecord<>( - result.topic(), - result.partition(), - result.offset(), - result.timestamp(), - TimestampType.CREATE_TIME, - result.checksum(), - result.serializedKeySize(), - result.serializedValueSize(), - result.key(), - result.value(), - incoming.headers()); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/ProcessorRecordContextHeadersAccess.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/ProcessorRecordContextHeadersAccess.java deleted file mode 100644 index 8cde81d5775..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/ProcessorRecordContextHeadersAccess.java +++ /dev/null @@ -1,30 +0,0 @@ -package datadog.trace.instrumentation.kafka_streams; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import org.apache.kafka.common.header.Headers; -import org.apache.kafka.streams.processor.internals.ProcessorRecordContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ProcessorRecordContextHeadersAccess { - private static final Logger log = - LoggerFactory.getLogger(ProcessorRecordContextHeadersAccess.class); - - public static final MethodHandle HEADERS_METHOD; - - static { - MethodHandle method; - try { - method = - MethodHandles.publicLookup() - .findVirtual( - ProcessorRecordContext.class, "headers", MethodType.methodType(Headers.class)); - } catch (Throwable e) { - log.debug("Exception loading MethodHandle", e); - method = null; - } - HEADERS_METHOD = method; - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/ProcessorRecordContextSetter.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/ProcessorRecordContextSetter.java deleted file mode 100644 index a444871f409..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/ProcessorRecordContextSetter.java +++ /dev/null @@ -1,30 +0,0 @@ -package datadog.trace.instrumentation.kafka_streams; - -import static datadog.trace.instrumentation.kafka_streams.ProcessorRecordContextHeadersAccess.HEADERS_METHOD; -import static java.nio.charset.StandardCharsets.UTF_8; - -import datadog.context.propagation.CarrierSetter; -import org.apache.kafka.common.header.Headers; -import org.apache.kafka.streams.processor.internals.ProcessorRecordContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ProcessorRecordContextSetter implements CarrierSetter { - private static final Logger log = LoggerFactory.getLogger(ProcessorRecordContextSetter.class); - - public static final ProcessorRecordContextSetter PR_SETTER = new ProcessorRecordContextSetter(); - - @Override - public void set(ProcessorRecordContext carrier, String key, String value) { - if (HEADERS_METHOD == null) { - return; - } - try { - Headers headers = (Headers) HEADERS_METHOD.invokeExact(carrier); - byte[] bytes = value.getBytes(UTF_8); - headers.remove(key).add(key, bytes); - } catch (Throwable e) { - log.debug("Unable to set value", e); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/ProcessorRecordContextVisitor.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/ProcessorRecordContextVisitor.java deleted file mode 100644 index e7aca6cea23..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/ProcessorRecordContextVisitor.java +++ /dev/null @@ -1,61 +0,0 @@ -package datadog.trace.instrumentation.kafka_streams; - -import static datadog.trace.instrumentation.kafka_streams.KafkaStreamsDecorator.KAFKA_PRODUCED_KEY; -import static datadog.trace.instrumentation.kafka_streams.ProcessorRecordContextHeadersAccess.HEADERS_METHOD; -import static java.nio.charset.StandardCharsets.UTF_8; - -import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; -import datadog.trace.bootstrap.instrumentation.api.AgentPropagation.ContextVisitor; -import java.nio.ByteBuffer; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.common.header.Headers; -import org.apache.kafka.streams.processor.internals.ProcessorRecordContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class ProcessorRecordContextVisitor implements ContextVisitor { - private static final Logger log = LoggerFactory.getLogger(ProcessorRecordContextVisitor.class); - - public static final ProcessorRecordContextVisitor PR_GETTER = new ProcessorRecordContextVisitor(); - - @Override - public void forEachKey( - ProcessorRecordContext carrier, AgentPropagation.KeyClassifier classifier) { - if (HEADERS_METHOD == null) { - return; - } - try { - Headers headers = (Headers) HEADERS_METHOD.invokeExact(carrier); - for (Header header : headers) { - String key = header.key(); - byte[] value = header.value(); - if (null != value) { - if (!classifier.accept(key, new String(header.value(), UTF_8))) { - return; - } - } - } - } catch (Throwable ex) { - log.debug("Exception getting headers", ex); - } - } - - public long extractTimeInQueueStart(ProcessorRecordContext carrier) { - if (HEADERS_METHOD == null) { - return 0; - } - try { - Headers headers = (Headers) HEADERS_METHOD.invokeExact(carrier); - Header header = headers.lastHeader(KAFKA_PRODUCED_KEY); - if (null != header) { - ByteBuffer buf = ByteBuffer.allocate(8); - buf.put(header.value()); - buf.flip(); - return buf.getLong(); - } - } catch (Throwable e) { - log.debug("Unable to get kafka produced time", e); - } - return 0; - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/StampedRecordContextSetter.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/StampedRecordContextSetter.java deleted file mode 100644 index 88d88cbb959..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/StampedRecordContextSetter.java +++ /dev/null @@ -1,19 +0,0 @@ -package datadog.trace.instrumentation.kafka_streams; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import datadog.context.propagation.CarrierSetter; -import org.apache.kafka.common.header.Headers; -import org.apache.kafka.streams.processor.internals.StampedRecord; - -public class StampedRecordContextSetter implements CarrierSetter { - public static final StampedRecordContextSetter SR_SETTER = new StampedRecordContextSetter(); - - @Override - public void set(StampedRecord carrier, String key, String value) { - Headers headers = carrier.value.headers(); - if (headers != null) { - headers.remove(key).add(key, value.getBytes(UTF_8)); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/StampedRecordContextVisitor.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/StampedRecordContextVisitor.java deleted file mode 100644 index 70a00f1f9e6..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/StampedRecordContextVisitor.java +++ /dev/null @@ -1,47 +0,0 @@ -package datadog.trace.instrumentation.kafka_streams; - -import static datadog.trace.instrumentation.kafka_streams.KafkaStreamsDecorator.KAFKA_PRODUCED_KEY; -import static java.nio.charset.StandardCharsets.UTF_8; - -import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; -import datadog.trace.bootstrap.instrumentation.api.AgentPropagation.ContextVisitor; -import java.nio.ByteBuffer; -import org.apache.kafka.common.header.Header; -import org.apache.kafka.streams.processor.internals.StampedRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class StampedRecordContextVisitor implements ContextVisitor { - - private static final Logger log = LoggerFactory.getLogger(StampedRecordContextVisitor.class); - - public static final StampedRecordContextVisitor SR_GETTER = new StampedRecordContextVisitor(); - - @Override - public void forEachKey(StampedRecord carrier, AgentPropagation.KeyClassifier classifier) { - for (Header header : carrier.value.headers()) { - String key = header.key(); - byte[] value = header.value(); - if (null != value) { - if (!classifier.accept(key, new String(header.value(), UTF_8))) { - return; - } - } - } - } - - public long extractTimeInQueueStart(StampedRecord carrier) { - try { - Header header = carrier.value.headers().lastHeader(KAFKA_PRODUCED_KEY); - if (null != header) { - ByteBuffer buf = ByteBuffer.allocate(8); - buf.put(header.value()); - buf.flip(); - return buf.getLong(); - } - } catch (Exception e) { - log.debug("Unable to get kafka produced time", e); - } - return 0; - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/StreamTaskContext.java b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/StreamTaskContext.java deleted file mode 100644 index 3e2009cdf8b..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/main/java/datadog/trace/instrumentation/kafka_streams/StreamTaskContext.java +++ /dev/null @@ -1,26 +0,0 @@ -package datadog.trace.instrumentation.kafka_streams; - -import datadog.trace.bootstrap.instrumentation.api.AgentScope; - -public class StreamTaskContext { - private AgentScope agentScope; - private String applicationId; - - public StreamTaskContext() {} - - public void setAgentScope(AgentScope agentScope) { - this.agentScope = agentScope; - } - - public AgentScope getAgentScope() { - return agentScope; - } - - public void setApplicationId(String applicationId) { - this.applicationId = applicationId; - } - - public String getApplicationId() { - return applicationId; - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/test/groovy/KafkaStreamsTestBase.groovy b/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/test/groovy/KafkaStreamsTestBase.groovy deleted file mode 100644 index d697e529f72..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-0.11/src/test/groovy/KafkaStreamsTestBase.groovy +++ /dev/null @@ -1,486 +0,0 @@ -import datadog.trace.agent.test.naming.VersionedNamingTestBase -import datadog.trace.api.DDTags -import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags -import datadog.trace.bootstrap.instrumentation.api.Tags -import datadog.trace.core.datastreams.StatsGroup -import org.apache.kafka.clients.consumer.ConsumerRecord -import org.apache.kafka.common.serialization.Serdes -import org.apache.kafka.streams.KafkaStreams -import org.apache.kafka.streams.StreamsConfig -import org.apache.kafka.streams.kstream.KStream -import org.apache.kafka.streams.kstream.KStreamBuilder -import org.apache.kafka.streams.kstream.ValueMapper -import org.springframework.kafka.core.DefaultKafkaConsumerFactory -import org.springframework.kafka.core.DefaultKafkaProducerFactory -import org.springframework.kafka.core.KafkaTemplate -import org.springframework.kafka.listener.KafkaMessageListenerContainer -import org.springframework.kafka.listener.MessageListener -import org.springframework.kafka.listener.config.ContainerProperties -import org.springframework.kafka.test.rule.KafkaEmbedded -import org.springframework.kafka.test.utils.ContainerTestUtils -import org.springframework.kafka.test.utils.KafkaTestUtils -import spock.lang.Ignore -import spock.lang.Shared - -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.TimeUnit - -abstract class KafkaStreamsTestBase extends VersionedNamingTestBase { - static final STREAM_PENDING = "test.pending" - static final STREAM_PROCESSED = "test.processed" - - @Shared - protected KafkaEmbedded embeddedKafka - - def setupSpec() { - embeddedKafka = new KafkaEmbedded(1, true, 1, STREAM_PENDING, STREAM_PROCESSED) - embeddedKafka.before() - } - - def cleanupSpec() { - embeddedKafka?.after() - } - - abstract boolean hasQueueSpan() - - abstract boolean splitByDestination() - - @Override - protected boolean isDataStreamsEnabled() { - return true - } - - @Override - int version() { - 0 - } - - @Override - String operation() { - return null - } - - String operationForProducer() { - "kafka.produce" - } - - String operationForConsumer() { - "kafka.consume" - } - - String serviceForTimeInQueue() { - "kafka" - } - - - @Ignore("Repeatedly fails with the wrong parent span https://github.com/DataDog/dd-trace-java/issues/3865") - def "test kafka produce and consume with streams in-between"() { - setup: - def config = new Properties() - def senderProps = KafkaTestUtils.senderProps(embeddedKafka.getBrokersAsString()) - config.putAll(senderProps) - config.put(StreamsConfig.APPLICATION_ID_CONFIG, "test-application") - config.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()) - config.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()) - - // CONFIGURE CONSUMER - def consumerFactory = new DefaultKafkaConsumerFactory(KafkaTestUtils.consumerProps("sender", "false", embeddedKafka)) - - def consumerContainer = new KafkaMessageListenerContainer<>(consumerFactory, new ContainerProperties(STREAM_PROCESSED)) - - // create a thread safe queue to store the processed message - def records = new LinkedBlockingQueue>() - - // setup a Kafka message listener - consumerContainer.setupMessageListener(new MessageListener() { - @Override - void onMessage(ConsumerRecord record) { - // ensure consistent ordering of traces - // this is the last processing step so we should see 2 traces here - TEST_WRITER.waitForTraces(2) - TEST_TRACER.activeSpan().setTag("testing", 123) - if (isDataStreamsEnabled()) { - TEST_DATA_STREAMS_WRITER.waitForGroups(1) - } - records.add(record) - } - }) - - // start the container and underlying message listener - consumerContainer.start() - - // wait until the container has the required number of assigned partitions - ContainerTestUtils.waitForAssignment(consumerContainer, embeddedKafka.getPartitionsPerTopic()) - - // CONFIGURE PROCESSOR - def builder = new KStreamBuilder() - KStream textLines = builder.stream(STREAM_PENDING) - def values = textLines - .mapValues(new ValueMapper() { - @Override - String apply(String textLine) { - TEST_WRITER.waitForTraces(1) // ensure consistent ordering of traces - TEST_TRACER.activeSpan().setTag("asdf", "testing") - if (isDataStreamsEnabled()) { - TEST_DATA_STREAMS_WRITER.waitForGroups(1) - } - return textLine.toLowerCase() - } - }) - - values.to(Serdes.String(), Serdes.String(), STREAM_PROCESSED) - KafkaStreams streams = new KafkaStreams(builder, config) - streams.start() - - // CONFIGURE PRODUCER - def producerFactory = new DefaultKafkaProducerFactory(senderProps) - def kafkaTemplate = new KafkaTemplate(producerFactory) - - when: - String greeting = "TESTING TESTING 123!" - kafkaTemplate.send(STREAM_PENDING, greeting) - - then: - // check that the message was received - def received = records.poll(10, TimeUnit.SECONDS) - received.value() == greeting.toLowerCase() - received.key() == null - - boolean hasQueueSpan = hasQueueSpan() - def firstProducerSpan, firstQueueSpan = null - def secondProducerSpan, secondQueueSpan = null - assertTraces(3) { - trace(1) { - // PRODUCER span 0 - span { - firstProducerSpan = it.span - serviceName service() - operationName operationForProducer() - resourceName "Produce Topic $STREAM_PENDING" - spanType "queue" - errored false - measured true - parent() - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_PRODUCER - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$STREAM_PENDING" - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" { it >= 0 } - "$InstrumentationTags.KAFKA_CLUSTER_ID" { String } - if ({ isDataStreamsEnabled() }){ - "$DDTags.PATHWAY_HASH" { String } - } - defaultTagsNoPeerService() - } - } - } - trace(hasQueueSpan ? 3 : 2) { - sortSpansByStart() - - if (hasQueueSpan) { - span { - firstQueueSpan = it.span - serviceName splitByDestination() ? "$STREAM_PENDING" : serviceForTimeInQueue() - operationName "kafka.deliver" - resourceName "$STREAM_PENDING" - spanType "queue" - errored false - measured true - childOf firstProducerSpan - tags { - "$Tags.COMPONENT" "java-kafka-streams" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_BROKER - defaultTags(true) - } - } - } - - // STREAMING span 0 - span { - serviceName service() - operationName operationForConsumer() - resourceName "Consume Topic $STREAM_PENDING" - spanType "queue" - errored false - measured true - childOf hasQueueSpan ? firstQueueSpan : firstProducerSpan - - tags { - "$Tags.COMPONENT" "java-kafka-streams" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_CONSUMER - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" 0 - "$InstrumentationTags.PROCESSOR_NAME" "KSTREAM-SOURCE-0000000000" - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$STREAM_PENDING" - "asdf" "testing" - if ({isDataStreamsEnabled()}) { - "$DDTags.PATHWAY_HASH" { String } - } - defaultTags(!hasQueueSpan) - } - } - - // STREAMING span 1 - span { - secondProducerSpan = it.span - serviceName service() - operationName operationForProducer() - resourceName "Produce Topic $STREAM_PROCESSED" - spanType "queue" - errored false - measured true - childOf span(hasQueueSpan ? 1 : 0) - - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_PRODUCER - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$STREAM_PROCESSED" - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" { it >= 0 } - "$InstrumentationTags.KAFKA_CLUSTER_ID" { String } - if ({isDataStreamsEnabled()}) { - "$DDTags.PATHWAY_HASH" { String } - } - defaultTagsNoPeerService() - } - } - } - trace(hasQueueSpan ? 2 : 1) { - sortSpansByStart() - - if (hasQueueSpan) { - span { - secondQueueSpan = it.span - serviceName splitByDestination() ? "$STREAM_PROCESSED" : serviceForTimeInQueue() - operationName "kafka.deliver" - resourceName "$STREAM_PROCESSED" - spanType "queue" - errored false - measured true - childOf secondProducerSpan - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_BROKER - defaultTags(true) - } - } - } - - // CONSUMER span 0 - span { - serviceName service() - operationName operationForConsumer() - resourceName "Consume Topic $STREAM_PROCESSED" - spanType "queue" - errored false - measured true - childOf hasQueueSpan ? secondQueueSpan : secondProducerSpan - tags { - "$Tags.COMPONENT" "java-kafka" - "$Tags.SPAN_KIND" Tags.SPAN_KIND_CONSUMER - "$InstrumentationTags.PARTITION" { it >= 0 } - "$InstrumentationTags.OFFSET" 0 - "$InstrumentationTags.CONSUMER_GROUP" "sender" - "$InstrumentationTags.KAFKA_CLUSTER_ID" { String } - "$InstrumentationTags.RECORD_QUEUE_TIME_MS" { it >= 0 } - "$InstrumentationTags.MESSAGING_DESTINATION_NAME" "$STREAM_PROCESSED" - "testing" 123 - if ({isDataStreamsEnabled()}) { - "$DDTags.PATHWAY_HASH" { String } - } - defaultTags(!hasQueueSpan) - } - } - } - } - - def headers = received.headers() - headers.iterator().hasNext() - new String(headers.headers("x-datadog-trace-id").iterator().next().value()) == "${TEST_WRITER[1][0].traceId}" - new String(headers.headers("x-datadog-parent-id").iterator().next().value()) == "${TEST_WRITER[1][0].spanId}" - - if (isDataStreamsEnabled()) { - StatsGroup originProducerPoint = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == 0 } - verifyAll(originProducerPoint) { - tags.hasAllTags("direction:out", "topic:$STREAM_PENDING", "type:kafka") - } - - StatsGroup kafkaStreamsConsumerPoint = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == originProducerPoint.hash } - verifyAll(kafkaStreamsConsumerPoint) { - tags.hasAllTags("direction:in", - "group:test-application", - "topic:$STREAM_PENDING".toString(), - "type:kafka") - } - - StatsGroup kafkaStreamsProducerPoint = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == kafkaStreamsConsumerPoint.hash } - verifyAll(kafkaStreamsProducerPoint) { - tags.hasAllTags("direction:out", "topic:$STREAM_PROCESSED", "type:kafka") - } - - StatsGroup finalConsumerPoint = TEST_DATA_STREAMS_WRITER.groups.find { it.parentHash == kafkaStreamsProducerPoint.hash } - verifyAll(finalConsumerPoint) { - tags.hasAllTags("direction:in", "group:sender", "topic:$STREAM_PROCESSED".toString(), "type:kafka") - } - } - - cleanup: - producerFactory?.stop() - streams?.close() - consumerContainer?.stop() - } -} - -abstract class KafkaStreamsForkedTest extends KafkaStreamsTestBase { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("dd.service", "KafkaStreamsTest") - } - - @Override - String service() { - return "KafkaStreamsTest" - } - - @Override - boolean hasQueueSpan() { - return true - } - - @Override - boolean splitByDestination() { - return false - } -} - -class KafkaStreamsV0ForkedTest extends KafkaStreamsForkedTest { -} - -class KafkaStreamsV1ForkedTest extends KafkaStreamsForkedTest { - @Override - int version() { - 1 - } - - @Override - String operationForProducer() { - "kafka.send" - } - - @Override - String operationForConsumer() { - return "kafka.process" - } - - @Override - String serviceForTimeInQueue() { - "kafka-queue" - } - - @Override - boolean hasQueueSpan() { - return false - } -} - -class KafkaStreamsSplitByDestinationForkedTest extends KafkaStreamsTestBase { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("dd.service", "KafkaStreamsTest") - injectSysConfig("dd.kafka.legacy.tracing.enabled", "false") - injectSysConfig("dd.message.broker.split-by-destination", "true") - } - - @Override - String service() { - return "KafkaStreamsTest" - } - - @Override - boolean hasQueueSpan() { - return true - } - - @Override - boolean splitByDestination() { - return true - } -} - -abstract class KafkaStreamsLegacyTracingForkedTest extends KafkaStreamsTestBase { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("dd.service", "KafkaStreamsLegacyTracingTest") - injectSysConfig("dd.kafka.legacy.tracing.enabled", "true") - } - - @Override - boolean hasQueueSpan() { - return false - } - - @Override - boolean splitByDestination() { - return false - } -} -class KafkaStreamsLegacyTracingV0ForkedTest extends KafkaStreamsLegacyTracingForkedTest { - @Override - String service() { - "kafka" - } -} - -class KafkaStreamsLegacyTracingV1ForkedTest extends KafkaStreamsLegacyTracingForkedTest { - @Override - String service() { - "KafkaStreamsLegacyTracingTest" - } - @Override - String operationForProducer() { - "kafka.send" - } - - @Override - String operationForConsumer() { - return "kafka.process" - } - - @Override - String serviceForTimeInQueue() { - "kafka-queue" - } -} - -class KafkaStreamsDataStreamsDisabledForkedTest extends KafkaStreamsTestBase { - @Override - void configurePreAgent() { - super.configurePreAgent() - injectSysConfig("dd.service", "KafkaStreamsDataStreamsDisabledForkedTest") - injectSysConfig("dd.kafka.legacy.tracing.enabled", "false") - } - - @Override - String service() { - return "KafkaStreamsDataStreamsDisabledForkedTest" - } - - @Override - boolean hasQueueSpan() { - return true - } - - @Override - boolean splitByDestination() { - return false - } - - @Override - boolean isDataStreamsEnabled() { - return false - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/build.gradle b/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/build.gradle deleted file mode 100644 index f476e4ad317..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/build.gradle +++ /dev/null @@ -1,20 +0,0 @@ -plugins { - id 'dd-trace-java.module.instrumentation' -} - -muzzle { - extraRepository("confluent-releases", "https://packages.confluent.io/maven/") - - pass { - group = "org.apache.kafka" - module = "kafka-streams" - versions = "[1.0.0.0,3)" - } -} - -addTestSuite('latestDepTest') - -dependencies { - compileOnly group: 'org.apache.kafka', name: 'kafka-streams', version: '1.0.0' - implementation project(':dd-java-agent:instrumentation:kafka:kafka-common') -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/gradle.lockfile b/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/gradle.lockfile deleted file mode 100644 index 9142797a4ee..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/gradle.lockfile +++ /dev/null @@ -1,135 +0,0 @@ -# This is a Gradle generated file for dependency locking. -# Manual edits can break the build and are not advised. -# This file is expected to be part of source control. -# To regenerate this file, run: ./gradlew :dd-java-agent:instrumentation:kafka:kafka-streams-1.0:dependencies --write-locks -cafe.cryptography:curve25519-elisabeth:0.1.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -cafe.cryptography:ed25519-elisabeth:0.1.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -ch.qos.logback:logback-classic:1.2.13=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -ch.qos.logback:logback-core:1.2.13=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.blogspot.mydailyjava:weak-lock-free:0.17=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq.okhttp3:okhttp:3.12.15=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq.okio:okio:1.17.6=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:dd-instrument-java:0.0.4=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:dd-javac-plugin-client:0.2.2=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -com.datadoghq:java-dogstatsd-client:4.4.5=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.datadoghq:sketches-java:0.8.3=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.fasterxml.jackson.core:jackson-annotations:2.9.0=compileClasspath -com.fasterxml.jackson.core:jackson-core:2.9.1=compileClasspath -com.fasterxml.jackson.core:jackson-databind:2.9.1=compileClasspath -com.github.javaparser:javaparser-core:3.25.6=codenarc -com.github.jnr:jffi:1.3.15=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-a64asm:1.0.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-constants:0.10.4=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-enxio:0.32.20=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-ffi:2.2.19=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-posix:3.1.22=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-unixsocket:0.38.25=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.jnr:jnr-x86asm:1.0.2=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.spotbugs:spotbugs-annotations:4.10.3=compileClasspath,spotbugs -com.github.spotbugs:spotbugs:4.10.3=spotbugs -com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs -com.google.auto.service:auto-service-annotations:1.1.1=annotationProcessor,compileClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,testAnnotationProcessor,testCompileClasspath -com.google.auto.service:auto-service:1.1.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.auto:auto-common:1.2.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,compileClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,spotbugs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath -com.google.code.gson:gson:2.14.0=spotbugs -com.google.errorprone:error_prone_annotations:2.18.0=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.errorprone:error_prone_annotations:2.47.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.errorprone:error_prone_annotations:2.48.0=spotbugs -com.google.guava:failureaccess:1.0.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.guava:failureaccess:1.0.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.guava:guava:32.0.1-jre=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.guava:guava:33.6.0-jre=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath -com.google.j2objc:j2objc-annotations:2.8=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -com.google.j2objc:j2objc-annotations:3.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.google.re2j:re2j:1.8=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.squareup.moshi:moshi:1.11.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.squareup.okhttp3:logging-interceptor:3.12.12=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.squareup.okhttp3:okhttp:3.12.12=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.squareup.okio:okio:1.17.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -com.thoughtworks.qdox:qdox:1.12.1=codenarc -commons-fileupload:commons-fileupload:1.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -commons-io:commons-io:2.11.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -commons-io:commons-io:2.21.0=spotbugs -de.thetaphi:forbiddenapis:3.10=compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.leangen.geantyref:geantyref:1.3.16=latestDepTestRuntimeClasspath,testRuntimeClasspath -io.sqreen:libsqreen:17.5.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -javax.servlet:javax.servlet-api:3.1.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -jaxen:jaxen:2.0.6=spotbugs -junit:junit:4.13.2=latestDepTestRuntimeClasspath,testRuntimeClasspath -net.bytebuddy:byte-buddy-agent:1.18.10=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -net.bytebuddy:byte-buddy:1.18.10=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath -net.java.dev.jna:jna-platform:5.8.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -net.java.dev.jna:jna:5.8.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -net.sf.saxon:Saxon-HE:12.10=spotbugs -org.apache.ant:ant-antlr:1.10.14=codenarc -org.apache.ant:ant-junit:1.10.14=codenarc -org.apache.bcel:bcel:6.12.0=spotbugs -org.apache.commons:commons-lang3:3.20.0=spotbugs -org.apache.commons:commons-text:1.15.0=spotbugs -org.apache.kafka:connect-api:1.0.0=compileClasspath -org.apache.kafka:connect-json:1.0.0=compileClasspath -org.apache.kafka:kafka-clients:1.0.0=compileClasspath -org.apache.kafka:kafka-streams:1.0.0=compileClasspath -org.apache.logging.log4j:log4j-api:2.26.1=spotbugs -org.apache.logging.log4j:log4j-core:2.26.1=spotbugs -org.apiguardian:apiguardian-api:1.1.2=latestDepTestCompileClasspath,testCompileClasspath -org.checkerframework:checker-qual:3.33.0=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor -org.codehaus.groovy:groovy-ant:3.0.23=codenarc -org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc -org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc -org.codehaus.groovy:groovy-json:3.0.23=codenarc -org.codehaus.groovy:groovy-json:3.0.25=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.codehaus.groovy:groovy-templates:3.0.23=codenarc -org.codehaus.groovy:groovy-xml:3.0.23=codenarc -org.codehaus.groovy:groovy:3.0.23=codenarc -org.codehaus.groovy:groovy:3.0.25=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.codenarc:CodeNarc:3.7.0=codenarc -org.dom4j:dom4j:2.2.0=spotbugs -org.gmetrics:GMetrics:2.1.0=codenarc -org.hamcrest:hamcrest-core:1.3=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.hamcrest:hamcrest:3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.jctools:jctools-core-jdk11:4.0.6=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.jctools:jctools-core:4.0.6=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.jspecify:jspecify:1.0.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-api:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-engine:5.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter-params:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.jupiter:junit-jupiter:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-commons:1.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-engine:1.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-launcher:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-runner:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-suite-api:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit.platform:junit-platform-suite-commons:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.junit:junit-bom:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit:junit-bom:6.1.2=spotbugs -org.lz4:lz4-java:1.4=compileClasspath -org.mockito:mockito-core:4.4.0=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.objenesis:objenesis:3.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.opentest4j:opentest4j:1.3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.ow2.asm:asm-analysis:9.10.1=spotbugs -org.ow2.asm:asm-analysis:9.7.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.ow2.asm:asm-commons:9.10.1=latestDepTestRuntimeClasspath,spotbugs,testRuntimeClasspath -org.ow2.asm:asm-tree:9.10.1=latestDepTestRuntimeClasspath,spotbugs,testRuntimeClasspath -org.ow2.asm:asm-util:9.10.1=spotbugs -org.ow2.asm:asm-util:9.7.1=latestDepTestRuntimeClasspath,testRuntimeClasspath -org.ow2.asm:asm:9.10.1=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath -org.rocksdb:rocksdbjni:5.7.3=compileClasspath -org.slf4j:jcl-over-slf4j:1.7.30=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:jul-to-slf4j:1.7.30=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:log4j-over-slf4j:1.7.30=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:1.7.30=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath -org.slf4j:slf4j-api:1.7.32=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.slf4j:slf4j-api:2.0.17=spotbugsSlf4j -org.slf4j:slf4j-api:2.0.18=spotbugs -org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j -org.snakeyaml:snakeyaml-engine:2.9=buildTimeInstrumentationPlugin,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testRuntimeClasspath -org.spockframework:spock-bom:2.4-groovy-3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.spockframework:spock-core:2.4-groovy-3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.tabletest:tabletest-junit:1.2.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.tabletest:tabletest-parser:1.2.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.xerial.snappy:snappy-java:1.1.4=compileClasspath -org.xmlresolver:xmlresolver:5.3.3=spotbugs -empty=spotbugsPlugins diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/src/main/java/datadog/trace/instrumentation/kafka_streams10/InternalTopologyBuilderInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/src/main/java/datadog/trace/instrumentation/kafka_streams10/InternalTopologyBuilderInstrumentation.java deleted file mode 100644 index 07271bb7904..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/src/main/java/datadog/trace/instrumentation/kafka_streams10/InternalTopologyBuilderInstrumentation.java +++ /dev/null @@ -1,48 +0,0 @@ -package datadog.trace.instrumentation.kafka_streams10; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.isPrivate; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import net.bytebuddy.asm.Advice; -import org.apache.kafka.streams.processor.internals.ProcessorTopology; - -@AutoService(InstrumenterModule.class) -public class InternalTopologyBuilderInstrumentation extends InstrumenterModule.Tracing - implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { - - public InternalTopologyBuilderInstrumentation() { - super("kafka", "kafka-streams"); - } - - @Override - public String instrumentedType() { - return "org.apache.kafka.streams.processor.internals.InternalTopologyBuilder"; - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".StreamingContextUpdater", - "datadog.trace.instrumentation.kafka_common.StreamingContext" - }; - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - transformer.applyAdvice( - isMethod().and(named("build")).and(isPrivate()).and(takesArguments(1)), - InternalTopologyBuilderInstrumentation.class.getName() + "$BuildAdvice"); - } - - public static class BuildAdvice { - @Advice.OnMethodExit(suppress = Throwable.class) - public static void exit(@Advice.Return final ProcessorTopology topology) { - StreamingContextUpdater.updateWithTopology(topology); - } - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/src/main/java/datadog/trace/instrumentation/kafka_streams10/StreamingContextUpdater.java b/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/src/main/java/datadog/trace/instrumentation/kafka_streams10/StreamingContextUpdater.java deleted file mode 100644 index 6dc766cdf5c..00000000000 --- a/dd-java-agent/instrumentation/kafka/kafka-streams-1.0/src/main/java/datadog/trace/instrumentation/kafka_streams10/StreamingContextUpdater.java +++ /dev/null @@ -1,26 +0,0 @@ -package datadog.trace.instrumentation.kafka_streams10; - -import static datadog.trace.instrumentation.kafka_common.StreamingContext.STREAMING_CONTEXT; - -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.kafka.streams.processor.StateStore; -import org.apache.kafka.streams.processor.internals.ProcessorTopology; - -public class StreamingContextUpdater { - - public static void updateWithTopology(ProcessorTopology topology) { - Set internalTopics = ConcurrentHashMap.newKeySet(); - Map storeToChanglogMap = topology.storeToChangelogTopic(); - - for (StateStore store : topology.stateStores()) { - if (storeToChanglogMap.containsKey(store.name())) { - internalTopics.add(storeToChanglogMap.get(store.name())); - } - } - - STREAMING_CONTEXT.registerTopics( - topology.sourceTopics(), topology.sinkTopics(), internalTopics); - } -} diff --git a/dd-java-agent/instrumentation/spring/spring-messaging-4.0/build.gradle b/dd-java-agent/instrumentation/spring/spring-messaging-4.0/build.gradle index b60941adfcf..f6cf2dd6507 100644 --- a/dd-java-agent/instrumentation/spring/spring-messaging-4.0/build.gradle +++ b/dd-java-agent/instrumentation/spring/spring-messaging-4.0/build.gradle @@ -83,7 +83,7 @@ dependencies { testImplementation "io.projectreactor:reactor-core:3.+" testRuntimeOnly project(':dd-java-agent:instrumentation:kotlin-coroutines-1.3') - testRuntimeOnly project(':dd-java-agent:instrumentation:kafka:kafka-clients-3.8') + // testRuntimeOnly project(':dd-java-agent:instrumentation:kafka:kafka-clients-3.8') latestDepTestImplementation group: 'org.springframework', name: 'spring-messaging', version: '6.+', { exclude group: 'org.slf4j', module: 'slf4j-api' diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/InstrumentationTags.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/InstrumentationTags.java index 0c1054e7776..db27761456b 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/InstrumentationTags.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/InstrumentationTags.java @@ -103,6 +103,7 @@ public class InstrumentationTags { public static final String HANDLER_TYPE = "handler.type"; public static final String KAFKA_BOOTSTRAP_SERVERS = "messaging.kafka.bootstrap.servers"; public static final String KAFKA_CLUSTER_ID = "messaging.kafka.cluster.id"; + public static final String MESSAGING_SYSTEM = "messaging.system"; public static final String MESSAGING_DESTINATION_NAME = "messaging.destination.name"; public static final String QUARTZ_JOB_NAME = "quartz.job.name"; public static final String QUARTZ_JOB_GROUP = "quartz.job.group"; diff --git a/settings.gradle.kts b/settings.gradle.kts index 14e988af3fe..da81d7748ea 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -461,11 +461,7 @@ include( ":dd-java-agent:instrumentation:junit:junit-5:junit-5-spock-2.0", ":dd-java-agent:instrumentation:junit:junit-5:junit-5.3", ":dd-java-agent:instrumentation:kafka:kafka-clients-0.11", - ":dd-java-agent:instrumentation:kafka:kafka-clients-3.8", ":dd-java-agent:instrumentation:kafka:kafka-common", - ":dd-java-agent:instrumentation:kafka:kafka-connect-0.11", - ":dd-java-agent:instrumentation:kafka:kafka-streams-0.11", - ":dd-java-agent:instrumentation:kafka:kafka-streams-1.0", ":dd-java-agent:instrumentation:karate:karate-1.0", ":dd-java-agent:instrumentation:karate:karate-2.0", ":dd-java-agent:instrumentation:kotlin-coroutines-1.3",