diff --git a/.gitlab/datasources/test-suites.yaml b/.gitlab/datasources/test-suites.yaml index d4eb86259..3184deaed 100644 --- a/.gitlab/datasources/test-suites.yaml +++ b/.gitlab/datasources/test-suites.yaml @@ -8,3 +8,4 @@ test_suites: - name: lmi-oom - name: payload-size - name: durable-cold-start + - name: apm-standalone diff --git a/bottlecap/src/bin/bottlecap/main.rs b/bottlecap/src/bin/bottlecap/main.rs index 39f352149..415126d5b 100644 --- a/bottlecap/src/bin/bottlecap/main.rs +++ b/bottlecap/src/bin/bottlecap/main.rs @@ -1300,6 +1300,17 @@ fn start_metrics_flushers( ) -> Vec { let mut flushers = Vec::new(); + // APM standalone ("traces only") mode: skip wiring up metrics flushers + // entirely, so no metrics (custom DogStatsD, enhanced, or process) reach + // intake. The DogStatsD server still runs and its aggregator is still drained + // on flush, but the drained data is discarded since there is no flusher to + // send it. Logs and OTLP signals are suppressed separately (see + // `apply_apm_standalone` and the logs flusher guard). See + // DD_APM_STANDALONE_ENABLED. + if config.ext.apm_standalone_enabled { + return flushers; + } + let metrics_intake_url = if !config.dd_url.is_empty() { let dd_dd_url = DdDdUrl::new(config.dd_url.clone()).expect("can't parse DD_DD_URL"); diff --git a/bottlecap/src/config/mod.rs b/bottlecap/src/config/mod.rs index db0a1f192..27d9f31d7 100644 --- a/bottlecap/src/config/mod.rs +++ b/bottlecap/src/config/mod.rs @@ -25,10 +25,35 @@ use serde::Deserialize; pub type Config = datadog_agent_config::Config; #[allow(clippy::module_name_repetitions)] -#[inline] #[must_use] pub fn get_config(config_directory: &Path) -> Config { - get_config_with_extension::(config_directory) + let mut config = get_config_with_extension::(config_directory); + apply_apm_standalone(&mut config); + config +} + +/// APM standalone ("traces only") mode: suppress every billable metrics and +/// logs egress path so the customer incurs no infrastructure-monitoring or +/// log-ingestion charges. This intentionally overrides any individually +/// configured metrics/logs toggles, since the guarantee must hold even if a +/// user also set, e.g., `DD_ENHANCED_METRICS=true`. Traces and APM trace stats +/// are unaffected. The custom `DogStatsD` egress is additionally disabled in +/// the metrics flusher wiring (see `start_metrics_flushers` in `main.rs`), and +/// log egress is guarded in the logs flusher. +fn apply_apm_standalone(config: &mut Config) { + if !config.ext.apm_standalone_enabled { + return; + } + + tracing::debug!( + "DD_APM_STANDALONE_ENABLED is set: forcing logs and all metrics off (traces-only mode)" + ); + + config.ext.serverless_logs_enabled = false; + config.ext.enhanced_metrics = false; + config.ext.lambda_proc_enhanced_metrics = false; + config.otlp_config_metrics_enabled = false; + config.otlp_config_logs_enabled = false; } // --------------------------------------------------------------------------- // LambdaConfig — bottlecap's `ConfigExtension` for the shared @@ -60,6 +85,12 @@ pub struct LambdaConfig { pub kms_api_key: String, pub api_key_ssm_arn: String, pub serverless_logs_enabled: bool, + /// When true, the extension operates in APM standalone ("traces only") + /// mode: logs and all metrics (enhanced, process, custom `DogStatsD`, and + /// OTLP) are suppressed at intake so that no infrastructure-monitoring or + /// log-ingestion charges are incurred. Traces and APM trace stats are + /// unaffected. Defaults to `false`. + pub apm_standalone_enabled: bool, pub serverless_flush_strategy: UpstreamFlushStrategy, pub enhanced_metrics: bool, pub lambda_proc_enhanced_metrics: bool, @@ -89,6 +120,7 @@ impl Default for LambdaConfig { kms_api_key: String::new(), api_key_ssm_arn: String::new(), serverless_logs_enabled: true, + apm_standalone_enabled: false, serverless_flush_strategy: UpstreamFlushStrategy::Default, enhanced_metrics: true, lambda_proc_enhanced_metrics: true, @@ -138,6 +170,13 @@ pub struct LambdaConfigSource { #[serde(deserialize_with = "deser_opt_bool")] pub logs_enabled: Option, + /// `DD_APM_STANDALONE_ENABLED` — run the extension in APM standalone + /// ("traces only") mode. When `true`, logs and all metrics (enhanced, + /// process, custom `DogStatsD`, and OTLP) are suppressed at intake. Traces + /// and APM trace stats are unaffected. Defaults to `false`. + #[serde(deserialize_with = "deser_opt_bool")] + pub apm_standalone_enabled: Option, + pub serverless_flush_strategy: Option, #[serde(deserialize_with = "deser_opt_bool")] @@ -193,6 +232,7 @@ impl DatadogConfigExtension for LambdaConfig { datadog_agent_config::merge_fields!(self, source, string: [api_key_secret_arn, kms_api_key, api_key_ssm_arn], value: [ + apm_standalone_enabled, serverless_flush_strategy, enhanced_metrics, lambda_proc_enhanced_metrics, @@ -260,6 +300,56 @@ mod lambda_config_tests { assert_eq!(config.ext, LambdaConfig::default()); } + #[test] + fn apm_standalone_defaults_off() { + let config = load(|_| Ok(())); + assert!(!config.ext.apm_standalone_enabled); + // Defaults remain unchanged when APM standalone is not set. + assert!(config.ext.serverless_logs_enabled); + assert!(config.ext.enhanced_metrics); + assert!(config.ext.lambda_proc_enhanced_metrics); + } + + #[test] + fn apm_standalone_from_yaml() { + // Guards the YAML key mapping (`apm_standalone_enabled:`) independently of + // the env-var path. The override application is covered by + // `apm_standalone_forces_metrics_and_logs_off` via `get_config`. + let config = load(|jail| { + jail.create_file("datadog.yaml", "apm_standalone_enabled: true\n")?; + Ok(()) + }); + assert!(config.ext.apm_standalone_enabled); + } + + #[test] + fn apm_standalone_forces_metrics_and_logs_off() { + // Exercised through `get_config` (not `load`) because the override is + // applied there, after the shared env/yaml merge. + Jail::expect_with(|jail| { + jail.clear_env(); + jail.set_env("DD_APM_STANDALONE_ENABLED", "true"); + // Even when a user explicitly enables these, APM standalone must + // override them so that no metrics or logs reach intake (billing + // guarantee). + jail.set_env("DD_SERVERLESS_LOGS_ENABLED", "true"); + jail.set_env("DD_ENHANCED_METRICS", "true"); + jail.set_env("DD_LAMBDA_PROC_ENHANCED_METRICS", "true"); + jail.set_env("DD_OTLP_CONFIG_METRICS_ENABLED", "true"); + jail.set_env("DD_OTLP_CONFIG_LOGS_ENABLED", "true"); + + let config = get_config(Path::new("")); + + assert!(config.ext.apm_standalone_enabled); + assert!(!config.ext.serverless_logs_enabled); + assert!(!config.ext.enhanced_metrics); + assert!(!config.ext.lambda_proc_enhanced_metrics); + assert!(!config.otlp_config_metrics_enabled); + assert!(!config.otlp_config_logs_enabled); + Ok(()) + }); + } + // ---- string fields from env / yaml ---- #[test] diff --git a/bottlecap/src/logs/flusher.rs b/bottlecap/src/logs/flusher.rs index 5f68a09c7..b18960a7d 100644 --- a/bottlecap/src/logs/flusher.rs +++ b/bottlecap/src/logs/flusher.rs @@ -259,6 +259,19 @@ impl LogsFlusher { &self, retry_request: Option, ) -> Vec { + // APM standalone ("traces only") mode: never send logs to intake. Logs are + // also dropped upstream (serverless_logs_enabled is forced off, so the + // processor never queues them), but this guard guarantees no log egress + // regardless of aggregator or redrive state. See DD_APM_STANDALONE_ENABLED. + if self.config.ext.apm_standalone_enabled { + // Drain and discard any queued batches so the aggregator stays bounded, + // mirroring the metrics path (which drains its aggregator and discards + // the data because no flusher is wired up). No requests are ever + // produced here, so nothing reaches intake. + let _ = self.aggregator_handle.get_batches().await; + return Vec::new(); + } + let mut failed_requests = Vec::new(); // If retry_request is provided, only process that request diff --git a/integration-tests/bin/app.ts b/integration-tests/bin/app.ts index 9c0ee5e76..7669e03fc 100644 --- a/integration-tests/bin/app.ts +++ b/integration-tests/bin/app.ts @@ -11,6 +11,7 @@ import {LmiOom} from '../lib/stacks/lmi-oom'; import {CustomMetrics} from '../lib/stacks/custom-metrics'; import {PayloadSize} from '../lib/stacks/payload-size'; import {DurableColdStart} from '../lib/stacks/durable-cold-start'; +import {ApmStandalone} from '../lib/stacks/apm-standalone'; import {AuthRoleStack} from '../lib/auth-role'; import {ACCOUNT, IDENTIFIER, REGION} from '../config'; import {CapacityProviderStack} from "../lib/capacity-provider"; @@ -58,6 +59,9 @@ const stacks = [ new DurableColdStart(app, `${IDENTIFIER}-durable-cold-start`, { env, }), + new ApmStandalone(app, `${IDENTIFIER}-apm-standalone`, { + env, + }), ] // Tag all stacks so we can easily clean them up diff --git a/integration-tests/lib/stacks/apm-standalone.ts b/integration-tests/lib/stacks/apm-standalone.ts new file mode 100644 index 000000000..4e0c2b7cb --- /dev/null +++ b/integration-tests/lib/stacks/apm-standalone.ts @@ -0,0 +1,57 @@ +import * as cdk from "aws-cdk-lib"; +import * as lambda from "aws-cdk-lib/aws-lambda"; +import { Construct } from "constructs"; +import { + createLogGroup, + defaultDatadogEnvVariables, + defaultDatadogSecretPolicy, + getExtensionLayer, + getDefaultNodeLayer, + defaultNodeRuntime, +} from "../util"; + +/** + * Two functions that emit the same telemetry (a trace, a custom DogStatsD + * metric, and logs). The baseline runs with the default config; the standalone + * function additionally sets DD_APM_STANDALONE_ENABLED=true. The test asserts + * that traces survive for both, while metrics and logs are suppressed for the + * standalone function — even though DD_SERVERLESS_LOGS_ENABLED=true is inherited + * from the default env, which APM standalone mode must override. + */ +export class ApmStandalone extends cdk.Stack { + constructor(scope: Construct, id: string, props: cdk.StackProps) { + super(scope, id, props); + + const extensionLayer = getExtensionLayer(this); + const nodeLayer = getDefaultNodeLayer(this); + + const makeFunction = (name: string, extraEnv: Record) => { + const fn = new lambda.Function(this, name, { + runtime: defaultNodeRuntime, + architecture: lambda.Architecture.ARM_64, + handler: "/opt/nodejs/node_modules/datadog-lambda-js/handler.handler", + code: lambda.Code.fromAsset("./lambda/custom-metrics-node"), + functionName: name, + timeout: cdk.Duration.seconds(30), + memorySize: 256, + environment: { + ...defaultDatadogEnvVariables, + DD_SERVICE: name, + DD_TRACE_ENABLED: "true", + DD_LAMBDA_HANDLER: "index.handler", + ...extraEnv, + }, + logGroup: createLogGroup(this, name), + }); + fn.addToRolePolicy(defaultDatadogSecretPolicy); + fn.addLayers(extensionLayer); + fn.addLayers(nodeLayer); + return fn; + }; + + makeFunction(`${id}-baseline-lambda`, {}); + makeFunction(`${id}-standalone-lambda`, { + DD_APM_STANDALONE_ENABLED: "true", + }); + } +} diff --git a/integration-tests/tests/apm-standalone.test.ts b/integration-tests/tests/apm-standalone.test.ts new file mode 100644 index 000000000..4f3d66ccd --- /dev/null +++ b/integration-tests/tests/apm-standalone.test.ts @@ -0,0 +1,131 @@ +import { + getInvocationTracesLogsByRequestId, + getMetricCount, + DatadogSpan, + DatadogTrace, +} from './utils/datadog'; +import { forceColdStart, invokeLambda } from './utils/lambda'; +import { IDENTIFIER, DEFAULT_DATADOG_INDEXING_WAIT_MS } from '../config'; + +const stackName = `${IDENTIFIER}-apm-standalone`; + +// Enhanced (billable) metric the extension emits per invocation in normal mode. +const ENHANCED_INVOCATIONS_METRIC = 'aws.lambda.enhanced.invocations'; +// Custom DogStatsD metric emitted by the custom-metrics-node handler. +const CUSTOM_METRIC = 'custom.exclude_tags_test'; + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +function hasAwsLambdaSpan(traces: DatadogTrace[] | undefined): boolean { + return (traces ?? []) + .flatMap((t: DatadogTrace) => t.spans) + .some((s: DatadogSpan) => s.attributes?.operation_name === 'aws.lambda'); +} + +describe('APM Standalone Integration Tests', () => { + const baselineFunctionName = `${stackName}-baseline-lambda`; + const standaloneFunctionName = `${stackName}-standalone-lambda`; + + let invocationStartTime: number; + let metricsEndTime: number; + let baseline: Awaited>; + let standalone: Awaited>; + + beforeAll(async () => { + const functionNames = [baselineFunctionName, standaloneFunctionName]; + + await Promise.all(functionNames.map(fn => forceColdStart(fn))); + + // Back up the metric query window so the rollup bucket (aligned to the + // interval boundary, often just before the invocation) falls in range. + invocationStartTime = Date.now() - 60_000; + + const [baselineInv, standaloneInv] = await Promise.all( + functionNames.map(fn => invokeLambda(fn)), + ); + + await sleep(DEFAULT_DATADOG_INDEXING_WAIT_MS); + metricsEndTime = Date.now(); + + baseline = await getInvocationTracesLogsByRequestId( + baselineFunctionName, + baselineInv.requestId, + ); + standalone = await getInvocationTracesLogsByRequestId( + standaloneFunctionName, + standaloneInv.requestId, + ); + + console.log('Invocation and telemetry collection complete'); + }, 1800000); + + // Traces are unaffected by APM standalone mode — both functions should + // produce a full trace with the inferred aws.lambda root span. + describe('traces (preserved in both modes)', () => { + it('baseline should have the aws.lambda root span', () => { + expect(hasAwsLambdaSpan(baseline.traces)).toBe(true); + }); + + it('standalone should still have the aws.lambda root span', () => { + expect(hasAwsLambdaSpan(standalone.traces)).toBe(true); + }); + }); + + describe('enhanced metrics', () => { + it('baseline should emit aws.lambda.enhanced.invocations', async () => { + const count = await getMetricCount( + ENHANCED_INVOCATIONS_METRIC, + baselineFunctionName, + invocationStartTime, + metricsEndTime, + ); + expect(count).toBeGreaterThan(0); + }); + + it('standalone should NOT emit aws.lambda.enhanced.invocations', async () => { + const count = await getMetricCount( + ENHANCED_INVOCATIONS_METRIC, + standaloneFunctionName, + invocationStartTime, + metricsEndTime, + ); + expect(count).toBe(0); + }); + }); + + describe('custom DogStatsD metrics', () => { + it('baseline should emit the custom metric', async () => { + const count = await getMetricCount( + CUSTOM_METRIC, + baselineFunctionName, + invocationStartTime, + metricsEndTime, + ); + expect(count).toBeGreaterThan(0); + }); + + it('standalone should NOT emit the custom metric', async () => { + const count = await getMetricCount( + CUSTOM_METRIC, + standaloneFunctionName, + invocationStartTime, + metricsEndTime, + ); + expect(count).toBe(0); + }); + }); + + // Logs must be suppressed even though DD_SERVERLESS_LOGS_ENABLED=true is + // inherited from the default env — APM standalone mode forces it off. + describe('logs', () => { + it('baseline should forward logs to Datadog', () => { + expect(baseline.logs?.length ?? 0).toBeGreaterThan(0); + }); + + it('standalone should NOT forward any logs to Datadog', () => { + expect(standalone.logs?.length ?? 0).toBe(0); + }); + }); +});