From 7f6278ef6eae765472bc51c04cd435299be1d290 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Tue, 5 May 2026 18:56:43 +0200 Subject: [PATCH 1/9] feat: add OpenTelemetry tracing with trust boundaries and probe filtering Behind ENABLE_OTEL=true: - NodeSDK with OTLP/HTTP exporter and ParentBased + ratio sampler so we honor upstream NGINX/Beyla sampling decisions instead of re-sampling them away. - HTTP instrumentation with two hooks: - ignoreIncomingRequestHook drops spans on probe / scrape paths (/live, /ready, /_/healthcheck, /_/healthcheck/deep, /metrics) and OPTIONS preflight. - requestHook strips traceparent/tracestate on outbound requests to hosts not in TRUSTED_HOSTS, so trace IDs do not leak to external destinations. The client span is preserved (we still observe the call) and tagged scality.trace.suppressed=true. - buildTrustedHosts derives the allowlist from cloudserver Config (vaultd, dataClient, metadataClient, bucketd, KMIP, KMS, scuba, utapi, mongodb, hdclient/sproxyd connectors from locationConfig, PUSH/MANAGEMENT_ENDPOINT env vars, plus loopback). A unit test asserts every Config host shape is covered so the derivation stays honest as new backends land. - shutdownOtel() helper for the server's cleanUp() to await the exporter flush before process.exit so in-flight traces are not lost on SIGTERM. - mongodb auto-instrumentation tuned for low cardinality; ioredis enabled with requireParentSpan; fs / redis (v2/v3/v4) / aws-sdk disabled. When ENABLE_OTEL is unset the SDK and @opentelemetry/* packages are not loaded at all - zero overhead off the OTEL path. Issue: CLDSRV-884 --- index.js | 3 + lib/otel.js | 277 +++++++++++++++++++ lib/tracing/healthPaths.js | 33 +++ package.json | 11 +- tests/unit/lib/otel.spec.js | 320 ++++++++++++++++++++++ yarn.lock | 511 +++++++++++++++++++++++++++++++++++- 6 files changed, 1150 insertions(+), 5 deletions(-) create mode 100644 lib/otel.js create mode 100644 lib/tracing/healthPaths.js create mode 100644 tests/unit/lib/otel.spec.js diff --git a/index.js b/index.js index f5fa36c2e2..d6de0f0a44 100644 --- a/index.js +++ b/index.js @@ -7,4 +7,7 @@ require('werelogs').stderrUtils.catchAndTimestampStderr( require('cluster').isPrimary ? 1 : null, ); +// Initialize OpenTelemetry SDK before everything else +require('./lib/otel.js'); + require('./lib/server.js')(); diff --git a/lib/otel.js b/lib/otel.js new file mode 100644 index 0000000000..5e70559e69 --- /dev/null +++ b/lib/otel.js @@ -0,0 +1,277 @@ +'use strict'; + +const enableOtel = process.env.ENABLE_OTEL === 'true'; + +/** + * Extract a bare hostname from a `host[:port]` string or a URL. IPv6 + * literals are only supported via URL form (e.g. `http://[::1]:8080`), + * not as bare `host:port` — none of cloudserver's config keys use + * IPv6 literals today. + * Returns undefined when the input is not a non-empty string. + */ +function extractHost(s) { + if (typeof s !== 'string' || s.length === 0) { + return undefined; + } + if (s.includes('://')) { + try { + return new URL(s).hostname.toLowerCase(); + } catch { + // fall through to plain host:port parsing + } + } + return s.split(':')[0].toLowerCase(); +} + +/** + * Factory for the OTEL HTTP instrumentation requestHook. Fires after + * propagation has injected trace headers on an outbound request but before + * the request is flushed on the wire. On outbound requests to hosts outside + * `trustedHosts`, strips `traceparent`/`tracestate` and annotates the + * client span with `scality.trace.suppressed=true` (span is preserved so + * we still observe the call — we only suppress the header leak). + * + * The hook receives both inbound server spans (`IncomingMessage`, which + * has no `getHeader` / `removeHeader`) and outbound client spans + * (`ClientRequest`). We short-circuit on inbound to avoid falsely + * tagging server spans as suppressed. + */ +function makeRequestHook(trustedHosts) { + return function requestHook(span, request) { + // Skip inbound server spans: IncomingMessage doesn't expose + // getHeader/removeHeader, and there's no outbound header to strip. + if (!request || typeof request.getHeader !== 'function') { + return; + } + const hostHeader = request.getHeader('host') || ''; + const host = hostHeader.toString().toLowerCase().split(':')[0]; + if (trustedHosts.has(host)) { + return; + } + if (typeof request.removeHeader === 'function') { + request.removeHeader('traceparent'); + request.removeHeader('tracestate'); + } + if (span && typeof span.setAttribute === 'function') { + span.setAttribute('scality.trace.suppressed', true); + } + }; +} + +/** + * Build the set of hosts we consider "trusted" for outbound trace context + * propagation. Any HTTP egress to a host not in this set will have its + * `traceparent`/`tracestate` headers stripped — the request still gets a + * client span (we still want to observe the call), but the trace ID is + * not leaked to an external destination. + * + * The set is derived from cloudserver's own Config.js, so it stays honest + * automatically as new backends are added. A unit test asserts this. + * + * @param {object} config - the cloudserver Config instance + * @returns {Set} + */ +function buildTrustedHosts(config) { + const hosts = new Set(['localhost', '127.0.0.1', '::1']); + + const add = v => { + const h = extractHost(v); + if (h) { + hosts.add(h); + } + }; + + if (!config) { + return hosts; + } + + add(config.vaultd?.host); + add(config.dataClient?.host); + add(config.metadataClient?.host); + add(config.pfsClient?.host); + add(config.cdmi?.host); + add(config.scuba?.host); + add(config.utapi?.host); + add(config.localCache?.host); + add(config.managementAgent?.host); + add(config.backbeat?.host); + add(config.kmsAWS?.endpoint); + + // bucketd bootstrap is an array of "host:port" strings. + config.bucketd?.bootstrap?.forEach(add); + + // KMIP transport is either a single object or an array of them. + if (config.kmip?.transport) { + const transports = Array.isArray(config.kmip.transport) + ? config.kmip.transport + : [config.kmip.transport]; + transports.forEach(t => add(t?.tls?.host)); + } + + // MongoDB replica set is one comma-separated "host:port,..." string. + if (typeof config.mongodb?.replicaSetHosts === 'string') { + config.mongodb.replicaSetHosts.split(',').forEach(add); + } + + // Management push/management endpoints are read from env vars by + // lib/management/index.js; Config.js does not carry them. + add(process.env.PUSH_ENDPOINT); + add(process.env.MANAGEMENT_ENDPOINT); + + // Storage connectors from locationConfig: only the two direct + // Scality-owned shapes are trusted. Every other locationType (aws_s3, + // azure, gcp, wasabi, do-spaces, ceph-radosgw, scality-ring-s3, + // *-archive, dmf, file, nfs-mount, mem) is either a separate cluster, + // an external cloud, or doesn't talk HTTP — those stay untrusted so + // trace context does not leak across cluster boundaries. + if (config.locationConstraints + && typeof config.locationConstraints === 'object') { + for (const loc of Object.values(config.locationConstraints)) { + loc?.details?.connector?.hdclient?.bootstrap?.forEach(add); + loc?.details?.connector?.sproxyd?.bootstrap?.forEach(add); + } + } + + return hosts; +} + +let sdk = null; + +if (enableOtel) { + const { diag, DiagConsoleLogger, DiagLogLevel } = require('@opentelemetry/api'); + const { NodeSDK } = require('@opentelemetry/sdk-node'); + const { resourceFromAttributes } = require('@opentelemetry/resources'); + const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http'); + const { HttpInstrumentation } = require('@opentelemetry/instrumentation-http'); + const { IORedisInstrumentation } = require('@opentelemetry/instrumentation-ioredis'); + const { MongoDBInstrumentation } = require('@opentelemetry/instrumentation-mongodb'); + const { + ParentBasedSampler, + TraceIdRatioBasedSampler, + } = require('@opentelemetry/sdk-trace-base'); + const { version } = require('../package.json'); + const { config } = require('./Config'); + const { isHealthPath } = require('./tracing/healthPaths'); + + // Surface SDK-internal warnings and errors (export failures, malformed + // headers, propagation issues) instead of letting them disappear. + diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.WARN); + + const exportUrl = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT || + 'http://otel-collector.default.svc.cluster.local:4318/v1/traces'; + + const traceExporter = new OTLPTraceExporter({ + url: exportUrl, + }); + + const parsedRatio = parseFloat(process.env.OTEL_SAMPLING_RATIO); + const samplingRatio = Number.isFinite(parsedRatio) ? parsedRatio : 0.01; + + const TRUSTED_HOSTS = buildTrustedHosts(config); + + // Filter for incoming requests: drop spans on probe/scrape paths and + // CORS preflight requests entirely — they're high-volume and + // zero-value for tracing. + const ignoreIncomingRequestHook = req => { + if (req.method === 'OPTIONS') { + return true; + } + return isHealthPath(req.url); + }; + + const requestHook = makeRequestHook(TRUSTED_HOSTS); + + sdk = new NodeSDK({ + resource: resourceFromAttributes({ + 'service.name': process.env.OTEL_SERVICE_NAME || 'cloudserver', + 'service.version': process.env.OTEL_SERVICE_VERSION || version, + 'service.namespace': process.env.OTEL_SERVICE_NAMESPACE || 'scality', + }), + traceExporter, + // We only ship traces. NodeSDK otherwise spins up OTLP log and + // metric exporters by default, causing unintended outbound + // connections to logs/metrics endpoints we never asked for. + logRecordProcessors: [], + metricReaders: [], + // Bound per-span memory in pathological cases (huge attribute + // values from a buggy handler or a crafted request). The OTEL + // spec leaves these unset by default. + spanLimits: { + attributeValueLengthLimit: 4096, + attributeCountLimit: 128, + eventCountLimit: 128, + linkCountLimit: 128, + }, + // Honor upstream sampling decisions when a parent context is present + // (e.g. traceparent extracted from NGINX/Beyla). Fall back to the + // ratio-based sampler for root spans originating in cloudserver. + sampler: new ParentBasedSampler({ + root: new TraceIdRatioBasedSampler(samplingRatio), + }), + instrumentations: [ + // Explicit allowlist: only the three instrumentations whose + // target modules cloudserver actually loads at runtime. We + // skip the @opentelemetry/auto-instrumentations-node bundle + // to keep the install footprint and audit surface tight. + new HttpInstrumentation({ + ignoreIncomingRequestHook, + requestHook, + }), + new IORedisInstrumentation({ + requireParentSpan: true, + }), + new MongoDBInstrumentation({ + // Mask leaf values in the captured db.statement (so it + // shows query shape — operators, fields — but not user + // data like object keys or filter values). Prevents PII + // from flowing to the trace backend. + enhancedDatabaseReporting: false, + }), + ], + }); + + sdk.start(); +} + +/** + * Flush and stop the OTEL SDK. Resolves once the exporter has drained + * its buffer (or rejected and been logged). Safe to call when OTEL is + * disabled — resolves immediately. Wired into the server's cleanUp() + * so trace context isn't lost on SIGTERM. + */ +// Cap the time spent flushing OTEL on shutdown. If the collector is +// unreachable, the BatchSpanProcessor's default 30s export timeout +// would otherwise block process.exit for the full 30s — at which +// point Kubernetes' default terminationGracePeriodSeconds (also 30s) +// is up and we get SIGKILL'd anyway, defeating the flush. +const OTEL_SHUTDOWN_DEADLINE_MS = 5000; + +async function shutdownOtel() { + if (!sdk) { + return; + } + try { + await Promise.race([ + sdk.shutdown(), + // .unref() so the timer doesn't keep the event loop alive + // when sdk.shutdown() resolves first. + new Promise(resolve => { + setTimeout(resolve, OTEL_SHUTDOWN_DEADLINE_MS).unref(); + }), + ]); + } catch (err) { + // We can't depend on the cloudserver logger here (this may run + // during a shutdown sequence where loggers are torn down), so + // log to stderr directly. + // eslint-disable-next-line no-console + console.error('OTEL shutdown failed', err); + } +} + +module.exports = { + sdk, + shutdownOtel, + buildTrustedHosts, + extractHost, + makeRequestHook, +}; diff --git a/lib/tracing/healthPaths.js b/lib/tracing/healthPaths.js new file mode 100644 index 0000000000..191890fa90 --- /dev/null +++ b/lib/tracing/healthPaths.js @@ -0,0 +1,33 @@ +'use strict'; + +/** + * Explicit set of HTTP paths that should never produce an OTEL span or + * be treated as request entries for tracing purposes. These are k8s probe + * endpoints and the Prometheus scrape path — high-frequency, zero-value + * noise that would otherwise dominate the trace stream. + */ +const HEALTH_PATHS = new Set([ + '/live', + '/ready', + '/_/healthcheck', + '/_/healthcheck/deep', + '/metrics', +]); + +/** + * Returns true when the given URL path (with any query string) corresponds + * to a probe/scrape endpoint that should bypass tracing. + * + * @param {string|undefined} url - the request URL or path, e.g. `/live?x=1` + * @returns {boolean} + */ +function isHealthPath(url) { + if (typeof url !== 'string' || url.length === 0) { + return false; + } + const qIdx = url.indexOf('?'); + const path = qIdx === -1 ? url : url.slice(0, qIdx); + return HEALTH_PATHS.has(path); +} + +module.exports = { isHealthPath }; diff --git a/package.json b/package.json index 1b42de351c..838b4765ca 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,14 @@ "@aws-sdk/signature-v4": "^3.374.0", "@azure/storage-blob": "^12.28.0", "@hapi/joi": "^17.1.1", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-http": "~0.216.0", + "@opentelemetry/instrumentation-http": "~0.216.0", + "@opentelemetry/instrumentation-ioredis": "~0.64.0", + "@opentelemetry/instrumentation-mongodb": "~0.69.0", + "@opentelemetry/resources": "^2.7.0", + "@opentelemetry/sdk-node": "~0.216.0", + "@opentelemetry/sdk-trace-base": "^2.7.0", "@smithy/node-http-handler": "^3.0.0", "arsenal": "git+https://github.com/scality/Arsenal#8.4.1", "async": "2.6.4", @@ -90,7 +98,8 @@ "nan": "v2.22.0", "fast-xml-parser": "^5.5.6", "ts-morph/**/brace-expansion": "^5.0.5", - "ts-morph/**/picomatch": "^4.0.4" + "ts-morph/**/picomatch": "^4.0.4", + "@opentelemetry/api": "^1.9.0" }, "countAsyncSourcePaths": [ "lib/**/*.js", diff --git a/tests/unit/lib/otel.spec.js b/tests/unit/lib/otel.spec.js new file mode 100644 index 0000000000..ff5f832e58 --- /dev/null +++ b/tests/unit/lib/otel.spec.js @@ -0,0 +1,320 @@ +'use strict'; + +const assert = require('assert'); + +const { + buildTrustedHosts, + extractHost, + makeRequestHook, +} = require('../../../lib/otel'); +const { isHealthPath } = require('../../../lib/tracing/healthPaths'); + +describe('otel.extractHost', () => { + it('extracts hostname from a plain host string', () => { + assert.strictEqual(extractHost('example.com'), 'example.com'); + }); + + it('extracts hostname from a host:port string', () => { + assert.strictEqual(extractHost('example.com:8500'), 'example.com'); + }); + + it('lower-cases the extracted hostname', () => { + assert.strictEqual(extractHost('Example.COM:8500'), 'example.com'); + }); + + it('extracts hostname from an http URL', () => { + assert.strictEqual( + extractHost('http://mongo.example.internal:27017/db'), + 'mongo.example.internal', + ); + }); + + it('extracts hostname from an https URL', () => { + assert.strictEqual( + extractHost('https://push.api.zenko.io/api/v1/instance'), + 'push.api.zenko.io', + ); + }); + + it('returns undefined for empty / non-string input', () => { + assert.strictEqual(extractHost(undefined), undefined); + assert.strictEqual(extractHost(''), undefined); + assert.strictEqual(extractHost(42), undefined); + }); +}); + +describe('otel.buildTrustedHosts', () => { + it('always contains loopback aliases', () => { + const hosts = buildTrustedHosts({}); + assert.ok(hosts.has('localhost')); + assert.ok(hosts.has('127.0.0.1')); + assert.ok(hosts.has('::1')); + }); + + it('handles a missing config gracefully', () => { + const hosts = buildTrustedHosts(); + assert.ok(hosts.has('localhost')); + assert.strictEqual(hosts.size, 3); + }); + + it('includes every host referenced in a full config', () => { + // Snapshot of every host-bearing config key that cloudserver + // consults. If a new config key is added without updating + // buildTrustedHosts, this test must fail — keep the derivation + // honest. + const config = { + vaultd: { host: 'vaultd.zenko.svc.cluster.local' }, + dataClient: { host: 'data.zenko.svc.cluster.local' }, + metadataClient: { host: 'bucketd.zenko.svc.cluster.local' }, + pfsClient: { host: 'pfs.zenko.svc.cluster.local' }, + cdmi: { host: 'cdmi.zenko.svc.cluster.local' }, + bucketd: { + bootstrap: [ + 'bucketd-a.zenko.svc.cluster.local:9000', + 'bucketd-b.zenko.svc.cluster.local:9000', + ], + }, + kmip: { + transport: [ + { tls: { host: 'kmip-a.zenko.svc.cluster.local' } }, + { tls: { host: 'kmip-b.zenko.svc.cluster.local' } }, + ], + }, + kmsAWS: { endpoint: 'https://aws-kms.example.com' }, + scuba: { host: 'scuba.zenko.svc.cluster.local' }, + utapi: { host: 'utapi.zenko.svc.cluster.local' }, + localCache: { host: 'redis.zenko.svc.cluster.local' }, + managementAgent: { host: 'localhost' }, + backbeat: { host: 'backbeat.zenko.svc.cluster.local' }, + mongodb: { + replicaSetHosts: + 'mongo-0.zenko.svc.cluster.local:27017,' + + 'mongo-1.zenko.svc.cluster.local:27017,' + + 'mongo-2.zenko.svc.cluster.local:27017', + }, + }; + + const saved = { + PUSH_ENDPOINT: process.env.PUSH_ENDPOINT, + MANAGEMENT_ENDPOINT: process.env.MANAGEMENT_ENDPOINT, + }; + process.env.PUSH_ENDPOINT = 'https://push.api.zenko.io'; + process.env.MANAGEMENT_ENDPOINT = 'https://api.zenko.io'; + try { + const hosts = buildTrustedHosts(config); + const expected = [ + 'vaultd.zenko.svc.cluster.local', + 'data.zenko.svc.cluster.local', + 'bucketd.zenko.svc.cluster.local', + 'pfs.zenko.svc.cluster.local', + 'cdmi.zenko.svc.cluster.local', + 'bucketd-a.zenko.svc.cluster.local', + 'bucketd-b.zenko.svc.cluster.local', + 'kmip-a.zenko.svc.cluster.local', + 'kmip-b.zenko.svc.cluster.local', + 'aws-kms.example.com', + 'scuba.zenko.svc.cluster.local', + 'utapi.zenko.svc.cluster.local', + 'redis.zenko.svc.cluster.local', + 'backbeat.zenko.svc.cluster.local', + 'mongo-0.zenko.svc.cluster.local', + 'mongo-1.zenko.svc.cluster.local', + 'mongo-2.zenko.svc.cluster.local', + 'push.api.zenko.io', + 'api.zenko.io', + ]; + for (const h of expected) { + assert.ok(hosts.has(h), `expected trusted host ${h}`); + } + } finally { + if (saved.PUSH_ENDPOINT === undefined) { + delete process.env.PUSH_ENDPOINT; + } else { + process.env.PUSH_ENDPOINT = saved.PUSH_ENDPOINT; + } + if (saved.MANAGEMENT_ENDPOINT === undefined) { + delete process.env.MANAGEMENT_ENDPOINT; + } else { + process.env.MANAGEMENT_ENDPOINT = saved.MANAGEMENT_ENDPOINT; + } + } + }); + + it('tolerates a single-transport KMIP config object', () => { + const hosts = buildTrustedHosts({ + kmip: { transport: { tls: { host: 'kmip-single.example.com' } } }, + }); + assert.ok(hosts.has('kmip-single.example.com')); + }); + + it('ignores undefined host values', () => { + const hosts = buildTrustedHosts({ + vaultd: {}, + dataClient: {}, + metadataClient: {}, + }); + assert.strictEqual(hosts.size, 3); // only the loopback aliases + }); + + it('includes hdclient and sproxyd connector hosts from locationConstraints, and only those', () => { + // Four shapes: hdclient (trusted), sproxyd (trusted), + // aws-s3 (untrusted), scality-ring-s3 (untrusted — separate + // cluster, its tracing stack does not cross-reference ours). + const hosts = buildTrustedHosts({ + locationConstraints: { + 'us-east-1': { + type: 'scality', + details: { + connector: { + hdclient: { + bootstrap: [ + 'hdproxy-a.xcore.svc:18888', + 'hdproxy-b.xcore.svc:18888', + ], + }, + }, + }, + }, + 'us-east-2': { + type: 'scality', + details: { + connector: { + sproxyd: { + bootstrap: [ + 'sproxyd-a.ring.svc:81', + 'sproxyd-b.ring.svc:81', + ], + }, + }, + }, + }, + 'aws-bucket': { + type: 'aws_s3', + details: { + awsEndpoint: 's3.us-west-2.amazonaws.com', + bucketName: 'external', + }, + }, + 'ring-remote': { + type: 'scality-ring-s3', + details: { + awsEndpoint: 's3.remote-ring.example.com', + bucketName: 'remote', + }, + }, + }, + }); + // Hdclient + sproxyd hosts are in the set. + assert.ok(hosts.has('hdproxy-a.xcore.svc')); + assert.ok(hosts.has('hdproxy-b.xcore.svc')); + assert.ok(hosts.has('sproxyd-a.ring.svc')); + assert.ok(hosts.has('sproxyd-b.ring.svc')); + // External / remote-cluster endpoints are NOT in the set. + assert.ok(!hosts.has('s3.us-west-2.amazonaws.com')); + assert.ok(!hosts.has('s3.remote-ring.example.com')); + }); + + it('tolerates locationConstraints entries without a connector', () => { + // file, mem, aws_s3 etc. — no connector at all must not throw. + assert.doesNotThrow(() => buildTrustedHosts({ + locationConstraints: { + local: { type: 'file', details: {} }, + mem: { type: 'mem', details: {} }, + }, + })); + }); +}); + +describe('otel.makeRequestHook', () => { + const trusted = new Set(['trusted.example.com', 'localhost']); + const hook = makeRequestHook(trusted); + + function fakeClientRequest(host) { + const removed = []; + return { + _removed: removed, + getHeader(name) { + return name.toLowerCase() === 'host' ? host : undefined; + }, + removeHeader(name) { removed.push(name); }, + }; + } + + function fakeSpan() { + const attrs = {}; + return { + _attrs: attrs, + setAttribute(k, v) { attrs[k] = v; }, + }; + } + + it('is a no-op on inbound IncomingMessage (no getHeader method)', () => { + const span = fakeSpan(); + // IncomingMessage shape: has .headers object, but no getHeader(). + const inbound = { headers: { host: 'untrusted.example.com' } }; + assert.doesNotThrow(() => hook(span, inbound)); + assert.strictEqual(span._attrs['scality.trace.suppressed'], undefined); + }); + + it('is a no-op on undefined request', () => { + const span = fakeSpan(); + assert.doesNotThrow(() => hook(span, undefined)); + assert.strictEqual(span._attrs['scality.trace.suppressed'], undefined); + }); + + it('leaves trusted outbound requests untouched', () => { + const span = fakeSpan(); + const req = fakeClientRequest('trusted.example.com:8500'); + hook(span, req); + assert.deepStrictEqual(req._removed, []); + assert.strictEqual(span._attrs['scality.trace.suppressed'], undefined); + }); + + it('strips trace headers and tags span on untrusted outbound requests', () => { + const span = fakeSpan(); + const req = fakeClientRequest('external.example.com'); + hook(span, req); + assert.deepStrictEqual(req._removed.sort(), ['traceparent', 'tracestate']); + assert.strictEqual(span._attrs['scality.trace.suppressed'], true); + }); + + it('handles missing host header by treating as untrusted', () => { + const span = fakeSpan(); + const req = fakeClientRequest(undefined); + hook(span, req); + assert.deepStrictEqual(req._removed.sort(), ['traceparent', 'tracestate']); + assert.strictEqual(span._attrs['scality.trace.suppressed'], true); + }); +}); + +describe('tracing.healthPaths', () => { + it('matches the canonical probe and scrape paths', () => { + [ + '/live', + '/ready', + '/_/healthcheck', + '/_/healthcheck/deep', + '/metrics', + ].forEach(p => assert.strictEqual(isHealthPath(p), true, p)); + }); + + it('matches when a query string is present', () => { + assert.strictEqual(isHealthPath('/live?token=x'), true); + assert.strictEqual(isHealthPath('/metrics?format=prom'), true); + }); + + it('does not match unrelated paths', () => { + [ + '/bucket/key', + '/_/backbeat/data/bucket/key', + '/livez', + '/metrics/custom', + ].forEach(p => assert.strictEqual(isHealthPath(p), false, p)); + }); + + it('returns false for non-string / empty input', () => { + assert.strictEqual(isHealthPath(undefined), false); + assert.strictEqual(isHealthPath(''), false); + assert.strictEqual(isHealthPath(42), false); + }); +}); diff --git a/yarn.lock b/yarn.lock index e454932745..840224be45 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3221,6 +3221,24 @@ "@eslint/core" "^0.12.0" levn "^0.4.1" +"@grpc/grpc-js@^1.14.3": + version "1.14.3" + resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.14.3.tgz#4c9b817a900ae4020ddc28515ae4b52c78cfb8da" + integrity sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA== + dependencies: + "@grpc/proto-loader" "^0.8.0" + "@js-sdsl/ordered-map" "^4.4.2" + +"@grpc/proto-loader@^0.8.0": + version "0.8.0" + resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.8.0.tgz#b6c324dd909c458a0e4aa9bfd3d69cf78a4b9bd8" + integrity sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ== + dependencies: + lodash.camelcase "^4.3.0" + long "^5.0.0" + protobufjs "^7.5.3" + yargs "^17.7.2" + "@hapi/address@^4.0.1": version "4.1.0" resolved "https://registry.yarnpkg.com/@hapi/address/-/address-4.1.0.tgz#d60c5c0d930e77456fdcde2598e77302e2955e1d" @@ -3395,6 +3413,11 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@js-sdsl/ordered-map@^4.4.2": + version "4.4.2" + resolved "https://registry.yarnpkg.com/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz#9299f82874bab9e4c7f9c48d865becbfe8d6907c" + integrity sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw== + "@js-sdsl/ordered-set@^4.4.2": version "4.4.2" resolved "https://registry.yarnpkg.com/@js-sdsl/ordered-set/-/ordered-set-4.4.2.tgz#ab857eb63cf358b5a0f74fdd458b4601423779b7" @@ -3432,16 +3455,390 @@ dependencies: semver "^7.3.5" -"@opentelemetry/api@^1.4.0": - version "1.9.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" - integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== +"@opentelemetry/api-logs@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api-logs/-/api-logs-0.216.0.tgz#7aa4b485ea2f2e21ffcb94120136c72f718c2eaf" + integrity sha512-KmGTgvxTJ0J01d4mOeX1wMV5NUTNf9HebIuOOGDfIn0a/IrnXIQbOnlylDyl9tkDv4h0DUpdI/GqCdLzfTkUXg== + dependencies: + "@opentelemetry/api" "^1.3.0" + +"@opentelemetry/api@^1.3.0", "@opentelemetry/api@^1.4.0", "@opentelemetry/api@^1.9.0": + version "1.9.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.1.tgz#c1b0346de336ba55af2d5a7970882037baedec05" + integrity sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q== + +"@opentelemetry/configuration@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/configuration/-/configuration-0.216.0.tgz#a50eec91107aed81bc189631099048768ad7ca1f" + integrity sha512-B7/LbHEIefF3ZartdrXSuTj1lRWrLfu+srV2Ts+xHrArvPs3U8y7l9i3lk0cjorlgt0lChKQm2XO4QoYI3uWyA== + dependencies: + "@opentelemetry/core" "2.7.1" + yaml "^2.0.0" + +"@opentelemetry/context-async-hooks@2.7.1": + version "2.7.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/context-async-hooks/-/context-async-hooks-2.7.1.tgz#1555a6fb269596416d8c626fd020c3f2c38e071f" + integrity sha512-OPFBYuXEn1E4ja3Y6eeA7O+ZnLBNcXTV5Cgsn1VaqBZ6hC5FnpZPLBNme1LJY8ZtF4aOujPKFoeWN4ik487KuQ== + +"@opentelemetry/core@2.7.1": + version "2.7.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-2.7.1.tgz#162bfab46d6ff4da1bef240ea52e23a926b0fdbc" + integrity sha512-QAqIj32AtK6+pEVNG7EOVxHdE06RP+FM5qpiEJ4RtDcFIqKUZHYhl7/7UY5efhwmwNAg7j8QbJVBLxMerc0+gw== + dependencies: + "@opentelemetry/semantic-conventions" "^1.29.0" + +"@opentelemetry/exporter-logs-otlp-grpc@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-logs-otlp-grpc/-/exporter-logs-otlp-grpc-0.216.0.tgz#55f13b06abeeb8ec046d9c28ee14cfcc6015a966" + integrity sha512-iyCkid5z3FUOB3MzHCeDYKv0MJ5JyL1PUgQDRfhK+HjFwB8PRSzizs5wr/+BdQOZzn1wTBaYwcgmzNcelK769g== + dependencies: + "@grpc/grpc-js" "^1.14.3" + "@opentelemetry/core" "2.7.1" + "@opentelemetry/otlp-exporter-base" "0.216.0" + "@opentelemetry/otlp-grpc-exporter-base" "0.216.0" + "@opentelemetry/otlp-transformer" "0.216.0" + "@opentelemetry/sdk-logs" "0.216.0" + +"@opentelemetry/exporter-logs-otlp-http@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-logs-otlp-http/-/exporter-logs-otlp-http-0.216.0.tgz#acf60f999bdd88f0415959fb8ba03a1a47249b6d" + integrity sha512-8SUzQY/aExKkz6Ab3vOf6gu690Xk4wHH90dGwXinejQzazn5HCIRR7yPVU/2fEuiZ73R92MU4qI3djHfYP7NJg== + dependencies: + "@opentelemetry/api-logs" "0.216.0" + "@opentelemetry/core" "2.7.1" + "@opentelemetry/otlp-exporter-base" "0.216.0" + "@opentelemetry/otlp-transformer" "0.216.0" + "@opentelemetry/sdk-logs" "0.216.0" + +"@opentelemetry/exporter-logs-otlp-proto@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-logs-otlp-proto/-/exporter-logs-otlp-proto-0.216.0.tgz#e2c2687e624d3b57b81bcc27edbd6d22bb0329af" + integrity sha512-fjnNDdsoG98yIcv4yCaw07+9aZeh28gyq1YPXDb0yBksaMWCMR11VGDKANd6CJHdgFloWv9G12x95symD7fq9g== + dependencies: + "@opentelemetry/api-logs" "0.216.0" + "@opentelemetry/core" "2.7.1" + "@opentelemetry/otlp-exporter-base" "0.216.0" + "@opentelemetry/otlp-transformer" "0.216.0" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/sdk-logs" "0.216.0" + "@opentelemetry/sdk-trace-base" "2.7.1" + +"@opentelemetry/exporter-metrics-otlp-grpc@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-metrics-otlp-grpc/-/exporter-metrics-otlp-grpc-0.216.0.tgz#254f8da8e78bfa4ad859a38e1872b75389b27700" + integrity sha512-62ZAduALHuMucuBpNGFhdxFJZ5IQafLW17UE0nVvPVuem3zNslLR0H+4R1xraU07/HCL11AbuicSXlqUkdkotA== + dependencies: + "@grpc/grpc-js" "^1.14.3" + "@opentelemetry/core" "2.7.1" + "@opentelemetry/exporter-metrics-otlp-http" "0.216.0" + "@opentelemetry/otlp-exporter-base" "0.216.0" + "@opentelemetry/otlp-grpc-exporter-base" "0.216.0" + "@opentelemetry/otlp-transformer" "0.216.0" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/sdk-metrics" "2.7.1" + +"@opentelemetry/exporter-metrics-otlp-http@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.216.0.tgz#5f2f0d547d774ac87e4142bf0f045d6c91311275" + integrity sha512-/4VRxjy3spitqFuSkAt9qNwICiDB5T3zqLr+DYd50O7HMMBgWAf9tAL8q98eTVbzwRyRIxsz5Kq1+U5xEyN6gA== + dependencies: + "@opentelemetry/core" "2.7.1" + "@opentelemetry/otlp-exporter-base" "0.216.0" + "@opentelemetry/otlp-transformer" "0.216.0" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/sdk-metrics" "2.7.1" + +"@opentelemetry/exporter-metrics-otlp-proto@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-metrics-otlp-proto/-/exporter-metrics-otlp-proto-0.216.0.tgz#2d903b46cae09272e4af287158917f43284f58f4" + integrity sha512-N7GCCXbw/le32/MrVL4Oj/FU9emFfHEHyGwubpcZLOtcuhUtFFAZWzPKJL1Etm0iNo37JA2JvG4W+5zNe/1NKQ== + dependencies: + "@opentelemetry/core" "2.7.1" + "@opentelemetry/exporter-metrics-otlp-http" "0.216.0" + "@opentelemetry/otlp-exporter-base" "0.216.0" + "@opentelemetry/otlp-transformer" "0.216.0" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/sdk-metrics" "2.7.1" + +"@opentelemetry/exporter-prometheus@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.216.0.tgz#a5719fe805d9008b7131df2986d28017247239f0" + integrity sha512-faltPHeLPyHCGm0MuSrQxv8UXvckZbWo9hUHNwGYiDPF687gaVj5UN24vHlz7VeADnBb6UXTfuw1t4MK4xmcrA== + dependencies: + "@opentelemetry/core" "2.7.1" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/sdk-metrics" "2.7.1" + "@opentelemetry/semantic-conventions" "^1.29.0" + +"@opentelemetry/exporter-trace-otlp-grpc@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.216.0.tgz#069ebcdc8efc6419d3ef3109f2286889aa42fce2" + integrity sha512-XTU//H/Gn+8F9LOWdOC9uyjgcIq/v7T+8aYMr+orBaOpzds05MpFD0jJASZ0mWimt0JWJTuQ8eto/k5/jvtwmw== + dependencies: + "@grpc/grpc-js" "^1.14.3" + "@opentelemetry/core" "2.7.1" + "@opentelemetry/otlp-exporter-base" "0.216.0" + "@opentelemetry/otlp-grpc-exporter-base" "0.216.0" + "@opentelemetry/otlp-transformer" "0.216.0" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/sdk-trace-base" "2.7.1" + +"@opentelemetry/exporter-trace-otlp-http@0.216.0", "@opentelemetry/exporter-trace-otlp-http@~0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.216.0.tgz#4f0c26d44dab8b2354aa07b494aa40345bfe51ab" + integrity sha512-DhWjvj0PUPFwFnhOEivpum8sJzj6FTuyx88zff+oHVLUhfd6cLyw4AIai/F4j0PZqYZBFuMT/OTMUd9wdXnBEQ== + dependencies: + "@opentelemetry/core" "2.7.1" + "@opentelemetry/otlp-exporter-base" "0.216.0" + "@opentelemetry/otlp-transformer" "0.216.0" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/sdk-trace-base" "2.7.1" + +"@opentelemetry/exporter-trace-otlp-proto@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-trace-otlp-proto/-/exporter-trace-otlp-proto-0.216.0.tgz#70bcf55a8b3eebf7263151fd17ad0e4589c45c2e" + integrity sha512-MlUFZlQCm2hWHADU1GntUIziy3A4QcqM9uSZfbqeEolZWk1QdbPQjO2t4LTE4QAA1niEXcYZC2SC23i/gVk8Pw== + dependencies: + "@opentelemetry/core" "2.7.1" + "@opentelemetry/otlp-exporter-base" "0.216.0" + "@opentelemetry/otlp-transformer" "0.216.0" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/sdk-trace-base" "2.7.1" + +"@opentelemetry/exporter-zipkin@2.7.1": + version "2.7.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-zipkin/-/exporter-zipkin-2.7.1.tgz#3b79d223adc8c097ba3323e4de4ed8abb83c789e" + integrity sha512-mfsD9bKAxcKrh5+y08TPodvClBO0CznBE3p79YAGnO81WI4LrdsGA65T53e4iTSbCalW4WaUpkbeJcbpyIUHfg== + dependencies: + "@opentelemetry/core" "2.7.1" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/sdk-trace-base" "2.7.1" + "@opentelemetry/semantic-conventions" "^1.29.0" + +"@opentelemetry/instrumentation-http@~0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-http/-/instrumentation-http-0.216.0.tgz#8c378aee54120455d100d816012394c8777df744" + integrity sha512-Ars2wAVCWIMnKIntxS1ohxKknJaw7i9xCP+8JjtT46vErNyNJ6ZHmhBBtF2dhQTk0XKbLif0NFDshUpT2LmaWw== + dependencies: + "@opentelemetry/core" "2.7.1" + "@opentelemetry/instrumentation" "0.216.0" + "@opentelemetry/semantic-conventions" "^1.29.0" + forwarded-parse "2.1.2" + +"@opentelemetry/instrumentation-ioredis@~0.64.0": + version "0.64.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.64.0.tgz#b02a214263d5f3a6848f6911fe23f505ff6a9181" + integrity sha512-GQ36/amPdO1rVPXgrRZNnd6MktqwDcYalzpMRe9m55b3EwX4pazq8VB3qfTH67xboElqm/B9J1tBEnbQmcvaww== + dependencies: + "@opentelemetry/instrumentation" "^0.216.0" + "@opentelemetry/redis-common" "^0.38.3" + "@opentelemetry/semantic-conventions" "^1.33.0" + +"@opentelemetry/instrumentation-mongodb@~0.69.0": + version "0.69.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.69.0.tgz#b03010de06c816f973038165cfe4cea852b3d43f" + integrity sha512-kj8w2FN2/z0VIXMqcdAdJYtc0udH41Sb485jC7tLl0X4+OD3KLjyhjVoZOXH/gxp+N+BQY6SKgMNC0yi8nok9A== + dependencies: + "@opentelemetry/instrumentation" "^0.216.0" + "@opentelemetry/semantic-conventions" "^1.33.0" + +"@opentelemetry/instrumentation@0.216.0", "@opentelemetry/instrumentation@^0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.216.0.tgz#048777113d0cb7f6b6613025055fc0d2f822bf49" + integrity sha512-BrY0b2K81OLgwBcFxY2wKgPFhq4DpindT+S83++zquc5Rtb2SuYLMkujgDRWMgZQDz+OT+dfvPnMGADPuw4FDw== + dependencies: + "@opentelemetry/api-logs" "0.216.0" + import-in-the-middle "^3.0.0" + require-in-the-middle "^8.0.0" + +"@opentelemetry/otlp-exporter-base@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.216.0.tgz#478f68a1a952adcf5b25fe726f7a0c593bfc2010" + integrity sha512-sSnvb5f+FYa4mfYxj03rmmUh+aDwo3jok62dgIWUDw8ZCUPzEbgtv/YhZyKUSlKNNey7Uc5xmJgmtTLLIV6UDQ== + dependencies: + "@opentelemetry/core" "2.7.1" + "@opentelemetry/otlp-transformer" "0.216.0" + +"@opentelemetry/otlp-grpc-exporter-base@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.216.0.tgz#49246bd0793c52cec0fb04d44e9bca332b064384" + integrity sha512-CrW+2cmZR6mcgtsncWK4WmAn7SC9RwVSHMLbi0IfOXfOYXBaSVKtCCkKYJQWa31VUg7aJFJSpD0n4ISVUN1jdQ== + dependencies: + "@grpc/grpc-js" "^1.14.3" + "@opentelemetry/core" "2.7.1" + "@opentelemetry/otlp-exporter-base" "0.216.0" + "@opentelemetry/otlp-transformer" "0.216.0" + +"@opentelemetry/otlp-transformer@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/otlp-transformer/-/otlp-transformer-0.216.0.tgz#cfef7cbc75e571c3875255e97f6220810bb27aee" + integrity sha512-g4Rb6sAsxQAo11eDjixfKxelruBsQFdJ8Wo23FCj7D6OXbidgXMu2xaRSYs4RdlomzAXSJuc86RcS3xmE8A6uA== + dependencies: + "@opentelemetry/api-logs" "0.216.0" + "@opentelemetry/core" "2.7.1" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/sdk-logs" "0.216.0" + "@opentelemetry/sdk-metrics" "2.7.1" + "@opentelemetry/sdk-trace-base" "2.7.1" + protobufjs "8.0.1" + +"@opentelemetry/propagator-b3@2.7.1": + version "2.7.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/propagator-b3/-/propagator-b3-2.7.1.tgz#107fe3e16d0728c489edbad221c402ee197514a4" + integrity sha512-RJid6E2CKyeGfKBzXKF21ejabGMHypFkPAh3qZ+NvI+SGjuIye79t3PmiqcDgtRzdKH6ynXzbfslQ8DfpRUg2A== + dependencies: + "@opentelemetry/core" "2.7.1" + +"@opentelemetry/propagator-jaeger@2.7.1": + version "2.7.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/propagator-jaeger/-/propagator-jaeger-2.7.1.tgz#e8ebf3f6c0e9aa525cf041893425889cf3e69125" + integrity sha512-KMjVBHzP4N60bOzxja76M1F1hZZ43lGPga5ix+mkv9+kk1nx9SbkxSvJsMbuVUxdPQmsPTqGShmhN8ulrMOg6Q== + dependencies: + "@opentelemetry/core" "2.7.1" + +"@opentelemetry/redis-common@^0.38.3": + version "0.38.3" + resolved "https://registry.yarnpkg.com/@opentelemetry/redis-common/-/redis-common-0.38.3.tgz#31a0464a48a991c29408614e3725d94db7c11aee" + integrity sha512-VCghU1JYs/4gP6Gqf/xro9MEsZ7LrMv2uONVsaESKL38ZOB9BqnI98FfS23wjMnHlpuE+TTaWSoAVNpTwYXzjw== + +"@opentelemetry/resources@2.7.1", "@opentelemetry/resources@^2.7.0": + version "2.7.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-2.7.1.tgz#3b2a9179f6119bb1f2cddefe41ba9b2855504a5d" + integrity sha512-DeT6KKolmC4e/dRQvMQ/RwlnzhaqeiFOXY5ngoOPJ07GgVVKxZOg9EcrNZb5aTzUn+iCrJldAgOfQm1O/QfPAQ== + dependencies: + "@opentelemetry/core" "2.7.1" + "@opentelemetry/semantic-conventions" "^1.29.0" + +"@opentelemetry/sdk-logs@0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-logs/-/sdk-logs-0.216.0.tgz#8b8be3027dd6f035e8381a0617473af95d78514d" + integrity sha512-KB3rcwQuitq0JbbsCcNdqMhRJX3kArAYz/ovb0jGRaBQAIrt2roik3xQXuhYxS37zx0jSkUZcJu1z3Y2UCxbDA== + dependencies: + "@opentelemetry/api-logs" "0.216.0" + "@opentelemetry/core" "2.7.1" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/semantic-conventions" "^1.29.0" + +"@opentelemetry/sdk-metrics@2.7.1": + version "2.7.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-metrics/-/sdk-metrics-2.7.1.tgz#b713f69dd67933ecc9c61357f1d452cdc9f4e281" + integrity sha512-MpDJdkiFDs3Pm1RHO3KByuZbuBdJEXEAkiC0+yJdsZGVCdf1RpHR6n+LHDcS7ffmfrt5kVCzJSCfm4z2C7v0uQ== + dependencies: + "@opentelemetry/core" "2.7.1" + "@opentelemetry/resources" "2.7.1" + +"@opentelemetry/sdk-node@~0.216.0": + version "0.216.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-node/-/sdk-node-0.216.0.tgz#aa9350d13add4aecd37b78f009d30c9539b01460" + integrity sha512-c2bPyD62yIhjS2STJVk5uSJMsiPZqJ747QIJQ0lAsxv6CjBlKPDO715dUjB+W5r9AI76wKhdRGVcG5dl06d65A== + dependencies: + "@opentelemetry/api-logs" "0.216.0" + "@opentelemetry/configuration" "0.216.0" + "@opentelemetry/context-async-hooks" "2.7.1" + "@opentelemetry/core" "2.7.1" + "@opentelemetry/exporter-logs-otlp-grpc" "0.216.0" + "@opentelemetry/exporter-logs-otlp-http" "0.216.0" + "@opentelemetry/exporter-logs-otlp-proto" "0.216.0" + "@opentelemetry/exporter-metrics-otlp-grpc" "0.216.0" + "@opentelemetry/exporter-metrics-otlp-http" "0.216.0" + "@opentelemetry/exporter-metrics-otlp-proto" "0.216.0" + "@opentelemetry/exporter-prometheus" "0.216.0" + "@opentelemetry/exporter-trace-otlp-grpc" "0.216.0" + "@opentelemetry/exporter-trace-otlp-http" "0.216.0" + "@opentelemetry/exporter-trace-otlp-proto" "0.216.0" + "@opentelemetry/exporter-zipkin" "2.7.1" + "@opentelemetry/instrumentation" "0.216.0" + "@opentelemetry/otlp-exporter-base" "0.216.0" + "@opentelemetry/propagator-b3" "2.7.1" + "@opentelemetry/propagator-jaeger" "2.7.1" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/sdk-logs" "0.216.0" + "@opentelemetry/sdk-metrics" "2.7.1" + "@opentelemetry/sdk-trace-base" "2.7.1" + "@opentelemetry/sdk-trace-node" "2.7.1" + "@opentelemetry/semantic-conventions" "^1.29.0" + +"@opentelemetry/sdk-trace-base@2.7.1", "@opentelemetry/sdk-trace-base@^2.7.0": + version "2.7.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.7.1.tgz#9160c3af9ef2219c26563abd136e22fb7d19b34f" + integrity sha512-NAYIlsF8MPUsKqJMiDQJTMPOmlbawC1Iz/omMLygZ1C9am8fTKYjTaI+OZM+WTY3t3Glo0wnOg/6/pac6RGPPw== + dependencies: + "@opentelemetry/core" "2.7.1" + "@opentelemetry/resources" "2.7.1" + "@opentelemetry/semantic-conventions" "^1.29.0" + +"@opentelemetry/sdk-trace-node@2.7.1": + version "2.7.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.7.1.tgz#54dedb8e77fa51a6d02fc2192097739266c82168" + integrity sha512-pCpQxU68lV+I9s9svqMyVu5iHdDDUnqUpSxqwyCU8A9ejEsSnMPCbearwsUO4yk08ZJzAIUCFuReMdVQvHrdvg== + dependencies: + "@opentelemetry/context-async-hooks" "2.7.1" + "@opentelemetry/core" "2.7.1" + "@opentelemetry/sdk-trace-base" "2.7.1" + +"@opentelemetry/semantic-conventions@^1.29.0", "@opentelemetry/semantic-conventions@^1.33.0": + version "1.40.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.40.0.tgz#10b2944ca559386590683392022a897eefd011d3" + integrity sha512-cifvXDhcqMwwTlTK04GBNeIe7yyo28Mfby85QXFe1Yk8nmi36Ab/5UQwptOx84SsoGNRg+EVSjwzfSZMy6pmlw== "@pkgjs/parseargs@^0.11.0": version "0.11.0" resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4", "@protobufjs/codegen@^2.0.5": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.5.tgz#d9315ad7cf3f30aac70bda3c068443dc6f143659" + integrity sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== + +"@protobufjs/inquire@^1.1.0", "@protobufjs/inquire@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.1.tgz#6cb936f4ac50965230af1e9d0bbfd57ea3675aa4" + integrity sha512-mnzgDV26ueAvk7rsbt9L7bE0SuAoqyuys/sMMrmVcN5x9VsxpcG3rqAUSgDyLp0UZlmNfIbQ4fHfCtreVBk8Ew== + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== + +"@protobufjs/utf8@^1.1.0", "@protobufjs/utf8@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.1.tgz#eaee5900122c110a3dbcb728c0597014a2621774" + integrity sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg== + "@rtsao/scc@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@rtsao/scc/-/scc-1.1.0.tgz#927dd2fae9bc3361403ac2c7a00c32ddce9ad7e8" @@ -5808,6 +6205,13 @@ dependencies: undici-types "~6.20.0" +"@types/node@>=13.7.0": + version "25.6.0" + resolved "https://registry.yarnpkg.com/@types/node/-/node-25.6.0.tgz#4e09bad9b469871f2d0f68140198cbd714f4edca" + integrity sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ== + dependencies: + undici-types "~7.19.0" + "@types/triple-beam@^1.3.2": version "1.3.5" resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.5.tgz#74fef9ffbaa198eb8b588be029f38b00299caa2c" @@ -5926,6 +6330,11 @@ accesscontrol@^2.2.1: dependencies: notation "^1.3.6" +acorn-import-attributes@^1.9.5: + version "1.9.5" + resolved "https://registry.yarnpkg.com/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz#7eb1557b1ba05ef18b5ed0ec67591bfab04688ef" + integrity sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ== + acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" @@ -5936,6 +6345,11 @@ acorn@^8.14.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.14.1.tgz#721d5dc10f7d5b5609a891773d47731796935dfb" integrity sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg== +acorn@^8.15.0: + version "8.16.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.16.0.tgz#4ce79c89be40afe7afe8f3adb902a1f1ce9ac08a" + integrity sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw== + agent-base@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.3.0.tgz#8165f01c436009bccad0b1d122f05ed770efc6ee" @@ -6706,6 +7120,11 @@ chownr@^3.0.0: resolved "https://registry.yarnpkg.com/chownr/-/chownr-3.0.0.tgz#9855e64ecd240a9cc4267ce8a4aa5d24a1da15e4" integrity sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g== +cjs-module-lexer@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz#b3ca5101843389259ade7d88c77bd06ce55849ca" + integrity sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ== + clean-stack@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" @@ -8010,6 +8429,11 @@ form-data@~2.3.2: combined-stream "^1.0.6" mime-types "^2.1.12" +forwarded-parse@2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/forwarded-parse/-/forwarded-parse-2.1.2.tgz#08511eddaaa2ddfd56ba11138eee7df117a09325" + integrity sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw== + forwarded@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" @@ -8534,6 +8958,16 @@ import-fresh@^3.2.1: parent-module "^1.0.0" resolve-from "^4.0.0" +import-in-the-middle@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-3.0.1.tgz#8a0a1230c9b865c0e12698171646ae1e3fff691d" + integrity sha512-pYkiyXVL2Mf3pozdlDGV6NAObxQx13Ae8knZk1UJRJ6uRW/ZRmTGHlQYtrsSl7ubuE5F8CD1z+s1n4RHNuTtuA== + dependencies: + acorn "^8.15.0" + acorn-import-attributes "^1.9.5" + cjs-module-lexer "^2.2.0" + module-details-from-path "^1.0.4" + imurmurhash@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" @@ -9503,6 +9937,11 @@ lodash-compat@^3.10.2: resolved "https://registry.yarnpkg.com/lodash-compat/-/lodash-compat-3.10.2.tgz#c6940128a9d30f8e902cd2cf99fd0cba4ecfc183" integrity sha512-k8SE/OwvWfYZqx3MA/Ry1SHBDWre8Z8tCs0Ba0bF5OqVNvymxgFZ/4VDtbTxzTvcoG11JpTMFsaeZp/yGYvFnA== +lodash.camelcase@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6" + integrity sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA== + lodash.defaults@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" @@ -9620,6 +10059,11 @@ long-timeout@0.1.1: resolved "https://registry.yarnpkg.com/long-timeout/-/long-timeout-0.1.1.tgz#9721d788b47e0bcb5a24c2e2bee1a0da55dab514" integrity sha512-BFRuQUqc7x2NWxfJBCyUrN8iYUYznzL9JROmRz1gZ6KlOIgmoD+njPVbb+VNn2nGMKggMsK79iUNErillsrx7w== +long@^5.0.0: + version "5.3.2" + resolved "https://registry.yarnpkg.com/long/-/long-5.3.2.tgz#1d84463095999262d7d7b7f8bfd4a8cc55167f83" + integrity sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA== + looper@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/looper/-/looper-2.0.0.tgz#66cd0c774af3d4fedac53794f742db56da8f09ec" @@ -9963,6 +10407,11 @@ mocha@^11.7.5: yargs-parser "^21.1.1" yargs-unparser "^2.0.0" +module-details-from-path@^1.0.3, module-details-from-path@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/module-details-from-path/-/module-details-from-path-1.0.4.tgz#b662fdcd93f6c83d3f25289da0ce81c8d9685b94" + integrity sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w== + moment@^2.30.1: version "2.30.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.30.1.tgz#f8c91c07b7a786e30c59926df530b4eac96974ae" @@ -10651,6 +11100,42 @@ promise-retry@^2.0.1: err-code "^2.0.2" retry "^0.12.0" +protobufjs@8.0.1: + version "8.0.1" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-8.0.1.tgz#c1781abf9a73812cbd483b32138ac59948223806" + integrity sha512-NWWCCscLjs+cOKF/s/XVNFRW7Yih0fdH+9brffR5NZCy8k42yRdl5KlWKMVXuI1vfCoy4o1z80XR/W/QUb3V3w== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/node" ">=13.7.0" + long "^5.0.0" + +protobufjs@^7.5.3: + version "7.5.6" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.5.6.tgz#11af832ebc4b4326f658a5b1308e6141eb57edfd" + integrity sha512-M71sTMB146U3u0di3yup8iM+zv8yPRNQVr1KK4tyBitl3qFvEGucq/rGDRShD2rsJhtN02RJaJ7j5X5hmy8SJg== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.5" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.1" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.1" + "@types/node" ">=13.7.0" + long "^5.0.0" + proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" @@ -10917,6 +11402,14 @@ require-directory@^2.1.1: resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== +require-in-the-middle@^8.0.0: + version "8.0.1" + resolved "https://registry.yarnpkg.com/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz#dbde2587f669398626d56b20c868ab87bf01cce4" + integrity sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ== + dependencies: + debug "^4.3.5" + module-details-from-path "^1.0.3" + require-main-filename@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" @@ -11980,6 +12473,11 @@ undici-types@~6.20.0: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.20.0.tgz#8171bf22c1f588d1554d55bf204bc624af388433" integrity sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg== +undici-types@~7.19.0: + version "7.19.2" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.19.2.tgz#1b67fc26d0f157a0cba3a58a5b5c1e2276b8ba2a" + integrity sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg== + unique-filename@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-4.0.0.tgz#a06534d370e7c977a939cd1d11f7f0ab8f1fed13" @@ -12443,6 +12941,11 @@ yallist@^5.0.0: resolved "https://registry.yarnpkg.com/yallist/-/yallist-5.0.0.tgz#00e2de443639ed0d78fd87de0d27469fbcffb533" integrity sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw== +yaml@^2.0.0: + version "2.8.4" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.8.4.tgz#4b5f411dd25f9544914d8673d4da7f29248e5e2e" + integrity sha512-ml/JPOj9fOQK8RNnWojA67GbZ0ApXAUlN2UQclwv2eVgTgn7O9gg9o7paZWKMp4g0H3nTLtS9LVzhkpOFIKzog== + yargs-parser@^18.1.2: version "18.1.3" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-18.1.3.tgz#be68c4975c6b2abf469236b0c870362fab09a7b0" From 540b777229fb3499f2fd08322ea0da184d7a46db Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Tue, 5 May 2026 18:57:04 +0200 Subject: [PATCH 2/9] feat: flush OTEL on shutdown Wire shutdownOtel() into the server's cleanUp() chain between closing HTTP servers and process.exit(0). Without this, async sdk.shutdown() fired by signal handlers can race the exit and lose buffered spans for whatever was still in flight at SIGTERM time. Inbound traceparent extraction is intentionally NOT done here: @opentelemetry/instrumentation-http already calls propagation.extract on every incoming request, creates a server span as a child of the remote parent, and sets that server span as the active context. A manual extract on top of that would replace the active context with the (non-recording) remote parent and demote downstream api spans to siblings - rather than children - of the HTTP server span, breaking the trace hierarchy in exactly the distributed-tracing scenarios the manual block was meant to support. Issue: CLDSRV-884 --- lib/server.js | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/server.js b/lib/server.js index a5bb364b61..affe0dac3a 100644 --- a/lib/server.js +++ b/lib/server.js @@ -6,6 +6,7 @@ const arsenal = require('arsenal'); const { setServerHeader } = arsenal.s3routes.routesUtils; const { RedisClient, StatsClient } = arsenal.metrics; const monitoringClient = require('./utilities/monitoringHandler'); +const { shutdownOtel } = require('./otel'); const logger = require('./utilities/logger'); const { internalHandlers } = require('./utilities/internalHandlers'); @@ -206,6 +207,7 @@ class S3Server { vault, }, }; + arsenal.s3routes.routes(req, res, params, logger, this.config); } @@ -334,21 +336,35 @@ class S3Server { } Promise.all(this.servers.map(server => new Promise(resolve => server.close(resolve)) - )).then(() => process.exit(0)); + )) + // Flush any buffered traces before exit so we don't lose + // the spans for whatever was in flight at shutdown. Use + // .finally so an unexpected rejection anywhere in the chain + // can't strand the process without an exit. + .then(() => shutdownOtel()) + .finally(() => process.exit(0)); } caughtExceptionShutdown() { if (!this.cluster) { - process.exit(1); + // Best-effort flush of in-flight traces before crashing. + // shutdownOtel() has its own short timeout so this can't + // hang the exit indefinitely. + shutdownOtel().finally(() => process.exit(1)); + return; } logger.error('shutdown of worker due to exception', { workerId: this.worker ? this.worker.id : undefined, workerPid: this.worker ? this.worker.process.pid : undefined, }); // Will close all servers, cause disconnect event on primary and kill - // worker process with 'SIGTERM'. + // worker process with 'SIGTERM'. Flush OTEL first — this.worker.kill() + // is graceful (closes servers + disconnects IPC) but does not fire + // our SIGTERM handler, so without an explicit flush the + // BatchSpanProcessor's buffered spans for the crashing worker + // would be lost. if (this.worker) { - this.worker.kill(); + shutdownOtel().finally(() => this.worker.kill()); } } From 5f3ca913a4b3f6d9a44d7e44e2025f04c40de191 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Tue, 5 May 2026 18:58:43 +0200 Subject: [PATCH 3/9] feat: instrument all S3 API handlers with OTEL spans Add lib/instrumentation/simple.js exporting instrumentApiMethod, a wrapper that surrounds an S3 handler invocation with an OTEL span named api.. The span owns the entire handler execution (auth, body parsing, metadata I/O, data path, finalizers) and becomes the parent for any auto-instrumentation spans (HTTP, MongoDB, ioredis) that fire underneath. Span name is the handler name verbatim - objectGetACL stays distinct from objectGet, objectPutTagging stays distinct from objectPut. ~70 handlers means ~70 distinct span names total, well within trace backend limits, and operators can tell variants apart without reading attributes. api.js applies the wrapper to every function-valued key in the api object except for the dispatcher (callApiMethod) and pure helpers (checkAuthResults, handleAuthorizationResults). New handlers added to the literal are automatically instrumented - no per-handler boilerplate to remember. The wrapper handles callback / promise / sync return paths, sets SpanStatusCode.OK or ERROR + recordException as appropriate, and sets cloudserver.error_code on the error path. When ENABLE_OTEL is unset @opentelemetry/api is not loaded and the wrapper returns the original function unchanged. Issue: CLDSRV-884 --- lib/api/api.js | 16 +++ lib/instrumentation/simple.js | 104 ++++++++++++++ tests/unit/lib/instrumentationSimple.spec.js | 134 +++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 lib/instrumentation/simple.js create mode 100644 tests/unit/lib/instrumentationSimple.spec.js diff --git a/lib/api/api.js b/lib/api/api.js index 0f6d39f5ef..4848684e93 100644 --- a/lib/api/api.js +++ b/lib/api/api.js @@ -80,6 +80,7 @@ const parseCopySource = require('./apiUtils/object/parseCopySource'); const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys'); const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks'); const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize'); +const { instrumentApiMethod } = require('../instrumentation/simple'); const constants = require('../../constants'); const { config } = require('../Config.js'); const metadata = require('../metadata/wrapper'); @@ -586,4 +587,19 @@ const api = { handleAuthorizationResults, }; +// Wrap every handler in the api object with an OTEL span. Skip the +// internal helpers (callApiMethod is the dispatcher; checkAuthResults +// and handleAuthorizationResults are pure helpers). New S3 handlers +// added to the literal above are automatically instrumented. +const NON_HANDLER_KEYS = new Set([ + 'callApiMethod', + 'checkAuthResults', + 'handleAuthorizationResults', +]); +for (const [name, handler] of Object.entries(api)) { + if (typeof handler === 'function' && !NON_HANDLER_KEYS.has(name)) { + api[name] = instrumentApiMethod(handler, name); + } +} + module.exports = api; diff --git a/lib/instrumentation/simple.js b/lib/instrumentation/simple.js new file mode 100644 index 0000000000..0092647688 --- /dev/null +++ b/lib/instrumentation/simple.js @@ -0,0 +1,104 @@ +'use strict'; + +// Lazy-load `@opentelemetry/api` only on the first OTEL-enabled call: +// when OTEL is off, the module is never required. The cache also lets +// direct callers of `instrumentApiMethod` (e.g. unit tests of this +// module) flip ENABLE_OTEL after require time and see the loaded +// module on the next call. This does NOT extend to api.js's wrapping +// loop, which runs once at require time — flipping the env var +// afterwards won't retroactively wrap already-dispatched handlers. +let otel = null; + +function loadOtel() { + if (otel) { + return otel; + } + if (process.env.ENABLE_OTEL !== 'true') { + return null; + } + const api = require('@opentelemetry/api'); + const { version } = require('../../package.json'); + otel = { + trace: api.trace, + context: api.context, + SpanStatusCode: api.SpanStatusCode, + SpanKind: api.SpanKind, + tracer: api.trace.getTracer('cloudserver-api', version), + }; + return otel; +} + +/** + * Wrap an S3 API handler so each invocation produces an OTEL span that + * owns the whole handler execution (auth, body parsing, metadata I/O, + * data path, finalizers). Auto-instrumentation spans (HTTP, MongoDB, + * ioredis) nest naturally beneath it. + * + * The span name is `api.` — one distinct name per S3 + * handler. Variants like objectGetACL / objectPutTagging stay distinct + * from objectGet / objectPut so flame graphs and queries can tell them + * apart without reading attributes. ~70 distinct names total — well + * within trace backend limits. + * + * Returns the original function unchanged when ENABLE_OTEL is unset + * at the time of the call, so there is zero overhead off the OTEL path. + */ +function instrumentApiMethod(apiMethod, methodName) { + const o = loadOtel(); + if (!o) { + return apiMethod; + } + + const spanName = `api.${methodName}`; + + return function instrumented(...args) { + const callbackIndex = args.findLastIndex(a => typeof a === 'function'); + if (callbackIndex === -1) { + // Cloudserver's S3 dispatch always passes a callback to the + // handler — every shape in callApiHandler has one. If a + // future call site routes here without one, fall through + // unwrapped rather than create a span we can't end. + return apiMethod.apply(this, args); + } + + const span = o.tracer.startSpan(spanName, { kind: o.SpanKind.INTERNAL }); + // Guard against double-ending: the wrapped callback may fire and + // end the span, then the outer apiMethod.apply could still throw + // synchronously (e.g. the callback itself threw). End-once + // semantics keeps the span and trace state consistent. + let spanEnded = false; + const endSpan = err => { + if (spanEnded) { + return; + } + spanEnded = true; + if (err) { + span.recordException(err); + span.setStatus({ code: o.SpanStatusCode.ERROR }); + if (err.code) { + span.setAttribute('cloudserver.error_code', err.code); + } + } else { + span.setStatus({ code: o.SpanStatusCode.OK }); + } + span.end(); + }; + + const originalCallback = args[callbackIndex]; + const wrappedArgs = [...args]; + wrappedArgs[callbackIndex] = function wrappedCallback(err, ...results) { + endSpan(err); + return originalCallback.call(this, err, ...results); + }; + + try { + return o.context.with(o.trace.setSpan(o.context.active(), span), () => + apiMethod.apply(this, wrappedArgs)); + } catch (error) { + endSpan(error); + throw error; + } + }; +} + +module.exports = { instrumentApiMethod }; diff --git a/tests/unit/lib/instrumentationSimple.spec.js b/tests/unit/lib/instrumentationSimple.spec.js new file mode 100644 index 0000000000..e0cd2cf4d1 --- /dev/null +++ b/tests/unit/lib/instrumentationSimple.spec.js @@ -0,0 +1,134 @@ +'use strict'; + +// Force the module under test onto its OTEL-on path. Must be set before +// any require pulls in lib/instrumentation/simple. +process.env.ENABLE_OTEL = 'true'; + +const assert = require('assert'); +const { trace, SpanStatusCode } = require('@opentelemetry/api'); +const { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, + AlwaysOnSampler, +} = require('@opentelemetry/sdk-trace-base'); + +// Capture spans into memory so tests can inspect them. Wire the provider +// up via the API surface (sdk-trace-base 2.x dropped provider.register). +const exporter = new InMemorySpanExporter(); +const provider = new BasicTracerProvider({ + sampler: new AlwaysOnSampler(), + spanProcessors: [new SimpleSpanProcessor(exporter)], +}); +trace.setGlobalTracerProvider(provider); + +const { instrumentApiMethod } = require('../../../lib/instrumentation/simple'); + +describe('instrumentApiMethod (OTEL on)', () => { + afterEach(() => exporter.reset()); + + it('wraps a callback handler and ends span on success', done => { + const handler = (a, b, cb) => cb(null, 'ok'); + const wrapped = instrumentApiMethod(handler, 'objectGet'); + + wrapped('foo', 'bar', (err, value) => { + assert.strictEqual(err, null); + assert.strictEqual(value, 'ok'); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + assert.strictEqual(spans[0].name, 'api.objectGet'); + assert.strictEqual(spans[0].status.code, SpanStatusCode.OK); + done(); + }); + }); + + it('ends span with ERROR when handler\'s callback fires with err', done => { + const handler = (a, cb) => cb(Object.assign(new Error('nope'), { code: 'NoSuchBucket' })); + const wrapped = instrumentApiMethod(handler, 'bucketHead'); + + wrapped('foo', err => { + assert.strictEqual(err.message, 'nope'); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + assert.strictEqual(spans[0].status.code, SpanStatusCode.ERROR); + assert.strictEqual(spans[0].attributes['cloudserver.error_code'], 'NoSuchBucket'); + done(); + }); + }); + + it('does not end the span twice when callback fires then handler throws', () => { + // Handler invokes the callback synchronously and then throws — + // tests the spanEnded guard. Without it the span would receive a + // second end() call and the recordException from the throw would + // overwrite the OK status set by the callback. + const handler = cb => { + cb(null, 'first'); + throw new Error('after-callback-boom'); + }; + const wrapped = instrumentApiMethod(handler, 'objectPut'); + + let cbErr = null; + let cbValue = null; + const cb = (err, val) => { cbErr = err; cbValue = val; }; + + assert.throws(() => wrapped(cb), /after-callback-boom/); + + assert.strictEqual(cbErr, null); + assert.strictEqual(cbValue, 'first'); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1, 'span ended exactly once'); + // The first endSpan call (from the callback) wins; status is OK, + // not ERROR — the post-callback throw is a programming bug, not + // an outcome of the API call. + assert.strictEqual(spans[0].status.code, SpanStatusCode.OK); + }); + + it('passes through unwrapped (no span) when no callback arg is found', () => { + // Defensive: cloudserver's S3 dispatch always passes a callback, + // so this branch is unreachable today. If a future caller ever + // routes here without one, the wrapper falls through to the raw + // handler rather than creating a span it can't end. + const handler = (a, b) => `${a}-${b}`; + const wrapped = instrumentApiMethod(handler, 'objectRestore'); + + assert.strictEqual(wrapped('x', 'y'), 'x-y'); + assert.strictEqual(exporter.getFinishedSpans().length, 0); + }); + + it('synchronous throw before callback fires ends span and re-throws', () => { + const handler = () => { + throw new Error('sync-boom'); + }; + const wrapped = instrumentApiMethod(handler, 'objectDelete'); + + assert.throws(() => wrapped(() => {}), /sync-boom/); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + assert.strictEqual(spans[0].status.code, SpanStatusCode.ERROR); + }); +}); + +describe('instrumentApiMethod with OTEL disabled', () => { + it('returns the original function unchanged', () => { + // We have to test this by re-requiring the module under a fresh + // ENABLE_OTEL=false. The module caches enableOtel at load time. + const path = require.resolve('../../../lib/instrumentation/simple'); + const cached = require.cache[path]; + delete require.cache[path]; + const savedEnv = process.env.ENABLE_OTEL; + process.env.ENABLE_OTEL = 'false'; + try { + const { instrumentApiMethod: gated } = + require('../../../lib/instrumentation/simple'); + const handler = () => 'identity'; + assert.strictEqual(gated(handler, 'foo'), handler); + } finally { + process.env.ENABLE_OTEL = savedEnv; + require.cache[path] = cached; + } + }); +}); From ee28d632c91bdffedf7ddccb040dd208ea0370b4 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Tue, 5 May 2026 18:59:41 +0200 Subject: [PATCH 4/9] chore: bump arsenal to ARSN-572 PoC branch for e2e trace context testing Temporarily point the arsenal dep at scality/Arsenal#improvement/ARSN-572/trace-context so we can validate end-to-end trace context propagation from cloudserver HTTP spans through to the MongoDB oplog on a test cluster. ARSN-572 adds traceContext plumbing on metadata writes; cloudserver needs no code change thanks to OTEL async context hooks. Yarn resolves the branch ref and pins the resolved commit hash in yarn.lock so installs are reproducible. Revert to a clean #8.x release tag once ARSN-572 ships. Issue: CLDSRV-884 --- package.json | 2 +- yarn.lock | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 838b4765ca..dba4c9ebca 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "@opentelemetry/sdk-node": "~0.216.0", "@opentelemetry/sdk-trace-base": "^2.7.0", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/Arsenal#8.4.1", + "arsenal": "git+https://github.com/scality/Arsenal#improvement/ARSN-572/trace-context", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/yarn.lock b/yarn.lock index 840224be45..c4f7a569d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6624,9 +6624,9 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.4.1": - version "8.4.1" - resolved "git+https://github.com/scality/Arsenal#6b3b58b152ac23d29176ab1f24f49f8eda3145b2" +"arsenal@git+https://github.com/scality/Arsenal#improvement/ARSN-572/trace-context": + version "8.3.11" + resolved "git+https://github.com/scality/Arsenal#86191514c86e1dba0532c14ec43824ee912e7d08" dependencies: "@aws-sdk/client-kms" "^3.975.0" "@aws-sdk/client-s3" "^3.975.0" @@ -6635,6 +6635,7 @@ arraybuffer.prototype.slice@^1.0.4: "@azure/identity" "^4.13.0" "@azure/storage-blob" "^12.31.0" "@js-sdsl/ordered-set" "^4.4.2" + "@opentelemetry/api" "^1.9.0" "@scality/hdclient" "^1.3.2" "@smithy/node-http-handler" "^4.3.0" "@smithy/protocol-http" "^5.3.5" From 8f65af3a43d8197fc4061be8cfd292ce1a9ac4fe Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Tue, 12 May 2026 16:50:29 +0200 Subject: [PATCH 5/9] fixup! feat: add OpenTelemetry tracing with trust boundaries and probe filtering --- index.js | 6 +- lib/otel.js | 277 ------------------ lib/tracing/healthPaths.js | 17 +- lib/tracing/index.js | 111 +++++++ lib/tracing/trustedHosts.js | 103 +++++++ tests/unit/lib/tracing/healthPaths.spec.js | 37 +++ .../trustedHosts.spec.js} | 57 +--- 7 files changed, 267 insertions(+), 341 deletions(-) delete mode 100644 lib/otel.js create mode 100644 lib/tracing/index.js create mode 100644 lib/tracing/trustedHosts.js create mode 100644 tests/unit/lib/tracing/healthPaths.spec.js rename tests/unit/lib/{otel.spec.js => tracing/trustedHosts.spec.js} (83%) diff --git a/index.js b/index.js index d6de0f0a44..f1d1d18c3f 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,9 @@ require('werelogs').stderrUtils.catchAndTimestampStderr( require('cluster').isPrimary ? 1 : null, ); -// Initialize OpenTelemetry SDK before everything else -require('./lib/otel.js'); +// Start tracing before requiring anything that hooks into HTTP, MongoDB, +// or ioredis — instrumentation patches modules on require, so anything +// loaded earlier than init() would run unpatched. +require('./lib/tracing').init(); require('./lib/server.js')(); diff --git a/lib/otel.js b/lib/otel.js deleted file mode 100644 index 5e70559e69..0000000000 --- a/lib/otel.js +++ /dev/null @@ -1,277 +0,0 @@ -'use strict'; - -const enableOtel = process.env.ENABLE_OTEL === 'true'; - -/** - * Extract a bare hostname from a `host[:port]` string or a URL. IPv6 - * literals are only supported via URL form (e.g. `http://[::1]:8080`), - * not as bare `host:port` — none of cloudserver's config keys use - * IPv6 literals today. - * Returns undefined when the input is not a non-empty string. - */ -function extractHost(s) { - if (typeof s !== 'string' || s.length === 0) { - return undefined; - } - if (s.includes('://')) { - try { - return new URL(s).hostname.toLowerCase(); - } catch { - // fall through to plain host:port parsing - } - } - return s.split(':')[0].toLowerCase(); -} - -/** - * Factory for the OTEL HTTP instrumentation requestHook. Fires after - * propagation has injected trace headers on an outbound request but before - * the request is flushed on the wire. On outbound requests to hosts outside - * `trustedHosts`, strips `traceparent`/`tracestate` and annotates the - * client span with `scality.trace.suppressed=true` (span is preserved so - * we still observe the call — we only suppress the header leak). - * - * The hook receives both inbound server spans (`IncomingMessage`, which - * has no `getHeader` / `removeHeader`) and outbound client spans - * (`ClientRequest`). We short-circuit on inbound to avoid falsely - * tagging server spans as suppressed. - */ -function makeRequestHook(trustedHosts) { - return function requestHook(span, request) { - // Skip inbound server spans: IncomingMessage doesn't expose - // getHeader/removeHeader, and there's no outbound header to strip. - if (!request || typeof request.getHeader !== 'function') { - return; - } - const hostHeader = request.getHeader('host') || ''; - const host = hostHeader.toString().toLowerCase().split(':')[0]; - if (trustedHosts.has(host)) { - return; - } - if (typeof request.removeHeader === 'function') { - request.removeHeader('traceparent'); - request.removeHeader('tracestate'); - } - if (span && typeof span.setAttribute === 'function') { - span.setAttribute('scality.trace.suppressed', true); - } - }; -} - -/** - * Build the set of hosts we consider "trusted" for outbound trace context - * propagation. Any HTTP egress to a host not in this set will have its - * `traceparent`/`tracestate` headers stripped — the request still gets a - * client span (we still want to observe the call), but the trace ID is - * not leaked to an external destination. - * - * The set is derived from cloudserver's own Config.js, so it stays honest - * automatically as new backends are added. A unit test asserts this. - * - * @param {object} config - the cloudserver Config instance - * @returns {Set} - */ -function buildTrustedHosts(config) { - const hosts = new Set(['localhost', '127.0.0.1', '::1']); - - const add = v => { - const h = extractHost(v); - if (h) { - hosts.add(h); - } - }; - - if (!config) { - return hosts; - } - - add(config.vaultd?.host); - add(config.dataClient?.host); - add(config.metadataClient?.host); - add(config.pfsClient?.host); - add(config.cdmi?.host); - add(config.scuba?.host); - add(config.utapi?.host); - add(config.localCache?.host); - add(config.managementAgent?.host); - add(config.backbeat?.host); - add(config.kmsAWS?.endpoint); - - // bucketd bootstrap is an array of "host:port" strings. - config.bucketd?.bootstrap?.forEach(add); - - // KMIP transport is either a single object or an array of them. - if (config.kmip?.transport) { - const transports = Array.isArray(config.kmip.transport) - ? config.kmip.transport - : [config.kmip.transport]; - transports.forEach(t => add(t?.tls?.host)); - } - - // MongoDB replica set is one comma-separated "host:port,..." string. - if (typeof config.mongodb?.replicaSetHosts === 'string') { - config.mongodb.replicaSetHosts.split(',').forEach(add); - } - - // Management push/management endpoints are read from env vars by - // lib/management/index.js; Config.js does not carry them. - add(process.env.PUSH_ENDPOINT); - add(process.env.MANAGEMENT_ENDPOINT); - - // Storage connectors from locationConfig: only the two direct - // Scality-owned shapes are trusted. Every other locationType (aws_s3, - // azure, gcp, wasabi, do-spaces, ceph-radosgw, scality-ring-s3, - // *-archive, dmf, file, nfs-mount, mem) is either a separate cluster, - // an external cloud, or doesn't talk HTTP — those stay untrusted so - // trace context does not leak across cluster boundaries. - if (config.locationConstraints - && typeof config.locationConstraints === 'object') { - for (const loc of Object.values(config.locationConstraints)) { - loc?.details?.connector?.hdclient?.bootstrap?.forEach(add); - loc?.details?.connector?.sproxyd?.bootstrap?.forEach(add); - } - } - - return hosts; -} - -let sdk = null; - -if (enableOtel) { - const { diag, DiagConsoleLogger, DiagLogLevel } = require('@opentelemetry/api'); - const { NodeSDK } = require('@opentelemetry/sdk-node'); - const { resourceFromAttributes } = require('@opentelemetry/resources'); - const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http'); - const { HttpInstrumentation } = require('@opentelemetry/instrumentation-http'); - const { IORedisInstrumentation } = require('@opentelemetry/instrumentation-ioredis'); - const { MongoDBInstrumentation } = require('@opentelemetry/instrumentation-mongodb'); - const { - ParentBasedSampler, - TraceIdRatioBasedSampler, - } = require('@opentelemetry/sdk-trace-base'); - const { version } = require('../package.json'); - const { config } = require('./Config'); - const { isHealthPath } = require('./tracing/healthPaths'); - - // Surface SDK-internal warnings and errors (export failures, malformed - // headers, propagation issues) instead of letting them disappear. - diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.WARN); - - const exportUrl = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT || - 'http://otel-collector.default.svc.cluster.local:4318/v1/traces'; - - const traceExporter = new OTLPTraceExporter({ - url: exportUrl, - }); - - const parsedRatio = parseFloat(process.env.OTEL_SAMPLING_RATIO); - const samplingRatio = Number.isFinite(parsedRatio) ? parsedRatio : 0.01; - - const TRUSTED_HOSTS = buildTrustedHosts(config); - - // Filter for incoming requests: drop spans on probe/scrape paths and - // CORS preflight requests entirely — they're high-volume and - // zero-value for tracing. - const ignoreIncomingRequestHook = req => { - if (req.method === 'OPTIONS') { - return true; - } - return isHealthPath(req.url); - }; - - const requestHook = makeRequestHook(TRUSTED_HOSTS); - - sdk = new NodeSDK({ - resource: resourceFromAttributes({ - 'service.name': process.env.OTEL_SERVICE_NAME || 'cloudserver', - 'service.version': process.env.OTEL_SERVICE_VERSION || version, - 'service.namespace': process.env.OTEL_SERVICE_NAMESPACE || 'scality', - }), - traceExporter, - // We only ship traces. NodeSDK otherwise spins up OTLP log and - // metric exporters by default, causing unintended outbound - // connections to logs/metrics endpoints we never asked for. - logRecordProcessors: [], - metricReaders: [], - // Bound per-span memory in pathological cases (huge attribute - // values from a buggy handler or a crafted request). The OTEL - // spec leaves these unset by default. - spanLimits: { - attributeValueLengthLimit: 4096, - attributeCountLimit: 128, - eventCountLimit: 128, - linkCountLimit: 128, - }, - // Honor upstream sampling decisions when a parent context is present - // (e.g. traceparent extracted from NGINX/Beyla). Fall back to the - // ratio-based sampler for root spans originating in cloudserver. - sampler: new ParentBasedSampler({ - root: new TraceIdRatioBasedSampler(samplingRatio), - }), - instrumentations: [ - // Explicit allowlist: only the three instrumentations whose - // target modules cloudserver actually loads at runtime. We - // skip the @opentelemetry/auto-instrumentations-node bundle - // to keep the install footprint and audit surface tight. - new HttpInstrumentation({ - ignoreIncomingRequestHook, - requestHook, - }), - new IORedisInstrumentation({ - requireParentSpan: true, - }), - new MongoDBInstrumentation({ - // Mask leaf values in the captured db.statement (so it - // shows query shape — operators, fields — but not user - // data like object keys or filter values). Prevents PII - // from flowing to the trace backend. - enhancedDatabaseReporting: false, - }), - ], - }); - - sdk.start(); -} - -/** - * Flush and stop the OTEL SDK. Resolves once the exporter has drained - * its buffer (or rejected and been logged). Safe to call when OTEL is - * disabled — resolves immediately. Wired into the server's cleanUp() - * so trace context isn't lost on SIGTERM. - */ -// Cap the time spent flushing OTEL on shutdown. If the collector is -// unreachable, the BatchSpanProcessor's default 30s export timeout -// would otherwise block process.exit for the full 30s — at which -// point Kubernetes' default terminationGracePeriodSeconds (also 30s) -// is up and we get SIGKILL'd anyway, defeating the flush. -const OTEL_SHUTDOWN_DEADLINE_MS = 5000; - -async function shutdownOtel() { - if (!sdk) { - return; - } - try { - await Promise.race([ - sdk.shutdown(), - // .unref() so the timer doesn't keep the event loop alive - // when sdk.shutdown() resolves first. - new Promise(resolve => { - setTimeout(resolve, OTEL_SHUTDOWN_DEADLINE_MS).unref(); - }), - ]); - } catch (err) { - // We can't depend on the cloudserver logger here (this may run - // during a shutdown sequence where loggers are torn down), so - // log to stderr directly. - // eslint-disable-next-line no-console - console.error('OTEL shutdown failed', err); - } -} - -module.exports = { - sdk, - shutdownOtel, - buildTrustedHosts, - extractHost, - makeRequestHook, -}; diff --git a/lib/tracing/healthPaths.js b/lib/tracing/healthPaths.js index 191890fa90..ec83cad2a0 100644 --- a/lib/tracing/healthPaths.js +++ b/lib/tracing/healthPaths.js @@ -1,11 +1,9 @@ 'use strict'; -/** - * Explicit set of HTTP paths that should never produce an OTEL span or - * be treated as request entries for tracing purposes. These are k8s probe - * endpoints and the Prometheus scrape path — high-frequency, zero-value - * noise that would otherwise dominate the trace stream. - */ +// Probe + scrape paths that should never produce a span. Filtered at +// ingest (not at the trace backend) because probe rate × pod count × +// always-on sampling overwhelms the exporter and storage with traffic +// nobody queries. const HEALTH_PATHS = new Set([ '/live', '/ready', @@ -14,13 +12,6 @@ const HEALTH_PATHS = new Set([ '/metrics', ]); -/** - * Returns true when the given URL path (with any query string) corresponds - * to a probe/scrape endpoint that should bypass tracing. - * - * @param {string|undefined} url - the request URL or path, e.g. `/live?x=1` - * @returns {boolean} - */ function isHealthPath(url) { if (typeof url !== 'string' || url.length === 0) { return false; diff --git a/lib/tracing/index.js b/lib/tracing/index.js new file mode 100644 index 0000000000..b8a54d774b --- /dev/null +++ b/lib/tracing/index.js @@ -0,0 +1,111 @@ +'use strict'; + +const { buildTrustedHosts, makeRequestHook } = require('./trustedHosts'); +const { isHealthPath } = require('./healthPaths'); + +let sdk = null; + +function isEnabled() { + return process.env.ENABLE_OTEL === 'true'; +} + +function init() { + if (!isEnabled() || sdk) { + return; + } + + const endpoint = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; + if (!endpoint) { + throw new Error( + 'ENABLE_OTEL=true but OTEL_EXPORTER_OTLP_TRACES_ENDPOINT is unset', + ); + } + + const { diag, DiagConsoleLogger, DiagLogLevel } = require('@opentelemetry/api'); + const { NodeSDK } = require('@opentelemetry/sdk-node'); + const { resourceFromAttributes } = require('@opentelemetry/resources'); + const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http'); + const { HttpInstrumentation } = require('@opentelemetry/instrumentation-http'); + const { IORedisInstrumentation } = require('@opentelemetry/instrumentation-ioredis'); + const { MongoDBInstrumentation } = require('@opentelemetry/instrumentation-mongodb'); + const { + ParentBasedSampler, + TraceIdRatioBasedSampler, + } = require('@opentelemetry/sdk-trace-base'); + const { version } = require('../../package.json'); + const { config } = require('../Config'); + + diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.WARN); + + const parsedRatio = parseFloat(process.env.OTEL_SAMPLING_RATIO); + const samplingRatio = Number.isFinite(parsedRatio) ? parsedRatio : 0.01; + + const trustedHosts = buildTrustedHosts(config); + + const ignoreIncomingRequestHook = req => + req.method === 'OPTIONS' || isHealthPath(req.url); + + sdk = new NodeSDK({ + resource: resourceFromAttributes({ + 'service.name': process.env.OTEL_SERVICE_NAME || 'cloudserver', + 'service.version': process.env.OTEL_SERVICE_VERSION || version, + 'service.namespace': process.env.OTEL_SERVICE_NAMESPACE || 'scality', + }), + traceExporter: new OTLPTraceExporter({ url: endpoint }), + logRecordProcessors: [], + metricReaders: [], + spanLimits: { + attributeValueLengthLimit: 4096, + attributeCountLimit: 128, + eventCountLimit: 128, + linkCountLimit: 128, + }, + sampler: new ParentBasedSampler({ + root: new TraceIdRatioBasedSampler(samplingRatio), + }), + instrumentations: [ + new HttpInstrumentation({ + ignoreIncomingRequestHook, + requestHook: makeRequestHook(trustedHosts), + }), + new IORedisInstrumentation({ requireParentSpan: true }), + // Mask leaf values in db.statement so query shape is captured + // without user data (object keys, filter values) flowing to + // the trace backend. + new MongoDBInstrumentation({ enhancedDatabaseReporting: false }), + ], + }); + + sdk.start(); +} + +// Cap the flush window. The BatchSpanProcessor's default 30s export +// timeout would otherwise block process.exit, and Kubernetes' default +// terminationGracePeriodSeconds is also 30s — we'd get SIGKILL'd +// before the flush ever completed. +const SHUTDOWN_DEADLINE_MS = 5000; + +async function close() { + if (!sdk) { + return; + } + try { + await Promise.race([ + sdk.shutdown(), + // .unref() so the timer doesn't pin the event loop open + // when sdk.shutdown() resolves first. + new Promise(resolve => { + setTimeout(resolve, SHUTDOWN_DEADLINE_MS).unref(); + }), + ]); + } catch (err) { + // Loggers may already be torn down at this point in shutdown; + // log to stderr directly. + // eslint-disable-next-line no-console + console.error('tracing close failed', err); + } finally { + sdk = null; + } +} + +module.exports = { init, close, isEnabled }; diff --git a/lib/tracing/trustedHosts.js b/lib/tracing/trustedHosts.js new file mode 100644 index 0000000000..520b60bdd4 --- /dev/null +++ b/lib/tracing/trustedHosts.js @@ -0,0 +1,103 @@ +'use strict'; + +function extractHost(s) { + if (typeof s !== 'string' || s.length === 0) { + return undefined; + } + if (s.includes('://')) { + try { + return new URL(s).hostname.toLowerCase(); + } catch { + // fall through to plain host:port parsing + } + } + return s.split(':')[0].toLowerCase(); +} + +// On outbound requests to hosts outside `trustedHosts`, strip +// traceparent/tracestate and tag the client span as suppressed — +// preserve the span (we still want to observe the call) without +// leaking trace IDs to external destinations. +function makeRequestHook(trustedHosts) { + return function requestHook(span, request) { + // IncomingMessage (inbound server spans) doesn't expose + // getHeader/removeHeader; only ClientRequest does. + if (!request || typeof request.getHeader !== 'function') { + return; + } + const host = extractHost((request.getHeader('host') || '').toString()); + if (trustedHosts.has(host)) { + return; + } + if (typeof request.removeHeader === 'function') { + request.removeHeader('traceparent'); + request.removeHeader('tracestate'); + } + if (span && typeof span.setAttribute === 'function') { + span.setAttribute('scality.trace.suppressed', true); + } + }; +} + +// Derived from cloudserver's Config so it stays honest as new backends +// land. A unit test asserts the set against a fixture Config. +function buildTrustedHosts(config) { + const hosts = new Set(['localhost', '127.0.0.1', '::1']); + + const add = v => { + const h = extractHost(v); + if (h) { + hosts.add(h); + } + }; + + if (!config) { + return hosts; + } + + add(config.vaultd?.host); + add(config.dataClient?.host); + add(config.metadataClient?.host); + add(config.pfsClient?.host); + add(config.cdmi?.host); + add(config.scuba?.host); + add(config.utapi?.host); + add(config.localCache?.host); + add(config.managementAgent?.host); + add(config.backbeat?.host); + add(config.kmsAWS?.endpoint); + + config.bucketd?.bootstrap?.forEach(add); + + if (config.kmip?.transport) { + const transports = Array.isArray(config.kmip.transport) + ? config.kmip.transport + : [config.kmip.transport]; + transports.forEach(t => add(t?.tls?.host)); + } + + if (typeof config.mongodb?.replicaSetHosts === 'string') { + config.mongodb.replicaSetHosts.split(',').forEach(add); + } + + // Read directly from env; lib/management/index.js sources these + // there too, they don't flow through Config.js. + add(process.env.PUSH_ENDPOINT); + add(process.env.MANAGEMENT_ENDPOINT); + + // Only the two Scality-owned connector shapes are trusted. Every + // other locationType (aws_s3, azure, gcp, *-archive, dmf, file, ...) + // is a separate cluster or external cloud — those stay untrusted + // so trace context doesn't leak across cluster boundaries. + if (config.locationConstraints + && typeof config.locationConstraints === 'object') { + for (const loc of Object.values(config.locationConstraints)) { + loc?.details?.connector?.hdclient?.bootstrap?.forEach(add); + loc?.details?.connector?.sproxyd?.bootstrap?.forEach(add); + } + } + + return hosts; +} + +module.exports = { extractHost, buildTrustedHosts, makeRequestHook }; diff --git a/tests/unit/lib/tracing/healthPaths.spec.js b/tests/unit/lib/tracing/healthPaths.spec.js new file mode 100644 index 0000000000..60386fd119 --- /dev/null +++ b/tests/unit/lib/tracing/healthPaths.spec.js @@ -0,0 +1,37 @@ +'use strict'; + +const assert = require('assert'); + +const { isHealthPath } = require('../../../../lib/tracing/healthPaths'); + +describe('tracing.isHealthPath', () => { + it('matches the canonical probe and scrape paths', () => { + [ + '/live', + '/ready', + '/_/healthcheck', + '/_/healthcheck/deep', + '/metrics', + ].forEach(p => assert.strictEqual(isHealthPath(p), true, p)); + }); + + it('matches when a query string is present', () => { + assert.strictEqual(isHealthPath('/live?token=x'), true); + assert.strictEqual(isHealthPath('/metrics?format=prom'), true); + }); + + it('does not match unrelated paths', () => { + [ + '/bucket/key', + '/_/backbeat/data/bucket/key', + '/livez', + '/metrics/custom', + ].forEach(p => assert.strictEqual(isHealthPath(p), false, p)); + }); + + it('returns false for non-string / empty input', () => { + assert.strictEqual(isHealthPath(undefined), false); + assert.strictEqual(isHealthPath(''), false); + assert.strictEqual(isHealthPath(42), false); + }); +}); diff --git a/tests/unit/lib/otel.spec.js b/tests/unit/lib/tracing/trustedHosts.spec.js similarity index 83% rename from tests/unit/lib/otel.spec.js rename to tests/unit/lib/tracing/trustedHosts.spec.js index ff5f832e58..c8a0bb6887 100644 --- a/tests/unit/lib/otel.spec.js +++ b/tests/unit/lib/tracing/trustedHosts.spec.js @@ -6,10 +6,9 @@ const { buildTrustedHosts, extractHost, makeRequestHook, -} = require('../../../lib/otel'); -const { isHealthPath } = require('../../../lib/tracing/healthPaths'); +} = require('../../../../lib/tracing/trustedHosts'); -describe('otel.extractHost', () => { +describe('tracing.extractHost', () => { it('extracts hostname from a plain host string', () => { assert.strictEqual(extractHost('example.com'), 'example.com'); }); @@ -43,7 +42,7 @@ describe('otel.extractHost', () => { }); }); -describe('otel.buildTrustedHosts', () => { +describe('tracing.buildTrustedHosts', () => { it('always contains loopback aliases', () => { const hosts = buildTrustedHosts({}); assert.ok(hosts.has('localhost')); @@ -58,10 +57,9 @@ describe('otel.buildTrustedHosts', () => { }); it('includes every host referenced in a full config', () => { - // Snapshot of every host-bearing config key that cloudserver - // consults. If a new config key is added without updating - // buildTrustedHosts, this test must fail — keep the derivation - // honest. + // Snapshot of every host-bearing config key cloudserver consults. + // If a new key is added without updating buildTrustedHosts, this + // test must fail. const config = { vaultd: { host: 'vaultd.zenko.svc.cluster.local' }, dataClient: { host: 'data.zenko.svc.cluster.local' }, @@ -153,13 +151,10 @@ describe('otel.buildTrustedHosts', () => { dataClient: {}, metadataClient: {}, }); - assert.strictEqual(hosts.size, 3); // only the loopback aliases + assert.strictEqual(hosts.size, 3); }); it('includes hdclient and sproxyd connector hosts from locationConstraints, and only those', () => { - // Four shapes: hdclient (trusted), sproxyd (trusted), - // aws-s3 (untrusted), scality-ring-s3 (untrusted — separate - // cluster, its tracing stack does not cross-reference ours). const hosts = buildTrustedHosts({ locationConstraints: { 'us-east-1': { @@ -204,18 +199,15 @@ describe('otel.buildTrustedHosts', () => { }, }, }); - // Hdclient + sproxyd hosts are in the set. assert.ok(hosts.has('hdproxy-a.xcore.svc')); assert.ok(hosts.has('hdproxy-b.xcore.svc')); assert.ok(hosts.has('sproxyd-a.ring.svc')); assert.ok(hosts.has('sproxyd-b.ring.svc')); - // External / remote-cluster endpoints are NOT in the set. assert.ok(!hosts.has('s3.us-west-2.amazonaws.com')); assert.ok(!hosts.has('s3.remote-ring.example.com')); }); it('tolerates locationConstraints entries without a connector', () => { - // file, mem, aws_s3 etc. — no connector at all must not throw. assert.doesNotThrow(() => buildTrustedHosts({ locationConstraints: { local: { type: 'file', details: {} }, @@ -225,7 +217,7 @@ describe('otel.buildTrustedHosts', () => { }); }); -describe('otel.makeRequestHook', () => { +describe('tracing.makeRequestHook', () => { const trusted = new Set(['trusted.example.com', 'localhost']); const hook = makeRequestHook(trusted); @@ -250,7 +242,6 @@ describe('otel.makeRequestHook', () => { it('is a no-op on inbound IncomingMessage (no getHeader method)', () => { const span = fakeSpan(); - // IncomingMessage shape: has .headers object, but no getHeader(). const inbound = { headers: { host: 'untrusted.example.com' } }; assert.doesNotThrow(() => hook(span, inbound)); assert.strictEqual(span._attrs['scality.trace.suppressed'], undefined); @@ -286,35 +277,3 @@ describe('otel.makeRequestHook', () => { assert.strictEqual(span._attrs['scality.trace.suppressed'], true); }); }); - -describe('tracing.healthPaths', () => { - it('matches the canonical probe and scrape paths', () => { - [ - '/live', - '/ready', - '/_/healthcheck', - '/_/healthcheck/deep', - '/metrics', - ].forEach(p => assert.strictEqual(isHealthPath(p), true, p)); - }); - - it('matches when a query string is present', () => { - assert.strictEqual(isHealthPath('/live?token=x'), true); - assert.strictEqual(isHealthPath('/metrics?format=prom'), true); - }); - - it('does not match unrelated paths', () => { - [ - '/bucket/key', - '/_/backbeat/data/bucket/key', - '/livez', - '/metrics/custom', - ].forEach(p => assert.strictEqual(isHealthPath(p), false, p)); - }); - - it('returns false for non-string / empty input', () => { - assert.strictEqual(isHealthPath(undefined), false); - assert.strictEqual(isHealthPath(''), false); - assert.strictEqual(isHealthPath(42), false); - }); -}); From 849cf07edbb667e260e16758c6acb601f8f4c705 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Tue, 12 May 2026 16:50:32 +0200 Subject: [PATCH 6/9] fixup! feat: flush OTEL on shutdown --- lib/server.js | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/lib/server.js b/lib/server.js index affe0dac3a..c0d9f31f82 100644 --- a/lib/server.js +++ b/lib/server.js @@ -6,7 +6,7 @@ const arsenal = require('arsenal'); const { setServerHeader } = arsenal.s3routes.routesUtils; const { RedisClient, StatsClient } = arsenal.metrics; const monitoringClient = require('./utilities/monitoringHandler'); -const { shutdownOtel } = require('./otel'); +const tracing = require('./tracing'); const logger = require('./utilities/logger'); const { internalHandlers } = require('./utilities/internalHandlers'); @@ -325,46 +325,32 @@ class S3Server { this.servers.push(server); } - /* - * This exits the running process properly. - */ cleanUp() { logger.info('server shutting down'); - // Stop token refill job if running if (this.config.rateLimiting?.enabled) { stopRefillJob(logger); } Promise.all(this.servers.map(server => new Promise(resolve => server.close(resolve)) )) - // Flush any buffered traces before exit so we don't lose - // the spans for whatever was in flight at shutdown. Use - // .finally so an unexpected rejection anywhere in the chain - // can't strand the process without an exit. - .then(() => shutdownOtel()) + .then(() => tracing.close()) .finally(() => process.exit(0)); } caughtExceptionShutdown() { if (!this.cluster) { - // Best-effort flush of in-flight traces before crashing. - // shutdownOtel() has its own short timeout so this can't - // hang the exit indefinitely. - shutdownOtel().finally(() => process.exit(1)); + tracing.close().finally(() => process.exit(1)); return; } logger.error('shutdown of worker due to exception', { workerId: this.worker ? this.worker.id : undefined, workerPid: this.worker ? this.worker.process.pid : undefined, }); - // Will close all servers, cause disconnect event on primary and kill - // worker process with 'SIGTERM'. Flush OTEL first — this.worker.kill() - // is graceful (closes servers + disconnects IPC) but does not fire - // our SIGTERM handler, so without an explicit flush the - // BatchSpanProcessor's buffered spans for the crashing worker - // would be lost. + // worker.kill() is graceful (closes servers, disconnects IPC) but + // does not fire our SIGTERM handler, so the BatchSpanProcessor + // would lose buffered spans without an explicit flush here. if (this.worker) { - shutdownOtel().finally(() => this.worker.kill()); + tracing.close().finally(() => this.worker.kill()); } } From 992c11d85bd85f37865fa0e6db7d0f9edc0a30e1 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Tue, 12 May 2026 16:50:35 +0200 Subject: [PATCH 7/9] fixup! feat: instrument all S3 API handlers with OTEL spans --- lib/api/api.js | 11 +- lib/instrumentation/simple.js | 96 ++++----- tests/unit/lib/instrumentationSimple.spec.js | 194 ++++++++++--------- 3 files changed, 148 insertions(+), 153 deletions(-) diff --git a/lib/api/api.js b/lib/api/api.js index 4848684e93..d426d54827 100644 --- a/lib/api/api.js +++ b/lib/api/api.js @@ -587,17 +587,16 @@ const api = { handleAuthorizationResults, }; -// Wrap every handler in the api object with an OTEL span. Skip the -// internal helpers (callApiMethod is the dispatcher; checkAuthResults -// and handleAuthorizationResults are pure helpers). New S3 handlers -// added to the literal above are automatically instrumented. -const NON_HANDLER_KEYS = new Set([ +// Denylist (not allowlist) so newly-added S3 handlers are auto-traced +// without a separate registration step. The three skipped keys are +// internal helpers, not S3 operations. +const NON_INSTRUMENTED_KEYS = new Set([ 'callApiMethod', 'checkAuthResults', 'handleAuthorizationResults', ]); for (const [name, handler] of Object.entries(api)) { - if (typeof handler === 'function' && !NON_HANDLER_KEYS.has(name)) { + if (typeof handler === 'function' && !NON_INSTRUMENTED_KEYS.has(name)) { api[name] = instrumentApiMethod(handler, name); } } diff --git a/lib/instrumentation/simple.js b/lib/instrumentation/simple.js index 0092647688..cfe4abfd59 100644 --- a/lib/instrumentation/simple.js +++ b/lib/instrumentation/simple.js @@ -1,71 +1,34 @@ 'use strict'; -// Lazy-load `@opentelemetry/api` only on the first OTEL-enabled call: -// when OTEL is off, the module is never required. The cache also lets -// direct callers of `instrumentApiMethod` (e.g. unit tests of this -// module) flip ENABLE_OTEL after require time and see the loaded -// module on the next call. This does NOT extend to api.js's wrapping -// loop, which runs once at require time — flipping the env var -// afterwards won't retroactively wrap already-dispatched handlers. -let otel = null; +const tracing = require('../tracing'); -function loadOtel() { - if (otel) { - return otel; +let tracer = null; +function getTracer() { + if (tracer) { + return tracer; } - if (process.env.ENABLE_OTEL !== 'true') { - return null; - } - const api = require('@opentelemetry/api'); + const { trace } = require('@opentelemetry/api'); const { version } = require('../../package.json'); - otel = { - trace: api.trace, - context: api.context, - SpanStatusCode: api.SpanStatusCode, - SpanKind: api.SpanKind, - tracer: api.trace.getTracer('cloudserver-api', version), - }; - return otel; + tracer = trace.getTracer('cloudserver-api', version); + return tracer; } -/** - * Wrap an S3 API handler so each invocation produces an OTEL span that - * owns the whole handler execution (auth, body parsing, metadata I/O, - * data path, finalizers). Auto-instrumentation spans (HTTP, MongoDB, - * ioredis) nest naturally beneath it. - * - * The span name is `api.` — one distinct name per S3 - * handler. Variants like objectGetACL / objectPutTagging stay distinct - * from objectGet / objectPut so flame graphs and queries can tell them - * apart without reading attributes. ~70 distinct names total — well - * within trace backend limits. - * - * Returns the original function unchanged when ENABLE_OTEL is unset - * at the time of the call, so there is zero overhead off the OTEL path. - */ function instrumentApiMethod(apiMethod, methodName) { - const o = loadOtel(); - if (!o) { + if (!tracing.isEnabled()) { return apiMethod; } + const api = require('@opentelemetry/api'); const spanName = `api.${methodName}`; return function instrumented(...args) { const callbackIndex = args.findLastIndex(a => typeof a === 'function'); - if (callbackIndex === -1) { - // Cloudserver's S3 dispatch always passes a callback to the - // handler — every shape in callApiHandler has one. If a - // future call site routes here without one, fall through - // unwrapped rather than create a span we can't end. - return apiMethod.apply(this, args); - } + const span = getTracer().startSpan(spanName, { kind: api.SpanKind.INTERNAL }); - const span = o.tracer.startSpan(spanName, { kind: o.SpanKind.INTERNAL }); - // Guard against double-ending: the wrapped callback may fire and - // end the span, then the outer apiMethod.apply could still throw - // synchronously (e.g. the callback itself threw). End-once - // semantics keeps the span and trace state consistent. + // End-once guard. Multiple termination paths can race: the + // wrapped callback may fire and then the handler may also throw + // synchronously, or a callback-and-Promise hybrid handler may + // resolve after firing the callback. let spanEnded = false; const endSpan = err => { if (spanEnded) { @@ -74,26 +37,39 @@ function instrumentApiMethod(apiMethod, methodName) { spanEnded = true; if (err) { span.recordException(err); - span.setStatus({ code: o.SpanStatusCode.ERROR }); + span.setStatus({ code: api.SpanStatusCode.ERROR }); if (err.code) { span.setAttribute('cloudserver.error_code', err.code); } } else { - span.setStatus({ code: o.SpanStatusCode.OK }); + span.setStatus({ code: api.SpanStatusCode.OK }); } span.end(); }; - const originalCallback = args[callbackIndex]; const wrappedArgs = [...args]; - wrappedArgs[callbackIndex] = function wrappedCallback(err, ...results) { - endSpan(err); - return originalCallback.call(this, err, ...results); - }; + if (callbackIndex !== -1) { + const originalCallback = args[callbackIndex]; + wrappedArgs[callbackIndex] = function wrappedCallback(err, ...results) { + endSpan(err); + return originalCallback.call(this, err, ...results); + }; + } + const ctx = api.trace.setSpan(api.context.active(), span); try { - return o.context.with(o.trace.setSpan(o.context.active(), span), () => + const result = api.context.with(ctx, () => apiMethod.apply(this, wrappedArgs)); + if (result && typeof result.then === 'function') { + return result.then( + value => { endSpan(); return value; }, + err => { endSpan(err); throw err; }, + ); + } + if (callbackIndex === -1) { + endSpan(); + } + return result; } catch (error) { endSpan(error); throw error; diff --git a/tests/unit/lib/instrumentationSimple.spec.js b/tests/unit/lib/instrumentationSimple.spec.js index e0cd2cf4d1..ed4d180e37 100644 --- a/tests/unit/lib/instrumentationSimple.spec.js +++ b/tests/unit/lib/instrumentationSimple.spec.js @@ -13,8 +13,6 @@ const { AlwaysOnSampler, } = require('@opentelemetry/sdk-trace-base'); -// Capture spans into memory so tests can inspect them. Wire the provider -// up via the API surface (sdk-trace-base 2.x dropped provider.register). const exporter = new InMemorySpanExporter(); const provider = new BasicTracerProvider({ sampler: new AlwaysOnSampler(), @@ -24,111 +22,133 @@ trace.setGlobalTracerProvider(provider); const { instrumentApiMethod } = require('../../../lib/instrumentation/simple'); -describe('instrumentApiMethod (OTEL on)', () => { - afterEach(() => exporter.reset()); +describe('instrumentApiMethod', () => { + describe('OTEL on', () => { + afterEach(() => exporter.reset()); - it('wraps a callback handler and ends span on success', done => { - const handler = (a, b, cb) => cb(null, 'ok'); - const wrapped = instrumentApiMethod(handler, 'objectGet'); + it('wraps a callback handler and ends span on success', done => { + const handler = (a, b, cb) => cb(null, 'ok'); + const wrapped = instrumentApiMethod(handler, 'objectGet'); - wrapped('foo', 'bar', (err, value) => { - assert.strictEqual(err, null); - assert.strictEqual(value, 'ok'); + wrapped('foo', 'bar', (err, value) => { + assert.strictEqual(err, null); + assert.strictEqual(value, 'ok'); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + assert.strictEqual(spans[0].name, 'api.objectGet'); + assert.strictEqual(spans[0].status.code, SpanStatusCode.OK); + done(); + }); + }); + + it('ends span with ERROR when handler\'s callback fires with err', done => { + const handler = (a, cb) => cb(Object.assign(new Error('nope'), { code: 'NoSuchBucket' })); + const wrapped = instrumentApiMethod(handler, 'bucketHead'); + + wrapped('foo', err => { + assert.strictEqual(err.message, 'nope'); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + assert.strictEqual(spans[0].status.code, SpanStatusCode.ERROR); + assert.strictEqual(spans[0].attributes['cloudserver.error_code'], 'NoSuchBucket'); + done(); + }); + }); + + it('ends span exactly once when callback fires then handler throws', () => { + // The first endSpan call (from the callback) wins; the + // post-callback throw is a programming bug, not an outcome + // of the API call — so status stays OK and we don't double- + // end (which would warn and corrupt span state). + const handler = cb => { + cb(null, 'first'); + throw new Error('after-callback-boom'); + }; + const wrapped = instrumentApiMethod(handler, 'objectPut'); + + let cbErr = null; + let cbValue = null; + const cb = (err, val) => { cbErr = err; cbValue = val; }; + + assert.throws(() => wrapped(cb), /after-callback-boom/); + + assert.strictEqual(cbErr, null); + assert.strictEqual(cbValue, 'first'); const spans = exporter.getFinishedSpans(); - assert.strictEqual(spans.length, 1); - assert.strictEqual(spans[0].name, 'api.objectGet'); + assert.strictEqual(spans.length, 1, 'span ended exactly once'); assert.strictEqual(spans[0].status.code, SpanStatusCode.OK); - done(); }); - }); - it('ends span with ERROR when handler\'s callback fires with err', done => { - const handler = (a, cb) => cb(Object.assign(new Error('nope'), { code: 'NoSuchBucket' })); - const wrapped = instrumentApiMethod(handler, 'bucketHead'); + it('synchronous throw before callback fires ends span and re-throws', () => { + const handler = () => { + throw new Error('sync-boom'); + }; + const wrapped = instrumentApiMethod(handler, 'objectDelete'); - wrapped('foo', err => { - assert.strictEqual(err.message, 'nope'); + assert.throws(() => wrapped(() => {}), /sync-boom/); const spans = exporter.getFinishedSpans(); assert.strictEqual(spans.length, 1); assert.strictEqual(spans[0].status.code, SpanStatusCode.ERROR); - assert.strictEqual(spans[0].attributes['cloudserver.error_code'], 'NoSuchBucket'); - done(); }); - }); - it('does not end the span twice when callback fires then handler throws', () => { - // Handler invokes the callback synchronously and then throws — - // tests the spanEnded guard. Without it the span would receive a - // second end() call and the recordException from the throw would - // overwrite the OK status set by the callback. - const handler = cb => { - cb(null, 'first'); - throw new Error('after-callback-boom'); - }; - const wrapped = instrumentApiMethod(handler, 'objectPut'); - - let cbErr = null; - let cbValue = null; - const cb = (err, val) => { cbErr = err; cbValue = val; }; - - assert.throws(() => wrapped(cb), /after-callback-boom/); - - assert.strictEqual(cbErr, null); - assert.strictEqual(cbValue, 'first'); - - const spans = exporter.getFinishedSpans(); - assert.strictEqual(spans.length, 1, 'span ended exactly once'); - // The first endSpan call (from the callback) wins; status is OK, - // not ERROR — the post-callback throw is a programming bug, not - // an outcome of the API call. - assert.strictEqual(spans[0].status.code, SpanStatusCode.OK); - }); + it('wraps an async handler and ends span on resolution', async () => { + const handler = async a => `async-${a}`; + const wrapped = instrumentApiMethod(handler, 'objectGetAsync'); - it('passes through unwrapped (no span) when no callback arg is found', () => { - // Defensive: cloudserver's S3 dispatch always passes a callback, - // so this branch is unreachable today. If a future caller ever - // routes here without one, the wrapper falls through to the raw - // handler rather than creating a span it can't end. - const handler = (a, b) => `${a}-${b}`; - const wrapped = instrumentApiMethod(handler, 'objectRestore'); + const value = await wrapped('x'); + assert.strictEqual(value, 'async-x'); - assert.strictEqual(wrapped('x', 'y'), 'x-y'); - assert.strictEqual(exporter.getFinishedSpans().length, 0); - }); + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + assert.strictEqual(spans[0].name, 'api.objectGetAsync'); + assert.strictEqual(spans[0].status.code, SpanStatusCode.OK); + }); + + it('wraps an async handler and ends span with ERROR on rejection', async () => { + const handler = async () => { + const err = new Error('async-nope'); + err.code = 'NoSuchKey'; + throw err; + }; + const wrapped = instrumentApiMethod(handler, 'objectGetAsync'); + + await assert.rejects(wrapped(), /async-nope/); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + assert.strictEqual(spans[0].status.code, SpanStatusCode.ERROR); + assert.strictEqual(spans[0].attributes['cloudserver.error_code'], 'NoSuchKey'); + }); - it('synchronous throw before callback fires ends span and re-throws', () => { - const handler = () => { - throw new Error('sync-boom'); - }; - const wrapped = instrumentApiMethod(handler, 'objectDelete'); + it('ends span on sync return when handler has no callback arg', () => { + const handler = (a, b) => `${a}-${b}`; + const wrapped = instrumentApiMethod(handler, 'objectRestore'); - assert.throws(() => wrapped(() => {}), /sync-boom/); + assert.strictEqual(wrapped('x', 'y'), 'x-y'); - const spans = exporter.getFinishedSpans(); - assert.strictEqual(spans.length, 1); - assert.strictEqual(spans[0].status.code, SpanStatusCode.ERROR); + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + assert.strictEqual(spans[0].status.code, SpanStatusCode.OK); + }); }); -}); -describe('instrumentApiMethod with OTEL disabled', () => { - it('returns the original function unchanged', () => { - // We have to test this by re-requiring the module under a fresh - // ENABLE_OTEL=false. The module caches enableOtel at load time. - const path = require.resolve('../../../lib/instrumentation/simple'); - const cached = require.cache[path]; - delete require.cache[path]; - const savedEnv = process.env.ENABLE_OTEL; - process.env.ENABLE_OTEL = 'false'; - try { - const { instrumentApiMethod: gated } = - require('../../../lib/instrumentation/simple'); - const handler = () => 'identity'; - assert.strictEqual(gated(handler, 'foo'), handler); - } finally { - process.env.ENABLE_OTEL = savedEnv; - require.cache[path] = cached; - } + describe('OTEL off', () => { + // tracing.isEnabled() is checked at each instrumentApiMethod call, + // so flipping ENABLE_OTEL between calls is enough — no require.cache + // dance needed. + it('returns the original function unchanged', () => { + const saved = process.env.ENABLE_OTEL; + process.env.ENABLE_OTEL = 'false'; + try { + const handler = () => 'identity'; + assert.strictEqual(instrumentApiMethod(handler, 'foo'), handler); + } finally { + process.env.ENABLE_OTEL = saved; + } + }); }); }); From 099c6789c4c891595f108763b26580d442092634 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Tue, 12 May 2026 18:05:10 +0200 Subject: [PATCH 8/9] fixup! feat: add OpenTelemetry tracing with trust boundaries and probe filtering race-safe close() --- lib/tracing/index.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/tracing/index.js b/lib/tracing/index.js index b8a54d774b..030ffddb86 100644 --- a/lib/tracing/index.js +++ b/lib/tracing/index.js @@ -86,12 +86,17 @@ function init() { const SHUTDOWN_DEADLINE_MS = 5000; async function close() { - if (!sdk) { + // Capture + clear before awaiting so concurrent callers (SIGTERM + // during an uncaught-exception flow, for example) don't both call + // sdk.shutdown() — the SDK doesn't guarantee idempotent shutdown. + const local = sdk; + if (!local) { return; } + sdk = null; try { await Promise.race([ - sdk.shutdown(), + local.shutdown(), // .unref() so the timer doesn't pin the event loop open // when sdk.shutdown() resolves first. new Promise(resolve => { @@ -103,8 +108,6 @@ async function close() { // log to stderr directly. // eslint-disable-next-line no-console console.error('tracing close failed', err); - } finally { - sdk = null; } } From 97b88f5c1b5d497c183600a20ab392d616304c13 Mon Sep 17 00:00:00 2001 From: Thomas Flament Date: Tue, 12 May 2026 18:05:11 +0200 Subject: [PATCH 9/9] fixup! feat: instrument all S3 API handlers with OTEL spans --- lib/instrumentation/simple.js | 17 ++++++---- tests/unit/lib/instrumentationSimple.spec.js | 35 ++++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/lib/instrumentation/simple.js b/lib/instrumentation/simple.js index cfe4abfd59..24e3b139fb 100644 --- a/lib/instrumentation/simple.js +++ b/lib/instrumentation/simple.js @@ -60,15 +60,20 @@ function instrumentApiMethod(apiMethod, methodName) { try { const result = api.context.with(ctx, () => apiMethod.apply(this, wrappedArgs)); - if (result && typeof result.then === 'function') { - return result.then( - value => { endSpan(); return value; }, - err => { endSpan(err); throw err; }, - ); - } if (callbackIndex === -1) { + if (result && typeof result.then === 'function') { + return result.then( + value => { endSpan(); return value; }, + err => { endSpan(err); throw err; }, + ); + } endSpan(); } + // Callback-style handler: the wrapped callback drives the + // span lifecycle. If the handler also returns a thenable + // (hybrid migration shape), pass it through untouched — + // attaching a second .then() chain would surface as an + // unhandled rejection in callback-only callers. return result; } catch (error) { endSpan(error); diff --git a/tests/unit/lib/instrumentationSimple.spec.js b/tests/unit/lib/instrumentationSimple.spec.js index ed4d180e37..c406b64638 100644 --- a/tests/unit/lib/instrumentationSimple.spec.js +++ b/tests/unit/lib/instrumentationSimple.spec.js @@ -124,6 +124,41 @@ describe('instrumentApiMethod', () => { assert.strictEqual(spans[0].attributes['cloudserver.error_code'], 'NoSuchKey'); }); + it('callback drives lifecycle even when handler also returns a Promise', done => { + // Hybrid shape (migration artifact): handler fires cb AND + // returns a Promise. The wrapper must NOT chain its own + // .then() onto that Promise — doing so would surface as an + // unhandled rejection in callback-only callers that discard + // the return value. + const handler = (a, cb) => { + cb(null, `cb-${a}`); + return Promise.resolve(`promise-${a}`); + }; + const wrapped = instrumentApiMethod(handler, 'hybridGet'); + + let cbErr; + let cbValue; + const returned = wrapped('x', (err, value) => { + cbErr = err; + cbValue = value; + }); + + assert.strictEqual(cbErr, null); + assert.strictEqual(cbValue, 'cb-x'); + + const spans = exporter.getFinishedSpans(); + assert.strictEqual(spans.length, 1); + assert.strictEqual(spans[0].status.code, SpanStatusCode.OK); + + // Caller still receives the handler's original Promise + // (not a chained wrapper) so they can choose to await it. + assert.ok(returned && typeof returned.then === 'function'); + returned.then(v => { + assert.strictEqual(v, 'promise-x'); + done(); + }); + }); + it('ends span on sync return when handler has no callback arg', () => { const handler = (a, b) => `${a}-${b}`; const wrapped = instrumentApiMethod(handler, 'objectRestore');