Skip to content

Commit 9b47930

Browse files
waleedlatif1waleedlatif1
authored
fix(observability): attach the real cause at three error-swallowing sites (#6336)
Three log sites discarded the underlying error, which blocked root-cause analysis in production. - Trace secret projection swallowed TraceSecretProjectionError in four catch blocks (per-field omission, whole-tree fallback, post-transform invariant, structural traversal) across ~30 distinct throw sites, so no warning said which invariant fired. Each now reports the failure. Only TraceSecretProjectionError messages are logged — they are fixed literals describing an invariant. A failure raised outside the module may quote trace content (a JSON parse error embeds the text it choked on), so those are reported by name only. - ExecutionLogger's unbilled-charge error logged `"error":{}` because a plain Error has non-enumerable message/stack. It now logs describeError. - WorkspaceFileStorage / FetchExternalUrl logged `saveError:{}` for the same reason, and the upload wrapper rethrew without a cause, so Drizzle's `Failed query:` wrapper dropped the Postgres SQLSTATE. The wrapper now chains the cause and both sites log describeError, which reports the deepest link's code. describeError additionally strips the `params:` tail Drizzle appends to its message, so bound parameter values never reach logs.
1 parent 2ba4556 commit 9b47930

10 files changed

Lines changed: 271 additions & 24 deletions

File tree

‎apps/sim/lib/logs/execution/logger.test.ts‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import type { SerializableExecutionState } from '@/executor/execution/types'
1616
afterAll(resetDbChainMock)
1717

1818
/** Flat logger whose withMetadata() children share one spy set, so log level is assertable. */
19-
const { mockLogger } = vi.hoisted(() => {
19+
const { mockLogger, statsLogErrorMock } = vi.hoisted(() => {
2020
const mockLogger: Record<string, ReturnType<typeof vi.fn>> = {
2121
info: vi.fn(),
2222
warn: vi.fn(),
@@ -27,7 +27,7 @@ const { mockLogger } = vi.hoisted(() => {
2727
}
2828
mockLogger.child = vi.fn(() => mockLogger)
2929
mockLogger.withMetadata = vi.fn(() => mockLogger)
30-
return { mockLogger }
30+
return { mockLogger, statsLogErrorMock: mockLogger.error }
3131
})
3232

3333
vi.mock('@sim/logger', () => ({
@@ -1230,4 +1230,32 @@ describe('recordExecutionUsage boundary-delta reconciliation', () => {
12301230
// The ledger INSERT participates in the locked transaction.
12311231
expect(vi.mocked(recordUsage).mock.calls[0][0]).toHaveProperty('tx')
12321232
})
1233+
1234+
test('reports the driver cause and SQLSTATE when the ledger write fails', async () => {
1235+
const driver = Object.assign(new Error('cannot execute INSERT in a read-only transaction'), {
1236+
code: '25006',
1237+
})
1238+
vi.mocked(recordUsage).mockRejectedValueOnce(
1239+
new Error('Failed query: insert into "usage_log"\nparams: user-1', { cause: driver })
1240+
)
1241+
1242+
await run(
1243+
costSummary({
1244+
models: {
1245+
'gpt-4o': { input: 0, output: 0, total: 1, tokens: { input: 0, output: 0, total: 0 } },
1246+
},
1247+
}),
1248+
[]
1249+
)
1250+
1251+
expect(statsLogErrorMock).toHaveBeenCalledWith(
1252+
'Failed to record execution usage to usage_log ledger; charge may be unbilled',
1253+
expect.objectContaining({
1254+
cause: expect.objectContaining({
1255+
code: '25006',
1256+
message: 'cannot execute INSERT in a read-only transaction',
1257+
}),
1258+
})
1259+
)
1260+
})
12331261
})

‎apps/sim/lib/logs/execution/logger.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010
workspace,
1111
} from '@sim/db/schema'
1212
import { createLogger } from '@sim/logger'
13-
import { getErrorMessage } from '@sim/utils/errors'
13+
import { describeError, getErrorMessage } from '@sim/utils/errors'
1414
import { generateId } from '@sim/utils/id'
1515
import { and, eq, inArray, sql } from 'drizzle-orm'
1616
import { checkUsageStatus as checkResolvedUsageStatus } from '@/lib/billing/calculations/usage-monitor'
@@ -1768,7 +1768,7 @@ export class ExecutionLogger implements IExecutionLoggerService {
17681768
statsLog.error(
17691769
'Failed to record execution usage to usage_log ledger; charge may be unbilled',
17701770
{
1771-
error,
1771+
cause: describeError(error),
17721772
actorUserId,
17731773
costSummary,
17741774
}

‎apps/sim/lib/logs/execution/trace-secret-projection.test.ts‎

Lines changed: 90 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,19 @@
44
import { createHash } from 'node:crypto'
55
import { beforeEach, describe, expect, it, vi } from 'vitest'
66

7-
const { materializeLargeValueRefMock, storeLargeValueMock } = vi.hoisted(() => ({
7+
const { materializeLargeValueRefMock, storeLargeValueMock, warnMock } = vi.hoisted(() => ({
88
materializeLargeValueRefMock: vi.fn(),
99
storeLargeValueMock: vi.fn(),
10+
warnMock: vi.fn(),
11+
}))
12+
13+
vi.mock('@sim/logger', () => ({
14+
createLogger: () => ({
15+
debug: vi.fn(),
16+
info: vi.fn(),
17+
warn: warnMock,
18+
error: vi.fn(),
19+
}),
1020
}))
1121

1222
vi.mock('@/lib/execution/payloads/store', () => ({
@@ -24,6 +34,8 @@ import {
2434
ResolvedSecretTraceRegistry,
2535
} from '@/executor/utils/resolved-secret-trace-registry'
2636

37+
const MAX_CONTENT_NODES = 100_000
38+
2739
const STORE = {
2840
workspaceId: 'workspace-1',
2941
workflowId: 'workflow-1',
@@ -438,6 +450,25 @@ describe('projectTraceSpansForSecrets', () => {
438450
expect(source[0].output).toEqual({ apiKey: '[REDACTED]' })
439451
})
440452

453+
it('names the invariant that forced the structural fallback', async () => {
454+
const source = [createSpan({ output: { apiKey: '[REDACTED]' } })]
455+
456+
await enforceTraceSpanSecretInvariant(source, {
457+
registry: createRegistry([{ plaintext: 'E', replacement: '{{X}}' }]),
458+
store: STORE,
459+
})
460+
461+
expect(warnMock).toHaveBeenCalledWith(
462+
'Trace secret invariant failed; retaining structural spans only',
463+
{
464+
failure: {
465+
name: 'TraceSecretProjectionError',
466+
reason: expect.any(String),
467+
},
468+
}
469+
)
470+
})
471+
441472
it('fails the final invariant closed when provenance is incomplete', async () => {
442473
const source = [createSpan({ output: { value: 'ordinary' } })]
443474

@@ -1149,6 +1180,64 @@ describe('projectTraceSpansForSecrets', () => {
11491180
expect(result[0].output).toEqual({ token: '{{API_SECRET}}' })
11501181
})
11511182

1183+
it('names the invariant that forced content to be omitted', async () => {
1184+
const output: Record<string, unknown> = { token: 'top-secret' }
1185+
output.self = output
1186+
1187+
const [result] = await projectTraceSpansForSecrets([createSpan({ output })], {
1188+
registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]),
1189+
store: STORE,
1190+
})
1191+
1192+
expect(result).not.toHaveProperty('output')
1193+
expect(warnMock).toHaveBeenCalledWith('Omitting trace content that could not be sanitized', {
1194+
failure: {
1195+
name: 'TraceSecretProjectionError',
1196+
reason: 'Trace content could not be sanitized',
1197+
},
1198+
})
1199+
})
1200+
1201+
it('withholds the message of a failure raised outside the projection module', async () => {
1202+
const descriptorSpy = vi.spyOn(Object, 'getOwnPropertyDescriptor').mockImplementation(() => {
1203+
throw new SyntaxError('Unexpected token in "sk-live-top-secret"')
1204+
})
1205+
1206+
try {
1207+
await projectTraceSpansForSecrets([createSpan({ output: { token: 'top-secret' } })], {
1208+
registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]),
1209+
store: STORE,
1210+
})
1211+
} finally {
1212+
descriptorSpy.mockRestore()
1213+
}
1214+
1215+
expect(warnMock).toHaveBeenCalledWith('Omitting trace content that could not be sanitized', {
1216+
failure: { name: 'SyntaxError' },
1217+
})
1218+
})
1219+
1220+
it('names the invariant that forced the whole-tree structural fallback', async () => {
1221+
const source = Array(MAX_CONTENT_NODES + 1).fill(
1222+
createSpan({ output: { token: 'top-secret' } })
1223+
)
1224+
1225+
await projectTraceSpansForSecrets(source, {
1226+
registry: createRegistry([{ plaintext: 'top-secret', replacement: '{{API_SECRET}}' }]),
1227+
store: STORE,
1228+
})
1229+
1230+
expect(warnMock).toHaveBeenCalledWith(
1231+
'Trace secret projection failed; retaining structural spans only',
1232+
{
1233+
failure: {
1234+
name: 'TraceSecretProjectionError',
1235+
reason: 'Trace structure array exceeds the projection limit',
1236+
},
1237+
}
1238+
)
1239+
})
1240+
11521241
it('uses bounded structural fallback when matcher construction fails', async () => {
11531242
let source = createSpan({ id: 'depth-150', output: { secret: 'raw' } })
11541243
for (let depth = 149; depth >= 0; depth -= 1) {

‎apps/sim/lib/logs/execution/trace-secret-projection.ts‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { createLogger } from '@sim/logger'
2+
import { toError } from '@sim/utils/errors'
23
import { isPlainRecord, omit } from '@sim/utils/object'
34
import {
45
isLargeArrayManifest,
@@ -128,6 +129,21 @@ class TraceSecretProjectionError extends Error {
128129
}
129130
}
130131

132+
/**
133+
* Diagnostic payload for a projection fallback.
134+
*
135+
* Every {@link TraceSecretProjectionError} message is a fixed literal describing
136+
* which invariant fired, so it is safe to log. Any other failure originates
137+
* outside this module and may embed trace content (a JSON parse error quotes the
138+
* text it choked on), so only its name is reported.
139+
*/
140+
function describeProjectionFailure(error: unknown): { name: string; reason?: string } {
141+
return {
142+
name: toError(error).name,
143+
...(error instanceof TraceSecretProjectionError ? { reason: error.message } : {}),
144+
}
145+
}
146+
131147
function createProjectionContext(
132148
matcher: ResolvedSecretMatcher,
133149
store: LargeValueStoreContext,
@@ -786,8 +802,10 @@ async function sanitizeContentField(
786802
): Promise<unknown | typeof OMIT> {
787803
try {
788804
return await sanitizeMaterializedValue(value, context)
789-
} catch {
790-
logger.warn('Omitting trace content that could not be sanitized')
805+
} catch (error) {
806+
logger.warn('Omitting trace content that could not be sanitized', {
807+
failure: describeProjectionFailure(error),
808+
})
791809
return OMIT
792810
}
793811
}
@@ -1182,8 +1200,10 @@ function projectBoundedTraceSpans(
11821200
function structuralOnlyTraceSpans(traceSpans: TraceSpan[]): TraceSpan[] {
11831201
try {
11841202
return projectBoundedTraceSpans(traceSpans, structuralOnlySpan)
1185-
} catch {
1186-
logger.warn('Trace structure could not be safely traversed; omitting projected spans')
1203+
} catch (error) {
1204+
logger.warn('Trace structure could not be safely traversed; omitting projected spans', {
1205+
failure: describeProjectionFailure(error),
1206+
})
11871207
return []
11881208
}
11891209
}
@@ -1522,8 +1542,10 @@ export async function enforceTraceSpanSecretInvariant(
15221542

15231543
await assertPostTransformTraceSpansAreSafe(traceSpans, matcher, options.store)
15241544
return traceSpans
1525-
} catch {
1526-
logger.warn('Trace secret invariant failed; retaining structural spans only')
1545+
} catch (error) {
1546+
logger.warn('Trace secret invariant failed; retaining structural spans only', {
1547+
failure: describeProjectionFailure(error),
1548+
})
15271549
return structuralOnlyTraceSpans(traceSpans)
15281550
}
15291551
}
@@ -1560,8 +1582,10 @@ export async function projectTraceSpansForSecrets(
15601582
}
15611583
assertTraceSpansContentIsSafe(projected, context)
15621584
return projected
1563-
} catch {
1564-
logger.warn('Trace secret projection failed; retaining structural spans only')
1585+
} catch (error) {
1586+
logger.warn('Trace secret projection failed; retaining structural spans only', {
1587+
failure: describeProjectionFailure(error),
1588+
})
15651589
return structuralOnlyTraceSpans(traceSpans)
15661590
}
15671591
}

‎apps/sim/lib/uploads/contexts/workspace/fetch-external-url.test.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,13 @@
99
* the module graph is fresh or reused.
1010
*/
1111
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'
12+
13+
const { warnMock } = vi.hoisted(() => ({ warnMock: vi.fn() }))
14+
15+
vi.mock('@sim/logger', () => ({
16+
createLogger: () => ({ debug: vi.fn(), info: vi.fn(), warn: warnMock, error: vi.fn() }),
17+
}))
18+
1219
import * as inputValidation from '@/lib/core/security/input-validation.server'
1320
import {
1421
ExternalUrlValidationError,
@@ -200,6 +207,33 @@ describe('fetchExternalUrlToWorkspace', () => {
200207
expect(result.savedWorkspaceFile).toBeUndefined()
201208
})
202209

210+
it('logs the driver cause and SQLSTATE behind a swallowed workspace save error', async () => {
211+
const driver = Object.assign(
212+
new Error('cannot execute SELECT FOR UPDATE in a read-only transaction'),
213+
{ code: '25006' }
214+
)
215+
secureFetchWithPinnedIPSpy.mockResolvedValue(makeResponse('bytes', 'text/plain'))
216+
uploadWorkspaceFileSpy.mockRejectedValueOnce(
217+
new Error('Failed to upload file: storage accounting failed', { cause: driver })
218+
)
219+
220+
await fetchExternalUrlToWorkspace({
221+
url: 'https://example.com/file.txt',
222+
userId: 'user-1',
223+
workspaceId: 'workspace-1',
224+
})
225+
226+
expect(warnMock).toHaveBeenCalledWith(
227+
'Failed to save fetched URL to workspace storage',
228+
expect.objectContaining({
229+
cause: expect.objectContaining({
230+
code: '25006',
231+
message: 'cannot execute SELECT FOR UPDATE in a read-only transaction',
232+
}),
233+
})
234+
)
235+
})
236+
203237
it('forwards custom headers to the fetch', async () => {
204238
secureFetchWithPinnedIPSpy.mockResolvedValue(makeResponse('bytes', 'text/plain'))
205239

‎apps/sim/lib/uploads/contexts/workspace/fetch-external-url.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import type { Buffer } from 'buffer'
22
import path from 'path'
33
import { createLogger } from '@sim/logger'
4+
import { describeError } from '@sim/utils/errors'
45
import {
56
secureFetchWithPinnedIP,
67
validateUrlWithDNS,
@@ -134,7 +135,7 @@ export async function fetchExternalUrlToWorkspace(
134135
logger.warn('Failed to save fetched URL to workspace storage', {
135136
workspaceId,
136137
filename,
137-
saveError,
138+
cause: describeError(saveError),
138139
})
139140
}
140141
} else if (permission === null) {

‎apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,12 @@ import { randomBytes } from 'crypto'
77
import { db } from '@sim/db'
88
import { workspaceFiles } from '@sim/db/schema'
99
import { createLogger } from '@sim/logger'
10-
import { getErrorMessage, getPostgresConstraintName, getPostgresErrorCode } from '@sim/utils/errors'
10+
import {
11+
describeError,
12+
getErrorMessage,
13+
getPostgresConstraintName,
14+
getPostgresErrorCode,
15+
} from '@sim/utils/errors'
1116
import { generateShortId } from '@sim/utils/id'
1217
import { and, eq, isNotNull, isNull, or, sql } from 'drizzle-orm'
1318
import type { ShareRecord } from '@/lib/api/contracts/public-shares'
@@ -446,15 +451,18 @@ export async function uploadWorkspaceFile(
446451
)
447452
continue
448453
}
449-
logger.error(`Failed to upload workspace file ${fileName}:`, error)
450-
throw new Error(`Failed to upload file: ${getErrorMessage(error, 'Unknown error')}`)
454+
logger.error(`Failed to upload workspace file ${fileName}:`, {
455+
cause: describeError(error),
456+
})
457+
throw new Error(`Failed to upload file: ${getErrorMessage(error, 'Unknown error')}`, {
458+
cause: error,
459+
})
451460
}
452461
}
453462

454-
logger.error(
455-
`Failed to upload workspace file after ${MAX_UPLOAD_UNIQUE_RETRIES} attempts`,
456-
lastError
457-
)
463+
logger.error(`Failed to upload workspace file after ${MAX_UPLOAD_UNIQUE_RETRIES} attempts`, {
464+
cause: describeError(lastError),
465+
})
458466
throw new FileConflictError(fileName)
459467
}
460468

0 commit comments

Comments
 (0)