diff --git a/apps/docs/content/docs/platform/self-hosting/networking.mdx b/apps/docs/content/docs/platform/self-hosting/networking.mdx index 1a995ac67c5..663f5314d41 100644 --- a/apps/docs/content/docs/platform/self-hosting/networking.mdx +++ b/apps/docs/content/docs/platform/self-hosting/networking.mdx @@ -278,7 +278,7 @@ This depends on the runtime. The published images run Bun. If you build the stan | Everything through the SSRF guard — SSO OIDC discovery, the HTTP block, tools, connectors, outbound webhooks, content fetches, MCP servers, and model endpoints typed into a block | Yes — see [Proxying guarded requests](#proxying-guarded-requests) | | SSO sign-in — the token, JWKS, and userinfo calls made while a user signs in | Yes — they use the default `fetch` | | SSO domain verification (TXT lookup) | No — DNS, sent to `1.1.1.1` and `8.8.8.8` unless `DOMAIN_VERIFICATION_NAMESERVERS` names your resolvers | -| Amazon Bedrock, object storage on S3, and every AWS integration (DynamoDB, SQS, Lambda, Textract, and the rest) | Yes — each AWS client gets a proxy-aware agent while a proxy is set. Bedrock then uses HTTP/1.1 instead of HTTP/2 | +| Amazon Bedrock, object storage on S3, and every AWS integration (DynamoDB, SQS, Lambda, Textract, and the rest) | Yes — each AWS service client gets a proxy-aware agent while a proxy is set, and STS calls made for its credentials inherit it. Bedrock then uses HTTP/1.1 instead of HTTP/2. IAM Identity Center (SSO) credential lookups do not use the proxy | | Email via SMTP, and the SMTP block | No — SMTP is not HTTP. Use a mail relay the app can reach directly, or Resend or SES | | Email via Amazon SES | Yes — the AWS SDK client above | | OTLP export from the server SDK | Yes — the exporters get the same proxy-aware agent | @@ -289,7 +289,7 @@ The practical consequence: a mandatory-egress-proxy environment can route LLM tr ### Proxying guarded requests -A request through the SSRF guard keeps its checks when it is proxied. Before each request, and again for each redirect hop, the app resolves the destination's hostname with its own DNS and refuses it if any policy in [Security](/platform/self-hosting/security#the-ssrf-boundary) blocks the address. It then hands the hostname, not the address, to the proxy, so a proxy that allowlists by domain sees the name it expects. +A request through the SSRF guard keeps its checks when it is proxied. Before each request, and again for each redirect hop, the app resolves the destination's hostname with its own DNS and refuses it if any policy in [Security](/platform/self-hosting/security#the-ssrf-boundary) blocks any of the addresses it resolves to, since the proxy may connect to any of them. It then hands the hostname, not the address, to the proxy, so a proxy that allowlists by domain sees the name it expects. - **DNS must still resolve inside the container.** A destination that the app cannot resolve is refused before the proxy is asked. - **The proxy resolves the name again.** The app cannot pin the address across a proxy, so the proxy's own egress policy is the last check. diff --git a/apps/docs/openapi-v2-resources.json b/apps/docs/openapi-v2-resources.json index 1a858de5f42..d1e521e3fbb 100644 --- a/apps/docs/openapi-v2-resources.json +++ b/apps/docs/openapi-v2-resources.json @@ -23929,7 +23929,7 @@ "type": "string", "minLength": 1, "maxLength": 4194304, - "description": "UTF-8 CSV contents, up to 4 MiB." + "description": "UTF-8 CSV contents, up to 4 MiB. Both files travel in one request body limited to 10 MiB after JSON encoding." } }, "required": ["filename", "content"], @@ -23949,7 +23949,7 @@ "type": "string", "minLength": 1, "maxLength": 4194304, - "description": "UTF-8 CSV contents, up to 4 MiB." + "description": "UTF-8 CSV contents, up to 4 MiB. Both files travel in one request body limited to 10 MiB after JSON encoding." } }, "required": ["filename", "content"], diff --git a/apps/sim/app/api/auth/oauth/token/route.test.ts b/apps/sim/app/api/auth/oauth/token/route.test.ts index f0784fcd428..9152d3bfa1d 100644 --- a/apps/sim/app/api/auth/oauth/token/route.test.ts +++ b/apps/sim/app/api/auth/oauth/token/route.test.ts @@ -7,6 +7,7 @@ import { createMockRequest, hybridAuthMockFns, } from '@sim/testing' +import { createRouteContext } from '@sim/testing/helpers/http' import { NextRequest } from 'next/server' import { beforeEach, describe, expect, it, vi } from 'vitest' @@ -36,11 +37,13 @@ vi.mock('@/lib/credentials/application/resolve-managed-oauth-token', () => ({ resolveManagedOAuthCredentialToken: { execute: mockResolveManagedOAuthCredentialToken }, })) +import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { createQuickBooksAccountId } from '@/lib/oauth/quickbooks' import { GET, POST } from '@/app/api/auth/oauth/token/route' import { getToolMetadata } from '@/tools/metadata' const mockGetToolMetadata = vi.mocked(getToolMetadata) +const mockCheckSessionOrInternalAuth = vi.mocked(checkSessionOrInternalAuth) describe('OAuth Token API Routes', () => { beforeEach(() => { @@ -290,6 +293,116 @@ describe('OAuth Token API Routes', () => { }) }) + describe('saved service-account credentials', () => { + const authorized = { + ok: true, + requesterUserId: 'test-user-id', + credentialOwnerUserId: 'test-user-id', + workspaceId: 'workspace-id', + } + + beforeEach(() => { + authOAuthUtilsMockFns.mockResolveOAuthAccountId.mockResolvedValueOnce({ + accountId: '', + credentialId: 'saved-credential-id', + credentialType: 'service_account', + providerId: 'vanta-service-account', + workspaceId: 'workspace-id', + usedCredentialTable: true, + }) + authOAuthUtilsMockFns.mockResolveServiceAccountToken.mockResolvedValue({ + accessToken: 'saved-secret', + }) + mockGetToolMetadata.mockReturnValue({ + id: 'vanta_list_tests', + params: {}, + oauth: { required: true, provider: 'vanta', credentialKind: 'service-account' }, + }) + }) + + it('refuses to release the secret to a session caller', async () => { + mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ + success: true, + authType: AuthType.SESSION, + userId: 'test-user-id', + }) + mockAuthorizeCredentialUse.mockResolvedValueOnce(authorized) + + const response = await POST( + createMockRequest('POST', { + credentialId: 'saved-credential-id', + toolId: 'vanta_list_tests', + }), + createRouteContext({}) + ) + + expect(response.status).toBe(403) + const body = await response.json() + expect(body).toMatchObject({ code: 'SERVICE_ACCOUNT_SESSION_FORBIDDEN' }) + expect(JSON.stringify(body)).not.toContain('saved-secret') + expect(authOAuthUtilsMockFns.mockResolveServiceAccountToken).not.toHaveBeenCalled() + }) + + it('resolves the secret for an internal-JWT execution caller', async () => { + mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ + success: true, + authType: AuthType.INTERNAL_JWT, + userId: 'test-user-id', + }) + mockAuthorizeCredentialUse.mockResolvedValueOnce(authorized) + + const response = await POST( + createMockRequest('POST', { + credentialId: 'saved-credential-id', + toolId: 'vanta_list_tests', + workflowId: 'workflow-id', + }), + createRouteContext({}) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + accessToken: 'saved-secret', + credentialType: 'service_account', + }) + }) + }) + + it('still returns an OAuth token to a session caller', async () => { + mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ + success: true, + authType: AuthType.SESSION, + userId: 'test-user-id', + }) + mockAuthorizeCredentialUse.mockResolvedValueOnce({ + ok: true, + requesterUserId: 'test-user-id', + credentialOwnerUserId: 'test-user-id', + }) + authOAuthUtilsMockFns.mockGetCredential.mockResolvedValueOnce({ + id: 'credential-id', + accessToken: 'test-token', + refreshToken: 'refresh-token', + accessTokenExpiresAt: new Date(Date.now() + 3600 * 1000), + providerId: 'google', + }) + authOAuthUtilsMockFns.mockRefreshTokenIfNeeded.mockResolvedValueOnce({ + accessToken: 'session-oauth-token', + refreshed: false, + }) + + const response = await POST( + createMockRequest('POST', { credentialId: 'credential-id' }), + createRouteContext({}) + ) + + expect(response.status).toBe(200) + await expect(response.json()).resolves.toMatchObject({ + accessToken: 'session-oauth-token', + credentialType: 'oauth', + }) + }) + describe('credentialAccountUserId + providerId path', () => { it('should reject internal JWT authentication', async () => { hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValueOnce({ diff --git a/apps/sim/lib/api/contracts/knowledge/gitlab-permissions.ts b/apps/sim/lib/api/contracts/knowledge/gitlab-permissions.ts index 6c254e2b4d2..6f3d390b14b 100644 --- a/apps/sim/lib/api/contracts/knowledge/gitlab-permissions.ts +++ b/apps/sim/lib/api/contracts/knowledge/gitlab-permissions.ts @@ -11,7 +11,9 @@ const gitLabCsvUploadSchema = z .string() .min(1) .max(GITLAB_CSV_MAX_BYTES) - .describe('UTF-8 CSV contents, up to 4 MiB.'), + .describe( + 'UTF-8 CSV contents, up to 4 MiB. Both files travel in one request body limited to 10 MiB after JSON encoding.' + ), }) .strict() export const gitLabPermissionInputSchema = z diff --git a/apps/sim/lib/core/network/environment-proxy.server.test.ts b/apps/sim/lib/core/network/environment-proxy.server.test.ts index f63b1ce0b8d..0082e3a4132 100644 --- a/apps/sim/lib/core/network/environment-proxy.server.test.ts +++ b/apps/sim/lib/core/network/environment-proxy.server.test.ts @@ -17,16 +17,17 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vites vi.mock('@/lib/core/network/config.server', () => networkConfigMock) vi.mock('@sim/security/dns', async (importOriginal) => { const actual = await importOriginal() - const records: Record = { - 'origin.invalid': '1.1.1.1', - 'internal.invalid': '10.0.0.5', + const records: Record = { + 'origin.invalid': ['1.1.1.1', '93.184.216.34'], + 'internal.invalid': ['10.0.0.5'], + 'split.invalid': ['93.184.216.34', '169.254.169.254'], } return { ...actual, resolveHostAddresses: async (host: string) => { - const address = records[host] - if (!address) throw new Error(`No addresses for ${host}`) - return { addresses: [address], preferred: address } + const addresses = records[host] + if (!addresses) throw new Error(`No addresses for ${host}`) + return { addresses, preferred: addresses[0] } }, } }) @@ -250,6 +251,23 @@ describe('guarded requests under HTTPS_PROXY', () => { expect(tunnels).toEqual([]) }) + it('refuses a name with any blocked record, since the proxy may dial any of them', async () => { + const guarded = createSsrfGuardedFetchWithDispatcher(options) + try { + await expect(guarded.fetch('https://split.invalid/')).rejects.toThrow(/metadata/i) + await expect(secureFetchWithValidation('https://split.invalid/', options)).rejects.toThrow( + /metadata/i + ) + } finally { + await guarded.dispatcher.destroy() + } + expect(tunnels).toEqual([]) + + const response = await secureFetchWithValidation('https://origin.invalid/', options) + expect(await response.text()).toBe('tls reached') + expect(tunnels.map((tunnel) => tunnel.authority)).toEqual(['origin.invalid:443']) + }) + it('judges each redirect hop again before proxying it', async () => { const pinned = createPinnedFetchWithDispatcher('1.1.1.1', options) try { diff --git a/apps/sim/lib/core/network/transport.server.ts b/apps/sim/lib/core/network/transport.server.ts index afea63b2d3e..cbe33094938 100644 --- a/apps/sim/lib/core/network/transport.server.ts +++ b/apps/sim/lib/core/network/transport.server.ts @@ -42,10 +42,10 @@ export function createOutboundTransport(options: OutboundTransportOptions): Outb ] /** * The proxy resolves the destination itself, so pinning cannot hold across it. Every request - * (each redirect hop included) is resolved and judged here first, so a name that points at - * a refused address is stopped before the proxy is asked to reach it. A name that changes - * its answer between this check and the proxy's own lookup is left to the proxy's egress - * policy, which is why the self-hosting docs make the proxy the final egress control. + * (each redirect hop included) is resolved and judged here first, and a name is refused when + * any of its records is refused, since the proxy may connect to any of them. A name that + * changes its answer between this check and the proxy's own lookup is left to the proxy's + * egress policy, which is why the self-hosting docs make the proxy the final egress control. */ const selectEnvironmentProxy = async ( target: string, @@ -56,6 +56,7 @@ export function createOutboundTransport(options: OutboundTransportOptions): Outb const validation = await validateEgressUrl(target, 'url', options.profile, { logDetails: false, signal, + requireEveryAddress: true, }) if (!validation.isValid) throw new Error(validation.error, { cause: validation.cause }) if (closed) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') diff --git a/apps/sim/lib/core/security/egress/validate.ts b/apps/sim/lib/core/security/egress/validate.ts index 54dfde0b72c..afc17ddc680 100644 --- a/apps/sim/lib/core/security/egress/validate.ts +++ b/apps/sim/lib/core/security/egress/validate.ts @@ -49,6 +49,8 @@ export interface EgressValidationOptions { logDetails?: boolean /** Cancels DNS validation before a guarded connection can begin. */ signal?: AbortSignal + /** Refuse when any record is refused; for transports that cannot pin (a forward proxy re-resolves). */ + requireEveryAddress?: boolean } type EgressDenial = Extract @@ -133,7 +135,7 @@ export async function validateEgressUrl( // Refused records are filtered rather than failing the whole host: pinning to a // surviving permitted address is just as safe as refusing outright, and a // split-horizon resolver that answers with a private record alongside the - // public one would otherwise be unusable. + // public one would otherwise be unusable. A caller that cannot pin opts out. let refusal: EgressDenial | undefined const usable = addresses.filter((address) => { const decision = evaluateAddress(parsed, address, policy) @@ -142,6 +144,8 @@ export async function validateEgressUrl( return false }) + if (refusal && options.requireEveryAddress) return fail(refusal, paramName, profile, options) + if (usable.length === 0) { return fail( refusal ?? { allowed: false, reason: 'address-blocked', detail: 'no addresses resolved' }, diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/http.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/http.test.ts index 5264ae1c622..b76b6ca4d01 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/http.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/http.test.ts @@ -9,6 +9,7 @@ vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock import { normalizeHttpBaseUrl, + refuseUnsafeTypedUrl, tokenCheckValidator, } from '@/lib/credentials/token-service-accounts/validators/http' @@ -159,8 +160,11 @@ describe('tokenCheckValidator with a typed host', () => { ) }) - it('refuses a base URL with a mistyped scheme instead of reading the scheme as a host', () => { + it('refuses a mistyped scheme instead of reading the scheme as a host', () => { expect(() => normalizeHttpBaseUrl('https:/grafana.example.com')).toThrow(/scheme/) + expect(() => refuseUnsafeTypedUrl('https:/evil.com')).toThrow(/scheme/) + expect(() => refuseUnsafeTypedUrl('HTTP:evil.com')).toThrow(/scheme/) + expect(refuseUnsafeTypedUrl('https://grafana.example.com')).toBe('https://grafana.example.com') expect(normalizeHttpBaseUrl('grafana.example.com:3000/')).toBe( 'https://grafana.example.com:3000' ) diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/http.ts b/apps/sim/lib/credentials/token-service-accounts/validators/http.ts index 0cb6143b455..0474ab705ad 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/http.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/http.ts @@ -121,6 +121,7 @@ export function normalizeHttpBaseUrl(raw: string | undefined): string { */ export function refuseUnsafeTypedUrl(raw: string | undefined): string { const typed = (raw ?? '').trim() + if (/^https?:(?!\/\/)/i.test(typed)) throw new Error('malformed scheme') if (/^(?!https?:\/\/)[a-z][a-z\d+.-]*:\/\//i.test(typed)) throw new Error('http(s) only') const authority = typed.replace(/^[a-z][a-z\d+.-]*:\/\//i, '').split(/[/?#]/)[0] if (authority.includes('@')) throw new Error('credentials in URL') diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/token-checks.ts b/apps/sim/lib/credentials/token-service-accounts/validators/token-checks.ts index 761003eb4e6..a6c30664cdd 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/token-checks.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/token-checks.ts @@ -420,7 +420,7 @@ export const validateDubServiceAccount = tokenCheckValidator({ */ export const validateDynatraceServiceAccount = tokenCheckValidator({ serviceLabel: 'Dynatrace', - domain: ({ domain }) => normalizeHttpBaseUrl(domain).replace(/\/api\/v[12]$/, ''), + domain: ({ domain }) => normalizeHttpBaseUrl(domain).replace(/\/api\/v[12]$/i, ''), egress: 'configuredEndpoint', request: ({ apiToken, domain = '' }) => ({ url: buildDynatraceUrl(domain, '/apiTokens/lookup'), diff --git a/apps/sim/lib/credentials/workload-identity/issuer.test.ts b/apps/sim/lib/credentials/workload-identity/issuer.test.ts index 68b7777731e..9428fb87ed4 100644 --- a/apps/sim/lib/credentials/workload-identity/issuer.test.ts +++ b/apps/sim/lib/credentials/workload-identity/issuer.test.ts @@ -70,6 +70,22 @@ describe('workload identity issuer', () => { ).rejects.toThrow(/an hour/) }) + it.each([ + ['another workspace', 'sim:workspace:ws-2:credential:abc123def456'], + ['an organization', 'sim:organization:ws-1:credential:abc123def456'], + ['a lookalike workspace id', 'sim:workspace:ws-10:credential:abc123def456'], + ])('refuses to mint a subject owned by %s', async (_owner, subject) => { + const issuer = await loadIssuer() + + await expect( + issuer.mintWorkloadIdentityToken({ + subject, + audience: 'sts.amazonaws.com', + owner: { workspaceId: 'ws-1' }, + }) + ).rejects.toThrow(/another owner/) + }) + it('rejects a token presented to a different audience', async () => { const issuer = await loadIssuer() const token = await issuer.mintWorkloadIdentityToken({ diff --git a/apps/sim/lib/credentials/workload-identity/issuer.ts b/apps/sim/lib/credentials/workload-identity/issuer.ts index 0c77debcc8d..d658467be05 100644 --- a/apps/sim/lib/credentials/workload-identity/issuer.ts +++ b/apps/sim/lib/credentials/workload-identity/issuer.ts @@ -4,6 +4,7 @@ import { generateId } from '@sim/utils/id' import { calculateJwkThumbprint, type JWK, SignJWT } from 'jose' import { env } from '@/lib/core/config/env' import { getBaseUrl } from '@/lib/core/utils/urls' +import { assertWorkloadIdentitySubjectOwner } from '@/lib/credentials/workload-identity/subject' /** * Sim's workload-identity issuer: a minimal OpenID Connect token issuer that @@ -194,6 +195,7 @@ export interface MintWorkloadIdentityTokenInput { export async function mintWorkloadIdentityToken( input: MintWorkloadIdentityTokenInput ): Promise { + assertWorkloadIdentitySubjectOwner(input.subject, input.owner) const ttl = input.ttlSeconds ?? WORKLOAD_IDENTITY_TOKEN_TTL_SECONDS if (!Number.isInteger(ttl) || ttl <= 0 || ttl > MAX_WORKLOAD_IDENTITY_TOKEN_TTL_SECONDS) { throw new Error('Workload identity token lifetime must be between 1 second and an hour') diff --git a/apps/sim/lib/credentials/workload-identity/subject.ts b/apps/sim/lib/credentials/workload-identity/subject.ts index 6a0fed7df19..ceaf840f7b1 100644 --- a/apps/sim/lib/credentials/workload-identity/subject.ts +++ b/apps/sim/lib/credentials/workload-identity/subject.ts @@ -40,12 +40,26 @@ function ownerSegment(owner: WorkloadIdentityOwner): string { throw new WorkloadIdentitySubjectError('A workload identity subject needs exactly one owner') } +function subjectPrefix(owner: WorkloadIdentityOwner): string { + return `sim:${ownerSegment(owner)}:credential:` +} + +/** Throws unless the subject was built for this owner. */ +export function assertWorkloadIdentitySubjectOwner( + subject: string, + owner: WorkloadIdentityOwner +): void { + if (!subject.startsWith(subjectPrefix(owner))) { + throw new WorkloadIdentitySubjectError('Workload identity subject belongs to another owner') + } +} + /** `sim:workspace::credential:` (or `sim:organization:…`). */ function buildWorkloadIdentitySubject(owner: WorkloadIdentityOwner, subjectId: string): string { if (!SUBJECT_ID_PATTERN.test(subjectId)) { throw new WorkloadIdentitySubjectError('Invalid workload identity subject') } - return `sim:${ownerSegment(owner)}:credential:${subjectId}` + return `${subjectPrefix(owner)}${subjectId}` } function proofKey(): Buffer { @@ -114,7 +128,7 @@ export function verifyWorkloadIdentitySubject( const expiresAt = Number(subjectProof.slice(0, separator)) const signature = subjectProof.slice(separator + 1) if (!Number.isSafeInteger(expiresAt) || expiresAt <= now) throw invalid() - if (!subject.startsWith(`sim:${ownerSegment(owner)}:credential:`)) throw invalid() + if (!subject.startsWith(subjectPrefix(owner))) throw invalid() if (!safeCompare(signature, sign(provider, owner, userId, subject, expiresAt))) throw invalid() } diff --git a/apps/sim/lib/oauth/token-resolution.ts b/apps/sim/lib/oauth/token-resolution.ts index ded642770db..f044f954876 100644 --- a/apps/sim/lib/oauth/token-resolution.ts +++ b/apps/sim/lib/oauth/token-resolution.ts @@ -7,7 +7,7 @@ import { type OAuthTokenResponse, } from '@/lib/api/contracts/oauth-connections' import { authorizeCredentialUseForAuth } from '@/lib/auth/credential-access' -import type { AuthResult } from '@/lib/auth/hybrid' +import { type AuthResult, AuthType } from '@/lib/auth/hybrid' import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { InvalidManagedOAuthDelegationError } from '@/lib/credentials/application/managed-oauth-delegation' import { resolveManagedOAuthCredentialToken } from '@/lib/credentials/application/resolve-managed-oauth-token' @@ -356,6 +356,15 @@ export async function resolveCredentialToken( if (!authz.ok) { return { ok: false, status: 403, error: authz.error || 'Unauthorized' } } + // A saved secret never leaves the server; only in-process execution may resolve it. + if (auth.authType === AuthType.SESSION) { + return { + ok: false, + status: 403, + code: 'SERVICE_ACCOUNT_SESSION_FORBIDDEN', + error: 'Saved credentials are resolved only during server-side execution', + } + } // A saved secret is released only to a tool of its own provider, never on a bare ID. if (!input.toolId) { diff --git a/apps/sim/lib/workflows/application/deployments.ts b/apps/sim/lib/workflows/application/deployments.ts index 219813bcdea..0455473b80b 100644 --- a/apps/sim/lib/workflows/application/deployments.ts +++ b/apps/sim/lib/workflows/application/deployments.ts @@ -5,7 +5,6 @@ import { toPrincipalActor, } from '@sim/auth/principal' import { createLogger } from '@sim/logger' -import { assertWorkflowMutable, WorkflowLockedError } from '@sim/platform-authz/workflow' import { getErrorMessage } from '@sim/utils/errors' import { OrchestrationError, type OrchestrationErrorCode } from '@/lib/core/orchestration/types' import { withinDeadline } from '@/lib/core/utils/deadline' @@ -16,6 +15,7 @@ import { requireWorkflowExecutionUserId } from '@/lib/workflows/application/auth import { defineAuthorizedWorkflowUseCase } from '@/lib/workflows/application/authorized-workflow-use-case' import { withWorkflowBlockScope } from '@/lib/workflows/application/block-scope' import type { ActiveWorkflowApplicationContext } from '@/lib/workflows/application/context' +import { requireMutableWorkflow } from '@/lib/workflows/application/mutability' import { workflowOperations } from '@/lib/workflows/application/operations' import { resolvePrincipalWorkflowContext } from '@/lib/workflows/application/principal-scope' import { checkNeedsRedeployment } from '@/lib/workflows/deployment-status' @@ -97,17 +97,6 @@ function throwDeploymentFailure( throw new OrchestrationError(result.errorCode, result.error ?? fallback) } -async function requireMutableWorkflow(workflowId: string): Promise { - try { - await assertWorkflowMutable(workflowId) - } catch (error) { - if (error instanceof WorkflowLockedError) { - throw new OrchestrationError('locked', error.message) - } - throw error - } -} - /** * How long a deploy waits for its lint. The lint reads credentials, tools, and * table schemas, so a slow lookup would otherwise hold a deploy that has diff --git a/apps/sim/tools/dynatrace/utils.ts b/apps/sim/tools/dynatrace/utils.ts index 413e5941c74..2536992f752 100644 --- a/apps/sim/tools/dynatrace/utils.ts +++ b/apps/sim/tools/dynatrace/utils.ts @@ -35,7 +35,7 @@ function normalizeEnvironmentUrl(environmentUrl: string): string { return environmentUrl .trim() .replace(/\/+$/, '') - .replace(/\/api\/v[12]$/, '') + .replace(/\/api\/v[12]$/i, '') } function buildUrl( diff --git a/apps/sim/tools/twilio_voice/list_calls.test.ts b/apps/sim/tools/twilio_voice/list_calls.test.ts new file mode 100644 index 00000000000..08547920cd7 --- /dev/null +++ b/apps/sim/tools/twilio_voice/list_calls.test.ts @@ -0,0 +1,97 @@ +import { jsonResponse } from '@sim/testing/helpers/http' +import { setupGlobalFetchMock } from '@sim/testing/mocks/fetch.mock' +import { beforeEach, describe, expect, it } from 'vitest' +import { listCallsTool } from '@/tools/twilio_voice/list_calls' + +let fetchMock: ReturnType + +function callsPage(recordingsPaths: string[]) { + return jsonResponse({ + calls: recordingsPaths.map((recordings, index) => ({ + sid: `CA${index}`, + from: '+15550000000', + to: '+15550000001', + status: 'completed', + direction: 'outbound-api', + duration: '12', + price: null, + price_unit: 'USD', + start_time: 'Mon, 09 Nov 2026 10:00:00 +0000', + end_time: null, + date_created: 'Mon, 09 Nov 2026 10:00:00 +0000', + subresource_uris: { recordings }, + })), + page: 0, + page_size: 50, + }) +} + +async function listCalls(recordingsPaths: string[], signal?: AbortSignal) { + if (!listCallsTool.transformResponse) throw new Error('Missing response transform') + return listCallsTool.transformResponse( + callsPage(recordingsPaths), + { accountSid: 'AC123', authToken: 'secret' }, + { signal } + ) +} + +describe('Twilio list calls recordings lookup', () => { + beforeEach(() => { + fetchMock = setupGlobalFetchMock() + fetchMock.mockImplementation(async () => jsonResponse({ recordings: [{ sid: 'RE1' }] })) + }) + + it.each(['@evil.test/x', '//evil.test/x', 'https://evil.test/x'])( + 'never sends the saved credentials off the Twilio API origin for %s', + async (path) => { + await listCalls([path]) + for (const [url] of fetchMock.mock.calls) { + expect(new URL(String(url)).origin).toBe('https://api.twilio.com') + } + } + ) + + it('fetches recordings for Twilio-relative paths', async () => { + const result = await listCalls(['/2010-04-01/Accounts/AC123/Calls/CA0/Recordings.json']) + expect(fetchMock).toHaveBeenCalledWith( + 'https://api.twilio.com/2010-04-01/Accounts/AC123/Calls/CA0/Recordings.json', + expect.anything() + ) + expect(result.output.calls?.[0]?.recordingSids).toEqual(['RE1']) + }) + + it('keeps at most ten recordings requests in flight', async () => { + let inFlight = 0 + let peak = 0 + fetchMock.mockImplementation(async () => { + inFlight++ + peak = Math.max(peak, inFlight) + await Promise.resolve() + inFlight-- + return jsonResponse({ recordings: [] }) + }) + + await listCalls(Array.from({ length: 35 }, (_, i) => `/Calls/CA${i}/Recordings.json`)) + + expect(fetchMock).toHaveBeenCalledTimes(35) + expect(peak).toBe(10) + }) + + it('starts no queued recordings requests once the run is cancelled', async () => { + const controller = new AbortController() + fetchMock.mockImplementation(async (_url: unknown, init?: RequestInit) => { + controller.abort() + expect(init?.signal).toBe(controller.signal) + return jsonResponse({ recordings: [] }) + }) + + await expect( + listCalls( + Array.from({ length: 35 }, (_, i) => `/Calls/CA${i}/Recordings.json`), + controller.signal + ) + ).rejects.toThrow() + + expect(fetchMock.mock.calls.length).toBeLessThanOrEqual(10) + }) +}) diff --git a/apps/sim/tools/twilio_voice/list_calls.ts b/apps/sim/tools/twilio_voice/list_calls.ts index 30e37ae624b..b1a6db6f353 100644 --- a/apps/sim/tools/twilio_voice/list_calls.ts +++ b/apps/sim/tools/twilio_voice/list_calls.ts @@ -1,10 +1,43 @@ import { createLogger } from '@sim/logger' +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { TWILIO_CREDENTIAL_OAUTH } from '@/tools/shared/saved-credentials' import type { TwilioListCallsOutput, TwilioListCallsParams } from '@/tools/twilio_voice/types' -import type { ToolConfig } from '@/tools/types' +import type { ToolConfig, ToolResponseContext } from '@/tools/types' const logger = createLogger('TwilioVoiceListCallsTool') +const TWILIO_API_ORIGIN = 'https://api.twilio.com' +const RECORDINGS_FETCH_CONCURRENCY = 10 + +interface TwilioCallRecord { + sid: string + from: string + to: string + status: string + direction: string + duration: string | null + price: string | null + price_unit: string + start_time: string + end_time: string | null + date_created: string + subresource_uris?: { recordings?: string } +} + +/** + * Resolves a Twilio-returned path on the Twilio API origin. The saved Basic auth + * rides along, so any other origin is skipped. + */ +function twilioApiUrl(path: string | undefined): string | null { + if (!path) return null + try { + const url = new URL(path, TWILIO_API_ORIGIN) + return url.origin === TWILIO_API_ORIGIN ? url.href : null + } catch { + return null + } +} + export const listCallsTool: ToolConfig = { id: 'twilio_voice_list_calls', name: 'Twilio Voice List Calls', @@ -100,7 +133,7 @@ export const listCallsTool: ToolConfig { + transformResponse: async (response, params, context?: ToolResponseContext) => { const data = await response.json() logger.info('Twilio List Calls Response:', { total: data.calls?.length || 0 }) @@ -119,23 +152,29 @@ export const listCallsTool: ToolConfig { + const calls = await mapWithConcurrency( + (data.calls || []) as TwilioCallRecord[], + RECORDINGS_FETCH_CONCURRENCY, + async (call) => { + context?.signal?.throwIfAborted() let recordingSids: string[] = [] - if (call.subresource_uris?.recordings) { + const recordingsUrl = twilioApiUrl(call.subresource_uris?.recordings) + if (recordingsUrl) { try { - const recordingsUrl = `https://api.twilio.com${call.subresource_uris.recordings}` const recordingsResponse = await fetch(recordingsUrl, { method: 'GET', headers: { Authorization: `Basic ${authToken}` }, + signal: context?.signal, }) if (recordingsResponse.ok) { - const recordingsData = await recordingsResponse.json() - recordingSids = (recordingsData.recordings || []).map((rec: any) => rec.sid) + const recordingsData: { recordings?: Array<{ sid: string }> } = + await recordingsResponse.json() + recordingSids = (recordingsData.recordings || []).map((rec) => rec.sid) } } catch (error) { - logger.warn(`Failed to fetch recordings for call ${call.sid}:`, error) + context?.signal?.throwIfAborted() + logger.warn('Failed to fetch recordings for call', { callSid: call.sid, error }) } } @@ -153,7 +192,7 @@ export const listCallsTool: ToolConfig=1.25.0-0" home: https://sim.ai diff --git a/helm/sim/README.md b/helm/sim/README.md index 5c24b0ee3c5..a534b3a84e3 100644 --- a/helm/sim/README.md +++ b/helm/sim/README.md @@ -340,7 +340,7 @@ Before installing in production, confirm each of the following: matchLabels: kubernetes.io/metadata.name: ingress-nginx ``` -* **Namespace hardening** — label the install namespace with Pod Security Standards `restricted` enforcement (`pod-security.kubernetes.io/enforce=restricted`). All workloads set `runAsNonRoot`, drop all Linux capabilities, disable privilege escalation, and set `seccompProfile: RuntimeDefault` — the four controls the Restricted profile requires. `readOnlyRootFilesystem` is intentionally **not** defaulted anywhere (Postgres/Ollama genuinely need a writable root; the stateless services — `realtime`, `pii`, `copilot` — could tolerate it but aren't pre-wired with a `/tmp` `emptyDir`). If your policy requires it, set `.securityContext.readOnlyRootFilesystem: true` and mount an `emptyDir` at `/tmp` yourself via `extraVolumes`/`extraVolumeMounts`. +* **Namespace hardening** — label the install namespace with Pod Security Standards `restricted` enforcement (`pod-security.kubernetes.io/enforce=restricted`). All workloads set `runAsNonRoot`, drop all Linux capabilities, disable privilege escalation, and set `seccompProfile: RuntimeDefault` — the four controls the Restricted profile requires. `readOnlyRootFilesystem` is intentionally **not** defaulted anywhere (Postgres/Ollama genuinely need a writable root; the stateless services — `realtime`, `pii`, `copilot` — could tolerate it but aren't pre-wired with a `/tmp` `emptyDir`). If your policy requires it, set `.securityContext.readOnlyRootFilesystem: true` and mount writable `emptyDir` volumes yourself via `extraVolumes`/`extraVolumeMounts`: `/tmp` for every component, plus `/app/apps/sim/.next/cache` and `/app/apps/sim/.next/server/route-cache` for `app`. * **Env validation** — keys under `app.env`, `realtime.env`, and `copilot.server.env` are passed through to the application and validated at startup. The JSON Schema intentionally does not enforce `additionalProperties: false` (would break custom user envs), so typos like `OPENA_API_KEY` (instead of `OPENAI_API_KEY`) surface as missing-key errors at runtime, not at `helm install` time. Review your env block carefully. * **Set public URLs** — `app.env.NEXT_PUBLIC_APP_URL` and `app.env.BETTER_AUTH_URL` must match your public origin (e.g. `https://sim.example.com`). Leaving them as `localhost` breaks sign-in. diff --git a/helm/sim/templates/_helpers.tpl b/helm/sim/templates/_helpers.tpl index fa54c75df87..c106dcbcbcc 100644 --- a/helm/sim/templates/_helpers.tpl +++ b/helm/sim/templates/_helpers.tpl @@ -540,7 +540,7 @@ securityContext: Container-level security context with Pod Security Standards "restricted" defaults. User-supplied `.securityContext` values override defaults (user wins). `readOnlyRootFilesystem` is intentionally NOT defaulted — set it per-workload in values -when the container can tolerate a read-only root (Next.js writes to `.next/cache`, +when the container can tolerate a read-only root (Next.js writes to `.next/cache` and `.next/server/route-cache`, Postgres writes to `/var/lib/postgresql/data`, so they're left writable by default). Usage: {{ include "sim.containerSecurityContext" .Values.app | nindent 10 }} */}} diff --git a/packages/sim-setup/README.md b/packages/sim-setup/README.md index 944115c75cc..97d576bef2b 100644 --- a/packages/sim-setup/README.md +++ b/packages/sim-setup/README.md @@ -17,7 +17,7 @@ other resources in a running Sim workspace, use the separate - Node.js 20 or newer - Docker with the Compose plugin -- Ports `3000`, `3002`, and, by default, `5432` available +- Ports `3000` and `3002` available - At least 12 GB of system memory and 20 GB of free disk for a small installation Allocate at least 8 GB of memory to Docker for reliable workflow execution. @@ -126,7 +126,7 @@ specific installation. | `npx sim-setup restart` | Restart the current containers | | `npx sim-setup update` | Refresh the managed Compose file, pull images, recreate services, and run migrations | | `npx sim-setup down` | Remove containers but keep data volumes | -| `npx sim-setup reset` | Archive `.env`, remove containers, and delete managed data volumes | +| `npx sim-setup reset` | Remove containers and delete managed data volumes, then archive `.env` | `down` and `reset` are deliberately different. Use `down` when you want to remove containers and bring the same installation back later. `reset` is a @@ -225,9 +225,9 @@ PostgreSQL data lives in a Docker volume. These commands preserve it: - `update` - `down` -`reset` deletes managed volumes. It archives `.env` beside the installation -before doing so, but that local archive is not a substitute for an off-machine -backup. +`reset` deletes managed volumes first, then archives `.env` beside the +installation. If Docker is not running, `reset` stops without changing anything. +The local archive is not a substitute for an off-machine backup. The default local file-storage fallback is different: uploaded files live inside the application container, not in a managed volume. They can be lost when the @@ -267,6 +267,11 @@ bun run sim-setup --mode k8s Standalone npm use supports Docker Compose mode only. Development and Kubernetes modes need source-only files and are rejected outside a complete checkout. +Dev mode publishes the managed Postgres and Redis containers on `127.0.0.1` +only. Containers created by an earlier version publish on every network +interface; run `bun run sim-setup down` and then re-run setup to rebind them. +Your data is kept. + ## Help ```bash diff --git a/packages/sim-setup/src/lifecycle.test.ts b/packages/sim-setup/src/lifecycle.test.ts index 46e104a9b3f..0bccc8a85ad 100644 --- a/packages/sim-setup/src/lifecycle.test.ts +++ b/packages/sim-setup/src/lifecycle.test.ts @@ -1,16 +1,66 @@ -import { mkdtempSync, rmSync } from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import path from 'node:path' -import { describe, expect, it } from 'vitest' +import { afterAll, describe, expect, it, vi } from 'vitest' + +const { fakeRoot, mockSpawnSync, mockConfirm } = vi.hoisted(() => { + const { mkdtempSync } = require('node:fs') as typeof import('node:fs') + const { tmpdir } = require('node:os') as typeof import('node:os') + const { join } = require('node:path') as typeof import('node:path') + return { + fakeRoot: mkdtempSync(join(tmpdir(), 'sim-setup-reset-')), + mockSpawnSync: vi.fn(), + mockConfirm: vi.fn(), + } +}) + +vi.mock('node:child_process', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, spawnSync: mockSpawnSync } +}) + +vi.mock('./context', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, SETUP_CONTEXT: { kind: 'source', root: fakeRoot } } +}) + +vi.mock('./prompter', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, confirm: mockConfirm } +}) + import { ensureProductionComposeFile } from './compose-asset' +import { SetupError } from './errors' import { composeInstallFromDirectory, composeServiceQuery, composeServiceState, type Install, + runLifecycle, serviceStatusRows, } from './lifecycle' +afterAll(() => { + rmSync(fakeRoot, { recursive: true, force: true }) +}) + +describe('reset with Docker installed but not running', () => { + it('refuses before archiving the env files the surviving volumes still need', async () => { + // The docker CLI resolves, but every daemon query fails, as with Docker Desktop stopped. + mockSpawnSync.mockReturnValue({ status: 1, stdout: '', stderr: 'daemon not running' }) + mockConfirm.mockResolvedValue(true) + const envFile = path.join(fakeRoot, 'apps/sim/.env') + mkdirSync(path.dirname(envFile), { recursive: true }) + writeFileSync(envFile, 'DATABASE_URL=postgresql://localhost/sim\n') + + await expect(runLifecycle('reset')).rejects.toBeInstanceOf(SetupError) + + expect(existsSync(envFile)).toBe(true) + expect(readdirSync(path.dirname(envFile))).toEqual(['.env']) + expect(mockConfirm).not.toHaveBeenCalled() + }) +}) + describe('setup lifecycle', () => { it('does not duplicate a running install restored from its directory', () => { const root = mkdtempSync(path.join(tmpdir(), 'sim-setup-lifecycle-')) diff --git a/packages/sim-setup/src/lifecycle.ts b/packages/sim-setup/src/lifecycle.ts index 3e38df95c7f..265d00d05ab 100644 --- a/packages/sim-setup/src/lifecycle.ts +++ b/packages/sim-setup/src/lifecycle.ts @@ -13,6 +13,7 @@ import { directoryOverride, resolveSetupContextAtRoot, SETUP_CONTEXT } from './c import { DB_CONTAINER, type Detection, REDIS_CONTAINER, runDetection } from './detect' import { archiveEnvFile, archiveFile, parseEnv, ROOT, upsertEnv, writeEnvFile } from './env-files' import { SetupError } from './errors' +import { executableExists } from './executables' import { forwardCommands, isLocalKubeContext } from './modes/k8s' import { httpHealth } from './probes' import * as p from './prompter' @@ -591,6 +592,14 @@ async function reset(install: Install | null): Promise { // cluster (or compose file / dev containers) is about to be wiped keeps a reset // from silently hitting the wrong same-named install after a context switch. const target = install ? ` ${describeInstall(install)} will be removed.` : '' + // With the daemon down nothing managed is visible, so a reset would archive the + // env files while the volumes they unlock survive. + if (install?.kind !== 'k8s' && executableExists('docker') && !dockerReachable()) { + throw new SetupError( + "Docker isn't reachable, so the managed containers and volumes can't be removed. Your env files were left in place.", + [`start Docker, then re-run ${theme.command('npx sim-setup reset')}`] + ) + } const ok = await p.confirm({ message: theme.error( `Reset archives your .env files and wipes managed data (volumes).${target} Continue?` diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index 375205acd29..47e93d1bdbc 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -950,7 +950,6 @@ "apps/sim/tools/thrive/update_audience.ts": 1, "apps/sim/tools/thrive/update_user.ts": 1, "apps/sim/tools/thrive/utils.ts": 2, - "apps/sim/tools/twilio_voice/list_calls.ts": 2, "apps/sim/tools/twilio_voice/make_call.ts": 2, "apps/sim/tools/typeform/create_form.ts": 1, "apps/sim/tools/typeform/types.ts": 20,