From 7169f79250c7bc1fedc87ef4a402cc2e6d01b9ca Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 12 Sep 2026 02:56:56 -0700 Subject: [PATCH 1/2] feat(network): add organization outbound routing --- .../docs/platform/enterprise/index.mdx | 1 + .../docs/platform/enterprise/meta.json | 1 + .../docs/platform/enterprise/security.mdx | 24 + .../platform/enterprise/session-policies.mdx | 2 +- .../docs/platform/self-hosting/networking.mdx | 6 +- apps/docs/content/docs/search/gitlab.mdx | 2 + apps/sim/app/api/auth/oauth/utils.test.ts | 5 +- .../cron/renew-subscriptions/route.test.ts | 197 +++- .../app/api/cron/renew-subscriptions/route.ts | 169 +-- .../app/api/knowledge/search/utils.test.ts | 6 + apps/sim/app/api/knowledge/utils.test.ts | 6 + .../api/organizations/[id]/network/route.ts | 24 + .../embeddings/models/route.test.ts | 15 +- .../knowledge/search/route.provenance.test.ts | 20 +- .../settings/[section]/settings.tsx | 10 +- .../settings/navigation.test.ts | 4 +- .../o/[organizationId]/settings/navigation.ts | 8 +- .../settings/[section]/page.test.tsx | 2 +- .../settings/[section]/settings.tsx | 14 +- .../settings-empty-state.tsx | 2 +- .../[workspaceId]/settings/navigation.test.ts | 5 +- .../[workspaceId]/settings/navigation.ts | 1 + .../components/settings/navigation.test.ts | 8 +- apps/sim/components/settings/navigation.ts | 23 +- .../settings/organization-security.tsx | 82 ++ apps/sim/connectors/asana/asana.test.ts | 4 + apps/sim/connectors/ashby/ashby.test.ts | 4 + .../connectors/bitbucket/bitbucket.test.ts | 4 + .../connectors/confluence/bulk-spaces.test.ts | 4 + .../connectors/confluence/confluence.test.ts | 7 + .../connectors/confluence/permissions.test.ts | 4 + .../connectors/confluence/preflight.test.ts | 4 + .../connectors/databricks/databricks.test.ts | 4 + apps/sim/connectors/databricks/databricks.ts | 7 +- .../connectors/fireflies/fireflies.test.ts | 3 + apps/sim/connectors/fireflies/fireflies.ts | 7 +- apps/sim/connectors/github/github.test.ts | 4 + apps/sim/connectors/github/pacing.test.ts | 4 + apps/sim/connectors/github/request.test.ts | 3 + apps/sim/connectors/github/request.ts | 7 +- .../google-calendar/company-crawl.test.ts | 4 + .../google-calendar/google-calendar.test.ts | 4 + .../google-chat/google-chat.test.ts | 4 + .../google-docs/google-docs.test.ts | 4 + .../google-drive/company-crawl.test.ts | 4 + .../connectors/google-drive/directory.test.ts | 4 + .../google-drive/google-drive-errors.ts | 7 +- .../google-drive/google-drive.test.ts | 4 + .../connectors/google-drive/shortcuts.test.ts | 4 + .../google-drive/workspace-drives.test.ts | 4 + .../google-meet/google-meet.test.ts | 4 + .../google-sheets/google-sheets.test.ts | 4 + .../google-workspace/company-crawl.test.ts | 4 + .../connectors/greenhouse/greenhouse.test.ts | 4 + .../connectors/incidentio/incidentio.test.ts | 4 + apps/sim/connectors/intercom/intercom.test.ts | 4 + apps/sim/connectors/jira/jira.test.ts | 7 +- apps/sim/connectors/outlook/outlook.test.ts | 4 + apps/sim/connectors/reddit/reddit.test.ts | 4 + .../connectors/servicenow/servicenow.test.ts | 4 + apps/sim/connectors/servicenow/servicenow.ts | 9 +- apps/sim/connectors/slack/slack.test.ts | 4 + apps/sim/connectors/typeform/typeform.test.ts | 4 + apps/sim/connectors/webflow/webflow.test.ts | 4 + .../connectors/wordpress/wordpress.test.ts | 4 + apps/sim/connectors/workday/workday.test.ts | 4 + apps/sim/connectors/workday/workday.ts | 9 +- apps/sim/connectors/youtube/youtube.test.ts | 4 + .../workflow/workflow-handler.test.ts | 36 +- .../handlers/workflow/workflow-handler.ts | 6 +- .../sim/hooks/queries/organization-network.ts | 22 + .../lib/api/contracts/organization-network.ts | 23 + apps/sim/lib/atlassian/assistant.test.ts | 6 + apps/sim/lib/atlassian/discovery.test.ts | 7 +- apps/sim/lib/atlassian/discovery.ts | 5 +- .../lib/copilot/generated/docs-manifest.ts | 1 + .../authorized-workspace-use-case.ts | 35 +- apps/sim/lib/core/config/appconfig.test.ts | 154 ++- apps/sim/lib/core/config/appconfig.ts | 128 ++- apps/sim/lib/core/config/env.ts | 4 + .../read-organization-network.test.ts | 107 ++ .../application/read-organization-network.ts | 52 + .../lib/core/network/aws-handler.server.ts | 61 + .../lib/core/network/config.server.test.ts | 119 ++ apps/sim/lib/core/network/config.server.ts | 227 ++++ .../lib/core/network/context.server.test.ts | 43 + apps/sim/lib/core/network/context.server.ts | 37 + .../network/environment-proxy.server.test.ts | 6 + .../fixtures/gateway-runtime.fixture.ts | 73 ++ apps/sim/lib/core/network/gateway-metadata.ts | 6 + .../lib/core/network/gateway.server.test.ts | 365 ++++++ apps/sim/lib/core/network/gateway.server.ts | 78 +- .../network/resource-scope.server.test.ts | 86 ++ .../lib/core/network/resource-scope.server.ts | 18 + apps/sim/lib/core/network/routing.test.ts | 48 + apps/sim/lib/core/network/routing.ts | 68 ++ .../lib/core/network/transport.server.test.ts | 147 +++ apps/sim/lib/core/network/transport.server.ts | 49 +- .../core/security/input-validation.server.ts | 2 + .../application/public-enrollment.test.ts | 43 + .../application/public-enrollment.ts | 3 +- .../lib/credential-groups/provider-adapter.ts | 2 +- .../slack-managed-users.test.ts | 10 +- .../credential-groups/slack-managed-users.ts | 11 +- .../standard-oauth-provider.test.ts | 4 + .../standard-oauth-provider.ts | 9 +- .../organization-credentials.test.ts | 43 + .../application/organization-credentials.ts | 10 +- .../atlassian-service-account.test.ts | 3 + .../credentials/atlassian-service-account.ts | 9 +- .../minters/box.test.ts | 4 + .../minters/netsuite.test.ts | 4 + .../minters/netsuite.ts | 7 +- .../minters/salesforce.test.ts | 4 + .../minters/zoho-desk.test.ts | 4 + .../minters/zoom.test.ts | 4 + .../token-service-accounts/errors.test.ts | 3 + .../token-service-accounts/errors.ts | 10 +- .../validators/airtable.test.ts | 4 + .../validators/asana.test.ts | 4 + .../validators/attio.test.ts | 4 + .../validators/calcom.test.ts | 4 + .../validators/harmonic.test.ts | 4 + .../validators/hubspot.test.ts | 4 + .../validators/linear.test.ts | 4 + .../validators/monday.test.ts | 4 + .../validators/notion.test.ts | 4 + .../validators/pipedrive.test.ts | 4 + .../validators/shopify.test.ts | 4 + .../validators/snowflake.test.ts | 4 + .../validators/trello.test.ts | 4 +- .../validators/wealthbox.test.ts | 4 + .../validators/webflow.test.ts | 4 + apps/sim/lib/embeddings/client.test.ts | 23 +- apps/sim/lib/embeddings/client.ts | 15 +- .../ollama-model-catalog.server.test.ts | 7 +- .../embeddings/ollama-model-catalog.server.ts | 16 +- .../openrouter-model-catalog.server.test.ts | 58 + .../openrouter-model-catalog.server.ts | 25 +- apps/sim/lib/execution/isolated-vm-worker.cjs | 17 +- apps/sim/lib/execution/isolated-vm.test.ts | 1 + apps/sim/lib/execution/isolated-vm.ts | 23 +- .../execution/remote-sandbox/provider.test.ts | 56 + .../lib/execution/remote-sandbox/provider.ts | 10 +- apps/sim/lib/internal/appconfig/client.ts | 3 + apps/sim/lib/internal/asana/client.test.ts | 8 + apps/sim/lib/internal/asana/client.ts | 5 +- .../sim/lib/internal/asana/operations.test.ts | 8 + .../sim/lib/internal/ashby/operations.test.ts | 4 +- apps/sim/lib/internal/ashby/operations.ts | 6 +- apps/sim/lib/internal/athena/client.ts | 2 + apps/sim/lib/internal/box/client.test.ts | 7 + apps/sim/lib/internal/box/client.ts | 5 +- apps/sim/lib/internal/brex/client.test.ts | 7 +- apps/sim/lib/internal/brex/client.ts | 6 +- .../browser-use/operations/run-task.test.ts | 8 + .../browser-use/operations/run-task.ts | 5 +- .../lib/internal/buffer/operations.test.ts | 7 + apps/sim/lib/internal/buffer/operations.ts | 5 +- apps/sim/lib/internal/clickup/client.test.ts | 7 + apps/sim/lib/internal/clickup/client.ts | 9 +- .../operations/get-zone-settings.ts | 9 +- .../sim/lib/internal/cloudformation/client.ts | 2 + apps/sim/lib/internal/cloudtrail/client.ts | 2 + .../sim/lib/internal/cloudtrail/operations.ts | 2 +- apps/sim/lib/internal/cloudwatch/client.ts | 3 + apps/sim/lib/internal/codepipeline/client.ts | 2 + .../lib/internal/confluence/client.test.ts | 7 + apps/sim/lib/internal/confluence/client.ts | 9 +- .../internal/confluence/operations.test.ts | 7 + .../lib/internal/crowdstrike/client.test.ts | 8 + apps/sim/lib/internal/crowdstrike/client.ts | 11 +- .../internal/crowdstrike/operations.test.ts | 8 + .../lib/internal/cursor/operations.test.ts | 4 +- apps/sim/lib/internal/cursor/operations.ts | 6 +- .../internal/datadog/operations/update-slo.ts | 7 +- apps/sim/lib/internal/daytona/operations.ts | 5 +- apps/sim/lib/internal/discord/client.test.ts | 7 + apps/sim/lib/internal/discord/client.ts | 28 +- apps/sim/lib/internal/docusign/client.test.ts | 8 + apps/sim/lib/internal/docusign/client.ts | 5 +- apps/sim/lib/internal/dropbox/client.test.ts | 5 + apps/sim/lib/internal/dropbox/client.ts | 37 +- apps/sim/lib/internal/dynamodb/client.ts | 3 + .../lib/internal/elevenlabs/client.test.ts | 8 + apps/sim/lib/internal/elevenlabs/client.ts | 5 +- .../lib/internal/firecrawl/operations.test.ts | 8 + apps/sim/lib/internal/firecrawl/operations.ts | 5 +- .../lib/internal/fireflies/operations.test.ts | 6 + apps/sim/lib/internal/fireflies/operations.ts | 6 +- apps/sim/lib/internal/gmail/client.ts | 5 +- .../sim/lib/internal/gmail/operations.test.ts | 7 + apps/sim/lib/internal/iam/client.ts | 6 +- .../lib/internal/identity-center/client.ts | 4 + .../sim/lib/internal/image/operations.test.ts | 32 +- apps/sim/lib/internal/image/operations.ts | 58 +- .../internal/instagram/execute-tool.test.ts | 7 + apps/sim/lib/internal/instagram/operations.ts | 5 +- .../lib/internal/instagram/publishing.test.ts | 7 + apps/sim/lib/internal/instagram/publishing.ts | 18 +- apps/sim/lib/internal/jira/client.test.ts | 7 + apps/sim/lib/internal/jira/client.ts | 9 +- apps/sim/lib/internal/jsm/client.test.ts | 8 + apps/sim/lib/internal/jsm/client.ts | 5 +- apps/sim/lib/internal/lambda/client.ts | 2 + .../sim/lib/internal/latex/operations.test.ts | 7 + apps/sim/lib/internal/latex/operations.ts | 5 +- apps/sim/lib/internal/linq/client.ts | 6 +- .../add-user-app-role-assignment.test.ts | 6 + .../add-user-app-role-assignment.ts | 9 +- .../lib/internal/microsoft-teams/client.ts | 5 +- .../microsoft-teams/operations.test.ts | 8 + .../internal/microsoft-teams/operations.ts | 7 +- .../internal/okta/operations/update-group.ts | 7 +- apps/sim/lib/internal/onepassword/client.ts | 2 + apps/sim/lib/internal/outlook/client.test.ts | 8 + apps/sim/lib/internal/outlook/client.ts | 9 +- .../lib/internal/persona/operations.test.ts | 5 + apps/sim/lib/internal/persona/operations.ts | 6 +- .../internal/quickbooks/operations.test.ts | 10 +- .../sim/lib/internal/quickbooks/operations.ts | 10 +- .../quickbooks/provider-operations.test.ts | 7 + .../quickbooks/provider-operations.ts | 13 +- apps/sim/lib/internal/quiver/client.test.ts | 9 + apps/sim/lib/internal/quiver/client.ts | 10 +- apps/sim/lib/internal/rds/client.ts | 2 + apps/sim/lib/internal/resend/client.ts | 10 +- apps/sim/lib/internal/resend/resend.test.ts | 6 + apps/sim/lib/internal/s3/client.ts | 2 + .../sim/lib/internal/sailpoint/client.test.ts | 8 + apps/sim/lib/internal/sailpoint/client.ts | 11 +- .../internal/sailpoint/execute-tool.test.ts | 8 + .../operations/update-custom-field.test.ts | 6 + .../operations/update-custom-field.ts | 17 +- .../lib/internal/secrets-manager/client.ts | 2 + apps/sim/lib/internal/sendgrid/client.ts | 6 +- apps/sim/lib/internal/ses/client.ts | 2 + apps/sim/lib/internal/slack/client.ts | 6 +- apps/sim/lib/internal/slack/oauth.test.ts | 11 +- apps/sim/lib/internal/slack/oauth.ts | 8 +- .../sim/lib/internal/slack/operations.test.ts | 11 +- .../lib/internal/slack/search-client.test.ts | 6 + apps/sim/lib/internal/sqs/client.ts | 2 + .../lib/internal/square/operations.test.ts | 7 + apps/sim/lib/internal/square/operations.ts | 5 +- apps/sim/lib/internal/ssm/client.ts | 2 + apps/sim/lib/internal/stagehand/operations.ts | 25 +- apps/sim/lib/internal/sts/client.ts | 3 + .../sim/lib/internal/stt/execute-tool.test.ts | 5 +- apps/sim/lib/internal/stt/operations.ts | 85 +- .../lib/internal/supabase/operations.test.ts | 18 +- apps/sim/lib/internal/supabase/operations.ts | 10 +- .../operations/storage-update-bucket.test.ts | 9 + .../operations/storage-update-bucket.ts | 7 +- .../lib/internal/telegram/operations.test.ts | 8 + apps/sim/lib/internal/telegram/operations.ts | 5 +- apps/sim/lib/internal/textract/operations.ts | 2 + .../lib/internal/tiktok/operations.test.ts | 8 + apps/sim/lib/internal/tiktok/operations.ts | 5 +- apps/sim/lib/internal/tiktok/upload.test.ts | 8 + apps/sim/lib/internal/tiktok/upload.ts | 5 +- apps/sim/lib/internal/tts/client.test.ts | 8 + apps/sim/lib/internal/tts/client.ts | 9 +- apps/sim/lib/internal/uptimerobot/client.ts | 9 +- apps/sim/lib/internal/vanta/client.test.ts | 8 + apps/sim/lib/internal/vanta/client.ts | 9 +- .../sim/lib/internal/vanta/operations.test.ts | 8 + apps/sim/lib/internal/vanta/operations.ts | 13 +- apps/sim/lib/internal/video/client.test.ts | 22 +- apps/sim/lib/internal/video/client.ts | 57 +- apps/sim/lib/internal/vision/client.test.ts | 6 +- apps/sim/lib/internal/vision/client.ts | 7 +- .../lib/internal/whatsapp/operations.test.ts | 6 +- apps/sim/lib/internal/whatsapp/operations.ts | 16 +- apps/sim/lib/internal/whatsapp/upload.test.ts | 7 + apps/sim/lib/internal/whatsapp/upload.ts | 5 +- apps/sim/lib/internal/wordpress/operations.ts | 5 +- apps/sim/lib/internal/workday/client.test.ts | 6 + apps/sim/lib/internal/workday/client.ts | 6 +- .../knowledge/access/confluence-site.test.ts | 18 +- .../lib/knowledge/access/confluence-site.ts | 8 +- .../access/github-installation.test.ts | 18 +- .../knowledge/access/github-installation.ts | 8 +- .../authorized-knowledge-use-case.ts | 13 +- .../connectors/external-group-sync.test.ts | 43 +- .../connectors/external-group-sync.ts | 129 ++- .../member-sync-engine.integration.test.ts | 37 +- .../connectors/member-sync-engine.ts | 1004 +++++++++-------- .../connectors/sync-content-pass.test.ts | 2 + .../knowledge/connectors/sync-engine.test.ts | 65 ++ .../lib/knowledge/connectors/sync-engine.ts | 977 ++++++++-------- .../document-processing-source.test.ts | 54 +- ...cument-processor-secret-provenance.test.ts | 6 + .../knowledge/documents/document-processor.ts | 10 +- .../documents/pdf-ocr-triage.test.ts | 6 + apps/sim/lib/knowledge/documents/service.ts | 882 ++++++++------- .../sim/lib/knowledge/documents/utils.test.ts | 7 +- apps/sim/lib/knowledge/documents/utils.ts | 9 +- apps/sim/lib/knowledge/reranker.test.ts | 6 + apps/sim/lib/knowledge/reranker.ts | 7 +- apps/sim/lib/media/falai.ts | 25 +- .../messaging/email/providers/gmail.test.ts | 7 +- .../lib/messaging/email/providers/gmail.ts | 8 +- apps/sim/lib/oauth/connection-network.test.ts | 151 +++ apps/sim/lib/oauth/credential-service.test.ts | 7 +- apps/sim/lib/oauth/credential-service.ts | 9 +- .../github-installation-credential.test.ts | 7 +- .../sim/lib/oauth/github-installation.test.ts | 4 +- apps/sim/lib/oauth/github-installation.ts | 7 +- .../sim/lib/oauth/github-repositories.test.ts | 6 +- apps/sim/lib/oauth/oauth.test.ts | 6 +- apps/sim/lib/oauth/oauth.ts | 992 +--------------- apps/sim/lib/oauth/quickbooks.test.ts | 3 + apps/sim/lib/oauth/quickbooks.ts | 9 +- apps/sim/lib/oauth/refresh-token.server.ts | 992 ++++++++++++++++ .../application/execute-selector.test.ts | 54 + .../selectors/application/execute-selector.ts | 4 +- .../server/providers/bigquery.test.ts | 4 + .../server/providers/bitbucket.test.ts | 4 + .../server/providers/confluence.test.ts | 4 + .../selectors/server/providers/google.test.ts | 4 + .../server/providers/harmonic.test.ts | 3 + .../selectors/server/providers/harmonic.ts | 7 +- .../server/providers/hubspot.test.ts | 4 + .../selectors/server/providers/jira.test.ts | 4 + .../selectors/server/providers/jsm.test.ts | 4 + .../server/providers/microsoft.test.ts | 4 + .../selectors/server/providers/monday.test.ts | 4 + .../selectors/server/providers/notion.test.ts | 4 + .../server/providers/pipedrive.test.ts | 4 + .../server/providers/provider-http.test.ts | 3 + .../server/providers/provider-http.ts | 7 +- .../server/providers/sharepoint.test.ts | 4 + .../server/providers/snowflake.test.ts | 3 + .../selectors/server/providers/snowflake.ts | 7 +- .../server/providers/webflow.test.ts | 4 + .../server/providers/zoho-desk.test.ts | 2 + .../selectors/server/providers/zoom.test.ts | 4 + .../sim/lib/table/application/folders.test.ts | 2 + apps/sim/lib/webhooks/polling/gmail.ts | 15 +- .../lib/webhooks/polling/google-calendar.ts | 7 +- apps/sim/lib/webhooks/polling/google-drive.ts | 9 +- .../sim/lib/webhooks/polling/google-sheets.ts | 11 +- apps/sim/lib/webhooks/polling/hubspot.test.ts | 3 + apps/sim/lib/webhooks/polling/hubspot.ts | 11 +- .../lib/webhooks/polling/orchestrator.test.ts | 97 ++ apps/sim/lib/webhooks/polling/orchestrator.ts | 15 +- .../lib/workflows/executor/execution-core.ts | 11 +- .../workspaces/admin-move-source-impact.ts | 2 +- apps/sim/package.json | 8 +- apps/sim/providers/anthropic/index.ts | 14 +- .../providers/azure-anthropic/index.test.ts | 9 +- apps/sim/providers/azure-anthropic/index.ts | 14 +- apps/sim/providers/azure-openai/index.test.ts | 15 +- apps/sim/providers/azure-openai/index.ts | 26 +- apps/sim/providers/baseten/index.ts | 6 + apps/sim/providers/bedrock/index.test.ts | 3 + apps/sim/providers/bedrock/index.ts | 7 +- apps/sim/providers/deepseek/index.ts | 6 + .../providers/file-attachments.server.test.ts | 6 + apps/sim/providers/file-attachments.server.ts | 9 +- apps/sim/providers/fireworks/index.ts | 6 + apps/sim/providers/index.ts | 2 + apps/sim/providers/kimi/index.ts | 6 + apps/sim/providers/litellm/index.ts | 16 +- apps/sim/providers/meta/index.ts | 6 + apps/sim/providers/mistral/index.ts | 6 + apps/sim/providers/nvidia/index.ts | 6 + apps/sim/providers/ollama-cloud/index.ts | 6 + apps/sim/providers/ollama/index.ts | 15 +- apps/sim/providers/openai/core.ts | 10 +- apps/sim/providers/openrouter/index.ts | 6 + apps/sim/providers/openrouter/utils.ts | 10 +- apps/sim/providers/sakana/index.ts | 6 + apps/sim/providers/together/index.ts | 6 + apps/sim/providers/vllm/index.test.ts | 11 +- apps/sim/providers/vllm/index.ts | 22 +- apps/sim/providers/xai/index.ts | 6 + apps/sim/providers/zai/index.ts | 6 + apps/sim/tools/apify/run_actor_async.ts | 9 +- .../tools/azure_devops/azure-devops.test.ts | 4 + .../azure_devops/get_work_items_batch.ts | 7 +- .../tools/azure_devops/query_work_items.ts | 7 +- apps/sim/tools/brightdata/discover.ts | 7 +- apps/sim/tools/cbinsights/cbinsights.test.ts | 4 + apps/sim/tools/cbinsights/utils.ts | 9 +- apps/sim/tools/cloudflare/cloudflare.test.ts | 7 + apps/sim/tools/datadog/datadog.test.ts | 4 + apps/sim/tools/dropbox/download.ts | 24 +- apps/sim/tools/dropcontact-hosting.test.ts | 4 + apps/sim/tools/dropcontact/enrich_contact.ts | 7 +- apps/sim/tools/enrichment-hosting.test.ts | 4 + apps/sim/tools/enrow-hosting.test.ts | 4 + apps/sim/tools/enrow/find_email.test.ts | 3 + apps/sim/tools/enrow/find_email.ts | 7 +- apps/sim/tools/enrow/verify_email.ts | 7 +- apps/sim/tools/exa/agent.ts | 7 +- .../file-message-provider-clients.test.ts | 5 + apps/sim/tools/firecrawl/agent.ts | 7 +- apps/sim/tools/firecrawl/batch-scrape.ts | 22 +- apps/sim/tools/firecrawl/crawl.ts | 7 +- apps/sim/tools/firecrawl/extract.ts | 22 +- apps/sim/tools/github/pr.test.ts | 3 + apps/sim/tools/github/pr.ts | 7 +- apps/sim/tools/gmail/read.test.ts | 3 + apps/sim/tools/gmail/read.ts | 9 +- apps/sim/tools/gmail/search.ts | 20 +- apps/sim/tools/gmail/utils.test.ts | 3 + apps/sim/tools/gmail/utils.ts | 9 +- apps/sim/tools/google_calendar/invite.ts | 7 +- apps/sim/tools/google_calendar/quick_add.ts | 9 +- apps/sim/tools/google_drive/create_folder.ts | 7 +- apps/sim/tools/google_drive/get_content.ts | 11 +- apps/sim/tools/google_slides/write.ts | 9 +- apps/sim/tools/icypeas-hosting.test.ts | 4 + apps/sim/tools/icypeas/find_email.ts | 24 +- apps/sim/tools/icypeas/verify_email.ts | 24 +- apps/sim/tools/jira/add_comment.ts | 7 +- apps/sim/tools/jira/add_watcher.ts | 7 +- apps/sim/tools/jira/add_worklog.ts | 7 +- apps/sim/tools/jira/assign_issue.ts | 7 +- .../tools/jira/attachment-downloads.test.ts | 4 + apps/sim/tools/jira/bulk_read.ts | 9 +- apps/sim/tools/jira/create_issue_link.ts | 9 +- apps/sim/tools/jira/delete_attachment.ts | 7 +- apps/sim/tools/jira/delete_comment.ts | 7 +- apps/sim/tools/jira/delete_issue.ts | 7 +- apps/sim/tools/jira/delete_issue_link.ts | 7 +- apps/sim/tools/jira/delete_worklog.ts | 7 +- apps/sim/tools/jira/get_attachments.ts | 7 +- apps/sim/tools/jira/get_comments.ts | 7 +- apps/sim/tools/jira/get_fields.ts | 7 +- apps/sim/tools/jira/get_project.ts | 7 +- apps/sim/tools/jira/get_transitions.ts | 22 +- apps/sim/tools/jira/get_users.ts | 7 +- apps/sim/tools/jira/get_worklogs.ts | 7 +- apps/sim/tools/jira/list_issue_types.ts | 7 +- apps/sim/tools/jira/list_projects.ts | 7 +- apps/sim/tools/jira/remove_watcher.ts | 7 +- apps/sim/tools/jira/retrieve.ts | 13 +- apps/sim/tools/jira/search_issues.ts | 7 +- apps/sim/tools/jira/search_users.ts | 7 +- apps/sim/tools/jira/transition_issue.ts | 11 +- apps/sim/tools/jira/update_comment.ts | 7 +- apps/sim/tools/jira/update_worklog.ts | 7 +- apps/sim/tools/linkedin/share_post.test.ts | 4 +- apps/sim/tools/linkedin/share_post.ts | 7 +- .../add_user_app_role_assignment.test.ts | 7 + apps/sim/tools/microsoft_ad/utils.ts | 7 +- .../sim/tools/microsoft_excel/format_range.ts | 7 +- apps/sim/tools/microsoft_excel/read.ts | 7 +- apps/sim/tools/microsoft_excel/utils.test.ts | 3 + apps/sim/tools/microsoft_excel/utils.ts | 7 +- .../attachment-downloads.test.ts | 4 + apps/sim/tools/microsoft_teams/utils.ts | 17 +- apps/sim/tools/netsuite/netsuite.test.ts | 4 + apps/sim/tools/netsuite/utils.test.ts | 3 + apps/sim/tools/netsuite/utils.ts | 7 +- apps/sim/tools/notion/read.ts | 9 +- apps/sim/tools/okta/tools.test.ts | 4 + apps/sim/tools/outlook/read.test.ts | 3 + apps/sim/tools/outlook/read.ts | 9 +- apps/sim/tools/parallel/deep_research.ts | 7 +- apps/sim/tools/parallel/parallel.test.ts | 4 + apps/sim/tools/quickbooks/client.ts | 7 +- apps/sim/tools/quickbooks/full_update.test.ts | 4 + apps/sim/tools/quickbooks/sales_utils.test.ts | 4 + apps/sim/tools/quickbooks/utils.ts | 9 +- apps/sim/tools/sharepoint/read_page.ts | 9 +- apps/sim/tools/slack/list_channels.test.ts | 7 + apps/sim/tools/slack/utils.ts | 7 +- apps/sim/tools/twilio_voice/list_calls.ts | 7 +- apps/sim/tools/wiza/individual_reveal.ts | 7 +- apps/sim/tools/x/read.ts | 7 +- bun.lock | 2 + scripts/check-egress-boundary.test.ts | 68 +- scripts/check-egress-boundary.ts | 110 +- ...check-tool-registry-boundary.baseline.json | 6 +- 478 files changed, 8419 insertions(+), 3214 deletions(-) create mode 100644 apps/docs/content/docs/platform/enterprise/security.mdx create mode 100644 apps/sim/app/api/organizations/[id]/network/route.ts create mode 100644 apps/sim/components/settings/organization-security.tsx create mode 100644 apps/sim/hooks/queries/organization-network.ts create mode 100644 apps/sim/lib/api/contracts/organization-network.ts create mode 100644 apps/sim/lib/core/network/application/read-organization-network.test.ts create mode 100644 apps/sim/lib/core/network/application/read-organization-network.ts create mode 100644 apps/sim/lib/core/network/aws-handler.server.ts create mode 100644 apps/sim/lib/core/network/config.server.test.ts create mode 100644 apps/sim/lib/core/network/config.server.ts create mode 100644 apps/sim/lib/core/network/context.server.test.ts create mode 100644 apps/sim/lib/core/network/context.server.ts create mode 100644 apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts create mode 100644 apps/sim/lib/core/network/gateway-metadata.ts create mode 100644 apps/sim/lib/core/network/gateway.server.test.ts create mode 100644 apps/sim/lib/core/network/resource-scope.server.test.ts create mode 100644 apps/sim/lib/core/network/resource-scope.server.ts create mode 100644 apps/sim/lib/core/network/routing.test.ts create mode 100644 apps/sim/lib/core/network/transport.server.test.ts create mode 100644 apps/sim/lib/execution/remote-sandbox/provider.test.ts create mode 100644 apps/sim/lib/oauth/connection-network.test.ts create mode 100644 apps/sim/lib/oauth/refresh-token.server.ts create mode 100644 apps/sim/lib/webhooks/polling/orchestrator.test.ts diff --git a/apps/docs/content/docs/platform/enterprise/index.mdx b/apps/docs/content/docs/platform/enterprise/index.mdx index a781bcc40d9..be354a1a18e 100644 --- a/apps/docs/content/docs/platform/enterprise/index.mdx +++ b/apps/docs/content/docs/platform/enterprise/index.mdx @@ -11,6 +11,7 @@ Sim Enterprise adds organization controls for access, provisioning, operations, | [Single sign-on](/platform/enterprise/sso) | Connect a SAML or OIDC identity provider | | [Verified domains](/platform/enterprise/verified-domains) | Prove domain ownership for SSO and provisioning | | [Directory provisioning](/platform/enterprise/scim) | Sync users and group-based access from your directory | +| [Security](/platform/enterprise/security) | Manage session policies and view configured outbound IP addresses | | [Session policies](/platform/enterprise/session-policies) | Set session lifetimes and revoke member sessions | | [Audit logs](/platform/enterprise/audit-logs) | Investigate configuration and security events | | [Usage tracking](/platform/enterprise/usage-tracking) | Review usage by member, workspace, model, and source | diff --git a/apps/docs/content/docs/platform/enterprise/meta.json b/apps/docs/content/docs/platform/enterprise/meta.json index 123eec90b29..c87d6bad42d 100644 --- a/apps/docs/content/docs/platform/enterprise/meta.json +++ b/apps/docs/content/docs/platform/enterprise/meta.json @@ -5,6 +5,7 @@ "sso", "scim", "verified-domains", + "security", "session-policies", "access-control", "custom-blocks", diff --git a/apps/docs/content/docs/platform/enterprise/security.mdx b/apps/docs/content/docs/platform/enterprise/security.mdx new file mode 100644 index 00000000000..1fd6c8024b7 --- /dev/null +++ b/apps/docs/content/docs/platform/enterprise/security.mdx @@ -0,0 +1,24 @@ +--- +title: Security +description: Manage organization session policies and view configured outbound IP addresses +--- + +Organization owners and admins open **Settings → Security** to manage session policies and view outbound IP addresses. [Single sign-on](/platform/enterprise/sso) remains a separate settings page for identity providers, verified domains, and provisioning. + +## Session policies + +Set **Max session lifetime** and **Idle timeout** in hours, then select **Save**. Leave either field empty to use its default behavior. **Discard** restores your saved values. + +**Sign out all members** opens a confirmation before revoking the organization's browser sessions, except your current session. It does not revoke API keys. + +See [Session policies](/platform/enterprise/session-policies) for limits, defaults, and how changes affect existing sessions. + +## Outbound IP addresses + +Copy your organization's configured addresses in `/32` format and allowlist **every listed address** on the destination firewall. These addresses apply to supported HTTPS connections from Sim and its background workers. + +Contact Sim support or your deployment administrator to configure dedicated IPs and confirm connection coverage. + +## Availability + +On Sim Cloud, Security settings require an Enterprise organization and an owner or admin role. On self-hosted deployments, the outbound IP section is available to organization administrators; session controls appear only when session policies are enabled. See [self-hosted enterprise configuration](/platform/enterprise/self-hosted). diff --git a/apps/docs/content/docs/platform/enterprise/session-policies.mdx b/apps/docs/content/docs/platform/enterprise/session-policies.mdx index 73ddd8fd2a4..c4474690879 100644 --- a/apps/docs/content/docs/platform/enterprise/session-policies.mdx +++ b/apps/docs/content/docs/platform/enterprise/session-policies.mdx @@ -11,7 +11,7 @@ Session Policies let organization owners and admins on Enterprise plans control ## Setup -Go to **Settings → Organization → Session policies** in your organization settings. +Go to **Settings → Security → Session policies** in your organization settings. The [Security page](/platform/enterprise/security) also shows configured outbound IP addresses; single sign-on remains a separate page. Both limits are optional. Leave a field empty to keep the default behavior: sessions last 30 days and extend automatically while a member stays active. diff --git a/apps/docs/content/docs/platform/self-hosting/networking.mdx b/apps/docs/content/docs/platform/self-hosting/networking.mdx index d81fd52e780..bcb0cbd1de3 100644 --- a/apps/docs/content/docs/platform/self-hosting/networking.mdx +++ b/apps/docs/content/docs/platform/self-hosting/networking.mdx @@ -302,7 +302,7 @@ A proxy body limit of 250 MB accommodates all three defaults. If you lower the a ## Outbound connectivity -The shared HTTP transport honors `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY` on both Bun and Node. This covers guarded HTTP requests, pinned model-provider clients, and webhook providers using Sim's fetch adapter. It does not configure a global dispatcher or route non-HTTP protocols. +The shared HTTP transport honors `HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY` on both Bun and Node. This covers guarded HTTP requests, model-provider clients using Sim's fetch adapter, and AWS clients using Sim's shared request handler. It does not configure a global dispatcher or route non-HTTP protocols. Use an `http://` or `https://` proxy URL. For proxy authentication, both the username and password must be nonempty; partial credentials are refused. Lowercase variables take precedence over uppercase ones. `HTTP_PROXY` also applies to HTTPS when `HTTPS_PROXY` is unset. `NO_PROXY` matches the original destination hostname, with optional ports; it accepts exact names, domain suffixes, and `*`, but not CIDR ranges. @@ -313,14 +313,14 @@ An environment proxy is trusted deployment configuration, so its own address can | Outbound path | Honors `HTTP_PROXY` / `HTTPS_PROXY` | |---|---| | Shared guarded HTTP transport: tools, connectors, outbound webhooks, content fetches, MCP HTTP connections, and model-provider clients using Sim's fetch adapter | Yes, on Bun and Node, subject to the CONNECT requirements above | -| Amazon Bedrock and AWS integration clients | No, these clients use their own AWS SDK transport | +| Amazon Bedrock and AWS integration clients using Sim's shared request handler | Yes, on Bun and Node | | Object storage on Azure Blob and GCS; Azure Communication Services email | Yes, through the SDK's own proxy support | | Native `fetch` paths and SDKs that use it, including Resend email, the desktop update feed, and the `/api/telemetry` relay | Yes on Bun; Node requires [`NODE_USE_ENV_PROXY=1`](https://nodejs.org/api/cli.html#node_use_env_proxy1) (Node 22.21+ / 24.0+) | | Application object storage on S3; application email via Amazon SES | No, these clients use their own AWS SDK transport | | Email via SMTP, Postgres, and Redis | No, these use raw TCP | | OTLP export from the server SDK | No | -An explicit HTTP-block `proxyUrl` takes precedence over environment proxy settings. +An assigned organization gateway takes precedence over environment proxy settings. An explicit HTTP-block `proxyUrl` takes precedence over environment settings on direct routes, and is refused when organization policy requires a gateway. Set `NO_PROXY` for every destination that is not on the public internet, not just model endpoints. The app reaches the realtime server (`SOCKET_SERVER_URL`), the Presidio PII service (`PII_URL`), and itself (`INTERNAL_API_BASE_URL`) over HTTP, alongside self-hosted Ollama, LiteLLM, and vLLM — so a proxy that cannot reach your internal network breaks live updates and PII redaction, not only inference. diff --git a/apps/docs/content/docs/search/gitlab.mdx b/apps/docs/content/docs/search/gitlab.mdx index 55848547f6c..4cf89cc24ab 100644 --- a/apps/docs/content/docs/search/gitlab.mdx +++ b/apps/docs/content/docs/search/gitlab.mdx @@ -25,6 +25,8 @@ Use a self-managed GitLab instance reachable by Sim over HTTPS. The administrato The CSV path checks the token's identity and project access. It does not require administrator directory access or a custom admin role. +If your GitLab instance restricts access by source IP, allowlist the configured addresses from [Settings → Security → Outbound IP addresses](/platform/enterprise/security#outbound-ip-addresses). Confirm connectivity from Sim and its background sync jobs before the initial sync. + CSV files define access in Sim. Each mapped user listed for the selected project can read all of that source's indexed, non-confidential content. Sim does not infer that user's GitLab role or feature restrictions in this path. Include only users who should have that access, and replace the files whenever memberships or email mappings change. diff --git a/apps/sim/app/api/auth/oauth/utils.test.ts b/apps/sim/app/api/auth/oauth/utils.test.ts index 13200bbf7f6..dfcf15afa45 100644 --- a/apps/sim/app/api/auth/oauth/utils.test.ts +++ b/apps/sim/app/api/auth/oauth/utils.test.ts @@ -7,9 +7,8 @@ import { redisConfigMockFns } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -vi.mock('@/lib/oauth/oauth', () => ({ +vi.mock('@/lib/oauth/refresh-token.server', () => ({ refreshOAuthToken: vi.fn(), - OAUTH_PROVIDERS: {}, })) const { mockDecryptSecret } = vi.hoisted(() => ({ mockDecryptSecret: vi.fn() })) @@ -30,7 +29,6 @@ import { NETSUITE_SERVICE_ACCOUNT_PROVIDER_ID, ZOOM_SERVICE_ACCOUNT_PROVIDER_ID, } from '@/lib/credentials/client-credential-accounts/descriptors' -import { refreshOAuthToken } from '@/lib/oauth' import { getCredential, refreshAccessTokenIfNeeded, @@ -38,6 +36,7 @@ import { resolveServiceAccountToken, } from '@/lib/oauth/credential-service' import { getOAuthRefreshCoordinationIdentity } from '@/lib/oauth/refresh-coordination' +import { refreshOAuthToken } from '@/lib/oauth/refresh-token.server' import { ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID, GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID, diff --git a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts index de6308d4d4d..a74a1524e97 100644 --- a/apps/sim/app/api/cron/renew-subscriptions/route.test.ts +++ b/apps/sim/app/api/cron/renew-subscriptions/route.test.ts @@ -3,27 +3,57 @@ * * @vitest-environment node */ + +import { webhook } from '@sim/db/schema' import { authOAuthUtilsMock, + authOAuthUtilsMockFns, createMockRequest, dbChainMockFns, + queueTableRows, redisConfigMockFns, resetDbChainMock, } from '@sim/testing' -import { sleep } from '@sim/utils/helpers' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockVerifyCronAuth } = vi.hoisted(() => ({ +const mocks = vi.hoisted(() => ({ mockVerifyCronAuth: vi.fn().mockReturnValue(null), + detached: vi.fn<(label: string, work: () => Promise) => void>(), + enabled: vi.fn(() => true), + workspace: vi.fn(), + route: vi.fn(async (organizationId: string | null | undefined) => ({ organizationId })), + fetch: vi.fn(), + credentialOwner: vi.fn(), })) vi.mock('@/lib/auth/internal', () => ({ - verifyCronAuth: mockVerifyCronAuth, + verifyCronAuth: mocks.mockVerifyCronAuth, })) vi.mock('@/lib/oauth/credential-service', () => authOAuthUtilsMock) +vi.mock('@/lib/core/utils/background', () => ({ runDetached: mocks.detached })) +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: mocks.enabled, + resolveOutboundRoute: mocks.route, +})) +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadActiveWorkspaceApplicationContext: mocks.workspace, +})) +vi.mock('@/lib/core/security/input-validation.server', () => ({ + secureFetchWithValidation: mocks.fetch, +})) +vi.mock('@/lib/webhooks/provider-subscription-utils', () => ({ + getCredentialOwner: mocks.credentialOwner, + getNotificationUrl: () => 'https://example.com/api/webhooks/trigger/teams', +})) + +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' +import { GET } from '@/app/api/cron/renew-subscriptions/route' -import { GET } from './route' +const NEW_EXPIRATION = '2030-01-04T00:00:00.000Z' function createRequest() { return createMockRequest( @@ -34,7 +64,30 @@ function createRequest() { ) } -const flushMicrotasks = () => sleep(0) +function expiringWebhook(id: string, workspaceId: string | null) { + return { + workspaceId, + webhook: { + id, + workflowId: `workflow-${id}`, + providerConfig: { + triggerId: 'microsoftteams_chat_subscription', + subscriptionExpiration: new Date(Date.now() + 60_000).toISOString(), + credentialId: 'shared-credential', + externalSubscriptionId: `subscription-${id}`, + chatId: 'chat-1', + }, + }, + } +} + +async function runBackground() { + expect(mocks.detached).toHaveBeenCalledExactlyOnceWith( + 'teams-subscription-renewal', + expect.any(Function) + ) + await mocks.detached.mock.calls[0][1]() +} describe('Teams subscription renewal route (fire-and-forget)', () => { beforeEach(() => { @@ -42,16 +95,28 @@ describe('Teams subscription renewal route (fire-and-forget)', () => { resetDbChainMock() redisConfigMockFns.mockAcquireLock.mockResolvedValue(true) redisConfigMockFns.mockReleaseLock.mockResolvedValue(true) - mockVerifyCronAuth.mockReturnValue(null) + mocks.mockVerifyCronAuth.mockReturnValue(null) + mocks.enabled.mockReturnValue(true) + mocks.workspace.mockResolvedValue({ workspaceOrganizationId: 'org-1' }) + mocks.credentialOwner.mockResolvedValue({ accountId: 'account-1', userId: 'credential-owner' }) + authOAuthUtilsMockFns.mockRefreshAccessTokenIfNeeded.mockImplementation(async () => { + await resolveCurrentOutboundRoute() + return 'access-token' + }) + mocks.fetch.mockImplementation(async () => { + await resolveCurrentOutboundRoute() + return Response.json({ expirationDateTime: NEW_EXPIRATION }) + }) }) it('returns the auth error when cron auth fails', async () => { - mockVerifyCronAuth.mockReturnValueOnce(new Response(null, { status: 401 }) as never) + mocks.mockVerifyCronAuth.mockReturnValueOnce(new Response(null, { status: 401 }) as never) const response = await GET(createRequest()) expect(response.status).toBe(401) expect(redisConfigMockFns.mockAcquireLock).not.toHaveBeenCalled() + expect(mocks.detached).not.toHaveBeenCalled() }) it('acknowledges with 202 and renews in the background after acquiring the lock', async () => { @@ -67,7 +132,8 @@ describe('Teams subscription renewal route (fire-and-forget)', () => { { reclaimOnFailure: true } ) - await flushMicrotasks() + expect(dbChainMockFns.select).not.toHaveBeenCalled() + await runBackground() expect(dbChainMockFns.select).toHaveBeenCalled() expect(redisConfigMockFns.mockReleaseLock).toHaveBeenCalledWith( 'teams-subscription-renewal-lock', @@ -84,5 +150,120 @@ describe('Teams subscription renewal route (fire-and-forget)', () => { const data = await response.json() expect(data).toMatchObject({ status: 'skip' }) expect(dbChainMockFns.select).not.toHaveBeenCalled() + expect(mocks.detached).not.toHaveBeenCalled() + }) + + it('scopes refresh and Graph calls by each canonical workspace, not the credential owner', async () => { + queueTableRows(webhook, [ + expiringWebhook('first', 'workspace-1'), + expiringWebhook('second', 'workspace-2'), + ]) + mocks.workspace + .mockResolvedValueOnce({ workspaceOrganizationId: 'org-1' }) + .mockResolvedValueOnce({ workspaceOrganizationId: null }) + + await GET(createRequest()) + await runWithOutboundOrganization('caller-org', runBackground) + + expect(mocks.workspace.mock.calls).toEqual([['workspace-1'], ['workspace-2']]) + expect(mocks.route.mock.calls).toEqual([['org-1'], ['org-1'], [null], [null]]) + expect(mocks.fetch.mock.calls.map(([url, init]) => [url, init?.method])).toEqual([ + ['https://graph.microsoft.com/v1.0/subscriptions/subscription-first', 'PATCH'], + ['https://graph.microsoft.com/v1.0/subscriptions/subscription-second', 'PATCH'], + ]) + expect(dbChainMockFns.set).toHaveBeenCalledTimes(2) + expect(await resolveCurrentOutboundRoute()).toEqual({ organizationId: undefined }) }) + + it.each([404, 410])( + 'recreates an expired subscription through the same scope after Graph returns %s', + async (status) => { + queueTableRows(webhook, [expiringWebhook('expired', 'workspace-1')]) + mocks.fetch + .mockImplementationOnce(async () => { + await resolveCurrentOutboundRoute() + return Response.json({ error: { message: 'Subscription expired' } }, { status }) + }) + .mockImplementationOnce(async () => { + await resolveCurrentOutboundRoute() + return Response.json({ id: 'replacement', expirationDateTime: NEW_EXPIRATION }) + }) + + await GET(createRequest()) + await runBackground() + + expect(mocks.route.mock.calls).toEqual([['org-1'], ['org-1'], ['org-1']]) + expect(mocks.fetch).toHaveBeenLastCalledWith( + 'https://graph.microsoft.com/v1.0/subscriptions', + expect.objectContaining({ method: 'POST' }) + ) + expect(dbChainMockFns.set).toHaveBeenCalledExactlyOnceWith({ + providerConfig: expect.objectContaining({ + externalSubscriptionId: 'replacement', + subscriptionExpiration: NEW_EXPIRATION, + }), + updatedAt: expect.any(Date), + }) + } + ) + + it.each([null, 'removed-workspace'])( + 'skips unresolved workspace %s without provider calls and renews the next webhook', + async (workspaceId) => { + queueTableRows(webhook, [ + expiringWebhook('unresolved', workspaceId), + expiringWebhook('valid', 'workspace-1'), + ]) + mocks.workspace.mockImplementation(async (id: string) => + id === 'workspace-1' ? { workspaceOrganizationId: 'org-1' } : null + ) + + await GET(createRequest()) + await runBackground() + + expect(authOAuthUtilsMockFns.mockRefreshAccessTokenIfNeeded).toHaveBeenCalledOnce() + expect(mocks.fetch).toHaveBeenCalledExactlyOnceWith( + 'https://graph.microsoft.com/v1.0/subscriptions/subscription-valid', + expect.objectContaining({ method: 'PATCH' }) + ) + expect(dbChainMockFns.set).toHaveBeenCalledOnce() + expect(redisConfigMockFns.mockReleaseLock).toHaveBeenCalledOnce() + } + ) + + it('continues after a routed provider failure and releases the lock', async () => { + queueTableRows(webhook, [ + expiringWebhook('failed', 'workspace-1'), + expiringWebhook('valid', 'workspace-2'), + ]) + mocks.workspace + .mockResolvedValueOnce({ workspaceOrganizationId: 'org-1' }) + .mockResolvedValueOnce({ workspaceOrganizationId: 'org-2' }) + mocks.fetch.mockImplementationOnce(async () => { + await resolveCurrentOutboundRoute() + throw new Error('Gateway unavailable') + }) + + await GET(createRequest()) + await runBackground() + + expect(mocks.route.mock.calls).toEqual([['org-1'], ['org-1'], ['org-2'], ['org-2']]) + expect(dbChainMockFns.set).toHaveBeenCalledOnce() + expect(redisConfigMockFns.mockReleaseLock).toHaveBeenCalledOnce() + }) + + it.each(['workspace-1', null])( + 'renews legacy workspace %s without an extra lookup when routing is unconfigured', + async (workspaceId) => { + mocks.enabled.mockReturnValue(false) + queueTableRows(webhook, [expiringWebhook('default', workspaceId)]) + + await GET(createRequest()) + await runBackground() + + expect(mocks.workspace).not.toHaveBeenCalled() + expect(mocks.fetch).toHaveBeenCalledOnce() + expect(dbChainMockFns.set).toHaveBeenCalledOnce() + } + ) }) diff --git a/apps/sim/app/api/cron/renew-subscriptions/route.ts b/apps/sim/app/api/cron/renew-subscriptions/route.ts index 08ba269b241..74f4dcc9f78 100644 --- a/apps/sim/app/api/cron/renew-subscriptions/route.ts +++ b/apps/sim/app/api/cron/renew-subscriptions/route.ts @@ -1,11 +1,14 @@ import { db } from '@sim/db' -import { webhook as webhookTable } from '@sim/db/schema' +import { webhook as webhookTable, workflow } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { generateShortId } from '@sim/utils/id' +import { isRecordLike } from '@sim/utils/object' import { and, eq, or } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { verifyCronAuth } from '@/lib/auth/internal' import { acquireLock, releaseLock } from '@/lib/core/config/redis' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { runDetached } from '@/lib/core/utils/background' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' @@ -30,7 +33,7 @@ const MAX_LIFETIME_MINUTES = 4230 */ async function recreateSubscription( webhook: Record, - config: Record, + config: Record, accessToken: string ): Promise<{ id: string; expirationDateTime: string } | null> { const chatId = config.chatId as string | undefined @@ -42,7 +45,9 @@ async function recreateSubscription( const notificationUrl = getNotificationUrl(webhook) const expirationDateTime = new Date(Date.now() + MAX_LIFETIME_MINUTES * 60 * 1000).toISOString() - const res = await fetch('https://graph.microsoft.com/v1.0/subscriptions', { + const res = await secureFetchWithValidation('https://graph.microsoft.com/v1.0/subscriptions', { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -63,13 +68,20 @@ async function recreateSubscription( const error = await res.json() logger.error(`Failed to recreate Teams subscription for webhook ${webhook.id}`, { status: res.status, - error: error.error, + error: isRecordLike(error) ? error.error : undefined, }) return null } const payload = await res.json() - return { id: payload.id as string, expirationDateTime: payload.expirationDateTime as string } + if ( + !isRecordLike(payload) || + typeof payload.id !== 'string' || + typeof payload.expirationDateTime !== 'string' + ) { + throw new Error('Invalid Teams subscription response') + } + return { id: payload.id, expirationDateTime: payload.expirationDateTime } } /** @@ -93,8 +105,10 @@ async function renewExpiringSubscriptions(): Promise<{ const webhooksWithWorkflows = await db .select({ webhook: webhookTable, + workspaceId: workflow.workspaceId, }) .from(webhookTable) + .leftJoin(workflow, eq(webhookTable.workflowId, workflow.id)) .where( and( deliverableWebhookPredicate(webhookTable, 'active_only'), @@ -112,8 +126,8 @@ async function renewExpiringSubscriptions(): Promise<{ /** Renew any subscription expiring within the next 48 hours. */ const renewalThreshold = new Date(Date.now() + 48 * 60 * 60 * 1000) - for (const { webhook } of webhooksWithWorkflows) { - const config = (webhook.providerConfig as Record) || {} + for (const { webhook, workspaceId } of webhooksWithWorkflows) { + const config = (webhook.providerConfig as Record) || {} if (config.triggerId !== 'microsoftteams_chat_subscription') continue @@ -140,86 +154,93 @@ async function renewExpiringSubscriptions(): Promise<{ continue } - const credentialOwner = await getCredentialOwner(credentialId, requestId) - if (!credentialOwner) { - logger.error(`Credential owner not found for credential ${credentialId}`) - totalFailed++ - continue - } - - const accessToken = await refreshAccessTokenIfNeeded( - credentialOwner.accountId, - credentialOwner.userId, - requestId - ) + await withResourceOutboundScope({ workspaceId }, async () => { + const credentialOwner = await getCredentialOwner(credentialId, requestId) + if (!credentialOwner) { + logger.error(`Credential owner not found for credential ${credentialId}`) + totalFailed++ + return + } - if (!accessToken) { - logger.error(`Failed to get access token for webhook ${webhook.id}`) - totalFailed++ - continue - } + const accessToken = await refreshAccessTokenIfNeeded( + credentialOwner.accountId, + credentialOwner.userId, + requestId + ) - const newExpirationDateTime = new Date( - Date.now() + MAX_LIFETIME_MINUTES * 60 * 1000 - ).toISOString() - - const res = await fetch( - `https://graph.microsoft.com/v1.0/subscriptions/${externalSubscriptionId}`, - { - method: 'PATCH', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ expirationDateTime: newExpirationDateTime }), + if (!accessToken) { + logger.error(`Failed to get access token for webhook ${webhook.id}`) + totalFailed++ + return } - ) - let newSubscriptionId: string | undefined - let newExpiration: string | undefined - - if (!res.ok) { - const error = await res.json() - logger.error( - `Failed to renew Teams subscription ${externalSubscriptionId} for webhook ${webhook.id}`, - { status: res.status, error: error.error } + const newExpirationDateTime = new Date( + Date.now() + MAX_LIFETIME_MINUTES * 60 * 1000 + ).toISOString() + + const res = await secureFetchWithValidation( + `https://graph.microsoft.com/v1.0/subscriptions/${externalSubscriptionId}`, + { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + method: 'PATCH', + headers: { + Authorization: `Bearer ${accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ expirationDateTime: newExpirationDateTime }), + } ) - if (res.status === 404 || res.status === 410) { - const recreated = await recreateSubscription(webhook, config, accessToken) - if (!recreated) { + let newSubscriptionId: string | undefined + let newExpiration: string | undefined + + if (!res.ok) { + const error = await res.json() + logger.error( + `Failed to renew Teams subscription ${externalSubscriptionId} for webhook ${webhook.id}`, + { status: res.status, error: isRecordLike(error) ? error.error : undefined } + ) + + if (res.status === 404 || res.status === 410) { + const recreated = await recreateSubscription(webhook, config, accessToken) + if (!recreated) { + totalFailed++ + return + } + newSubscriptionId = recreated.id + newExpiration = recreated.expirationDateTime + logger.info( + `Recreated Teams subscription for webhook ${webhook.id} after the previous one expired (new id: ${newSubscriptionId})` + ) + } else { totalFailed++ - continue + return } - newSubscriptionId = recreated.id - newExpiration = recreated.expirationDateTime - logger.info( - `Recreated Teams subscription for webhook ${webhook.id} after the previous one expired (new id: ${newSubscriptionId})` - ) } else { - totalFailed++ - continue + const payload = await res.json() + if (!isRecordLike(payload) || typeof payload.expirationDateTime !== 'string') { + throw new Error('Invalid Teams subscription response') + } + newExpiration = payload.expirationDateTime } - } else { - const payload = await res.json() - newExpiration = payload.expirationDateTime as string - } - const updatedConfig = { - ...config, - ...(newSubscriptionId ? { externalSubscriptionId: newSubscriptionId } : {}), - subscriptionExpiration: newExpiration, - } + const updatedConfig = { + ...config, + ...(newSubscriptionId ? { externalSubscriptionId: newSubscriptionId } : {}), + subscriptionExpiration: newExpiration, + } - await db - .update(webhookTable) - .set({ providerConfig: updatedConfig, updatedAt: new Date() }) - .where(eq(webhookTable.id, webhook.id)) + await db + .update(webhookTable) + .set({ providerConfig: updatedConfig, updatedAt: new Date() }) + .where(eq(webhookTable.id, webhook.id)) - logger.info( - `Successfully renewed Teams subscription for webhook ${webhook.id}. New expiration: ${newExpiration}` - ) - totalRenewed++ + logger.info( + `Successfully renewed Teams subscription for webhook ${webhook.id}. New expiration: ${newExpiration}` + ) + totalRenewed++ + }) } catch (error) { logger.error(`Error renewing subscription for webhook ${webhook.id}:`, error) totalFailed++ diff --git a/apps/sim/app/api/knowledge/search/utils.test.ts b/apps/sim/app/api/knowledge/search/utils.test.ts index 2c0a4abaf59..d4edc704a4c 100644 --- a/apps/sim/app/api/knowledge/search/utils.test.ts +++ b/apps/sim/app/api/knowledge/search/utils.test.ts @@ -6,6 +6,7 @@ */ import { dbChainMockFns, + inputValidationMock, mockNextFetchResponse, queueTableRows, resetDbChainMock, @@ -18,6 +19,11 @@ import * as documentsUtilsModule from '@/lib/knowledge/documents/utils' import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + vi.mock('@/lib/core/rate-limiter/provider-admission', () => ({ PROVIDER_QUOTA_COOLDOWN_MS: 300_000, ProviderQuotaExhaustedError: class ProviderQuotaExhaustedError extends Error {}, diff --git a/apps/sim/app/api/knowledge/utils.test.ts b/apps/sim/app/api/knowledge/utils.test.ts index d957ff866c3..d899473b6d8 100644 --- a/apps/sim/app/api/knowledge/utils.test.ts +++ b/apps/sim/app/api/knowledge/utils.test.ts @@ -9,6 +9,7 @@ import { dbChainMockFns, defaultMockEnv, + inputValidationMock, queueTableRows, resetDbChainMock, schemaMock, @@ -20,6 +21,11 @@ import { env } from '@/lib/core/config/env' import * as documentsUtilsModule from '@/lib/knowledge/documents/utils' import * as workspacesUtilsModule from '@/lib/workspaces/utils' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + vi.mock('@/lib/core/rate-limiter/provider-admission', () => ({ PROVIDER_QUOTA_COOLDOWN_MS: 300_000, ProviderQuotaExhaustedError: class ProviderQuotaExhaustedError extends Error {}, diff --git a/apps/sim/app/api/organizations/[id]/network/route.ts b/apps/sim/app/api/organizations/[id]/network/route.ts new file mode 100644 index 00000000000..a6e87d2cb75 --- /dev/null +++ b/apps/sim/app/api/organizations/[id]/network/route.ts @@ -0,0 +1,24 @@ +import { getOrganizationNetworkContract } from '@/lib/api/contracts/organization-network' +import { + defineInternalJsonRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { + readOrganizationNetwork, + readOrganizationNetworkOperation, +} from '@/lib/core/network/application/read-organization-network' + +export const dynamic = 'force-dynamic' + +export const GET = defineInternalJsonRoute({ + contract: getOrganizationNetworkContract, + auth: internalSessionAuth, + operation: readOrganizationNetworkOperation, + rateLimit: internalRateLimits.user({ bucketName: 'organization-network-read' }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: ({ params }) => ({ organizationId: params.id }), + useCase: readOrganizationNetwork, + staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, +}) diff --git a/apps/sim/app/api/providers/openrouter/embeddings/models/route.test.ts b/apps/sim/app/api/providers/openrouter/embeddings/models/route.test.ts index 6807800c062..b97adc4ab5a 100644 --- a/apps/sim/app/api/providers/openrouter/embeddings/models/route.test.ts +++ b/apps/sim/app/api/providers/openrouter/embeddings/models/route.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { createMockRequest } from '@sim/testing' +import { createMockRequest, inputValidationMock } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockFilterBlacklistedModels, mockIsProviderBlacklisted } = vi.hoisted(() => ({ @@ -10,6 +10,11 @@ const { mockFetch, mockFilterBlacklistedModels, mockIsProviderBlacklisted } = vi mockIsProviderBlacklisted: vi.fn(), })) +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: mockFetch, +})) + vi.mock('@/providers/utils', () => ({ filterBlacklistedModels: mockFilterBlacklistedModels, isProviderBlacklisted: mockIsProviderBlacklisted, @@ -18,17 +23,19 @@ vi.mock('@/providers/utils', () => ({ import { GET } from '@/app/api/providers/openrouter/embeddings/models/route' const request = () => createMockRequest('GET') +let clock = 0 describe('GET /api/providers/openrouter/embeddings/models', () => { beforeEach(() => { vi.clearAllMocks() - vi.stubGlobal('fetch', mockFetch) + clock += 300_001 + vi.spyOn(performance, 'now').mockImplementation(() => clock) mockIsProviderBlacklisted.mockReturnValue(false) mockFilterBlacklistedModels.mockImplementation((models: string[]) => models) }) afterAll(() => { - vi.unstubAllGlobals() + vi.restoreAllMocks() }) it('returns every unique embedding model with the OpenRouter prefix', async () => { @@ -50,7 +57,7 @@ describe('GET /api/providers/openrouter/embeddings/models', () => { }) expect(mockFetch).toHaveBeenCalledWith( 'https://openrouter.ai/api/v1/embeddings/models', - expect.objectContaining({ next: { revalidate: 300 } }) + expect.objectContaining({ profile: 'configuredEndpoint', maxResponseBytes: 4 * 1024 * 1024 }) ) }) diff --git a/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts b/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts index 7e847cb2797..6122e555bfe 100644 --- a/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts +++ b/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts @@ -2,6 +2,16 @@ * @vitest-environment node */ +import { + inputValidationMock, + queueTableRows, + resetDbChainMock, + V2_OPERATION_RATE_LIMIT_ALLOWED, + V2_PREAUTH_RATE_LIMIT_ALLOWED, + v2ApiKeyAuthModuleMock, + v2RateLimiterModuleMock, + v2RouteMocks, +} from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -96,15 +106,6 @@ const knowledgeBase = { import { document, embedding } from '@sim/db/schema' import { sha256Hex } from '@sim/security/hash' -import { - queueTableRows, - resetDbChainMock, - V2_OPERATION_RATE_LIMIT_ALLOWED, - V2_PREAUTH_RATE_LIMIT_ALLOWED, - v2ApiKeyAuthModuleMock, - v2RateLimiterModuleMock, - v2RouteMocks, -} from '@sim/testing' import { NextRequest } from 'next/server' import { env } from '@/lib/core/config/env' import { @@ -116,6 +117,7 @@ import { POST } from '@/app/api/v2/knowledge/search/route' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const provider = vi.hoisted(() => ({ fetch: vi.fn(), decrypt: vi.fn() })) +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) vi.mock('@/lib/api/server/routes/v2-api-key-auth', () => v2ApiKeyAuthModuleMock) vi.mock('@/lib/core/rate-limiter', () => v2RateLimiterModuleMock) vi.mock('@/lib/core/rate-limiter/storage/factory', () => ({ diff --git a/apps/sim/app/o/[organizationId]/settings/[section]/settings.tsx b/apps/sim/app/o/[organizationId]/settings/[section]/settings.tsx index f5adfa63f19..94221fc31fc 100644 --- a/apps/sim/app/o/[organizationId]/settings/[section]/settings.tsx +++ b/apps/sim/app/o/[organizationId]/settings/[section]/settings.tsx @@ -51,11 +51,6 @@ const AuditLogs = dynamic(() => import('@/ee/audit-logs/components/audit-logs').then((m) => m.AuditLogs) ) const SSO = dynamic(() => import('@/ee/sso/components/sso-settings').then((m) => m.SSO)) -const SessionPolicySettings = dynamic(() => - import('@/ee/session-policy/components/session-policy-settings').then( - (m) => m.SessionPolicySettings - ) -) const DataRetentionSettings = dynamic(() => import('@/ee/data-retention/components/data-retention-settings').then( (m) => m.DataRetentionSettings @@ -64,6 +59,9 @@ const DataRetentionSettings = dynamic(() => const DataDrainsSettings = dynamic(() => import('@/ee/data-drains/components/data-drains-settings').then((m) => m.DataDrainsSettings) ) +const OrganizationSecuritySettings = dynamic(() => + import('@/components/settings/organization-security').then((m) => m.OrganizationSecuritySettings) +) const UsageMonitoring = dynamic(() => import('@/ee/organization-usage/components/usage-monitoring').then((m) => m.UsageMonitoring) ) @@ -113,7 +111,7 @@ export function OrganizationSettings({ section }: OrganizationSettingsProps) { /> )} {section === 'sso' && } - {section === 'sessions' && } + {section === 'security' && } {section === 'data-retention' && } {section === 'data-drains' && } {section === 'whitelabeling' && } diff --git a/apps/sim/app/o/[organizationId]/settings/navigation.test.ts b/apps/sim/app/o/[organizationId]/settings/navigation.test.ts index 36c23895f8c..94ec37b962a 100644 --- a/apps/sim/app/o/[organizationId]/settings/navigation.test.ts +++ b/apps/sim/app/o/[organizationId]/settings/navigation.test.ts @@ -73,6 +73,8 @@ describe('organization settings navigation', () => { ) expect(resolveOrganizationSettingsSection('subscription')).toBe('billing') expect(resolveOrganizationSettingsSection('domains')).toBe('sso') + expect(resolveOrganizationSettingsSection('sessions')).toBe('security') + expect(resolveOrganizationSettingsSection('/o/one/settings/network')).toBeNull() expect(resolveOrganizationSettingsSection('skills')).toBeNull() expect(buildOrganizationNavItems('org', true).map(({ id }) => id)).toEqual([ 'home', @@ -92,7 +94,7 @@ describe('organization settings navigation', () => { 'governance:audit-logs', 'governance:access-control', 'governance:sso', - 'governance:sessions', + 'governance:security', 'governance:data-retention', 'governance:data-drains', 'sim-search:integrations', diff --git a/apps/sim/app/o/[organizationId]/settings/navigation.ts b/apps/sim/app/o/[organizationId]/settings/navigation.ts index 124ff827666..0a96e8a0754 100644 --- a/apps/sim/app/o/[organizationId]/settings/navigation.ts +++ b/apps/sim/app/o/[organizationId]/settings/navigation.ts @@ -37,7 +37,13 @@ export function resolveOrganizationSettingsSection( path, items: ORGANIZATION_SETTINGS_ITEMS, defaultSection: null, - aliases: { organization: 'members', team: 'members', subscription: 'billing', domains: 'sso' }, + aliases: { + organization: 'members', + team: 'members', + subscription: 'billing', + domains: 'sso', + sessions: 'security', + }, }) } diff --git a/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.test.tsx b/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.test.tsx index 034c639a179..bfd7f13b9f7 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.test.tsx @@ -51,7 +51,7 @@ vi.mock('@/app/workspace/[workspaceId]/settings/navigation', () => ({ 'access-control', 'audit-logs', 'sso', - 'sessions', + 'security', 'data-retention', 'data-drains', 'whitelabeling', diff --git a/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx b/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx index 1247284618e..ae626e7a9d9 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx @@ -93,11 +93,6 @@ const AuditLogs = dynamic(() => import('@/ee/audit-logs/components/audit-logs').then((m) => m.AuditLogs) ) const SSO = dynamic(() => import('@/ee/sso/components/sso-settings').then((m) => m.SSO)) -const SessionPolicySettings = dynamic(() => - import('@/ee/session-policy/components/session-policy-settings').then( - (m) => m.SessionPolicySettings - ) -) const DataRetentionSettings = dynamic(() => import('@/ee/data-retention/components/data-retention-settings').then( (m) => m.DataRetentionSettings @@ -106,6 +101,9 @@ const DataRetentionSettings = dynamic(() => const DataDrainsSettings = dynamic(() => import('@/ee/data-drains/components/data-drains-settings').then((m) => m.DataDrainsSettings) ) +const OrganizationSecuritySettings = dynamic(() => + import('@/components/settings/organization-security').then((m) => m.OrganizationSecuritySettings) +) const UsageMonitoring = dynamic(() => import('@/ee/organization-usage/components/usage-monitoring').then((m) => m.UsageMonitoring) ) @@ -201,15 +199,15 @@ export function SettingsPage({ section }: SettingsPageProps) { /> )} {effectiveSection === 'sso' && organizationId && } - {effectiveSection === 'sessions' && organizationId && ( - - )} {effectiveSection === 'data-retention' && organizationId && ( )} {effectiveSection === 'data-drains' && organizationId && ( )} + {effectiveSection === 'security' && organizationId && ( + + )} {effectiveSection === 'whitelabeling' && organizationId && ( )} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/settings-empty-state/settings-empty-state.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/settings-empty-state/settings-empty-state.tsx index ec541844f65..cb1ab6b89e7 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/settings-empty-state/settings-empty-state.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/settings-empty-state/settings-empty-state.tsx @@ -55,7 +55,7 @@ export function SettingsQueryErrorState({ return (
- {getErrorMessage(error, fallback)} + {getErrorMessage(error, fallback)} {isRetrying ? 'Retrying…' : 'Try again'} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts b/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts index 42172436605..fdad1a693a0 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts @@ -44,7 +44,7 @@ describe('unified settings navigation', () => { { id: 'recently-deleted', label: 'Recently deleted', section: 'workspace' }, { id: 'self-host', label: 'Self hosting', section: 'platform' }, { id: 'sso', label: 'Single sign-on', section: 'organization' }, - { id: 'sessions', label: 'Session policies', section: 'organization' }, + { id: 'security', label: 'Security', section: 'organization' }, { id: 'data-retention', label: 'Data retention', section: 'organization' }, { id: 'data-drains', label: 'Data drains', section: 'organization' }, { id: 'whitelabeling', label: 'White-labeling', section: 'organization' }, @@ -90,7 +90,7 @@ describe('unified settings navigation', () => { 'audit-logs', 'whitelabeling', 'sso', - 'sessions', + 'security', 'data-retention', 'data-drains', ]) @@ -124,6 +124,7 @@ describe('resolveSettingsSection', () => { team: 'organization', 'api-keys': 'apikeys', domains: 'sso', + sessions: 'security', } as const it('keeps legacy section links working', () => { diff --git a/apps/sim/app/workspace/[workspaceId]/settings/navigation.ts b/apps/sim/app/workspace/[workspaceId]/settings/navigation.ts index 8427439f40a..973e5b78fb5 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/navigation.ts +++ b/apps/sim/app/workspace/[workspaceId]/settings/navigation.ts @@ -44,6 +44,7 @@ const SECTION_ALIASES: Readonly> = { 'api-keys': 'apikeys', /** Verified domains moved into the SSO page. */ domains: 'sso', + sessions: 'security', } export interface ResolvedSettingsSection { diff --git a/apps/sim/components/settings/navigation.test.ts b/apps/sim/components/settings/navigation.test.ts index 7e29f8ca48c..e469f904048 100644 --- a/apps/sim/components/settings/navigation.test.ts +++ b/apps/sim/components/settings/navigation.test.ts @@ -116,7 +116,7 @@ describe('settings navigation boundaries', () => { 'recently-deleted', 'self-host', 'sso', - 'sessions', + 'security', 'data-retention', 'data-drains', 'whitelabeling', @@ -237,7 +237,7 @@ describe('settings navigation boundaries', () => { 'access-control': false, 'audit-logs': false, sso: true, - sessions: true, + security: true, 'data-retention': false, 'data-drains': false, usage: true, @@ -301,7 +301,7 @@ describe('settings navigation boundaries', () => { 'data-drains', 'data-retention', 'organization', - 'sessions', + 'security', 'sso', 'usage', 'whitelabeling', @@ -319,7 +319,7 @@ describe('settings navigation boundaries', () => { 'access-control': 'access-control', 'audit-logs': 'audit-logs', sso: 'sso', - sessions: 'sessions', + security: 'security', 'data-retention': 'data-retention', 'data-drains': 'data-drains', whitelabeling: 'whitelabeling', diff --git a/apps/sim/components/settings/navigation.ts b/apps/sim/components/settings/navigation.ts index 2665b09e256..7721f1c7841 100644 --- a/apps/sim/components/settings/navigation.ts +++ b/apps/sim/components/settings/navigation.ts @@ -2,7 +2,6 @@ import type { ComponentType } from 'react' import { ChartColumn, ClipboardList, - Clock, Credit, Database, Globe, @@ -55,7 +54,7 @@ export type OrganizationSettingsSection = | 'access-control' | 'audit-logs' | 'sso' - | 'sessions' + | 'security' | 'data-retention' | 'data-drains' | 'whitelabeling' @@ -115,7 +114,7 @@ export type UnifiedSettingsSection = | 'inbox' | 'sandboxes' | 'admin' - | 'sessions' + | 'security' | 'data-retention' | 'data-drains' | 'mothership' @@ -701,18 +700,18 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] }, }, { - label: 'Session policies', - icon: Clock, - docsLink: 'https://docs.sim.ai/platform/enterprise/session-policies', + label: 'Security', + icon: Lock, + docsLink: 'https://docs.sim.ai/platform/enterprise/security', unified: { - id: 'sessions', - description: 'Limit session lifetimes and sign out members org-wide.', + id: 'security', + description: 'Manage session policies and view outbound IP addresses.', group: 'organization', order: 8, requiresHosted: true, requiresEnterprise: true, - selfHostedOverride: 'sessionPolicies', - organizationSection: 'sessions', + selfHostedOverride: 'always', + organizationSection: 'security', }, }, { @@ -905,7 +904,7 @@ const ORGANIZATION_SECTION_GROUPS: Record + {hosted || features.sessionPolicies ? ( + + + + ) : ( + + )} + +
+ ) +} + +function OrganizationNetworkSection({ organizationId }: OrganizationSecuritySettingsProps) { + const { data, error, isPending, isFetching, refetch } = useOrganizationNetwork(organizationId) + + return ( + + {isPending ? ( + + Loading network settings… + + ) : error || data?.mode === 'unavailable' ? ( + void refetch()} + variant='inline' + /> + ) : data?.mode === 'gateway' && data.publicIps.length > 0 ? ( +
+

+ Allowlist every address below for supported HTTPS connections. +

+
+ {data.publicIps.map((ip) => ( + + ))} +
+
+ ) : data ? ( +

+ {data.mode === 'blocked' + ? 'Outbound connections are paused. Contact your administrator.' + : data.mode === 'gateway' + ? 'Your outbound IP addresses aren’t available yet.' + : 'Outbound IP addresses are not configured for this organization.'} +

+ ) : null} + + ) +} diff --git a/apps/sim/connectors/asana/asana.test.ts b/apps/sim/connectors/asana/asana.test.ts index aa005fa3a30..968b8d784dc 100644 --- a/apps/sim/connectors/asana/asana.test.ts +++ b/apps/sim/connectors/asana/asana.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { asanaConnector, diff --git a/apps/sim/connectors/ashby/ashby.test.ts b/apps/sim/connectors/ashby/ashby.test.ts index e1f061f1564..276cfe8a955 100644 --- a/apps/sim/connectors/ashby/ashby.test.ts +++ b/apps/sim/connectors/ashby/ashby.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { beforeEach, describe, expect, it, vi } from 'vitest' import { ashbyConnector } from '@/connectors/ashby/ashby' import { ashbyConnectorMeta } from '@/connectors/ashby/meta' diff --git a/apps/sim/connectors/bitbucket/bitbucket.test.ts b/apps/sim/connectors/bitbucket/bitbucket.test.ts index 3c43cdb8850..9ee60118dc4 100644 --- a/apps/sim/connectors/bitbucket/bitbucket.test.ts +++ b/apps/sim/connectors/bitbucket/bitbucket.test.ts @@ -7,6 +7,10 @@ * Every path that shortens the listing has to leave `syncContext.listingCapped` set, * because the sync engine hard-deletes whatever a full listing omits. */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { bitbucketConnector } from '@/connectors/bitbucket/bitbucket' diff --git a/apps/sim/connectors/confluence/bulk-spaces.test.ts b/apps/sim/connectors/confluence/bulk-spaces.test.ts index f2e670030a6..0dc0d48987a 100644 --- a/apps/sim/connectors/confluence/bulk-spaces.test.ts +++ b/apps/sim/connectors/confluence/bulk-spaces.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { confluenceConnector } from '@/connectors/confluence/confluence' diff --git a/apps/sim/connectors/confluence/confluence.test.ts b/apps/sim/connectors/confluence/confluence.test.ts index f4c3601b9a9..7ddafb41f03 100644 --- a/apps/sim/connectors/confluence/confluence.test.ts +++ b/apps/sim/connectors/confluence/confluence.test.ts @@ -1,6 +1,13 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { AtlassianSiteNotAccessibleError, diff --git a/apps/sim/connectors/confluence/permissions.test.ts b/apps/sim/connectors/confluence/permissions.test.ts index 41558041873..da1d284f088 100644 --- a/apps/sim/connectors/confluence/permissions.test.ts +++ b/apps/sim/connectors/confluence/permissions.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { beforeEach, describe, expect, it, vi } from 'vitest' import { getReadRestriction, diff --git a/apps/sim/connectors/confluence/preflight.test.ts b/apps/sim/connectors/confluence/preflight.test.ts index 5bcf03e534a..c83b123ae30 100644 --- a/apps/sim/connectors/confluence/preflight.test.ts +++ b/apps/sim/connectors/confluence/preflight.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { confluenceConnector } from '@/connectors/confluence/confluence' diff --git a/apps/sim/connectors/databricks/databricks.test.ts b/apps/sim/connectors/databricks/databricks.test.ts index d062264a537..c8079f3e4c6 100644 --- a/apps/sim/connectors/databricks/databricks.test.ts +++ b/apps/sim/connectors/databricks/databricks.test.ts @@ -11,6 +11,10 @@ * bulk of what is asserted here, alongside the stub/`getDocument` hash equality * that keeps every sync from re-indexing every notebook. */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { databricksConnector } from '@/connectors/databricks/databricks' import { databricksConnectorMeta } from '@/connectors/databricks/meta' diff --git a/apps/sim/connectors/databricks/databricks.ts b/apps/sim/connectors/databricks/databricks.ts index 88f1adfdc80..fde4077b524 100644 --- a/apps/sim/connectors/databricks/databricks.ts +++ b/apps/sim/connectors/databricks/databricks.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { truncate } from '@sim/utils/string' import { validateDatabricksWorkspaceHost } from '@/lib/core/security/input-validation' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { decodeTextBuffer } from '@/lib/file-parsers/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { @@ -21,6 +22,10 @@ import { takeIndexableWithinCap, } from '@/connectors/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('DatabricksConnector') /** Page size for `GET /api/2.0/sql/queries`. */ @@ -231,7 +236,7 @@ async function databricksGet( Accept: 'application/json', }, }, - retryOptions + { ...retryOptions, fetcher: providerFetch } ) } diff --git a/apps/sim/connectors/fireflies/fireflies.test.ts b/apps/sim/connectors/fireflies/fireflies.test.ts index 43470dfd395..96ab78422c5 100644 --- a/apps/sim/connectors/fireflies/fireflies.test.ts +++ b/apps/sim/connectors/fireflies/fireflies.test.ts @@ -1,12 +1,15 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/components/icons', () => ({ FirefliesIcon: () => null })) import { firefliesConnector } from '@/connectors/fireflies/fireflies' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + beforeEach(() => { vi.useRealTimers() vi.unstubAllGlobals() diff --git a/apps/sim/connectors/fireflies/fireflies.ts b/apps/sim/connectors/fireflies/fireflies.ts index 47a7de32aa2..ddc17ea7ff5 100644 --- a/apps/sim/connectors/fireflies/fireflies.ts +++ b/apps/sim/connectors/fireflies/fireflies.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { z } from 'zod' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { isRetryableError, @@ -19,6 +20,10 @@ import { parseTagDate, } from '@/connectors/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('FirefliesConnector') const FIREFLIES_GRAPHQL_URL = 'https://api.fireflies.ai/graphql' @@ -200,7 +205,7 @@ async function firefliesGraphQL( return retryWithExponentialBackoff( async () => { /** One retry layer owns transport, HTTP, and GraphQL semantic failures. */ - const response = await fetch(FIREFLIES_GRAPHQL_URL, { + const response = await providerFetch(FIREFLIES_GRAPHQL_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/connectors/github/github.test.ts b/apps/sim/connectors/github/github.test.ts index 32ec8e9f9b0..6ae24cb1e98 100644 --- a/apps/sim/connectors/github/github.test.ts +++ b/apps/sim/connectors/github/github.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, describe, expect, it, vi } from 'vitest' import { beginListingCheckpoint, diff --git a/apps/sim/connectors/github/pacing.test.ts b/apps/sim/connectors/github/pacing.test.ts index b9114f943b6..74dd5b5a243 100644 --- a/apps/sim/connectors/github/pacing.test.ts +++ b/apps/sim/connectors/github/pacing.test.ts @@ -1,4 +1,8 @@ /** @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mutate } = vi.hoisted(() => ({ mutate: vi.fn() })) diff --git a/apps/sim/connectors/github/request.test.ts b/apps/sim/connectors/github/request.test.ts index d68e49f3b80..9bdbc3fdcbe 100644 --- a/apps/sim/connectors/github/request.test.ts +++ b/apps/sim/connectors/github/request.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { acquire, settle } = vi.hoisted(() => ({ acquire: vi.fn(), settle: vi.fn() })) @@ -10,6 +11,8 @@ import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider- import { isRateLimitError } from '@/lib/knowledge/documents/utils' import { fetchGitHubWithRetry } from '@/connectors/github/request' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const URL = 'https://api.github.com/repos/example/repository/git/blobs/blob-id' const OPTIONS = { headers: { Authorization: 'Bearer private-token' } } diff --git a/apps/sim/connectors/github/request.ts b/apps/sim/connectors/github/request.ts index 13198b79ae5..1b475356d5e 100644 --- a/apps/sim/connectors/github/request.ts +++ b/apps/sim/connectors/github/request.ts @@ -6,6 +6,7 @@ import { ProviderCapacityDeferredError, } from '@/lib/core/rate-limiter/provider-capacity-error' import type { ProviderCapacityQuota } from '@/lib/core/rate-limiter/provider-capacity-state' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { fetchWithRetry, @@ -14,6 +15,10 @@ import { resolveRetryDelayMs, } from '@/lib/knowledge/documents/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('GitHubConnectorRequest') const REQUEST_BUDGET_MS = 150_000 const ADMISSION_WAIT_MS = 120_000 @@ -119,7 +124,7 @@ export async function fetchGitHubWithRetry( } try { - const response = await fetch(input, init) + const response = await providerFetch(input, init) quota = readRequestQuota(response.headers) let secondaryLimit = false let forbiddenBody: string | undefined diff --git a/apps/sim/connectors/google-calendar/company-crawl.test.ts b/apps/sim/connectors/google-calendar/company-crawl.test.ts index d1e49c9d82c..63fa2269348 100644 --- a/apps/sim/connectors/google-calendar/company-crawl.test.ts +++ b/apps/sim/connectors/google-calendar/company-crawl.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockListUsers, mockGetUser } = vi.hoisted(() => ({ diff --git a/apps/sim/connectors/google-calendar/google-calendar.test.ts b/apps/sim/connectors/google-calendar/google-calendar.test.ts index 782fd3a66e5..61f267be9ef 100644 --- a/apps/sim/connectors/google-calendar/google-calendar.test.ts +++ b/apps/sim/connectors/google-calendar/google-calendar.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { googleCalendarConnector } from '@/connectors/google-calendar/google-calendar' import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' diff --git a/apps/sim/connectors/google-chat/google-chat.test.ts b/apps/sim/connectors/google-chat/google-chat.test.ts index 2b68fb36b39..37aa1630ef9 100644 --- a/apps/sim/connectors/google-chat/google-chat.test.ts +++ b/apps/sim/connectors/google-chat/google-chat.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { googleChatConnector } from '@/connectors/google-chat/google-chat' import { googleChatConnectorMeta } from '@/connectors/google-chat/meta' diff --git a/apps/sim/connectors/google-docs/google-docs.test.ts b/apps/sim/connectors/google-docs/google-docs.test.ts index e2e86fec321..33bbabc5d44 100644 --- a/apps/sim/connectors/google-docs/google-docs.test.ts +++ b/apps/sim/connectors/google-docs/google-docs.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/components/icons', () => ({ diff --git a/apps/sim/connectors/google-drive/company-crawl.test.ts b/apps/sim/connectors/google-drive/company-crawl.test.ts index 1633a255f98..fcc23c7d31f 100644 --- a/apps/sim/connectors/google-drive/company-crawl.test.ts +++ b/apps/sim/connectors/google-drive/company-crawl.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { validateGoogleCompanyConfig } from '@/connectors/google-drive/company-crawl' import { googleDriveConnector as drive } from '@/connectors/google-drive/google-drive' diff --git a/apps/sim/connectors/google-drive/directory.test.ts b/apps/sim/connectors/google-drive/directory.test.ts index 58bcc4c55c3..3646974762d 100644 --- a/apps/sim/connectors/google-drive/directory.test.ts +++ b/apps/sim/connectors/google-drive/directory.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { beforeEach, describe, expect, it, vi } from 'vitest' import { listDomainGroups, openGoogleDirectory } from '@/connectors/google-drive/directory' diff --git a/apps/sim/connectors/google-drive/google-drive-errors.ts b/apps/sim/connectors/google-drive/google-drive-errors.ts index db9c993b5e5..d3f2cdf9a46 100644 --- a/apps/sim/connectors/google-drive/google-drive-errors.ts +++ b/apps/sim/connectors/google-drive/google-drive-errors.ts @@ -1,3 +1,4 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { attachRetryHeaders, isRetryableError, @@ -11,6 +12,10 @@ import { } from '@/connectors/source-error' import { readBodyWithLimit } from '@/connectors/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const GOOGLE_ERROR_BODY_MAX_BYTES = 64 * 1024 const GOOGLE_ERROR_REASON_MAX_COUNT = 16 @@ -202,7 +207,7 @@ export async function fetchGoogleDriveWithRetry( ): Promise { return retryWithExponentialBackoff( async () => { - const response = await fetch(url, options) + const response = await providerFetch(url, options) if (response.ok) return response const error = await readGoogleDriveApiError(response) diff --git a/apps/sim/connectors/google-drive/google-drive.test.ts b/apps/sim/connectors/google-drive/google-drive.test.ts index 06444b02cf4..89d0d3e1051 100644 --- a/apps/sim/connectors/google-drive/google-drive.test.ts +++ b/apps/sim/connectors/google-drive/google-drive.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import * as XLSX from 'xlsx' diff --git a/apps/sim/connectors/google-drive/shortcuts.test.ts b/apps/sim/connectors/google-drive/shortcuts.test.ts index 16cebd03556..42710bd5653 100644 --- a/apps/sim/connectors/google-drive/shortcuts.test.ts +++ b/apps/sim/connectors/google-drive/shortcuts.test.ts @@ -1,4 +1,8 @@ /** @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { fetchMock } = vi.hoisted(() => ({ fetchMock: vi.fn() })) diff --git a/apps/sim/connectors/google-drive/workspace-drives.test.ts b/apps/sim/connectors/google-drive/workspace-drives.test.ts index 79e1a8ebcf5..0892ebd1f36 100644 --- a/apps/sim/connectors/google-drive/workspace-drives.test.ts +++ b/apps/sim/connectors/google-drive/workspace-drives.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { GoogleDriveApiError } from '@/connectors/google-drive/google-drive-errors' import { listGoogleWorkspaceDrives } from '@/connectors/google-drive/workspace-drives' diff --git a/apps/sim/connectors/google-meet/google-meet.test.ts b/apps/sim/connectors/google-meet/google-meet.test.ts index 3baf293ceeb..f4045a8c45a 100644 --- a/apps/sim/connectors/google-meet/google-meet.test.ts +++ b/apps/sim/connectors/google-meet/google-meet.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { googleMeetConnector } from '@/connectors/google-meet/google-meet' import { googleMeetConnectorMeta } from '@/connectors/google-meet/meta' diff --git a/apps/sim/connectors/google-sheets/google-sheets.test.ts b/apps/sim/connectors/google-sheets/google-sheets.test.ts index 8279753c025..42ff578f066 100644 --- a/apps/sim/connectors/google-sheets/google-sheets.test.ts +++ b/apps/sim/connectors/google-sheets/google-sheets.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/components/icons', () => ({ diff --git a/apps/sim/connectors/google-workspace/company-crawl.test.ts b/apps/sim/connectors/google-workspace/company-crawl.test.ts index 1e34062abfc..cad86c52573 100644 --- a/apps/sim/connectors/google-workspace/company-crawl.test.ts +++ b/apps/sim/connectors/google-workspace/company-crawl.test.ts @@ -1,4 +1,8 @@ /** @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { getGoogleWorkspaceDocument, diff --git a/apps/sim/connectors/greenhouse/greenhouse.test.ts b/apps/sim/connectors/greenhouse/greenhouse.test.ts index aec03f5dab0..8d71a3ef493 100644 --- a/apps/sim/connectors/greenhouse/greenhouse.test.ts +++ b/apps/sim/connectors/greenhouse/greenhouse.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { greenhouseConnector } from '@/connectors/greenhouse/greenhouse' diff --git a/apps/sim/connectors/incidentio/incidentio.test.ts b/apps/sim/connectors/incidentio/incidentio.test.ts index 5ecfafce8fe..577c8c21dee 100644 --- a/apps/sim/connectors/incidentio/incidentio.test.ts +++ b/apps/sim/connectors/incidentio/incidentio.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { buildStatusCategoryParams, diff --git a/apps/sim/connectors/intercom/intercom.test.ts b/apps/sim/connectors/intercom/intercom.test.ts index 0e83ed6101f..388e78c18a4 100644 --- a/apps/sim/connectors/intercom/intercom.test.ts +++ b/apps/sim/connectors/intercom/intercom.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { intercomConnector } from '@/connectors/intercom/intercom' diff --git a/apps/sim/connectors/jira/jira.test.ts b/apps/sim/connectors/jira/jira.test.ts index a2c08e7a0ff..8a2abedf490 100644 --- a/apps/sim/connectors/jira/jira.test.ts +++ b/apps/sim/connectors/jira/jira.test.ts @@ -1,7 +1,12 @@ /** * @vitest-environment node */ -import { createMockResponse } from '@sim/testing' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + +import { createMockResponse, inputValidationMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { AtlassianSiteNotMatchedError, clearAtlassianCloudIdCache } from '@/lib/atlassian/discovery' import { diff --git a/apps/sim/connectors/outlook/outlook.test.ts b/apps/sim/connectors/outlook/outlook.test.ts index 7452c2969c3..b02b5878411 100644 --- a/apps/sim/connectors/outlook/outlook.test.ts +++ b/apps/sim/connectors/outlook/outlook.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { DEFAULT_MAX_CONVERSATIONS } from '@/connectors/outlook/meta' import { diff --git a/apps/sim/connectors/reddit/reddit.test.ts b/apps/sim/connectors/reddit/reddit.test.ts index efee0a3707d..110633ea8cf 100644 --- a/apps/sim/connectors/reddit/reddit.test.ts +++ b/apps/sim/connectors/reddit/reddit.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { redditConnector } from '@/connectors/reddit/reddit' import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' diff --git a/apps/sim/connectors/servicenow/servicenow.test.ts b/apps/sim/connectors/servicenow/servicenow.test.ts index 65a3c5fa85c..87f4a815831 100644 --- a/apps/sim/connectors/servicenow/servicenow.test.ts +++ b/apps/sim/connectors/servicenow/servicenow.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@/components/icons', () => ({ diff --git a/apps/sim/connectors/servicenow/servicenow.ts b/apps/sim/connectors/servicenow/servicenow.ts index bf223f89f21..0d479dd6e14 100644 --- a/apps/sim/connectors/servicenow/servicenow.ts +++ b/apps/sim/connectors/servicenow/servicenow.ts @@ -1,11 +1,16 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { validateServiceNowInstanceUrl } from '@/lib/core/security/input-validation' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_ITEMS, servicenowConnectorMeta } from '@/connectors/servicenow/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, parseTagDate } from '@/connectors/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('ServiceNowConnector') const PAGE_SIZE = 100 @@ -255,7 +260,7 @@ async function serviceNowApiGet( 'Content-Type': 'application/json', }, }, - retryOptions + { ...retryOptions, fetcher: providerFetch } ) if (!response.ok) { @@ -309,7 +314,7 @@ async function serviceNowApiGetById( 'Content-Type': 'application/json', }, }, - retryOptions + { ...retryOptions, fetcher: providerFetch } ) if (response.status === 404) { diff --git a/apps/sim/connectors/slack/slack.test.ts b/apps/sim/connectors/slack/slack.test.ts index 3353e691137..75045457352 100644 --- a/apps/sim/connectors/slack/slack.test.ts +++ b/apps/sim/connectors/slack/slack.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { slackConnectorMeta } from '@/connectors/slack/meta' import { slackConnector } from '@/connectors/slack/slack' diff --git a/apps/sim/connectors/typeform/typeform.test.ts b/apps/sim/connectors/typeform/typeform.test.ts index 973fe9bac0e..f130e4b491c 100644 --- a/apps/sim/connectors/typeform/typeform.test.ts +++ b/apps/sim/connectors/typeform/typeform.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { typeformConnector } from '@/connectors/typeform/typeform' diff --git a/apps/sim/connectors/webflow/webflow.test.ts b/apps/sim/connectors/webflow/webflow.test.ts index 3b757e019ec..36ef73e6a05 100644 --- a/apps/sim/connectors/webflow/webflow.test.ts +++ b/apps/sim/connectors/webflow/webflow.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { isCurrentItem, webflowConnector } from '@/connectors/webflow/webflow' diff --git a/apps/sim/connectors/wordpress/wordpress.test.ts b/apps/sim/connectors/wordpress/wordpress.test.ts index 580354b4e5a..dc263c3b8a0 100644 --- a/apps/sim/connectors/wordpress/wordpress.test.ts +++ b/apps/sim/connectors/wordpress/wordpress.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { wordpressConnector } from '@/connectors/wordpress/wordpress' diff --git a/apps/sim/connectors/workday/workday.test.ts b/apps/sim/connectors/workday/workday.test.ts index 3bd51ea630e..ee73b18ff09 100644 --- a/apps/sim/connectors/workday/workday.test.ts +++ b/apps/sim/connectors/workday/workday.test.ts @@ -7,6 +7,10 @@ * with the `listingCapped` flag the sync engine reads before hard-deleting the * documents a partial listing left out. */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { workdayConnector } from '@/connectors/workday/workday' diff --git a/apps/sim/connectors/workday/workday.ts b/apps/sim/connectors/workday/workday.ts index a19b9bf0ef6..e9876d2a003 100644 --- a/apps/sim/connectors/workday/workday.ts +++ b/apps/sim/connectors/workday/workday.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { validatePathSegment, validateWorkdayTenantUrl } from '@/lib/core/security/input-validation' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { RetryOptions } from '@/lib/knowledge/documents/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -13,6 +14,10 @@ import { } from '@/connectors/utils' import { workdayConnectorMeta } from '@/connectors/workday/meta' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('WorkdayConnector') /** @@ -216,7 +221,7 @@ async function getBearerToken( refresh_token: refreshToken, }).toString(), }, - retryOptions + { ...retryOptions, fetcher: providerFetch } ) if (!response.ok) { @@ -289,7 +294,7 @@ async function workdayGet( fetchWithRetry( url, { method: 'GET', headers: { Authorization: `Bearer ${bearer}`, Accept: 'application/json' } }, - retryOptions + { ...retryOptions, fetcher: providerFetch } ) const response = await send(await getBearerToken(accessToken, wd, syncContext, retryOptions)) diff --git a/apps/sim/connectors/youtube/youtube.test.ts b/apps/sim/connectors/youtube/youtube.test.ts index 2eb8fe7242d..40f9ebb5900 100644 --- a/apps/sim/connectors/youtube/youtube.test.ts +++ b/apps/sim/connectors/youtube/youtube.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { youtubeConnector } from '@/connectors/youtube/youtube' diff --git a/apps/sim/executor/handlers/workflow/workflow-handler.test.ts b/apps/sim/executor/handlers/workflow/workflow-handler.test.ts index 6b1d4283ef3..380ac6c1ab4 100644 --- a/apps/sim/executor/handlers/workflow/workflow-handler.test.ts +++ b/apps/sim/executor/handlers/workflow/workflow-handler.test.ts @@ -7,6 +7,10 @@ import { } from '@sim/testing' import { afterAll, beforeAll, beforeEach, describe, expect, it, type Mock, vi } from 'vitest' import { createTimeoutAbortController, getExecutionDeadlineAt } from '@/lib/core/execution-limits' +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { getBlock } from '@/blocks/registry' import { BlockType } from '@/executor/constants' @@ -28,6 +32,21 @@ const mockWorkflowLogger = vi.mocked(loggerMock.createLogger).mock.results[ vi.mocked(createLogger).mock.calls.findIndex(([name]) => name === 'WorkflowBlockHandler') ].value +const outboundMocks = vi.hoisted(() => ({ + enabled: vi.fn(() => false), + workspace: vi.fn(), + route: vi.fn(async (organizationId: string | null | undefined) => ({ organizationId })), +})) +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: outboundMocks.enabled, + resolveOutboundRoute: outboundMocks.route, +})) +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadActiveWorkspaceApplicationContext: outboundMocks.workspace, +})) + +beforeEach(() => outboundMocks.enabled.mockReturnValue(false)) + const { mockExecutorExecute, mockCreateSnapshot, @@ -632,7 +651,9 @@ describe('WorkflowBlockHandler', () => { expect(mockGetPersonalAndWorkspaceEnv).not.toHaveBeenCalled() }) - it('resolves a source-scoped billing attribution for custom block children', async () => { + it('resolves source billing and routing for custom block children', async () => { + outboundMocks.enabled.mockReturnValue(true) + outboundMocks.workspace.mockResolvedValue({ workspaceOrganizationId: 'source-org' }) const consumerAttribution = { actorUserId: 'consumer-1', workspaceId: 'workspace-consumer' } const sourceAttribution = { actorUserId: 'owner-9', workspaceId: 'workspace-source' } const customBlock = { @@ -688,9 +709,18 @@ describe('WorkflowBlockHandler', () => { } }) mockCreateSnapshot.mockResolvedValue({ snapshot: { id: 'snapshot-1' } }) - mockExecutorExecute.mockResolvedValue({ success: true, output: { data: 'ok' } }) + mockExecutorExecute.mockImplementationOnce(async () => { + await resolveCurrentOutboundRoute() + expect(outboundMocks.route).toHaveBeenLastCalledWith('source-org') + return { success: true, output: { data: 'ok' } } + }) - await handler.execute(ctx, customBlock, {}) + await runWithOutboundOrganization('consumer-org', async () => { + await handler.execute(ctx, customBlock, {}) + await resolveCurrentOutboundRoute() + expect(outboundMocks.route).toHaveBeenLastCalledWith('consumer-org') + }) + expect(outboundMocks.workspace).toHaveBeenCalledExactlyOnceWith('workspace-source') expect(mockReadWorkflowDefinitionAsExecutor).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/apps/sim/executor/handlers/workflow/workflow-handler.ts b/apps/sim/executor/handlers/workflow/workflow-handler.ts index 3eb08c62779..961606ac8a1 100644 --- a/apps/sim/executor/handlers/workflow/workflow-handler.ts +++ b/apps/sim/executor/handlers/workflow/workflow-handler.ts @@ -5,6 +5,7 @@ import { isRecordLike } from '@sim/utils/object' import type { Variable, WorkflowState } from '@sim/workflow-types/workflow' import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' import { getExecutionDeadlineAt } from '@/lib/core/execution-limits' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { asOrchestrationError } from '@/lib/core/orchestration/types' import { getExecutionEnvironment } from '@/lib/environment/utils' import { buildNextCallChain, validateCallChain } from '@/lib/execution/call-chain' @@ -908,7 +909,10 @@ export class WorkflowBlockHandler implements BlockHandler { const startTime = performance.now() - const result = await subExecutor.execute(workflowId) + const executeChild = () => subExecutor.execute(workflowId) + const result = await (isCustomBlock + ? withResourceOutboundScope({ workspaceId: childWorkspaceId }, executeChild) + : executeChild()) const executionResult = this.toExecutionResult(result) const duration = performance.now() - startTime diff --git a/apps/sim/hooks/queries/organization-network.ts b/apps/sim/hooks/queries/organization-network.ts new file mode 100644 index 00000000000..12d34343cff --- /dev/null +++ b/apps/sim/hooks/queries/organization-network.ts @@ -0,0 +1,22 @@ +import { useQuery } from '@tanstack/react-query' +import { requestJson } from '@/lib/api/client/request' +import { getOrganizationNetworkContract } from '@/lib/api/contracts/organization-network' + +export const ORGANIZATION_NETWORK_STALE_TIME = 30_000 + +export const organizationNetworkKeys = { + all: ['organization-network'] as const, + details: () => [...organizationNetworkKeys.all, 'detail'] as const, + detail: (organizationId: string) => + [...organizationNetworkKeys.details(), organizationId] as const, +} + +export function useOrganizationNetwork(organizationId: string) { + return useQuery({ + queryKey: organizationNetworkKeys.detail(organizationId), + queryFn: ({ signal }) => + requestJson(getOrganizationNetworkContract, { params: { id: organizationId }, signal }), + enabled: Boolean(organizationId), + staleTime: ORGANIZATION_NETWORK_STALE_TIME, + }) +} diff --git a/apps/sim/lib/api/contracts/organization-network.ts b/apps/sim/lib/api/contracts/organization-network.ts new file mode 100644 index 00000000000..bd82581d40a --- /dev/null +++ b/apps/sim/lib/api/contracts/organization-network.ts @@ -0,0 +1,23 @@ +import { z } from 'zod' +import { organizationIdSchema } from '@/lib/api/contracts/primitives' +import { defineRouteContract } from '@/lib/api/contracts/types' +import { gatewayPublicMetadataSchema } from '@/lib/core/network/gateway-metadata' + +export const organizationNetworkSchema = z.discriminatedUnion('mode', [ + z.object({ mode: z.literal('direct') }), + z.object({ + mode: z.literal('gateway'), + publicIps: gatewayPublicMetadataSchema.shape.publicIps, + }), + z.object({ mode: z.literal('blocked') }), + z.object({ mode: z.literal('unavailable') }), +]) + +export const getOrganizationNetworkContract = defineRouteContract({ + method: 'GET', + path: '/api/organizations/[id]/network', + params: z.object({ id: organizationIdSchema }), + response: { mode: 'json', schema: organizationNetworkSchema }, +}) + +export type OrganizationNetwork = z.output diff --git a/apps/sim/lib/atlassian/assistant.test.ts b/apps/sim/lib/atlassian/assistant.test.ts index 1a940b9dd28..874c87fbbed 100644 --- a/apps/sim/lib/atlassian/assistant.test.ts +++ b/apps/sim/lib/atlassian/assistant.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { clearAtlassianCloudIdCache } from '@/lib/atlassian/discovery' import { @@ -15,6 +16,11 @@ import { getToolIds } from '@/tools/tool-ids' vi.unmock('@/tools/metadata') vi.unmock('@/tools/tool-ids') +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const CLOUD_ID = '12345678-1234-1234-1234-123456789012' const OTHER_CLOUD_ID = '12345678-1234-1234-1234-123456789013' const DOMAIN = 'selected.atlassian.net' diff --git a/apps/sim/lib/atlassian/discovery.test.ts b/apps/sim/lib/atlassian/discovery.test.ts index 5fcc307125f..166bc23298b 100644 --- a/apps/sim/lib/atlassian/discovery.test.ts +++ b/apps/sim/lib/atlassian/discovery.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { createMockResponse } from '@sim/testing' +import { createMockResponse, inputValidationMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { AtlassianSiteNotMatchedError, @@ -10,6 +10,11 @@ import { resolveAtlassianCloudId, } from '@/lib/atlassian/discovery' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const SITE = 'https://acme.atlassian.net' const CLOUD_ID = 'cloud-abc' diff --git a/apps/sim/lib/atlassian/discovery.ts b/apps/sim/lib/atlassian/discovery.ts index 33c9aa4f96a..6dd524747b5 100644 --- a/apps/sim/lib/atlassian/discovery.ts +++ b/apps/sim/lib/atlassian/discovery.ts @@ -1,6 +1,7 @@ import { sha256Hex } from '@sim/security/hash' import { parseRetryAfter } from '@sim/utils/retry' import { LRUCache } from 'lru-cache' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { type HTTPError, isRetryableError, @@ -158,7 +159,9 @@ export function fetchAtlassianDiscoveryJson( ): Promise { return retryWithExponentialBackoff( async () => { - const response = await fetch(url, { + const response = await secureFetchWithValidation(url, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, method: 'GET', headers, signal: AbortSignal.timeout(DISCOVERY_REQUEST_TIMEOUT_MS), diff --git a/apps/sim/lib/copilot/generated/docs-manifest.ts b/apps/sim/lib/copilot/generated/docs-manifest.ts index 69d6daa5aab..bced482121a 100644 --- a/apps/sim/lib/copilot/generated/docs-manifest.ts +++ b/apps/sim/lib/copilot/generated/docs-manifest.ts @@ -383,6 +383,7 @@ export const DOCS_MANIFEST: readonly string[] = [ 'platform/enterprise/scim.mdx', 'platform/enterprise/scim/entra.mdx', 'platform/enterprise/scim/okta.mdx', + 'platform/enterprise/security.mdx', 'platform/enterprise/self-hosted.mdx', 'platform/enterprise/session-policies.mdx', 'platform/enterprise/sso.mdx', diff --git a/apps/sim/lib/core/application/authorized-workspace-use-case.ts b/apps/sim/lib/core/application/authorized-workspace-use-case.ts index c3a8b2e1d3d..19ed9ff3965 100644 --- a/apps/sim/lib/core/application/authorized-workspace-use-case.ts +++ b/apps/sim/lib/core/application/authorized-workspace-use-case.ts @@ -12,6 +12,7 @@ import type { PrincipalForOperation, WorkspaceOperation, } from '@/lib/core/application/workspace-operation' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import type { OrchestrationRequestContext } from '@/lib/core/orchestration/types' import type { ResourcePolicyBinding } from '@/lib/resource-policies/registry' @@ -205,23 +206,25 @@ export function defineAuthorizedWorkspaceUseCase< async execute(args) { const executionContext = await authorizePhase(args) const { principal, context, request } = executionContext - const result = await definition.execute(executionContext) - const resultContext = { ...executionContext, result } - const projectedAudit = definition.projectAudit?.(resultContext) - if (projectedAudit !== undefined) { - const auditEntries = Array.isArray(projectedAudit) ? projectedAudit : [projectedAudit] - if (auditEntries.length > 0) { - recordProjectedUseCaseAuditEntries( - definition.operation, - context.workspaceId, - principal, - request, - auditEntries - ) + return runWithOutboundOrganization(context.workspaceOrganizationId, async () => { + const result = await definition.execute(executionContext) + const resultContext = { ...executionContext, result } + const projectedAudit = definition.projectAudit?.(resultContext) + if (projectedAudit !== undefined) { + const auditEntries = Array.isArray(projectedAudit) ? projectedAudit : [projectedAudit] + if (auditEntries.length > 0) { + recordProjectedUseCaseAuditEntries( + definition.operation, + context.workspaceId, + principal, + request, + auditEntries + ) + } } - } - await definition.afterSuccess?.(resultContext) - return result + await definition.afterSuccess?.(resultContext) + return result + }) }, } } diff --git a/apps/sim/lib/core/config/appconfig.test.ts b/apps/sim/lib/core/config/appconfig.test.ts index fb497fe1788..e5f797d665f 100644 --- a/apps/sim/lib/core/config/appconfig.test.ts +++ b/apps/sim/lib/core/config/appconfig.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockSend } = vi.hoisted(() => ({ mockSend: vi.fn(), @@ -21,7 +21,7 @@ vi.mock('@aws-sdk/client-appconfigdata', () => ({ }, })) -import { fetchAppConfigProfile } from '@/lib/core/config/appconfig' +import { fetchAppConfigProfile, fetchAppConfigSnapshot } from '@/lib/core/config/appconfig' const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value)) @@ -133,4 +133,154 @@ describe('fetchAppConfigProfile', () => { expect(b).toEqual({ x: 1 }) expect(mockSend.mock.calls.map(([c]) => c.__type)).toEqual(['start', 'get']) }) + + it('honors the server poll interval and serves warm values during one shared refresh', async () => { + vi.useFakeTimers() + vi.setSystemTime(100_000) + try { + mockSend.mockImplementation((command: { __type: string }) => + Promise.resolve( + command.__type === 'start' + ? { InitialConfigurationToken: 'token' } + : { + Configuration: encode({ revision: 'first' }), + NextPollConfigurationToken: 'next', + NextPollIntervalInSeconds: 60, + } + ) + ) + const ids = uniqueIds() + const parse = (value: unknown) => value + expect(await fetchAppConfigProfile(ids, parse)).toEqual({ revision: 'first' }) + vi.setSystemTime(130_001) + await fetchAppConfigProfile(ids, parse) + expect(mockSend).toHaveBeenCalledTimes(2) + + let finish: (value: unknown) => void = () => {} + mockSend.mockReturnValueOnce( + new Promise((resolve) => { + finish = resolve + }) + ) + vi.setSystemTime(161_000) + expect( + await Promise.all([fetchAppConfigProfile(ids, parse), fetchAppConfigProfile(ids, parse)]) + ).toEqual([{ revision: 'first' }, { revision: 'first' }]) + expect(mockSend).toHaveBeenCalledTimes(3) + finish({ Configuration: encode({ revision: 'second' }), NextPollConfigurationToken: 'next' }) + await vi.waitFor(async () => { + expect(await fetchAppConfigProfile(ids, parse)).toEqual({ revision: 'second' }) + }) + } finally { + vi.useRealTimers() + } + }) +}) + +describe('fetchAppConfigSnapshot freshness', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + vi.setSystemTime(100_000) + }) + afterEach(() => vi.useRealTimers()) + + it('evicts old profiles and requires fresh evidence when they are requested again', async () => { + mockSend.mockImplementation((command: { __type: string }) => + Promise.resolve( + command.__type === 'start' + ? { InitialConfigurationToken: 'token' } + : { Configuration: encode({ revision: 'valid' }), NextPollConfigurationToken: 'next' } + ) + ) + const oldest = uniqueIds() + await fetchAppConfigSnapshot(oldest, (value) => value) + const results = await Promise.all( + Array.from({ length: 65 }, () => fetchAppConfigSnapshot(uniqueIds(), (value) => value)) + ) + expect(results.every((result) => result.value !== null)).toBe(true) + mockSend.mockRejectedValueOnce(new Error('unavailable')) + expect(await fetchAppConfigSnapshot(oldest, (value) => value)).toEqual({ + value: null, + validatedAt: null, + }) + expect(mockSend.mock.calls.at(-1)?.[0].__type).toBe('start') + }) + + it('does not turn an empty first response or a cold failure into a valid snapshot', async () => { + mockSend.mockRejectedValueOnce(new Error('unavailable')) + expect(await fetchAppConfigSnapshot(uniqueIds(), (value) => value)).toEqual({ + value: null, + validatedAt: null, + }) + mockSend.mockImplementation((command: { __type: string }) => + Promise.resolve( + command.__type === 'start' + ? { InitialConfigurationToken: 'token' } + : { Configuration: new Uint8Array(), NextPollConfigurationToken: 'next' } + ) + ) + expect(await fetchAppConfigSnapshot(uniqueIds(), (value) => value)).toEqual({ + value: null, + validatedAt: null, + }) + }) + + it('does not renew an old snapshot after a rejected revision followed by unchanged polls', async () => { + let payload = encode({ revision: 'valid' }) + mockSend.mockImplementation((command: { __type: string }) => + Promise.resolve( + command.__type === 'start' + ? { InitialConfigurationToken: 'token' } + : { + Configuration: payload, + NextPollConfigurationToken: 'next', + NextPollIntervalInSeconds: 30, + } + ) + ) + const ids = uniqueIds() + const parse = (value: unknown) => value + const first = await fetchAppConfigSnapshot(ids, parse) + expect(first.validatedAt).toBe(100_000) + payload = new TextEncoder().encode('invalid json') + vi.setSystemTime(130_001) + expect(await fetchAppConfigSnapshot(ids, parse)).toEqual(first) + payload = new Uint8Array() + vi.setSystemTime(160_002) + expect(await fetchAppConfigSnapshot(ids, parse)).toEqual(first) + payload = encode({ revision: 'replacement' }) + vi.setSystemTime(190_003) + expect(await fetchAppConfigSnapshot(ids, parse)).toEqual({ + value: { revision: 'replacement' }, + validatedAt: 190_003, + }) + }) + + it('renews a validated unchanged revision and deduplicates due polls', async () => { + let payload = encode({ revision: 'valid' }) + mockSend.mockImplementation((command: { __type: string }) => + Promise.resolve( + command.__type === 'start' + ? { InitialConfigurationToken: 'token' } + : { + Configuration: payload, + NextPollConfigurationToken: 'next', + NextPollIntervalInSeconds: 30, + } + ) + ) + const ids = uniqueIds() + const parse = (value: unknown) => value + await fetchAppConfigSnapshot(ids, parse) + payload = new Uint8Array() + vi.setSystemTime(130_001) + const results = await Promise.all([ + fetchAppConfigSnapshot(ids, parse), + fetchAppConfigSnapshot(ids, parse), + ]) + expect(results[0].validatedAt).toBe(130_001) + expect(results[1]).toEqual(results[0]) + expect(mockSend.mock.calls).toHaveLength(3) + }) }) diff --git a/apps/sim/lib/core/config/appconfig.ts b/apps/sim/lib/core/config/appconfig.ts index 61aa093c375..8ebfc5eb954 100644 --- a/apps/sim/lib/core/config/appconfig.ts +++ b/apps/sim/lib/core/config/appconfig.ts @@ -6,6 +6,7 @@ import { } from '@aws-sdk/client-appconfigdata' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { LRUCache } from 'lru-cache' import { getAwsCredentialsFromEnv } from '@/lib/core/config/aws' import { env } from '@/lib/core/config/env' @@ -22,16 +23,43 @@ export interface AppConfigProfileIdentifiers { interface CacheEntry { /** Last successfully parsed value, or `null` if the config is empty/unseeded. */ value: T | null - /** True once any poll has completed (success, empty payload, or error). */ - loaded: boolean /** Token for the next `GetLatestConfiguration` poll, rotated on each call. */ nextToken: string | undefined - expiresAt: number - /** In-flight poll, shared so concurrent callers don't each hit AppConfig. */ - inflight: Promise | null + validatedAt: number | null + remoteMatchesValue: boolean + strict: boolean } -const cache = new Map>() +export interface AppConfigSnapshot { + readonly value: T | null + readonly validatedAt: number | null +} + +interface PollContext { + ids: AppConfigProfileIdentifiers + parse: (json: unknown) => unknown + strict: boolean +} + +const cache = new LRUCache, PollContext>({ + max: 64, + ttl: DEFAULT_TTL_MS, + ttlResolution: 0, + ignoreFetchAbort: true, + /** Poll intervals and snapshot freshness share the same clock. */ + perf: { now: () => Date.now() }, + fetchMethod: async (_key, stale, { context, options }) => { + const entry = stale ?? { + value: null, + nextToken: undefined, + validatedAt: null, + remoteMatchesValue: false, + strict: context.strict, + } + options.ttl = await poll(context.ids, context.parse, entry) + return entry + }, +}) let client: AppConfigDataClient | null = null @@ -58,68 +86,69 @@ function cacheKey(ids: AppConfigProfileIdentifiers): string { * Run one AppConfig poll for `entry`: starts a session if no token is held, then * calls `GetLatestConfiguration`. An empty payload means "unchanged" (or an * unseeded profile) and the previous value is kept. Any error is logged and the - * last good value is retained. Marks the entry `loaded` on any outcome so callers - * never re-block on the cold path, and honors AppConfig's `NextPollInterval` so we + * last good value is retained. Returns AppConfig's `NextPollInterval` so we * don't poll faster than the server allows (which would throttle). */ async function poll( ids: AppConfigProfileIdentifiers, parse: (json: unknown) => T, entry: CacheEntry -): Promise { +): Promise { let response: GetLatestConfigurationCommandOutput try { const dataClient = getClient() if (!entry.nextToken) { + entry.remoteMatchesValue = false const session = await dataClient.send( new StartConfigurationSessionCommand({ ApplicationIdentifier: ids.application, EnvironmentIdentifier: ids.environment, ConfigurationProfileIdentifier: ids.profile, - }) + }), + { abortSignal: AbortSignal.timeout(5000) } ) entry.nextToken = session.InitialConfigurationToken } response = await dataClient.send( - new GetLatestConfigurationCommand({ ConfigurationToken: entry.nextToken }) + new GetLatestConfigurationCommand({ ConfigurationToken: entry.nextToken }), + { abortSignal: AbortSignal.timeout(5000) } ) entry.nextToken = response.NextPollConfigurationToken ?? entry.nextToken } catch (error) { - // Network/session failure: drop the token so the next attempt starts a fresh - // session (handles expired or invalid tokens). Mark loaded + back off so we - // serve the fallback and retry in the background rather than blocking every - // request during an outage. + /** A failed or expired session retries after backoff without renewing snapshot freshness. */ entry.nextToken = undefined - entry.expiresAt = Date.now() + DEFAULT_TTL_MS - entry.loaded = true logger.error('AppConfig fetch failed; serving last known value', { profile: cacheKey(ids), error: getErrorMessage(error), }) - return entry.value + return DEFAULT_TTL_MS } - // Parse outside the network try: a decode/parse error must NOT discard the - // already-rotated session token — the round trip succeeded, so the next poll - // can reuse it instead of opening a new session. Keep the last good value. + /** Decode failures retain the rotated session token and last validated value. */ try { if (response.Configuration && response.Configuration.length > 0) { + entry.remoteMatchesValue = false + if (entry.strict && response.Configuration.length > 1_048_576) { + throw new Error('Configuration exceeds the maximum size') + } const text = new TextDecoder().decode(response.Configuration) entry.value = parse(JSON.parse(text)) + entry.remoteMatchesValue = true + } + if (entry.remoteMatchesValue && entry.value !== null) { + entry.validatedAt = Date.now() } } catch (error) { logger.error('AppConfig response parse failed; serving last known value', { profile: cacheKey(ids), - error: getErrorMessage(error), + error: entry.strict ? 'Configuration rejected' : getErrorMessage(error), }) } const intervalMs = (response.NextPollIntervalInSeconds ?? 60) * 1000 - entry.expiresAt = Date.now() + Math.max(DEFAULT_TTL_MS, intervalMs) - entry.loaded = true - return entry.value + return Math.max(DEFAULT_TTL_MS, intervalMs) } /** @@ -137,31 +166,28 @@ export async function fetchAppConfigProfile( ids: AppConfigProfileIdentifiers, parse: (json: unknown) => T ): Promise { - const key = cacheKey(ids) - const entry = (cache.get(key) as CacheEntry | undefined) ?? { - value: null, - loaded: false, - nextToken: undefined, - expiresAt: 0, - inflight: null, - } - cache.set(key, entry) - - // Cold: never polled — await a single shared poll so concurrent callers don't - // each hit AppConfig (and don't race the rotating session token). - if (!entry.loaded) { - entry.inflight ??= poll(ids, parse, entry).finally(() => { - entry.inflight = null - }) - return entry.inflight - } - - // Warm but stale: serve cached value, refresh once in the background. - if (Date.now() >= entry.expiresAt && !entry.inflight) { - entry.inflight = poll(ids, parse, entry).finally(() => { - entry.inflight = null - }) - } + const entry = await cache.fetch(cacheKey(ids), { + context: { ids, parse, strict: false }, + allowStale: true, + }) + return (entry?.value ?? null) as T | null +} - return entry.value +/** + * Security-sensitive callers receive freshness evidence instead of an implicit fallback. + * Due polls are awaited and deduplicated. Rejected remote revisions cannot renew an old + * snapshot through subsequent unchanged responses. The caller owns its maximum stale age. + */ +export async function fetchAppConfigSnapshot( + ids: AppConfigProfileIdentifiers, + parse: (json: unknown) => T +): Promise> { + const entry = await cache.fetch(`strict:${cacheKey(ids)}`, { + context: { ids, parse, strict: true }, + allowStale: false, + }) + return Object.freeze({ + value: (entry?.value ?? null) as T | null, + validatedAt: entry?.validatedAt ?? null, + }) } diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index efece380bdb..72c47f9a279 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -107,6 +107,10 @@ export const env = createEnv({ skipValidation: true, server: { + OUTBOUND_ROUTING_SOURCE: z.enum(['env', 'appconfig']).optional(), + OUTBOUND_ROUTING_CONFIG: z.string().optional(), + OUTBOUND_GATEWAYS: z.string().optional(), + OUTBOUND_GATEWAY_CREDENTIALS: z.string().optional(), // Core Database & Authentication DATABASE_URL: z.string().url(), // Primary database connection string DATABASE_REPLICA_URL: z.string().url().optional(), // Read-replica connection string; opt-in reads fall back to the primary when unset diff --git a/apps/sim/lib/core/network/application/read-organization-network.test.ts b/apps/sim/lib/core/network/application/read-organization-network.test.ts new file mode 100644 index 00000000000..bb97d4736e8 --- /dev/null +++ b/apps/sim/lib/core/network/application/read-organization-network.test.ts @@ -0,0 +1,107 @@ +/** @vitest-environment node */ +import type { Principal } from '@sim/auth/principal' +import { db } from '@sim/db' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + membership: vi.fn(), + section: vi.fn(), + route: vi.fn(), +})) +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfigForOrganization: vi.fn(async () => null), +})) +vi.mock('@/lib/settings/application/organization-section-access', () => ({ + authorizeOrganizationSettingsSection: mocks.section, +})) +vi.mock('@/lib/core/network/config.server', () => ({ resolveOutboundRoute: mocks.route })) + +import { readOrganizationNetwork } from '@/lib/core/network/application/read-organization-network' +import { OutboundRoutingError } from '@/lib/core/network/routing' + +const principal: Principal = { + kind: 'session', + userId: 'user_example', + sessionId: 'session_example', +} +const input = { organizationId: 'org_example' } + +beforeEach(() => { + vi.clearAllMocks() + mocks.membership.mockResolvedValue([{ role: 'admin' }]) + mocks.section.mockResolvedValue(true) + mocks.route.mockResolvedValue({ kind: 'direct' }) + const query = { from: vi.fn(), where: vi.fn(), limit: mocks.membership } + query.from.mockReturnValue(query) + query.where.mockReturnValue(query) + vi.mocked(db.select).mockReturnValue(query as ReturnType) +}) + +describe('organization network settings', () => { + it.each([[], [{ role: 'member' }]])('withholds routing from non-admins', async (rows) => { + mocks.membership.mockResolvedValue(rows) + await expect(readOrganizationNetwork.execute({ principal, input })).rejects.toThrow() + expect(mocks.route).not.toHaveBeenCalled() + expect(mocks.section).not.toHaveBeenCalled() + }) + + it('rejects workspace credentials before looking up the organization', async () => { + const workspacePrincipal: Principal = { + kind: 'workspace_api_key', + workspaceId: 'workspace_example', + apiKeyId: 'key_example', + } + await expect( + readOrganizationNetwork.execute({ principal: workspacePrincipal, input }) + ).rejects.toThrow() + expect(mocks.membership).not.toHaveBeenCalled() + expect(mocks.route).not.toHaveBeenCalled() + }) + + it('checks the target organization entitlement before reading its network', async () => { + mocks.section.mockResolvedValue(false) + await expect(readOrganizationNetwork.execute({ principal, input })).rejects.toThrow( + 'Network settings are not available' + ) + expect(mocks.route).not.toHaveBeenCalled() + expect(mocks.section).toHaveBeenCalledWith( + expect.objectContaining({ organizationId: 'org_example', section: 'security' }) + ) + }) + + it('projects only published addresses, never credentials or dial addresses', async () => { + mocks.route.mockResolvedValue({ + kind: 'gateway', + revision: 'revision_example', + scopeKey: 'org_example', + gateway: { + id: 'gateway_example', + url: 'https://private.example.invalid', + token: 'synthetic-secret', + ca: 'synthetic-ca', + publicIps: ['192.0.2.10', '192.0.2.20'], + }, + }) + await expect(readOrganizationNetwork.execute({ principal, input })).resolves.toEqual({ + mode: 'gateway', + publicIps: ['192.0.2.10', '192.0.2.20'], + }) + expect(mocks.route).toHaveBeenCalledWith('org_example') + }) + + it.each([ + ['ROUTE_BLOCKED', 'blocked'], + ['CONFIGURATION_UNAVAILABLE', 'unavailable'], + ['INVALID_CONFIGURATION', 'unavailable'], + ] as const)('reports %s without claiming direct fallback', async (code, mode) => { + mocks.route.mockRejectedValue(new OutboundRoutingError(code)) + await expect(readOrganizationNetwork.execute({ principal, input })).resolves.toEqual({ mode }) + }) + + it('propagates unexpected failures instead of treating them as default routing', async () => { + mocks.route.mockRejectedValue(new Error('unexpected failure')) + await expect(readOrganizationNetwork.execute({ principal, input })).rejects.toThrow( + 'unexpected failure' + ) + }) +}) diff --git a/apps/sim/lib/core/network/application/read-organization-network.ts b/apps/sim/lib/core/network/application/read-organization-network.ts new file mode 100644 index 00000000000..b2bc8f9404b --- /dev/null +++ b/apps/sim/lib/core/network/application/read-organization-network.ts @@ -0,0 +1,52 @@ +import type { OrganizationNetwork } from '@/lib/api/contracts/organization-network' +import type { OperationUseCase } from '@/lib/core/application/operation' +import { authorizeOrganizationOperation } from '@/lib/core/application/organization-authorization' +import { defineOrganizationOperation } from '@/lib/core/application/organization-operation' +import { resolveOutboundRoute } from '@/lib/core/network/config.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { authorizeOrganizationSettingsSection } from '@/lib/settings/application/organization-section-access' + +/** + * permission-group-exempt: Network settings are governed by organization administrator membership. + */ +export const readOrganizationNetworkOperation = defineOrganizationOperation({ + id: 'organization.network.read', + minimumRole: 'admin', + principalKinds: ['session'], + capability: 'none', +}) + +export const readOrganizationNetwork: OperationUseCase< + typeof readOrganizationNetworkOperation, + { organizationId: string }, + OrganizationNetwork +> = { + operation: readOrganizationNetworkOperation, + async execute({ principal, input }) { + const context = await authorizeOrganizationOperation( + principal, + readOrganizationNetworkOperation, + input + ) + if ( + !(await authorizeOrganizationSettingsSection({ + ...context, + section: 'security', + })) + ) { + throw new OrchestrationError('forbidden', 'Network settings are not available') + } + try { + const route = await resolveOutboundRoute(context.organizationId) + if (route.kind === 'direct') return { mode: 'direct' } + return { + mode: 'gateway', + publicIps: [...(route.gateway.publicIps ?? [])], + } + } catch (error) { + if (!(error instanceof OutboundRoutingError)) throw error + return { mode: error.code === 'ROUTE_BLOCKED' ? 'blocked' : 'unavailable' } + } + }, +} diff --git a/apps/sim/lib/core/network/aws-handler.server.ts b/apps/sim/lib/core/network/aws-handler.server.ts new file mode 100644 index 00000000000..d38706c0060 --- /dev/null +++ b/apps/sim/lib/core/network/aws-handler.server.ts @@ -0,0 +1,61 @@ +import { isIP } from 'node:net' +import { buildQueryString, HttpResponse } from '@smithy/core/protocols' +import { NodeHttpHandler } from '@smithy/node-http-handler' +import type { Dispatcher } from 'undici/index.js' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { + createOutboundTransport, + requestWithOutboundDispatcher, +} from '@/lib/core/network/transport.server' + +/** Cached AWS clients resolve organization routing per request, using the same transport on both runtimes. */ +export function createOutboundAwsHttpHandler() { + const direct = new NodeHttpHandler() + const transport = createOutboundTransport({ profile: 'configuredEndpoint' }) + return { + metadata: direct.metadata, + async handle(...[request, options]: Parameters) { + const dispatcher = await transport.selectDispatcher() + if (!dispatcher) return direct.handle(request, options) + const hostname = isIP(request.hostname) === 6 ? `[${request.hostname}]` : request.hostname + const port = request.port ? `:${request.port}` : '' + const target = `${request.protocol}//${hostname}${port}` + try { + const query = buildQueryString(request.query ?? {}) + const result = await requestWithOutboundDispatcher( + `${target}${request.path}${query ? `?${query}` : ''}`, + { + dispatcher, + method: request.method as Dispatcher.HttpMethod, + headers: request.headers, + body: request.body, + signal: options?.abortSignal as AbortSignal | undefined, + } + ) + const responseHeaders: Record = {} + for (const [name, value] of Object.entries(result.headers)) { + if (value !== undefined) + responseHeaders[name] = Array.isArray(value) ? value.join(',') : value + } + return { + response: new HttpResponse({ + statusCode: result.statusCode, + headers: responseHeaders, + body: result.body, + }), + } + } catch { + if (options?.abortSignal?.aborted) throw new DOMException('Request aborted', 'AbortError') + throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } + }, + destroy() { + direct.destroy() + void transport.destroy() + }, + updateHttpClientConfig(...args: Parameters) { + direct.updateHttpClientConfig(...args) + }, + httpHandlerConfigs: () => direct.httpHandlerConfigs(), + } +} diff --git a/apps/sim/lib/core/network/config.server.test.ts b/apps/sim/lib/core/network/config.server.test.ts new file mode 100644 index 00000000000..a93f4232e35 --- /dev/null +++ b/apps/sim/lib/core/network/config.server.test.ts @@ -0,0 +1,119 @@ +/** @vitest-environment node */ +import { describe, expect, it, vi } from 'vitest' +import { createOutboundRoutingReader } from '@/lib/core/network/config.server' + +const document = { + schemaVersion: 1, + revision: 'revision-1', + defaultRoute: { kind: 'direct' }, + organizations: { org_a: { kind: 'gateway', gatewayId: 'gateway_a' } }, +} +const catalog = { + gateway_a: { + url: 'https://proxy.example.invalid/', + credentialId: 'credential_a', + generation: 'generation-1', + }, +} +const credentials = { credential_a: { token: 'synthetic-test-token-0000000000000000' } } +const options = { + source: 'env', + configuration: JSON.stringify(document), + gateways: JSON.stringify(catalog), + credentials: JSON.stringify(credentials), +} +const dependencies = { now: () => 400_000, readSnapshot: vi.fn() } + +describe('outbound configuration', () => { + it('keeps unconfigured OSS deployments independent of AWS and organization context', async () => { + const readSnapshot = vi.fn() + const reader = createOutboundRoutingReader({}, { ...dependencies, readSnapshot }) + expect(reader.enabled).toBe(false) + expect(await reader.resolve(undefined)).toEqual({ kind: 'direct' }) + expect(readSnapshot).not.toHaveBeenCalled() + }) + + it('does not treat missing context as a personal workspace', async () => { + const reader = createOutboundRoutingReader(options, dependencies) + await expect(reader.resolve(undefined)).rejects.toThrow('MISSING_SCOPE') + expect(await reader.resolve(null)).toEqual({ kind: 'direct' }) + expect(await reader.resolve('org_a')).toMatchObject({ + kind: 'gateway', + scopeKey: 'org_a', + gateway: { id: 'gateway_a' }, + }) + }) + + it('keeps personal scope distinct from an organization named personal', async () => { + const gatewayRoute = { kind: 'gateway', gatewayId: 'gateway_a' } + const reader = createOutboundRoutingReader( + { + ...options, + configuration: JSON.stringify({ + ...document, + defaultRoute: gatewayRoute, + organizations: { personal: gatewayRoute }, + }), + }, + dependencies + ) + expect(await reader.resolve(null)).toMatchObject({ kind: 'gateway', scopeKey: null }) + expect(await reader.resolve('personal')).toMatchObject({ + kind: 'gateway', + scopeKey: 'personal', + }) + }) + + it.each([ + { ...options, source: undefined }, + { ...options, configuration: undefined }, + { ...options, gateways: '{}' }, + { ...options, credentials: '{}' }, + { + ...options, + gateways: JSON.stringify({ + gateway_a: { ...catalog.gateway_a, url: 'http://proxy.example.invalid/' }, + }), + }, + { + ...options, + gateways: JSON.stringify({ + gateway_a: { ...catalog.gateway_a, url: 'https://user:password@proxy.example.invalid/' }, + }), + }, + ])('rejects incomplete or unsafe operator configuration', (input) => { + expect(() => createOutboundRoutingReader(input, dependencies)).toThrow('INVALID_CONFIGURATION') + }) + + it('never falls back to direct routing on cold, stale or future snapshots', async () => { + for (const validatedAt of [null, 100_000, 500_000]) { + const reader = createOutboundRoutingReader( + { ...options, source: 'appconfig', configuration: undefined }, + { + ...dependencies, + readSnapshot: async (parse) => ({ value: parse(document), validatedAt }), + } + ) + await expect(reader.resolve('org_a')).rejects.toThrow('CONFIGURATION_UNAVAILABLE') + await expect(reader.resolve('org_other')).rejects.toThrow('CONFIGURATION_UNAVAILABLE') + } + }) + + it('keeps a still-valid snapshot and re-resolves current routing for each operation', async () => { + let current: unknown = document + const reader = createOutboundRoutingReader( + { ...options, source: 'appconfig', configuration: undefined }, + { + ...dependencies, + readSnapshot: async (parse) => ({ value: parse(current), validatedAt: 350_000 }), + } + ) + expect(await reader.resolve('org_a')).toMatchObject({ kind: 'gateway' }) + current = { + ...document, + revision: 'revision-2', + organizations: { org_a: { kind: 'blocked' } }, + } + await expect(reader.resolve('org_a')).rejects.toThrow('ROUTE_BLOCKED') + }) +}) diff --git a/apps/sim/lib/core/network/config.server.ts b/apps/sim/lib/core/network/config.server.ts new file mode 100644 index 00000000000..4f8c9902441 --- /dev/null +++ b/apps/sim/lib/core/network/config.server.ts @@ -0,0 +1,227 @@ +import { isIP } from 'node:net' +import { z } from 'zod' +import type { AppConfigSnapshot } from '@/lib/core/config/appconfig' +import { env } from '@/lib/core/config/env' +import { gatewayPublicMetadataSchema } from '@/lib/core/network/gateway-metadata' +import { + type OutboundRoutingConfig, + OutboundRoutingError, + parseOutboundJson, + parseOutboundRoutingConfig, + selectOutboundRoute, +} from '@/lib/core/network/routing' + +const MAX_STALE_MS = 300_000 +const keySchema = z + .string() + .min(1) + .max(128) + .regex(/^[a-zA-Z0-9_-]+$/) +const gatewaySchema = z.strictObject({ + url: z.string().url().max(2048), + servername: z + .string() + .min(1) + .max(253) + .regex(/^[a-zA-Z0-9.-]+$/) + .optional(), + credentialId: keySchema, + generation: keySchema, + ...gatewayPublicMetadataSchema.partial().shape, +}) +const gatewaysSchema = z + .record(keySchema, gatewaySchema) + .refine((value) => Object.keys(value).length <= 100) +const credentialSchema = z.strictObject({ + token: z + .string() + .min(32) + .max(4096) + .regex(/^[a-zA-Z0-9_=-]+$/), + ca: z.string().min(1).max(65_536).optional(), +}) +const credentialsSchema = z + .record(keySchema, credentialSchema) + .refine((value) => Object.keys(value).length <= 100) + +export interface OutboundGateway { + readonly id: string + readonly url: string + readonly servername: string + readonly generation: string + readonly token: string + readonly ca?: string + readonly publicIps?: readonly string[] +} + +export type ResolvedOutboundRoute = + | { readonly kind: 'direct' } + | { + readonly kind: 'gateway' + readonly gateway: OutboundGateway + readonly revision: string + readonly scopeKey: string | null + } + +interface OutboundConfigurationOptions { + source?: string + configuration?: string + gateways?: string + credentials?: string +} + +interface OutboundConfigurationDependencies { + readSnapshot( + parse: (value: unknown) => OutboundRoutingConfig + ): Promise> + now(): number +} + +/** A configured reader is immutable; failures cannot switch it to deployment defaults. */ +export function createOutboundRoutingReader( + options: OutboundConfigurationOptions, + dependencies: OutboundConfigurationDependencies +): { + enabled: boolean + resolve(organizationId: string | null | undefined): Promise +} { + if (options.source === undefined) { + if ( + options.configuration !== undefined || + options.gateways !== undefined || + options.credentials !== undefined + ) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + return { enabled: false, resolve: async () => ({ kind: 'direct' }) } + } + if (options.source !== 'env' && options.source !== 'appconfig') { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + const parsedGateways = gatewaysSchema.safeParse(parseOutboundJson(options.gateways ?? '{}')) + const parsedCredentials = credentialsSchema.safeParse( + parseOutboundJson(options.credentials ?? '{}') + ) + if (!parsedGateways.success || !parsedCredentials.success) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + const gateways = new Map() + for (const [id, entry] of Object.entries(parsedGateways.data)) { + const url = new URL(entry.url) + const hostname = url.hostname.replace(/^\[|\]$/g, '') + if ( + url.protocol !== 'https:' || + url.username || + url.password || + url.pathname !== '/' || + url.search || + url.hash + ) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + const servername = entry.servername ?? hostname + if ( + isIP(servername) || + !servername || + !Object.hasOwn(parsedCredentials.data, entry.credentialId) + ) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + const credential = parsedCredentials.data[entry.credentialId] + gateways.set( + id, + Object.freeze({ + id, + url: url.href, + servername, + generation: entry.generation, + ...credential, + ...(entry.publicIps ? { publicIps: Object.freeze([...new Set(entry.publicIps)]) } : {}), + }) + ) + } + const parse = (value: unknown): OutboundRoutingConfig => { + const config = parseOutboundRoutingConfig(value) + for (const route of [config.defaultRoute, ...Object.values(config.organizations)]) { + if (route.kind === 'gateway' && !gateways.has(route.gatewayId)) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + } + return config + } + const staticConfig = + options.source === 'env' ? parse(parseOutboundJson(options.configuration ?? '')) : null + if (options.source === 'appconfig' && options.configuration !== undefined) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + return { + enabled: true, + async resolve(organizationId) { + if (organizationId === undefined) throw new OutboundRoutingError('MISSING_SCOPE') + let config = staticConfig + if (!config) { + const snapshot = await dependencies.readSnapshot(parse) + const age = + snapshot.validatedAt === null + ? Number.POSITIVE_INFINITY + : dependencies.now() - snapshot.validatedAt + if (!snapshot.value || age < 0 || age >= MAX_STALE_MS) { + throw new OutboundRoutingError('CONFIGURATION_UNAVAILABLE') + } + config = snapshot.value + } + const route = selectOutboundRoute(config, organizationId) + if (route.kind === 'direct') return { kind: 'direct' } + if (route.kind === 'blocked') throw new OutboundRoutingError('ROUTE_BLOCKED') + const gateway = gateways.get(route.gatewayId) + if (!gateway) throw new OutboundRoutingError('INVALID_CONFIGURATION') + return { + kind: 'gateway', + gateway, + revision: config.revision, + scopeKey: organizationId, + } + }, + } +} + +let reader: ReturnType | undefined + +function getReader() { + reader ??= createOutboundRoutingReader( + { + source: env.OUTBOUND_ROUTING_SOURCE, + configuration: env.OUTBOUND_ROUTING_CONFIG, + gateways: env.OUTBOUND_GATEWAYS, + credentials: env.OUTBOUND_GATEWAY_CREDENTIALS, + }, + { + now: Date.now, + async readSnapshot(parse) { + if (!env.APPCONFIG_APPLICATION || !env.APPCONFIG_ENVIRONMENT) { + throw new OutboundRoutingError('CONFIGURATION_UNAVAILABLE') + } + const { fetchAppConfigSnapshot } = await import('@/lib/core/config/appconfig') + return fetchAppConfigSnapshot( + { + application: env.APPCONFIG_APPLICATION, + environment: env.APPCONFIG_ENVIRONMENT, + profile: 'outbound-routing', + }, + parse + ) + }, + } + ) + return reader +} + +/** True only for deployments with explicitly configured managed routing. */ +export function isOutboundRoutingEnabled(): boolean { + return getReader().enabled +} + +/** Credentials remain inside the transport layer; never serialize the result. */ +export function resolveOutboundRoute(organizationId: string | null | undefined) { + return getReader().resolve(organizationId) +} diff --git a/apps/sim/lib/core/network/context.server.test.ts b/apps/sim/lib/core/network/context.server.test.ts new file mode 100644 index 00000000000..dbec224f615 --- /dev/null +++ b/apps/sim/lib/core/network/context.server.test.ts @@ -0,0 +1,43 @@ +/** @vitest-environment node */ +import { describe, expect, it, vi } from 'vitest' + +const { resolve } = vi.hoisted(() => ({ + resolve: vi.fn(async (organizationId: string | null | undefined) => ({ + kind: 'direct', + organizationId, + })), +})) +vi.mock('@/lib/core/network/config.server', () => ({ resolveOutboundRoute: resolve })) + +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' + +describe('outbound execution context', () => { + it('isolates interleaved organizations and restores the parent after nested calls', async () => { + let release: () => void = () => {} + const gate = new Promise((done) => { + release = done + }) + const first = runWithOutboundOrganization('org_a', async () => { + await gate + await resolveCurrentOutboundRoute() + await runWithOutboundOrganization('org_child', resolveCurrentOutboundRoute) + return resolveCurrentOutboundRoute() + }) + const second = runWithOutboundOrganization('org_b', async () => { + await resolveCurrentOutboundRoute() + release() + return resolveCurrentOutboundRoute() + }) + expect(await Promise.all([first, second])).toEqual([ + { kind: 'direct', organizationId: 'org_a' }, + { kind: 'direct', organizationId: 'org_b' }, + ]) + expect(await resolveCurrentOutboundRoute()).toEqual({ + kind: 'direct', + organizationId: undefined, + }) + }) +}) diff --git a/apps/sim/lib/core/network/context.server.ts b/apps/sim/lib/core/network/context.server.ts new file mode 100644 index 00000000000..b9b5f4b6002 --- /dev/null +++ b/apps/sim/lib/core/network/context.server.ts @@ -0,0 +1,37 @@ +import { AsyncLocalStorage } from 'node:async_hooks' +import { resolveOutboundRoute } from '@/lib/core/network/config.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' + +interface OutboundScope { + readonly organizationId: string | null +} + +const storage = new AsyncLocalStorage() + +/** + * Established by authorized operations or jobs after loading canonical ownership. + * Null means a verified personal workspace. Never pass a requested organization. + */ +export function runWithOutboundOrganization(organizationId: string | null, run: () => T): T { + if (organizationId !== null && (typeof organizationId !== 'string' || !organizationId)) { + throw new OutboundRoutingError('MISSING_SCOPE') + } + return storage.run(Object.freeze({ organizationId }), run) +} + +/** Resolves current policy per operation, rather than freezing policy for a long-running job. */ +export function resolveCurrentOutboundRoute() { + return resolveOutboundRoute(storage.getStore()?.organizationId) +} + +/** Unsupported transports cannot silently escape a required gateway. */ +export async function requireDirectOutboundTransport(): Promise { + const route = await resolveCurrentOutboundRoute() + if (route.kind !== 'direct') throw new OutboundRoutingError('UNSUPPORTED_TRANSPORT') +} + +/** Captures only the outbound scope for deferred callbacks; route policy is still read per call. */ +export function captureOutboundScope(): (run: () => T) => T { + const scope = storage.getStore() + return (run: () => T): T => (scope ? storage.run(scope, run) : storage.exit(run)) +} diff --git a/apps/sim/lib/core/network/environment-proxy.server.test.ts b/apps/sim/lib/core/network/environment-proxy.server.test.ts index 2f09c10c241..22c352f9394 100644 --- a/apps/sim/lib/core/network/environment-proxy.server.test.ts +++ b/apps/sim/lib/core/network/environment-proxy.server.test.ts @@ -12,7 +12,12 @@ import { fileURLToPath } from 'node:url' import { promisify } from 'node:util' import { resolveHostAddresses } from '@sim/security/dns' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' +import type { ResolvedOutboundRoute } from '@/lib/core/network/config.server' +const { resolveRoute } = vi.hoisted(() => ({ + resolveRoute: vi.fn<() => Promise>(), +})) +vi.mock('@/lib/core/network/context.server', () => ({ resolveCurrentOutboundRoute: resolveRoute })) vi.mock('@sim/security/dns', { spy: true }) import { @@ -138,6 +143,7 @@ beforeEach(() => { ]) { vi.stubEnv(name, '') } + resolveRoute.mockResolvedValue({ kind: 'direct' }) rejectConnections = false connections.length = 0 requests.length = 0 diff --git a/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts b/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts new file mode 100644 index 00000000000..da628d53834 --- /dev/null +++ b/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts @@ -0,0 +1,73 @@ +/** Real runtime probe invoked by gateway.server.test.ts against its local TLS servers. */ +import { readFileSync } from 'node:fs' +import { HttpRequest } from '@smithy/core/protocols' +import { request as undiciRequest } from 'undici/index.js' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { createGatewayDispatcher } from '@/lib/core/network/gateway.server' +import { secureFetchWithPinnedIP } from '@/lib/core/security/input-validation.server' + +const [proxyPort, originPort, certificatePath] = process.argv.slice(2) +if (!proxyPort || !originPort || !certificatePath) + throw new Error('Local fixture ports and certificate are required') +const gateway = { + id: 'synthetic', + url: `https://127.0.0.1:${proxyPort}`, + servername: 'gateway.invalid', + generation: 'test', + token: 'a'.repeat(48), + ca: readFileSync(certificatePath, 'utf8'), +} +const dispatcher = createGatewayDispatcher(gateway, { + profile: 'selfHostedService', + resolvedIP: '127.0.0.1', +}) +const handler = createOutboundAwsHttpHandler() +try { + const response = await undiciRequest(`https://localhost:${originPort}`, { + dispatcher, + signal: AbortSignal.timeout(10_000), + }) + if (response.headers['x-via-proxy'] !== 'yes') throw new Error('Undici bypassed proxy') + if ((await response.body.text()) !== 'tls reached') throw new Error('Undici response mismatch') + + await runWithOutboundOrganization('org_test', async () => { + const pinned = await secureFetchWithPinnedIP(`https://localhost:${originPort}`, '127.0.0.1', { + profile: 'selfHostedService', + timeout: 10_000, + }) + if (pinned.headers.get('x-via-proxy') !== 'yes') throw new Error('Pinned fetch bypassed proxy') + if ((await pinned.text()) !== 'tls reached') throw new Error('Pinned fetch response mismatch') + + const { response: aws } = await handler.handle( + new HttpRequest({ + protocol: 'https:', + hostname: 'localhost', + port: Number(originPort), + method: 'GET', + path: '/', + headers: {}, + query: {}, + }) + ) + if (aws.headers['x-via-proxy'] !== 'yes') throw new Error('AWS handler bypassed proxy') + const chunks: Buffer[] = [] + for await (const chunk of aws.body) chunks.push(Buffer.from(chunk)) + if (Buffer.concat(chunks).toString() !== 'tls reached') throw new Error('AWS response mismatch') + }) + + let rejected = false + try { + await undiciRequest(`https://127.0.0.1:${originPort}`, { + dispatcher, + headers: { host: `localhost:${originPort}` }, + signal: AbortSignal.timeout(10_000), + }) + } catch { + rejected = true + } + if (!rejected) throw new Error('Mismatched upstream certificate was accepted') +} finally { + handler.destroy() + await dispatcher.destroy() +} diff --git a/apps/sim/lib/core/network/gateway-metadata.ts b/apps/sim/lib/core/network/gateway-metadata.ts new file mode 100644 index 00000000000..d7b82b7f820 --- /dev/null +++ b/apps/sim/lib/core/network/gateway-metadata.ts @@ -0,0 +1,6 @@ +import { z } from 'zod' + +/** Public operator-published metadata; never includes transport or credential configuration. */ +export const gatewayPublicMetadataSchema = z.object({ + publicIps: z.array(z.ipv4()).max(16), +}) diff --git a/apps/sim/lib/core/network/gateway.server.test.ts b/apps/sim/lib/core/network/gateway.server.test.ts new file mode 100644 index 00000000000..0c6c5a4f8e1 --- /dev/null +++ b/apps/sim/lib/core/network/gateway.server.test.ts @@ -0,0 +1,365 @@ +/** @vitest-environment node */ +import { execFile } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { createServer as httpServer } from 'node:http' +import { createServer as httpsServer } from 'node:https' +import { type AddressInfo, connect, type Socket } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { getCACertificates, setDefaultCACertificates } from 'node:tls' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import type { OutboundGateway, ResolvedOutboundRoute } from '@/lib/core/network/config.server' + +const { state } = vi.hoisted(() => ({ + state: { gateways: new Map(), revision: 'v1' }, +})) +vi.mock('@/lib/core/network/config.server', () => ({ + async resolveOutboundRoute( + organizationId: string | null | undefined + ): Promise { + const gateway = organizationId ? state.gateways.get(organizationId) : undefined + if (!gateway || !organizationId) throw new Error('No configured route') + return { kind: 'gateway', gateway, scopeKey: organizationId, revision: state.revision } + }, +})) + +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { openGatewayTunnel } from '@/lib/core/network/gateway.server' +import { + createPinnedFetchWithDispatcher, + secureFetchWithPinnedIP, +} from '@/lib/core/security/input-validation.server' + +const certificateDirectory = mkdtempSync(join(tmpdir(), 'gateway-tls-')) +const certificatePath = join(certificateDirectory, 'cert.pem') +const keyPath = join(certificateDirectory, 'key.pem') +let cert = '' +const sockets = new Set() +const admissions: Array<{ + token: string | undefined + destination: string | undefined + sni: string | undefined +}> = [] +let originRequests = 0 +let receivedHeaders: Record = {} +const origin = httpServer(async (req, res) => { + originRequests++ + receivedHeaders = req.headers + if (req.url === '/redirect') { + res.writeHead(302, { location: '/done' }) + res.end() + return + } + if (req.url === '/wait') return + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + res.end(req.method === 'POST' ? Buffer.concat(chunks) : 'reached') +}) +const proxiedPorts = new Set() +const secureOrigin = httpsServer((req, res) => { + res.setHeader('x-via-proxy', proxiedPorts.has(req.socket.remotePort ?? 0) ? 'yes' : 'no') + res.end('tls reached') +}) +const proxy = httpsServer() +proxy.on('connect', (req, socket, head) => { + socket.on('error', () => {}) + const tlsSocket = req.socket as Socket & { servername?: string } + admissions.push({ + token: req.headers['proxy-authorization'], + destination: req.url, + sni: tlsSocket.servername, + }) + if (!req.headers['proxy-authorization']?.match(/^Bearer (alpha|bravo|a{48})$/)) { + socket.end('HTTP/1.1 407 Proxy Authentication Required\r\nContent-Length: 0\r\n\r\n') + return + } + const target = new URL(`http://${req.url}`) + const upstream = connect(Number(target.port), target.hostname, () => { + if (upstream.localPort) proxiedPorts.add(upstream.localPort) + socket.write('HTTP/1.1 200 Connection Established\r\n\r\n') + if (head.length) upstream.write(head) + socket.pipe(upstream).pipe(socket) + }) + upstream.on('error', () => socket.destroy()) + socket.on('error', () => upstream.destroy()) + socket.on('close', () => upstream.destroy()) + sockets.add(upstream) + upstream.on('close', () => sockets.delete(upstream)) +}) +let originPort = 0 +let securePort = 0 +const trust = getCACertificates('default') +beforeAll(async () => { + await promisify(execFile)('openssl', [ + 'req', + '-x509', + '-newkey', + 'rsa:2048', + '-sha256', + '-nodes', + '-keyout', + keyPath, + '-out', + certificatePath, + '-days', + '2', + '-subj', + '/CN=gateway.invalid', + '-addext', + 'subjectAltName=DNS:gateway.invalid,DNS:localhost', + '-addext', + 'extendedKeyUsage=serverAuth', + ]) + cert = readFileSync(certificatePath, 'utf8') + const key = readFileSync(keyPath, 'utf8') + for (const server of [secureOrigin, proxy]) server.setSecureContext({ cert, key }) + setDefaultCACertificates([...trust, cert]) + for (const server of [origin, secureOrigin, proxy]) { + server.on('connection', (socket) => { + sockets.add(socket) + socket.on('close', () => sockets.delete(socket)) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + } + originPort = (origin.address() as AddressInfo).port + securePort = (secureOrigin.address() as AddressInfo).port + const proxyPort = (proxy.address() as AddressInfo).port + for (const [id, token] of [ + ['org_a', 'alpha'], + ['org_b', 'bravo'], + ]) + state.gateways.set(id, { + id, + token, + url: `https://127.0.0.1:${proxyPort}`, + servername: 'gateway.invalid', + generation: '1', + ca: cert, + }) +}) +afterAll(async () => { + for (const socket of sockets) socket.destroy() + await Promise.all( + [origin, secureOrigin, proxy].map( + (server) => new Promise((resolve) => server.close(() => resolve())) + ) + ) + setDefaultCACertificates(trust) + rmSync(certificateDirectory, { recursive: true, force: true }) +}) +const url = () => `http://localhost:${originPort}` +const options = { profile: 'selfHostedService' as const } + +describe('organization gateways over real TLS CONNECT sockets', () => { + it('keeps an upgraded tunnel usable after five minutes', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + let tunnel: Socket | undefined + try { + tunnel = await openGatewayTunnel(state.gateways.get('org_a')!, '127.0.0.1', originPort) + await vi.advanceTimersByTimeAsync(6 * 60_000) + expect(tunnel.destroyed).toBe(false) + tunnel.write('GET /done HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + const chunks: Buffer[] = [] + for await (const chunk of tunnel) chunks.push(Buffer.from(chunk)) + expect(Buffer.concat(chunks).toString()).toContain('reached') + } finally { + tunnel?.destroy() + vi.useRealTimers() + } + }) + + it('pins guarded requests and keeps proxy credentials out of origin headers', async () => { + const response = await runWithOutboundOrganization('org_a', () => + secureFetchWithPinnedIP(url(), '127.0.0.1', { + ...options, + headers: { 'Proxy-Authorization': 'must-not-reach-origin' }, + }) + ) + expect(await response.text()).toBe('reached') + expect(admissions.at(-1)).toEqual({ + token: 'Bearer alpha', + destination: `127.0.0.1:${originPort}`, + sni: 'gateway.invalid', + }) + expect(receivedHeaders['proxy-authorization']).toBeUndefined() + expect(receivedHeaders.host).toBe(`localhost:${originPort}`) + }) + it('isolates concurrent requests to the same origin and rotates pools on revision change', async () => { + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + const start = admissions.length + try { + expect( + await Promise.all( + ['org_a', 'org_b'].map((id) => + runWithOutboundOrganization(id, async () => (await transport.fetch(url())).text()) + ) + ) + ).toEqual(['reached', 'reached']) + expect( + admissions + .slice(start) + .map((item) => item.token) + .sort() + ).toEqual(['Bearer alpha', 'Bearer bravo']) + state.revision = 'v2' + await runWithOutboundOrganization('org_a', async () => (await transport.fetch(url())).text()) + expect(admissions.length).toBe(start + 3) + } finally { + await transport.dispatcher.destroy() + } + }) + it('preserves upstream TLS identity on both transports and rejects the wrong hostname', async () => { + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + try { + await runWithOutboundOrganization('org_a', async () => { + expect(await (await transport.fetch(`https://localhost:${securePort}`)).text()).toBe( + 'tls reached' + ) + expect( + await ( + await secureFetchWithPinnedIP(`https://localhost:${securePort}`, '127.0.0.1', options) + ).text() + ).toBe('tls reached') + await expect(transport.fetch(`https://127.0.0.1:${securePort}`)).rejects.toThrow() + await expect( + transport.fetch(`https://127.0.0.1:${securePort}`, { + headers: { host: `localhost:${securePort}` }, + }) + ).rejects.toThrow() + await expect( + secureFetchWithPinnedIP(`https://127.0.0.1:${securePort}`, '127.0.0.1', options) + ).rejects.toThrow() + }) + } finally { + await transport.dispatcher.destroy() + } + }) + it('streams Request bodies, encodes multipart data, and honors manual redirects', async () => { + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + try { + await runWithOutboundOrganization('org_a', async () => { + expect( + await ( + await transport.fetch(new Request(url(), { method: 'POST', body: 'request body' })) + ).text() + ).toBe('request body') + const form = new FormData() + form.set('field', 'multipart body') + expect( + await (await transport.fetch(url(), { method: 'POST', body: form })).text() + ).toContain('multipart body') + const redirect = await transport.fetch(`${url()}/redirect`, { redirect: 'manual' }) + expect(redirect.status).toBe(302) + await redirect.body?.cancel() + }) + } finally { + await transport.dispatcher.destroy() + } + }) + it('fails closed on denied proxy credentials without contacting the origin', async () => { + state.gateways.set('denied', { ...state.gateways.get('org_a')!, token: 'denied' }) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + const start = originRequests + try { + await runWithOutboundOrganization('denied', async () => { + await expect(transport.fetch(url())).rejects.toThrow() + await expect(secureFetchWithPinnedIP(url(), '127.0.0.1', options)).rejects.toThrow() + }) + expect(originRequests).toBe(start) + } finally { + await transport.dispatcher.destroy() + } + }) + it('routes a cached AWS SDK handler separately for each organization', async () => { + const handler = createOutboundAwsHttpHandler() + const start = admissions.length + try { + for (const id of ['org_a', 'org_b']) { + await runWithOutboundOrganization(id, async () => { + const { response } = await handler.handle({ + protocol: 'https:', + hostname: 'localhost', + port: securePort, + method: 'GET', + path: '/', + headers: {}, + query: {}, + }) + const chunks: Buffer[] = [] + for await (const chunk of response.body) chunks.push(Buffer.from(chunk)) + expect(Buffer.concat(chunks).toString()).toBe('tls reached') + }) + } + expect(admissions.slice(start).map((item) => item.token)).toEqual([ + 'Bearer alpha', + 'Bearer bravo', + ]) + } finally { + handler.destroy() + } + }) + + it('routes and verifies TLS under the actual Bun runtime', async () => { + const start = admissions.length + await promisify(execFile)( + 'bun', + [ + '--no-env-file', + 'run', + fileURLToPath(new URL('./fixtures/gateway-runtime.fixture.ts', import.meta.url)), + String((proxy.address() as AddressInfo).port), + String(securePort), + certificatePath, + ], + { + timeout: 15_000, + env: { + ...process.env, + /** This local tunnel fixture must not inherit hosted-mode loopback restrictions. */ + NEXT_PUBLIC_APP_URL: 'http://localhost:3000', + NEXT_PUBLIC_FORCE_HOSTED: 'false', + NODE_EXTRA_CA_CERTS: certificatePath, + OUTBOUND_ROUTING_SOURCE: 'env', + OUTBOUND_ROUTING_CONFIG: JSON.stringify({ + schemaVersion: 1, + revision: 'test', + defaultRoute: { kind: 'blocked' }, + organizations: { org_test: { kind: 'gateway', gatewayId: 'synthetic' } }, + }), + OUTBOUND_GATEWAYS: JSON.stringify({ + synthetic: { + url: `https://127.0.0.1:${(proxy.address() as AddressInfo).port}`, + servername: 'gateway.invalid', + credentialId: 'synthetic', + generation: 'test', + }, + }), + OUTBOUND_GATEWAY_CREDENTIALS: JSON.stringify({ + synthetic: { token: 'a'.repeat(48), ca: cert }, + }), + }, + } + ) + expect(admissions.length).toBe(start + 4) + expect( + admissions.slice(start).every((entry) => entry.token === `Bearer ${'a'.repeat(48)}`) + ).toBe(true) + }) + + it('supports cancellation without a direct retry', async () => { + const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) + const controller = new AbortController() + try { + const promise = runWithOutboundOrganization('org_a', () => + transport.fetch(`${url()}/wait`, { signal: controller.signal }) + ) + controller.abort() + await expect(promise).rejects.toThrow() + } finally { + await transport.dispatcher.destroy() + } + }) +}) diff --git a/apps/sim/lib/core/network/gateway.server.ts b/apps/sim/lib/core/network/gateway.server.ts index 40a012b0235..b5138d1ab02 100644 --- a/apps/sim/lib/core/network/gateway.server.ts +++ b/apps/sim/lib/core/network/gateway.server.ts @@ -1,6 +1,10 @@ -import { isIP, type Socket } from 'node:net' +import { isIP, Socket } from 'node:net' import { checkServerIdentity, connect as connectTls } from 'node:tls' +import { Agent, Client } from 'undici/index.js' +import type { OutboundGateway } from '@/lib/core/network/config.server' import { OutboundRoutingError } from '@/lib/core/network/routing' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' +import { checkResolvedEgress, validateEgressUrl } from '@/lib/core/security/egress/validate' const CONNECT_TIMEOUT_MS = 10_000 @@ -45,3 +49,75 @@ export async function secureOutboundTunnel( }) }) } + +/** Uses the npm HTTP parser on both runtimes; Bun's Node HTTP shim cannot send CONNECT. */ +export async function openGatewayTunnel( + gateway: OutboundGateway, + address: string, + port: number +): Promise { + if (!isIP(address) || !Number.isInteger(port) || port < 1 || port > 65535) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + const authority = `${isIP(address) === 6 ? `[${address}]` : address}:${port}` + const client = new Client(gateway.url, { + allowH2: false, + maxHeaderSize: 16_384, + connectTimeout: CONNECT_TIMEOUT_MS, + connect: { servername: gateway.servername, ca: gateway.ca, rejectUnauthorized: true }, + }) + try { + const { statusCode, socket } = await client.connect({ + path: authority, + signal: AbortSignal.timeout(CONNECT_TIMEOUT_MS), + headers: { host: authority, 'proxy-authorization': `Bearer ${gateway.token}` }, + }) + if (statusCode !== 200 || !(socket instanceof Socket)) { + socket.destroy() + throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } + return socket + } catch { + throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } finally { + /** An upgraded socket is detached from the dispatcher and now belongs to its caller. */ + await client.destroy() + } +} + +/** A dispatcher belongs to one organization and gateway revision, never a process-wide default. */ +export function createGatewayDispatcher( + gateway: OutboundGateway, + options: { profile: EgressProfile; resolvedIP?: string; maxResponseSize?: number } +): Agent { + return new Agent({ + allowH2: false, + ...(options.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), + connect(connection, callback) { + const port = Number(connection.port || (connection.protocol === 'https:' ? 443 : 80)) + const hostname = connection.hostname.replace(/^\[|\]$/g, '') + const authority = isIP(hostname) === 6 ? `[${hostname}]` : hostname + const url = new URL(`${connection.protocol}//${authority}:${port}`) + void (async () => { + let address = options.resolvedIP + if (address) { + if (!checkResolvedEgress(url, address, options.profile).allowed) { + throw new OutboundRoutingError('ROUTE_BLOCKED') + } + } else { + const result = await validateEgressUrl(url.href, 'url', options.profile, { + logDetails: false, + }) + if (!result.isValid) throw new OutboundRoutingError('ROUTE_BLOCKED') + address = result.resolvedIP + } + const socket = await openGatewayTunnel(gateway, address, port) + if (connection.protocol !== 'https:') return socket + return secureOutboundTunnel(socket, hostname, port) + })().then( + (socket) => callback(null, socket), + () => callback(new OutboundRoutingError('GATEWAY_UNAVAILABLE'), null) + ) + }, + }) +} diff --git a/apps/sim/lib/core/network/resource-scope.server.test.ts b/apps/sim/lib/core/network/resource-scope.server.test.ts new file mode 100644 index 00000000000..916e969ba99 --- /dev/null +++ b/apps/sim/lib/core/network/resource-scope.server.test.ts @@ -0,0 +1,86 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + enabled: vi.fn(() => true), + workspace: vi.fn(), + route: vi.fn(async (organizationId: string | null | undefined) => ({ organizationId })), +})) +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: mocks.enabled, + resolveOutboundRoute: mocks.route, +})) +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadActiveWorkspaceApplicationContext: mocks.workspace, +})) + +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' + +describe('canonical resource outbound scope', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.enabled.mockReturnValue(true) + }) + + it.each(['publisher-org', null])( + 'uses current workspace ownership %s and restores its caller', + async (organizationId) => { + mocks.workspace.mockResolvedValue({ workspaceOrganizationId: organizationId }) + await runWithOutboundOrganization('caller-org', async () => { + expect( + await withResourceOutboundScope( + { workspaceId: 'source-workspace' }, + resolveCurrentOutboundRoute + ) + ).toEqual({ organizationId }) + expect(await resolveCurrentOutboundRoute()).toEqual({ organizationId: 'caller-org' }) + }) + expect(mocks.workspace).toHaveBeenCalledExactlyOnceWith('source-workspace') + } + ) + + it('rejects a missing or archived workspace before executing provider work', async () => { + mocks.workspace.mockResolvedValue(null) + await expect( + withResourceOutboundScope({ workspaceId: 'removed' }, resolveCurrentOutboundRoute) + ).rejects.toMatchObject({ code: 'MISSING_SCOPE' }) + expect(mocks.route).not.toHaveBeenCalled() + }) + + it('uses organization ownership without loading a workspace or retaining caller scope', async () => { + await runWithOutboundOrganization('caller-org', async () => { + expect( + await withResourceOutboundScope( + { workspaceId: null, organizationId: 'owner-org' }, + resolveCurrentOutboundRoute + ) + ).toEqual({ organizationId: 'owner-org' }) + expect(await resolveCurrentOutboundRoute()).toEqual({ organizationId: 'caller-org' }) + }) + expect(mocks.workspace).not.toHaveBeenCalled() + }) + + it('adds no workspace query when routing is unconfigured', async () => { + mocks.enabled.mockReturnValue(false) + const run = vi.fn(async () => 'done') + expect(await withResourceOutboundScope({ workspaceId: null }, run)).toBe('done') + expect(mocks.workspace).not.toHaveBeenCalled() + expect(run).toHaveBeenCalledOnce() + }) + + it('rejects ambiguous or missing ownership when routing is configured', async () => { + const run = vi.fn(async () => 'done') + await expect(withResourceOutboundScope({}, run)).rejects.toThrow( + 'Resource requires exactly one workspace or organization owner' + ) + await expect( + withResourceOutboundScope({ workspaceId: 'workspace', organizationId: 'org' }, run) + ).rejects.toThrow('Resource requires exactly one workspace or organization owner') + expect(run).not.toHaveBeenCalled() + expect(mocks.workspace).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/core/network/resource-scope.server.ts b/apps/sim/lib/core/network/resource-scope.server.ts new file mode 100644 index 00000000000..13c2ce09fde --- /dev/null +++ b/apps/sim/lib/core/network/resource-scope.server.ts @@ -0,0 +1,18 @@ +import { isOutboundRoutingEnabled } from '@/lib/core/network/config.server' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope' +import { loadActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' + +/** Establishes routing after resource authorization, reloading current workspace ownership when needed. */ +export async function withResourceOutboundScope( + owner: ResourceOwner, + run: () => Promise +): Promise { + if (!isOutboundRoutingEnabled()) return run() + const scope = resourceScopeFromOwner(owner) + if (scope.kind === 'organization') return runWithOutboundOrganization(scope.organizationId, run) + const workspace = await loadActiveWorkspaceApplicationContext(scope.workspaceId) + if (!workspace) throw new OutboundRoutingError('MISSING_SCOPE') + return runWithOutboundOrganization(workspace.workspaceOrganizationId, run) +} diff --git a/apps/sim/lib/core/network/routing.test.ts b/apps/sim/lib/core/network/routing.test.ts new file mode 100644 index 00000000000..0324764837e --- /dev/null +++ b/apps/sim/lib/core/network/routing.test.ts @@ -0,0 +1,48 @@ +/** @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { + parseOutboundJson, + parseOutboundRoutingConfig, + selectOutboundRoute, +} from '@/lib/core/network/routing' + +const document = { + schemaVersion: 1, + revision: 'revision-1', + defaultRoute: { kind: 'direct' }, + organizations: { org_a: { kind: 'gateway', gatewayId: 'gateway_a' }, org_b: { kind: 'blocked' } }, +} + +describe('outbound routing policy', () => { + it('selects exact organization bindings, including a deliberate stop', () => { + const policy = parseOutboundRoutingConfig(document) + expect(selectOutboundRoute(policy, 'org_a')).toEqual({ + kind: 'gateway', + gatewayId: 'gateway_a', + }) + expect(selectOutboundRoute(policy, 'org_c')).toEqual({ kind: 'direct' }) + expect(selectOutboundRoute(policy, null)).toEqual({ kind: 'direct' }) + expect(() => selectOutboundRoute(policy, 'org_b')).toThrow('ROUTE_BLOCKED') + }) + + it('cannot be mutated or inherit a binding from Object.prototype', () => { + const policy = parseOutboundRoutingConfig(document) + expect(Object.isFrozen(policy.organizations.org_a)).toBe(true) + expect(Object.isFrozen(policy.organizations)).toBe(true) + expect(selectOutboundRoute(policy, 'constructor')).toEqual({ kind: 'direct' }) + }) + + it.each([ + { ...document, schemaVersion: 2 }, + { ...document, organizations: { org_a: { kind: 'gateway' } } }, + { ...document, defaultRoute: { kind: 'gateway', gatewayId: 'a', fallback: true } }, + { ...document, extra: true }, + ])('rejects malformed policy without exposing its content', (input) => { + expect(() => parseOutboundRoutingConfig(input)).toThrow('INVALID_CONFIGURATION') + }) + + it('bounds serialized configuration and does not expose JSON syntax errors', () => { + expect(() => parseOutboundJson('sensitive-value')).toThrow('INVALID_CONFIGURATION') + expect(() => parseOutboundJson(' '.repeat(1_048_577))).toThrow('INVALID_CONFIGURATION') + }) +}) diff --git a/apps/sim/lib/core/network/routing.ts b/apps/sim/lib/core/network/routing.ts index b4caa437859..b94f6d89738 100644 --- a/apps/sim/lib/core/network/routing.ts +++ b/apps/sim/lib/core/network/routing.ts @@ -1,3 +1,33 @@ +import { z } from 'zod' + +const MAX_CONFIG_BYTES = 1_048_576 +const identifier = z + .string() + .min(1) + .max(128) + .regex(/^[a-zA-Z0-9_-]+$/) +const routeSchema = z.discriminatedUnion('kind', [ + z.strictObject({ kind: z.literal('direct') }), + z.strictObject({ kind: z.literal('gateway'), gatewayId: identifier }), + z.strictObject({ kind: z.literal('blocked') }), +]) +const configSchema = z.strictObject({ + schemaVersion: z.literal(1), + revision: identifier, + defaultRoute: routeSchema, + organizations: z + .record(identifier, routeSchema) + .refine((value) => Object.keys(value).length <= 10_000), +}) + +export type OutboundRoute = Readonly> +export interface OutboundRoutingConfig { + readonly schemaVersion: 1 + readonly revision: string + readonly defaultRoute: OutboundRoute + readonly organizations: Readonly> +} + export type OutboundRoutingErrorCode = | 'CONFIGURATION_UNAVAILABLE' | 'INVALID_CONFIGURATION' @@ -13,3 +43,41 @@ export class OutboundRoutingError extends Error { this.name = 'OutboundRoutingError' } } + +/** Bounds operator configuration before parsing and hides secret-bearing parser errors. */ +export function parseOutboundJson(value: string): unknown { + if (Buffer.byteLength(value, 'utf8') > MAX_CONFIG_BYTES) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + try { + return JSON.parse(value) + } catch { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } +} + +/** Produces an immutable routing snapshot without reading deployment state. */ +export function parseOutboundRoutingConfig(value: unknown): OutboundRoutingConfig { + const parsed = configSchema.safeParse(value) + if (!parsed.success) throw new OutboundRoutingError('INVALID_CONFIGURATION') + for (const route of Object.values(parsed.data.organizations)) Object.freeze(route) + Object.freeze(parsed.data.defaultRoute) + Object.freeze(parsed.data.organizations) + return Object.freeze(parsed.data) +} + +/** A null organization is a verified personal scope; undefined is never an implicit default. */ +export function selectOutboundRoute( + config: OutboundRoutingConfig, + organizationId: string | null +): OutboundRoute { + if (organizationId !== null && !identifier.safeParse(organizationId).success) { + throw new OutboundRoutingError('MISSING_SCOPE') + } + const route = + organizationId !== null && Object.hasOwn(config.organizations, organizationId) + ? config.organizations[organizationId] + : config.defaultRoute + if (route.kind === 'blocked') throw new OutboundRoutingError('ROUTE_BLOCKED') + return route +} diff --git a/apps/sim/lib/core/network/transport.server.test.ts b/apps/sim/lib/core/network/transport.server.test.ts new file mode 100644 index 00000000000..b1a8b67b196 --- /dev/null +++ b/apps/sim/lib/core/network/transport.server.test.ts @@ -0,0 +1,147 @@ +/** @vitest-environment node */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { ResolvedOutboundRoute } from '@/lib/core/network/config.server' + +const { resolveRoute, createGateway } = vi.hoisted(() => ({ + resolveRoute: vi.fn<() => Promise>(), + createGateway: vi.fn(), +})) +vi.mock('@/lib/core/network/context.server', () => ({ resolveCurrentOutboundRoute: resolveRoute })) +vi.mock('@/lib/core/network/gateway.server', () => ({ createGatewayDispatcher: createGateway })) + +import { createOutboundTransport } from '@/lib/core/network/transport.server' + +const route = (scopeKey: string | null = 'org_a', revision = 'v1'): ResolvedOutboundRoute => ({ + kind: 'gateway', + scopeKey, + revision, + gateway: { + id: 'gateway', + url: 'https://gateway.invalid', + servername: 'gateway.invalid', + generation: 'v1', + token: 'synthetic', + }, +}) + +beforeEach(() => { + vi.clearAllMocks() + for (const name of [ + 'http_proxy', + 'https_proxy', + 'no_proxy', + 'HTTP_PROXY', + 'HTTPS_PROXY', + 'NO_PROXY', + ]) { + vi.stubEnv(name, '') + } + resolveRoute.mockResolvedValue(route()) + createGateway.mockImplementation(() => ({ + close: vi.fn(async () => {}), + destroy: vi.fn(async () => {}), + })) +}) + +afterEach(() => vi.unstubAllEnvs()) + +describe('shared outbound transport ownership', () => { + it('uses the organization gateway even when environment proxy configuration is invalid', async () => { + vi.stubEnv('http_proxy', 'socks5://operator:synthetic@proxy.invalid') + const owner = createOutboundTransport({ profile: 'configuredEndpoint' }) + try { + expect(await owner.selectDispatcher()).toBe(createGateway.mock.results[0].value) + expect(createGateway).toHaveBeenCalledOnce() + } finally { + await owner.destroy() + } + }) + + it('rejects an explicit proxy when organization policy requires its gateway', async () => { + const owner = createOutboundTransport({ + profile: 'configuredEndpoint', + proxyUrl: 'http://proxy.invalid', + }) + try { + await expect(owner.selectDispatcher()).rejects.toThrow('UNSUPPORTED_TRANSPORT') + expect(createGateway).not.toHaveBeenCalled() + } finally { + await owner.destroy() + } + }) + + it('never shares personal and organization pools for the same gateway', async () => { + const owner = createOutboundTransport({ profile: 'configuredEndpoint' }) + try { + resolveRoute.mockResolvedValue(route(null)) + const personal = await owner.selectDispatcher() + resolveRoute.mockResolvedValue(route('personal')) + const organization = await owner.selectDispatcher() + expect(organization).not.toBe(personal) + resolveRoute.mockResolvedValue(route(null)) + expect(await owner.selectDispatcher()).toBe(personal) + expect(createGateway).toHaveBeenCalledTimes(2) + } finally { + await owner.destroy() + } + }) + + it('reuses only matching organization and revision pools and disposes all of them', async () => { + const owner = createOutboundTransport({ profile: 'configuredEndpoint' }) + const first = await owner.selectDispatcher() + expect(await owner.selectDispatcher()).toBe(first) + resolveRoute.mockResolvedValue(route('org_b')) + expect(await owner.selectDispatcher()).not.toBe(first) + resolveRoute.mockResolvedValue(route('org_a', 'v2')) + expect(await owner.selectDispatcher()).not.toBe(first) + await owner.destroy() + for (const result of createGateway.mock.results) + expect(result.value.destroy).toHaveBeenCalledOnce() + await expect(owner.selectDispatcher()).rejects.toThrow('GATEWAY_UNAVAILABLE') + }) + + it('delegates explicit direct routes but propagates policy failure without a fallback', async () => { + const owner = createOutboundTransport({ profile: 'configuredEndpoint' }) + resolveRoute.mockResolvedValue({ kind: 'direct' }) + expect(await owner.selectDispatcher()).toBeNull() + resolveRoute.mockRejectedValue(new Error('policy unavailable')) + await expect(owner.selectDispatcher()).rejects.toThrow('policy unavailable') + expect(createGateway).not.toHaveBeenCalled() + }) + + it('refuses to create a pool when its owner is destroyed during policy resolution', async () => { + let resolve: (route: ResolvedOutboundRoute) => void = () => {} + resolveRoute.mockReturnValue( + new Promise((done) => { + resolve = done + }) + ) + const owner = createOutboundTransport({ profile: 'configuredEndpoint' }) + const pending = owner.selectDispatcher() + await owner.destroy() + resolve(route()) + await expect(pending).rejects.toThrow('GATEWAY_UNAVAILABLE') + expect(createGateway).not.toHaveBeenCalled() + }) + + it('bounds active and draining pools when revisions change while streams stay open', async () => { + const drain: Array<() => void> = [] + createGateway.mockImplementation(() => ({ + close: vi.fn(() => new Promise((done) => drain.push(done))), + destroy: vi.fn(async () => {}), + })) + const owner = createOutboundTransport({ profile: 'configuredEndpoint' }) + try { + for (let index = 0; index < 32; index++) { + resolveRoute.mockResolvedValue(route('org_a', `v${index}`)) + expect(await owner.selectDispatcher()).not.toBeNull() + } + resolveRoute.mockResolvedValue(route('org_a', 'overflow')) + await expect(owner.selectDispatcher()).rejects.toThrow('GATEWAY_UNAVAILABLE') + expect(createGateway).toHaveBeenCalledTimes(32) + } finally { + await owner.destroy() + for (const finish of drain) finish() + } + }) +}) diff --git a/apps/sim/lib/core/network/transport.server.ts b/apps/sim/lib/core/network/transport.server.ts index 7be2ec4decc..ecc2f6befbc 100644 --- a/apps/sim/lib/core/network/transport.server.ts +++ b/apps/sim/lib/core/network/transport.server.ts @@ -10,7 +10,8 @@ import { Pool, request, } from 'undici/index.js' -import { secureOutboundTunnel } from '@/lib/core/network/gateway.server' +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' +import { createGatewayDispatcher, secureOutboundTunnel } from '@/lib/core/network/gateway.server' import { OutboundRoutingError } from '@/lib/core/network/routing' import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { checkResolvedEgress, validateEgressUrl } from '@/lib/core/security/egress/validate' @@ -203,30 +204,66 @@ interface OutboundTransportOwner { } /** - * Owns direct and environment-proxy connection lifetimes for every HTTP adapter. - * Destination provenance and optional pinning are immutable for this owner. + * Owns routing and connection lifetimes for every HTTP adapter. Destination provenance + * and optional pinning are immutable for this owner; policy is resolved per operation. */ export function createOutboundTransport(options: OutboundTransportOptions): OutboundTransportOwner { + const pools = new Map() + const retired = new Set() let closed = false let environment: Dispatcher | null | undefined const allPools = () => [ ...(options.direct ? [options.direct] : []), ...(environment ? [environment] : []), + ...pools.values(), + ...retired, ] return { async selectDispatcher() { + const route = await resolveCurrentOutboundRoute() if (closed) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') - if (options.proxyUrl) return options.direct ?? null - if (environment === undefined) environment = createEnvironmentProxyDispatcher(options) - return environment ?? options.direct ?? null + if (route.kind === 'direct') { + if (options.proxyUrl) return options.direct ?? null + if (environment === undefined) environment = createEnvironmentProxyDispatcher(options) + return environment ?? options.direct ?? null + } + if (options.proxyUrl) throw new OutboundRoutingError('UNSUPPORTED_TRANSPORT') + const key = JSON.stringify([ + route.scopeKey, + route.gateway.id, + route.gateway.generation, + route.revision, + ]) + let pool = pools.get(key) + if (!pool) { + if (retired.size >= 16) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + if (pools.size >= 16) { + const oldest = pools.entries().next().value + if (oldest) { + pools.delete(oldest[0]) + retired.add(oldest[1]) + void oldest[1] + .close() + .catch(() => {}) + .finally(() => retired.delete(oldest[1])) + } + } + pool = createGatewayDispatcher(route.gateway, options) + pools.set(key, pool) + } + return pool }, async close() { closed = true await Promise.all(allPools().map((pool) => pool.close())) + pools.clear() + retired.clear() }, async destroy() { closed = true await Promise.all(allPools().map((pool) => pool.destroy())) + pools.clear() + retired.clear() }, } } diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 6d29ee1524d..1d25b08d4e9 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -17,6 +17,7 @@ import { errors, type RequestInit as UndiciRequestInit, } from 'undici/index.js' +import { requireDirectOutboundTransport } from '@/lib/core/network/context.server' import { OutboundRoutingError } from '@/lib/core/network/routing' import { createOutboundTransport, @@ -158,6 +159,7 @@ export async function validateDatabaseHost( paramName = 'host', options: { logDetails?: boolean } = {} ): Promise { + await requireDirectOutboundTransport() if (!host) { return { isValid: false, error: `${paramName} is required` } } diff --git a/apps/sim/lib/credential-groups/application/public-enrollment.test.ts b/apps/sim/lib/credential-groups/application/public-enrollment.test.ts index 2a979fcd80a..963b10dfc7c 100644 --- a/apps/sim/lib/credential-groups/application/public-enrollment.test.ts +++ b/apps/sim/lib/credential-groups/application/public-enrollment.test.ts @@ -20,6 +20,17 @@ const mocks = vi.hoisted(() => ({ completeMcpOAuth: vi.fn(), startMcpOAuth: vi.fn(), startOAuth: vi.fn(), + routingEnabled: vi.fn(() => false), + workspace: vi.fn(), + route: vi.fn(async (organizationId: string | null | undefined) => ({ organizationId })), +})) + +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: mocks.routingEnabled, + resolveOutboundRoute: mocks.route, +})) +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadActiveWorkspaceApplicationContext: mocks.workspace, })) vi.mock('@/lib/organizations/settings-access', () => ({ @@ -53,6 +64,10 @@ vi.mock('@/lib/credential-groups/trigger', () => ({ fireCredentialGroupTrigger: mocks.fireTrigger, })) +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' import { completePublicCredentialGroupEnrollment, completePublicCredentialGroupMcpOAuth, @@ -101,6 +116,7 @@ const oauthAttempt = { describe('public Credential Group enrollment application operations', () => { beforeEach(() => { vi.clearAllMocks() + mocks.routingEnabled.mockReturnValue(false) mocks.bind.mockResolvedValue(undefined) mocks.memberAccess.mockResolvedValue({ isMember: false }) mocks.searchAvailable.mockResolvedValue(true) @@ -333,6 +349,33 @@ describe('public Credential Group enrollment application operations', () => { expect.objectContaining({ event: 'credential_reconnected', enrollmentStatus: 'completed' }) ) }) + it.each(['organization', 'workspace'] as const)( + 'scopes OAuth callbacks to the authorized %s and restores the caller', + async (kind) => { + mocks.routingEnabled.mockReturnValue(true) + mocks.workspace.mockResolvedValue({ workspaceOrganizationId: 'workspace-org' }) + const owner = + kind === 'organization' + ? { organizationId: 'org-1', workspaceId: undefined } + : { workspaceId: 'workspace-1', organizationId: undefined } + const completion = await mocks.completeOAuth() + mocks.completeOAuth.mockImplementationOnce(async () => { + await resolveCurrentOutboundRoute() + return completion + }) + await runWithOutboundOrganization('caller-org', async () => { + await completePublicCredentialGroupOAuth.execute({ + principal: { ...principal, ...owner }, + input: { attempt: { ...oauthAttempt, ...owner }, code: 'code' }, + }) + expect(mocks.route).toHaveBeenLastCalledWith( + kind === 'organization' ? 'org-1' : 'workspace-org' + ) + await resolveCurrentOutboundRoute() + expect(mocks.route).toHaveBeenLastCalledWith('caller-org') + }) + } + ) it('rejects a consumed attempt after invitation rotation before exchanging its code', async () => { mocks.bind.mockRejectedValue(new Error('Invitation is invalid or expired')) await expect( diff --git a/apps/sim/lib/credential-groups/application/public-enrollment.ts b/apps/sim/lib/credential-groups/application/public-enrollment.ts index 0c0f388e82b..a45f06f9135 100644 --- a/apps/sim/lib/credential-groups/application/public-enrollment.ts +++ b/apps/sim/lib/credential-groups/application/public-enrollment.ts @@ -2,6 +2,7 @@ import type { CredentialGroupEnrollmentPrincipal, Principal } from '@sim/auth/pr import { safeCompare } from '@sim/security/compare' import { sha256Hex } from '@sim/security/hash' import type { OperationUseCase } from '@/lib/core/application' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { resourceScopeFields, @@ -91,7 +92,7 @@ function defineAuthorizedCredentialGroupEnrollmentUseCase< }, async execute({ principal, input }) { const authorized = await authorize(principal, input) - return definition.execute(authorized) + return withResourceOutboundScope(authorized.context, () => definition.execute(authorized)) }, } } diff --git a/apps/sim/lib/credential-groups/provider-adapter.ts b/apps/sim/lib/credential-groups/provider-adapter.ts index 897bcf9282b..592a81556e1 100644 --- a/apps/sim/lib/credential-groups/provider-adapter.ts +++ b/apps/sim/lib/credential-groups/provider-adapter.ts @@ -4,8 +4,8 @@ import type { CredentialGroupOAuthContext } from '@/lib/credential-groups/enroll import type { CredentialGroupOAuthAttempt } from '@/lib/credential-groups/oauth-state' import type { CredentialGroupProvider } from '@/lib/credential-groups/providers' import type { DbOrTx } from '@/lib/db/types' -import type { RefreshTokenResult } from '@/lib/oauth' import type { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' +import type { RefreshTokenResult } from '@/lib/oauth/refresh-token.server' export interface CredentialGroupProviderPolicy { provider: CredentialGroupProvider diff --git a/apps/sim/lib/credential-groups/slack-managed-users.test.ts b/apps/sim/lib/credential-groups/slack-managed-users.test.ts index 3e32054c599..a94f916cbc4 100644 --- a/apps/sim/lib/credential-groups/slack-managed-users.test.ts +++ b/apps/sim/lib/credential-groups/slack-managed-users.test.ts @@ -1,7 +1,13 @@ /** * @vitest-environment node */ -import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { + dbChainMockFns, + inputValidationMock, + queueTableRows, + resetDbChainMock, + schemaMock, +} from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { attempts, redis } = vi.hoisted(() => { @@ -70,6 +76,8 @@ import { verifySlackUserIdentity, } from '@/lib/credential-groups/slack-managed-users' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + function slackResponse(value: Record): Response { return new Response(JSON.stringify(value), { status: 200, diff --git a/apps/sim/lib/credential-groups/slack-managed-users.ts b/apps/sim/lib/credential-groups/slack-managed-users.ts index 5bd4b689c59..aee1d4c7528 100644 --- a/apps/sim/lib/credential-groups/slack-managed-users.ts +++ b/apps/sim/lib/credential-groups/slack-managed-users.ts @@ -16,6 +16,7 @@ import { getRedisClient } from '@/lib/core/config/redis' import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getBaseUrl } from '@/lib/core/utils/urls' import { credentialGroupScopePolicyVersion } from '@/lib/credential-groups/provider-adapter' import { @@ -33,6 +34,10 @@ import { resolveSlackAppCredentials } from '@/lib/slack-search/app-configuration import { requireSlackSearchAppAvailable } from '@/lib/slack-search/shared-app' import { getSharedSlackSearchAppConfiguration } from '@/lib/slack-search/shared-app-env' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('SlackManagedUsers') const SLACK_MANAGED_USERS_ATTEMPT_TTL_MS = 10 * 60 * 1000 const SLACK_MANAGED_USERS_ATTEMPT_VERSION = 4 as const @@ -293,7 +298,7 @@ function parseSlackOAuthResponse(value: unknown): SlackOAuthSuccess { } export async function revokeSlackToken(token: string): Promise { - const response = await fetch('https://slack.com/api/auth.revoke', { + const response = await providerFetch('https://slack.com/api/auth.revoke', { method: 'POST', headers: { Authorization: `Bearer ${token}`, @@ -319,7 +324,7 @@ export async function revokeSlackToken(token: string): Promise { async function callSlackApi(method: string, accessToken: string, body?: URLSearchParams) { let response: Response try { - response = await fetch(`https://slack.com/api/${method}`, { + response = await providerFetch(`https://slack.com/api/${method}`, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, @@ -435,7 +440,7 @@ export async function exchangeSlackUserAuthorization(params: { const body = new URLSearchParams({ code: params.code, redirect_uri: params.redirectUri }) let response: Response try { - response = await fetch('https://slack.com/api/oauth.v2.access', { + response = await providerFetch('https://slack.com/api/oauth.v2.access', { method: 'POST', headers: { Authorization: `Basic ${basicAuth}`, diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts index ac06e6f834b..53e997ca569 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts @@ -1,7 +1,9 @@ /** * @vitest-environment node */ + import { createHash } from 'node:crypto' +import { inputValidationMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { CredentialGroupOAuthContext } from '@/lib/credential-groups/enrollments' import type { CredentialGroupOAuthAttempt } from '@/lib/credential-groups/oauth-state' @@ -81,6 +83,8 @@ vi.mock('@/lib/auth/connectors/managed-oauth', () => ({ import { createStandardOAuthCredentialGroupProviderAdapter } from '@/lib/credential-groups/standard-oauth-provider' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const adapter = createStandardOAuthCredentialGroupProviderAdapter('google-calendar') const jiraAdapter = createStandardOAuthCredentialGroupProviderAdapter('jira') diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.ts index 4492ad3812d..47793f843e1 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.ts @@ -12,6 +12,7 @@ import { getManagedOAuthConnectorProviderConfig, type ManagedOAuthConnectorConfig, } from '@/lib/auth/connectors/managed-oauth' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { credentialGroupOAuthNonceMatches } from '@/lib/credential-groups/oauth-state' import type { @@ -25,8 +26,12 @@ import { } from '@/lib/credential-groups/provider-adapter' import type { CredentialGroupStandardOAuthProvider } from '@/lib/credential-groups/providers' import { getCredentialGroupProviderService } from '@/lib/credential-groups/providers' -import { refreshOAuthToken } from '@/lib/oauth' import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' +import { refreshOAuthToken } from '@/lib/oauth/refresh-token.server' + +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) const OAUTH_DISCOVERY_TIMEOUT_MS = 10_000 const OAUTH_DISCOVERY_MAX_BYTES = 256 * 1024 @@ -81,7 +86,7 @@ async function resolveOAuthEndpoints( if (connector.discoveryUrl) { let response: Response try { - response = await fetch(connector.discoveryUrl, { + response = await providerFetch(connector.discoveryUrl, { headers: connector.discoveryHeaders, signal: AbortSignal.timeout(OAUTH_DISCOVERY_TIMEOUT_MS), }) diff --git a/apps/sim/lib/credentials/application/organization-credentials.test.ts b/apps/sim/lib/credentials/application/organization-credentials.test.ts index d38ab7f6370..cbd93daf02b 100644 --- a/apps/sim/lib/credentials/application/organization-credentials.test.ts +++ b/apps/sim/lib/credentials/application/organization-credentials.test.ts @@ -19,6 +19,11 @@ const mocks = vi.hoisted(() => ({ resolveToken: vi.fn(), ownedManaged: vi.fn(), resolveManaged: vi.fn(), + route: vi.fn(async (organizationId: string | null | undefined) => ({ organizationId })), +})) +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: () => true, + resolveOutboundRoute: mocks.route, })) vi.mock('@sim/audit', () => auditMock) vi.mock('@/lib/core/application/organization-authorization', () => ({ @@ -59,6 +64,10 @@ vi.mock('@/lib/oauth/utils', () => ({ getServiceConfigByProviderId: () => ({ serviceAccountProviderId: 'google-service-account' }), })) +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' import { createOrganizationCredential, launchOrganizationCredentialConnection, @@ -228,6 +237,40 @@ describe('organization connection application boundary', () => { }) ) }) + it.each(['create', 'update'] as const)( + 'routes provider verification for %s through the credential organization', + async (operation) => { + const verify = operation === 'create' ? mocks.create : mocks.update + verify.mockImplementationOnce(async () => { + expect(await resolveCurrentOutboundRoute()).toEqual({ organizationId: 'org-1' }) + return { success: true, created: true, credential: row, updatedFields: ['description'] } + }) + + await runWithOutboundOrganization('caller-org', async () => { + if (operation === 'create') { + await createOrganizationCredential.execute({ + principal, + input: { + organizationId: 'org-1', + type: 'service_account', + providerId: 'google-service-account', + }, + }) + } else { + await updateOrganizationCredential.execute({ + principal, + input: { + organizationId: 'org-1', + credentialId: row.id, + description: 'Updated', + }, + }) + } + expect(await resolveCurrentOutboundRoute()).toEqual({ organizationId: 'caller-org' }) + }) + expect(verify).toHaveBeenCalledOnce() + } + ) it.each([ { providerId: undefined, diff --git a/apps/sim/lib/credentials/application/organization-credentials.ts b/apps/sim/lib/credentials/application/organization-credentials.ts index 6117032c586..97ea3207545 100644 --- a/apps/sim/lib/credentials/application/organization-credentials.ts +++ b/apps/sim/lib/credentials/application/organization-credentials.ts @@ -17,6 +17,7 @@ import { } from '@/lib/core/application/organization-authorization' import { defineOrganizationOperation } from '@/lib/core/application/organization-operation' import { PrincipalKindAuthorizationError } from '@/lib/core/application/workspace-authorization' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { throwCredentialMutationFailure } from '@/lib/credentials/application/credential-crud' @@ -216,9 +217,8 @@ export const createOrganizationCredential: OperationUseCase< ) requireAvailableOAuthCredentialProvider(catalog, input.providerId ?? '') } - const result = await createCredentialRecord( - { ...input, userId: context.userId }, - { authorizeWorkspace: false } + const result = await withResourceOutboundScope(context, () => + createCredentialRecord({ ...input, userId: context.userId }, { authorizeWorkspace: false }) ) if (!result.success) throwCredentialMutationFailure(result) if (!result.credential) throw new Error('Credential creation returned no credential') @@ -409,7 +409,9 @@ export const updateOrganizationCredential: OperationUseCase< (row.type === 'oauth' && row.createdBy !== context.userId) ) throw new OrchestrationError('not_found', 'Credential not found') - const result = await updateCredentialRecord({ ...input, credential: row }) + const result = await withResourceOutboundScope(row, () => + updateCredentialRecord({ ...input, credential: row }) + ) if (!result.success) throwCredentialMutationFailure(result) const updated = await getOrganizationCredential(input.organizationId, input.credentialId) if (!updated) throw new OrchestrationError('not_found', 'Credential not found') diff --git a/apps/sim/lib/credentials/atlassian-service-account.test.ts b/apps/sim/lib/credentials/atlassian-service-account.test.ts index bf80b3860b4..c8de71d0daa 100644 --- a/apps/sim/lib/credentials/atlassian-service-account.test.ts +++ b/apps/sim/lib/credentials/atlassian-service-account.test.ts @@ -1,7 +1,10 @@ /** @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { validateAtlassianServiceAccount } from '@/lib/credentials/atlassian-service-account' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const fetchMock = vi.fn() describe('Atlassian service-account identity verification', () => { diff --git a/apps/sim/lib/credentials/atlassian-service-account.ts b/apps/sim/lib/credentials/atlassian-service-account.ts index a3ac12ede1e..ddaaf414753 100644 --- a/apps/sim/lib/credentials/atlassian-service-account.ts +++ b/apps/sim/lib/credentials/atlassian-service-account.ts @@ -1,6 +1,11 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { AtlassianProduct } from '@/lib/credentials/service-account-fields' import { parseAtlassianErrorMessage } from '@/tools/jira/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + /** * Discrete validation failure codes returned to the client. The UI maps each * code to a human message; raw Atlassian response bodies stay in server logs. @@ -94,7 +99,7 @@ export async function validateAtlassianServiceAccount( }> { assertAtlassianCloudHost(domain) - const tenantInfoRes = await fetch(`https://${domain}/_edge/tenant_info`, { + const tenantInfoRes = await providerFetch(`https://${domain}/_edge/tenant_info`, { headers: { Accept: 'application/json' }, }) if (tenantInfoRes.status === 404) { @@ -112,7 +117,7 @@ export async function validateAtlassianServiceAccount( const cloudId = tenantInfo.cloudId const identityPath = product === 'confluence' ? 'wiki/rest/api/user/current' : 'rest/api/3/myself' - const myselfRes = await fetch( + const myselfRes = await providerFetch( `https://api.atlassian.com/ex/${product}/${cloudId}/${identityPath}`, { headers: { Authorization: `Bearer ${apiToken}`, Accept: 'application/json' }, diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts index 7b09b9b2ad5..66fc1e94bab 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/box.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { mintBoxServiceAccountToken } from '@/lib/credentials/client-credential-accounts/minters/box' diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts index 68d5f57052c..0d49be1d00d 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.test.ts @@ -1,11 +1,15 @@ /** * @vitest-environment node */ + import { generateKeyPairSync } from 'node:crypto' +import { inputValidationMock } from '@sim/testing' import { jwtVerify } from 'jose' import { afterEach, describe, expect, it, vi } from 'vitest' import { mintNetSuiteServiceAccountToken } from '@/lib/credentials/client-credential-accounts/minters/netsuite' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const ORIGIN = 'https://1234567-sb1.suitetalk.api.netsuite.com' const TOKEN_URL = `${ORIGIN}/services/rest/auth/oauth2/v1/token` const rsaKeyPair = generateKeyPairSync('rsa', { modulusLength: 3072 }) diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts index d0737e1d68a..b8b2c3e7f6a 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/netsuite.ts @@ -2,6 +2,7 @@ import { createPrivateKey, type KeyObject } from 'node:crypto' import { generateId } from '@sim/utils/id' import { truncate } from '@sim/utils/string' import { SignJWT } from 'jose' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -19,6 +20,10 @@ import { TokenServiceAccountValidationError, } from '@/lib/credentials/token-service-accounts/errors' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const TOKEN_PATH = '/services/rest/auth/oauth2/v1/token' const TOKEN_EXCHANGE_TIMEOUT_MS = 30_000 const TOKEN_RESPONSE_MAX_BYTES = 1024 * 1024 @@ -108,7 +113,7 @@ async function exchangeNetSuiteToken( const signal = AbortSignal.timeout(TOKEN_EXCHANGE_TIMEOUT_MS) let response: Response try { - response = await fetch(tokenUrl, { + response = await providerFetch(tokenUrl, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.test.ts index 01bc6f68131..1009bd4ffc2 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/salesforce.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { createVerify, generateKeyPairSync } from 'crypto' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { mintSalesforceServiceAccountToken } from '@/lib/credentials/client-credential-accounts/minters/salesforce' diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts index 861497837ee..7f37b019eda 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoho-desk.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockGetCanonicalScopesForProvider, mockLoggerWarn } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.test.ts b/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.test.ts index afa900ed610..13f760d6a06 100644 --- a/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.test.ts +++ b/apps/sim/lib/credentials/client-credential-accounts/minters/zoom.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { mintZoomServiceAccountToken } from '@/lib/credentials/client-credential-accounts/minters/zoom' diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.test.ts b/apps/sim/lib/credentials/token-service-accounts/errors.test.ts index d2b15fe2107..60a1a148947 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { fetchProvider, @@ -10,6 +11,8 @@ import { throwForProviderResponse, } from '@/lib/credentials/token-service-accounts/errors' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const mockFetch = vi.fn() const PROVIDER_URL = 'https://api.example-provider.com/v1/self' diff --git a/apps/sim/lib/credentials/token-service-accounts/errors.ts b/apps/sim/lib/credentials/token-service-accounts/errors.ts index bdc53a52a2d..f2e6519dab5 100644 --- a/apps/sim/lib/credentials/token-service-accounts/errors.ts +++ b/apps/sim/lib/credentials/token-service-accounts/errors.ts @@ -1,5 +1,10 @@ import { getErrorMessage } from '@sim/utils/errors' import { truncate } from '@sim/utils/string' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' + +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) /** * Discrete validation failure codes returned to the client for token @@ -98,7 +103,10 @@ export async function fetchProvider( options?: FetchProviderOptions ): Promise { try { - return await fetch(url, { ...init, signal: AbortSignal.timeout(PROVIDER_FETCH_TIMEOUT_MS) }) + return await providerFetch(url, { + ...init, + signal: AbortSignal.timeout(PROVIDER_FETCH_TIMEOUT_MS), + }) } catch (error) { const causeCode = (error as { cause?: { code?: unknown } })?.cause?.code // Only ENOTFOUND proves the host doesn't exist; EAI_AGAIN is a transient diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/airtable.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/airtable.test.ts index b71becb916b..be070305e0c 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/airtable.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/airtable.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { validateAirtableServiceAccount } from '@/lib/credentials/token-service-accounts/validators/airtable' diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/asana.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/asana.test.ts index c1c57c7f815..77b9849ab4f 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/asana.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/asana.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { validateAsanaServiceAccount } from '@/lib/credentials/token-service-accounts/validators/asana' diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/attio.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/attio.test.ts index 92002f1752c..c2e85382e4f 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/attio.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/attio.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { validateAttioServiceAccount } from '@/lib/credentials/token-service-accounts/validators/attio' diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/calcom.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/calcom.test.ts index 78b0b7bbcd2..df814d585bc 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/calcom.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/calcom.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { validateCalcomServiceAccount } from '@/lib/credentials/token-service-accounts/validators/calcom' diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/harmonic.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/harmonic.test.ts index f31cb9d2939..48ae15b0e97 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/harmonic.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/harmonic.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { getTokenServiceAccountDescriptor, diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/hubspot.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/hubspot.test.ts index c37e2ad1a53..fb802a3b172 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/hubspot.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/hubspot.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { validateHubspotServiceAccount } from '@/lib/credentials/token-service-accounts/validators/hubspot' diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/linear.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/linear.test.ts index 37ef8fc156c..9dae29fc15f 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/linear.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/linear.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { validateLinearServiceAccount } from '@/lib/credentials/token-service-accounts/validators/linear' diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/monday.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/monday.test.ts index 590d425c79d..b6549c04238 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/monday.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/monday.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { validateMondayServiceAccount } from '@/lib/credentials/token-service-accounts/validators/monday' diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/notion.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/notion.test.ts index adffbfb5a1f..dfe3248dc0b 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/notion.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/notion.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { validateNotionServiceAccount } from '@/lib/credentials/token-service-accounts/validators/notion' diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/pipedrive.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/pipedrive.test.ts index 4e75faeb57f..56745b99d5e 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/pipedrive.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/pipedrive.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { validatePipedriveServiceAccount } from '@/lib/credentials/token-service-accounts/validators/pipedrive' diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/shopify.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/shopify.test.ts index 46d42d993b4..a932f629536 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/shopify.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/shopify.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { validateShopifyServiceAccount } from '@/lib/credentials/token-service-accounts/validators/shopify' diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/snowflake.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/snowflake.test.ts index 6870785dc4b..460cc956c8d 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/snowflake.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/snowflake.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { validateSnowflakeServiceAccount } from '@/lib/credentials/token-service-accounts/validators/snowflake' diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/trello.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/trello.test.ts index 82c76793363..c1717c0bd07 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/trello.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/trello.test.ts @@ -1,7 +1,9 @@ /** * @vitest-environment node */ -import { resetEnvMock, setEnv } from '@sim/testing' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + +import { inputValidationMock, resetEnvMock, setEnv } from '@sim/testing' import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' beforeAll(() => { diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/wealthbox.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/wealthbox.test.ts index 153faef2706..8499afdbdfe 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/wealthbox.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/wealthbox.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { validateWealthboxServiceAccount } from '@/lib/credentials/token-service-accounts/validators/wealthbox' diff --git a/apps/sim/lib/credentials/token-service-accounts/validators/webflow.test.ts b/apps/sim/lib/credentials/token-service-accounts/validators/webflow.test.ts index 109f862070b..3e11c81a83c 100644 --- a/apps/sim/lib/credentials/token-service-accounts/validators/webflow.test.ts +++ b/apps/sim/lib/credentials/token-service-accounts/validators/webflow.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { TokenServiceAccountValidationError } from '@/lib/credentials/token-service-accounts/errors' import { validateWebflowServiceAccount } from '@/lib/credentials/token-service-accounts/validators/webflow' diff --git a/apps/sim/lib/embeddings/client.test.ts b/apps/sim/lib/embeddings/client.test.ts index 5efc3104276..176727468be 100644 --- a/apps/sim/lib/embeddings/client.test.ts +++ b/apps/sim/lib/embeddings/client.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ -import { resetEnvMock, setEnv } from '@sim/testing' +import { inputValidationMock, resetEnvMock, setEnv } from '@sim/testing' import { interruptibleSleep } from '@sim/utils/helpers' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' @@ -24,6 +24,11 @@ import { MAX_EMBEDDING_SUCCESS_RESPONSE_BYTES, } from '@/lib/embeddings/client' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const { mockGetBYOKKey } = vi.hoisted(() => ({ mockGetBYOKKey: vi.fn(), })) @@ -295,6 +300,14 @@ describe('embed', () => { const sentCounts = fetchMock.mock.calls.map( ([, init]) => JSON.parse((init as RequestInit).body as string).requests.length ) + expect(fetchMock.mock.calls[0][1]).toMatchObject({ + headers: { 'x-goog-api-key': 'g-test' }, + redirectPolicy: { + mode: 'standard', + sendCredentialsOnCrossOriginRedirect: false, + sensitiveHeaders: ['api-key', 'x-goog-api-key'], + }, + }) expect(sentCounts).toEqual([100, 100, 50]) expect(result.embeddings).toHaveLength(250) // Native dimensionality means no reduction, so values pass through unnormalized. @@ -1110,6 +1123,14 @@ describe('knowledge embedding transport fallback', () => { expect(fetchMock.mock.calls[0][0]).toBe( 'https://example.openai.azure.com/openai/deployments/kb-embedding-deployment/embeddings?api-version=2024-10-21' ) + expect(fetchMock.mock.calls[0][1]).toMatchObject({ + headers: { 'api-key': 'azure-test' }, + redirectPolicy: { + mode: 'standard', + sendCredentialsOnCrossOriginRedirect: false, + sensitiveHeaders: ['api-key', 'x-goog-api-key'], + }, + }) expect(result.modelName).toBe('kb-embedding-deployment') }) diff --git a/apps/sim/lib/embeddings/client.ts b/apps/sim/lib/embeddings/client.ts index 846ba759f0d..401e9d784d4 100644 --- a/apps/sim/lib/embeddings/client.ts +++ b/apps/sim/lib/embeddings/client.ts @@ -16,6 +16,7 @@ import { waitForProviderAdmission, } from '@/lib/core/rate-limiter/provider-admission' import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { DEFAULT_MAX_ERROR_BODY_BYTES, @@ -285,7 +286,10 @@ function isQuotaExhaustionBody(errorText: string): boolean { } /** Reads a bounded provider body only for internal quota classification. */ -async function readEmbeddingErrorBody(response: Response, signal?: AbortSignal): Promise { +async function readEmbeddingErrorBody( + response: Parameters[0], + signal?: AbortSignal +): Promise { try { return await readResponseTextWithLimit(response, { maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, @@ -587,7 +591,14 @@ async function callEmbeddingAPI( const timeout = setTimeout(() => controller.abort(), EMBEDDING_REQUEST_TIMEOUT_MS) try { - const response = await fetch(request.apiUrl, { + const response = await secureFetchWithValidation(request.apiUrl, { + profile: providerId === 'ollama' ? 'selfHostedService' : 'configuredEndpoint', + maxRedirects: 20, + redirectPolicy: { + mode: 'standard', + sendCredentialsOnCrossOriginRedirect: false, + sensitiveHeaders: ['api-key', 'x-goog-api-key'], + }, method: 'POST', headers: request.headers, body: JSON.stringify(request.body), diff --git a/apps/sim/lib/embeddings/ollama-model-catalog.server.test.ts b/apps/sim/lib/embeddings/ollama-model-catalog.server.test.ts index d563ff76a8c..0d13e3f7cb5 100644 --- a/apps/sim/lib/embeddings/ollama-model-catalog.server.test.ts +++ b/apps/sim/lib/embeddings/ollama-model-catalog.server.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { resetEnvMock, setEnv } from '@sim/testing' +import { inputValidationMock, resetEnvMock, setEnv } from '@sim/testing' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { fetchOllamaEmbeddingModelCatalog, @@ -11,6 +11,11 @@ import { OllamaUnreachableError, } from '@/lib/embeddings/ollama-model-catalog.server' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const fetchMock = vi.fn() /** Mirrors an Ollama server: `/api/tags` lists everything, `/api/show` classifies it. */ diff --git a/apps/sim/lib/embeddings/ollama-model-catalog.server.ts b/apps/sim/lib/embeddings/ollama-model-catalog.server.ts index 75008e1f4be..f12cfcafc09 100644 --- a/apps/sim/lib/embeddings/ollama-model-catalog.server.ts +++ b/apps/sim/lib/embeddings/ollama-model-catalog.server.ts @@ -5,6 +5,10 @@ import { ollamaUpstreamResponseSchema, } from '@/lib/api/contracts/providers' import { isHosted } from '@/lib/core/config/env-flags' +import { + type SecureFetchOptions, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { getOllamaUrl, isOllamaUrlConfigured } from '@/lib/core/utils/urls' @@ -69,8 +73,16 @@ export class OllamaEmbeddingWidthUnknownError extends Error { } } -async function fetchOllamaJson(path: string, init: RequestInit, signal?: AbortSignal) { - const response = await fetch(`${getOllamaUrl().replace(/\/+$/, '')}${path}`, { +async function fetchOllamaJson( + path: string, + init: Pick, + signal?: AbortSignal +) { + const response = await secureFetchWithValidation(`${getOllamaUrl().replace(/\/+$/, '')}${path}`, { + profile: 'selfHostedService', + maxResponseBytes: MAX_OLLAMA_CATALOG_BYTES, + maxRedirects: 20, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, headers: { 'Content-Type': 'application/json' }, signal, ...init, diff --git a/apps/sim/lib/embeddings/openrouter-model-catalog.server.test.ts b/apps/sim/lib/embeddings/openrouter-model-catalog.server.test.ts index 1e6ed46e09d..0a2d06fd284 100644 --- a/apps/sim/lib/embeddings/openrouter-model-catalog.server.test.ts +++ b/apps/sim/lib/embeddings/openrouter-model-catalog.server.test.ts @@ -1,22 +1,33 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' import { + fetchOpenRouterEmbeddingModelCatalog, getOpenRouterEmbeddingModelMetadata, OpenRouterEmbeddingModelNotFoundError, } from '@/lib/embeddings/openrouter-model-catalog.server' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const fetchMock = vi.fn() +let clock = 0 describe('OpenRouter embedding model catalog', () => { beforeEach(() => { vi.clearAllMocks() + clock += 300_001 + vi.spyOn(performance, 'now').mockImplementation(() => clock) vi.stubGlobal('fetch', fetchMock) }) afterAll(() => { vi.unstubAllGlobals() + vi.restoreAllMocks() }) it('resolves a prefixed model with its live input ceiling', async () => { @@ -49,4 +60,51 @@ describe('OpenRouter embedding model catalog', () => { getOpenRouterEmbeddingModelMetadata('openrouter/example/missing-context') ).rejects.toThrow('Invalid input') }) + + it('reuses the public catalog for five minutes without sharing mutable results', async () => { + fetchMock.mockImplementation(async () => + Response.json({ data: [{ id: 'qwen/embed', context_length: 32768 }] }) + ) + const first = await fetchOpenRouterEmbeddingModelCatalog() + first[0].maxInputTokens = 1 + first.length = 0 + + clock += 299_999 + await expect(fetchOpenRouterEmbeddingModelCatalog()).resolves.toEqual([ + { id: 'openrouter/qwen/embed', maxInputTokens: 32768 }, + ]) + expect(fetchMock).toHaveBeenCalledOnce() + + clock += 2 + await fetchOpenRouterEmbeddingModelCatalog() + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + + it('retries a rejected catalog without caching the failure', async () => { + const cancelBody = vi.fn() + fetchMock + .mockResolvedValueOnce( + new Response(new ReadableStream({ cancel: cancelBody }), { status: 503 }) + ) + .mockResolvedValueOnce(Response.json({ data: [] })) + + await expect(fetchOpenRouterEmbeddingModelCatalog()).rejects.toThrow('503') + expect(cancelBody).toHaveBeenCalledOnce() + await expect(fetchOpenRouterEmbeddingModelCatalog()).resolves.toEqual([]) + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + + it('forwards cancellation on a miss and rejects an aborted caller on a cache hit', async () => { + fetchMock.mockResolvedValueOnce(Response.json({ data: [] })) + const controller = new AbortController() + await fetchOpenRouterEmbeddingModelCatalog(controller.signal) + expect(fetchMock).toHaveBeenCalledWith( + 'https://openrouter.ai/api/v1/embeddings/models', + expect.objectContaining({ profile: 'configuredEndpoint', signal: controller.signal }) + ) + + controller.abort() + await expect(fetchOpenRouterEmbeddingModelCatalog(controller.signal)).rejects.toThrow() + expect(fetchMock).toHaveBeenCalledOnce() + }) }) diff --git a/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts b/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts index b65f499b892..db731d55e34 100644 --- a/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts +++ b/apps/sim/lib/embeddings/openrouter-model-catalog.server.ts @@ -1,4 +1,6 @@ +import { LRUCache } from 'lru-cache' import { openRouterEmbeddingModelsUpstreamResponseSchema } from '@/lib/api/contracts/providers' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { toOpenRouterEmbeddingModelId, @@ -13,6 +15,13 @@ export interface OpenRouterEmbeddingModelMetadata { maxInputTokens: number } +/** Runtime-local caching also works in workers, where Next's fetch cache is unavailable. */ +const modelCatalogCache = new LRUCache({ + max: 1, + ttl: 300_000, + ttlResolution: 0, +}) + export class OpenRouterEmbeddingModelNotFoundError extends Error { constructor(model: string) { super(`Unsupported OpenRouter embedding model: ${model}`) @@ -24,12 +33,20 @@ export class OpenRouterEmbeddingModelNotFoundError extends Error { export async function fetchOpenRouterEmbeddingModelCatalog( signal?: AbortSignal ): Promise { - const response = await fetch(OPENROUTER_EMBEDDING_MODELS_URL, { + signal?.throwIfAborted() + const cached = modelCatalogCache.get(OPENROUTER_EMBEDDING_MODELS_URL) + if (cached) return structuredClone(cached) + + const response = await secureFetchWithValidation(OPENROUTER_EMBEDDING_MODELS_URL, { + profile: 'configuredEndpoint', + maxResponseBytes: MAX_OPENROUTER_EMBEDDING_CATALOG_BYTES, + maxRedirects: 20, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, headers: { 'Content-Type': 'application/json' }, - next: { revalidate: 300 }, signal, }) if (!response.ok) { + await response.body?.cancel().catch(() => {}) throw new Error( `Failed to fetch OpenRouter embedding models: ${response.status} ${response.statusText}` ) @@ -47,7 +64,9 @@ export async function fetchOpenRouterEmbeddingModelCatalog( const id = toOpenRouterEmbeddingModelId(model.id) models.set(id, { id, maxInputTokens: model.context_length }) } - return Array.from(models.values()) + const catalog = Array.from(models.values()) + modelCatalogCache.set(OPENROUTER_EMBEDDING_MODELS_URL, structuredClone(catalog)) + return catalog } /** Resolves and validates one selected model against OpenRouter's live catalog. */ diff --git a/apps/sim/lib/execution/isolated-vm-worker.cjs b/apps/sim/lib/execution/isolated-vm-worker.cjs index ac19d487b21..a5eeb2e6dd0 100644 --- a/apps/sim/lib/execution/isolated-vm-worker.cjs +++ b/apps/sim/lib/execution/isolated-vm-worker.cjs @@ -263,13 +263,16 @@ async function executeCode(request, executionId) { resolve(JSON.stringify({ error: 'Parent process disconnected' })) return } - sendIpcRequest({ type: 'fetch', fetchId, requestId, url, optionsJson }, (err) => { - const pending = pendingFetches.get(fetchId) - if (!pending) return - clearTimeout(pending.timeout) - pendingFetches.delete(fetchId) - pending.resolve(JSON.stringify({ error: `Fetch IPC send failed: ${err.message}` })) - }) + sendIpcRequest( + { type: 'fetch', fetchId, executionId, requestId, url, optionsJson }, + (err) => { + const pending = pendingFetches.get(fetchId) + if (!pending) return + clearTimeout(pending.timeout) + pendingFetches.delete(fetchId) + pending.resolve(JSON.stringify({ error: `Fetch IPC send failed: ${err.message}` })) + } + ) }) }) await jail.set('__fetchRef', fetchCallback) diff --git a/apps/sim/lib/execution/isolated-vm.test.ts b/apps/sim/lib/execution/isolated-vm.test.ts index 717f9ebaa2e..27dc37b469d 100644 --- a/apps/sim/lib/execution/isolated-vm.test.ts +++ b/apps/sim/lib/execution/isolated-vm.test.ts @@ -143,6 +143,7 @@ function createReadyFetchProxyProc(fetchMessage: { url: string; optionsJson?: st setImmediate(() => { proc.emit('message', { type: 'fetch', + executionId: currentExecutionId, fetchId: 1, requestId: msg.request?.requestId ?? 'fetch-test', url: fetchMessage.url, diff --git a/apps/sim/lib/execution/isolated-vm.ts b/apps/sim/lib/execution/isolated-vm.ts index 93bbf55fedb..d257c08c8fa 100644 --- a/apps/sim/lib/execution/isolated-vm.ts +++ b/apps/sim/lib/execution/isolated-vm.ts @@ -9,6 +9,7 @@ import { randomFloat } from '@sim/utils/random' import { env } from '@/lib/core/config/env' import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server' import { getRedisClient } from '@/lib/core/config/redis' +import { captureOutboundScope } from '@/lib/core/network/context.server' import { type SecureFetchOptions, secureFetchWithValidation, @@ -160,6 +161,7 @@ const QUEUE_RETRY_DELAY_MS = 1000 const DISTRIBUTED_LEASE_GRACE_MS = 30000 interface PendingExecution { + runInOutboundScope: ReturnType resolve: (result: IsolatedVMExecutionResult) => void timeout: ReturnType ownerKey: string @@ -199,6 +201,7 @@ interface QueuedExecution { * against the queue-to-worker handoff. */ interface ExecutionState { + runInOutboundScope: ReturnType cancelled: boolean queueId?: number workerId?: number @@ -730,7 +733,7 @@ function handleBrokerMessage( } Promise.resolve() - .then(() => handler(args)) + .then(() => pending.runInOutboundScope(() => handler(args))) .then((resultValue) => { if (pending.cancelled) { sendResponse({ error: 'Execution cancelled' }) @@ -809,6 +812,18 @@ function handleWorkerMessage(workerId: number, message: unknown) { } if (msg.type === 'fetch') { + const pending = + typeof msg.executionId === 'number' + ? workerInfo?.pendingExecutions.get(msg.executionId) + : undefined + if (!pending || pending.cancelled) { + workerInfo?.process.send({ + type: 'fetchResponse', + fetchId: msg.fetchId, + response: JSON.stringify({ error: 'Execution no longer active' }), + }) + return + } const { fetchId, requestId, url, optionsJson } = msg as { fetchId: number requestId: string @@ -847,7 +862,8 @@ function handleWorkerMessage(workerId: number, message: unknown) { return } } - secureFetch(requestId, url, options) + pending + .runInOutboundScope(() => secureFetch(requestId, url, options)) .then((response) => { try { workerInfo?.process.send({ type: 'fetchResponse', fetchId, response }) @@ -1200,6 +1216,7 @@ function dispatchToWorker( }, req.timeoutMs + 1000) workerInfo.pendingExecutions.set(execId, { + runInOutboundScope: state.runInOutboundScope, resolve, timeout, ownerKey: ownerState.ownerKey, @@ -1472,7 +1489,7 @@ export async function executeInIsolatedVM( // An undetermined lease cannot reject the execution: the per-process pool and // the per-owner active/queued limits above still bound this work. - const state: ExecutionState = { cancelled: false } + const state: ExecutionState = { cancelled: false, runInOutboundScope: captureOutboundScope() } return new Promise((resolve) => { let abortListener: (() => void) | null = null diff --git a/apps/sim/lib/execution/remote-sandbox/provider.test.ts b/apps/sim/lib/execution/remote-sandbox/provider.test.ts new file mode 100644 index 00000000000..241efb35007 --- /dev/null +++ b/apps/sim/lib/execution/remote-sandbox/provider.test.ts @@ -0,0 +1,56 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OutboundRoutingError } from '@/lib/core/network/routing' + +const mocks = vi.hoisted(() => ({ + route: vi.fn(), + create: vi.fn(), + startBuild: vi.fn(), +})) + +vi.mock('@/lib/core/config/env-capabilities.server', () => ({ + getSelectedSandboxProviderId: () => 'e2b', +})) +vi.mock('@/lib/core/network/config.server', () => ({ + resolveOutboundRoute: mocks.route, +})) +vi.mock('@/lib/execution/remote-sandbox/e2b', () => ({ + e2bProvider: { id: 'e2b', create: mocks.create, images: { startBuild: mocks.startBuild } }, +})) +vi.mock('@/lib/execution/remote-sandbox/daytona', () => ({ + daytonaProvider: { id: 'daytona' }, +})) + +import { resolveProvider } from '@/lib/execution/remote-sandbox/provider' + +describe('sandbox provider routing boundary', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.route.mockRejectedValue(new OutboundRoutingError('MISSING_SCOPE')) + }) + + it('builds deployment-shared images without requiring an organization scope', async () => { + const image = { imageRef: 'image', buildId: 'build' } + mocks.startBuild.mockResolvedValue(image) + + await expect( + resolveProvider().images!.startBuild( + { language: 'python', dependencies: [], cliTools: [], systemPackages: [] }, + 'spec-hash', + { + rendererRevision: 1, + generation: 1, + imageRefPrefix: 'image:', + baseImageRef: 'base', + } + ) + ).resolves.toBe(image) + expect(mocks.startBuild).toHaveBeenCalledOnce() + expect(mocks.route).not.toHaveBeenCalled() + }) + + it('still requires a resolved route before creating an execution sandbox', async () => { + await expect(resolveProvider().create('code')).rejects.toMatchObject({ code: 'MISSING_SCOPE' }) + expect(mocks.create).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/execution/remote-sandbox/provider.ts b/apps/sim/lib/execution/remote-sandbox/provider.ts index 8eebe70aa9d..00aebf5b001 100644 --- a/apps/sim/lib/execution/remote-sandbox/provider.ts +++ b/apps/sim/lib/execution/remote-sandbox/provider.ts @@ -1,4 +1,5 @@ import { getSelectedSandboxProviderId } from '@/lib/core/config/env-capabilities.server' +import { requireDirectOutboundTransport } from '@/lib/core/network/context.server' import { daytonaProvider } from '@/lib/execution/remote-sandbox/daytona' import { e2bProvider } from '@/lib/execution/remote-sandbox/e2b' import type { SandboxProvider, SandboxProviderId } from '@/lib/execution/remote-sandbox/types' @@ -25,5 +26,12 @@ const PROVIDERS: Record = { */ export function resolveProvider(): SandboxProvider { const configured = getSelectedSandboxProviderId() - return PROVIDERS[configured] + const provider = PROVIDERS[configured] + return { + ...provider, + async create(kind, options) { + await requireDirectOutboundTransport() + return provider.create(kind, options) + }, + } } diff --git a/apps/sim/lib/internal/appconfig/client.ts b/apps/sim/lib/internal/appconfig/client.ts index 38a5e285031..7cb999e6703 100644 --- a/apps/sim/lib/internal/appconfig/client.ts +++ b/apps/sim/lib/internal/appconfig/client.ts @@ -30,10 +30,12 @@ import { GetLatestConfigurationCommand, StartConfigurationSessionCommand, } from '@aws-sdk/client-appconfigdata' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' import type { AppConfigConnectionConfig } from '@/tools/appconfig/types' export function createAppConfigClient(config: AppConfigConnectionConfig): AppConfigClient { return new AppConfigClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, @@ -44,6 +46,7 @@ export function createAppConfigClient(config: AppConfigConnectionConfig): AppCon export function createAppConfigDataClient(config: AppConfigConnectionConfig): AppConfigDataClient { return new AppConfigDataClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/asana/client.test.ts b/apps/sim/lib/internal/asana/client.test.ts index 790ebd8bebe..1d5c8c61827 100644 --- a/apps/sim/lib/internal/asana/client.test.ts +++ b/apps/sim/lib/internal/asana/client.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { AsanaClient } from '@/lib/internal/asana/client' import { AsanaOperationError } from '@/lib/internal/asana/errors' diff --git a/apps/sim/lib/internal/asana/client.ts b/apps/sim/lib/internal/asana/client.ts index 1ea4e1b025d..1f6b08dfe14 100644 --- a/apps/sim/lib/internal/asana/client.ts +++ b/apps/sim/lib/internal/asana/client.ts @@ -1,6 +1,9 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { AsanaOperationError } from '@/lib/internal/asana/errors' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const ASANA_API_BASE_URL = 'https://app.asana.com/api/1.0' const ASANA_RESPONSE_MAX_BYTES = 10 * 1024 * 1024 @@ -42,7 +45,7 @@ export class AsanaClient { private async fetch(path: string, init: RequestInit, signal?: AbortSignal): Promise { signal?.throwIfAborted() - return fetch(this.url(path), { + return providerFetch(this.url(path), { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/asana/operations.test.ts b/apps/sim/lib/internal/asana/operations.test.ts index af0dd3f50f9..a51d7408608 100644 --- a/apps/sim/lib/internal/asana/operations.test.ts +++ b/apps/sim/lib/internal/asana/operations.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { AsanaOperationError } from '@/lib/internal/asana/errors' import { executeAsanaAddComment, diff --git a/apps/sim/lib/internal/ashby/operations.test.ts b/apps/sim/lib/internal/ashby/operations.test.ts index 54b3ec0ee5b..acd6a53571c 100644 --- a/apps/sim/lib/internal/ashby/operations.test.ts +++ b/apps/sim/lib/internal/ashby/operations.test.ts @@ -17,7 +17,9 @@ vi.mock('@/app/api/files/authorization', () => ({ vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ downloadServableFileFromStorage: mocks.downloadServableFileFromStorage, })) -vi.mock('@/lib/core/security/input-validation.server', () => ({ +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, validateUrlWithDNS: mocks.validateUrlWithDNS, })) diff --git a/apps/sim/lib/internal/ashby/operations.ts b/apps/sim/lib/internal/ashby/operations.ts index ee327909f80..e41a9e99662 100644 --- a/apps/sim/lib/internal/ashby/operations.ts +++ b/apps/sim/lib/internal/ashby/operations.ts @@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { secureFetchWithPinnedIP, + secureFetchWithValidation, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { @@ -40,7 +41,10 @@ async function ashbyPost( onBehalfOfUserId: string | null | undefined, signal?: AbortSignal ): Promise { - const response = await fetch(`https://api.ashbyhq.com/${path}`, { + const response = await secureFetchWithValidation(`https://api.ashbyhq.com/${path}`, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_ASHBY_JSON_BYTES, method: 'POST', headers: ashbyAuthHeaders(apiKey, onBehalfOfUserId ?? undefined), body: JSON.stringify(body), diff --git a/apps/sim/lib/internal/athena/client.ts b/apps/sim/lib/internal/athena/client.ts index c584509c627..116e09ac354 100644 --- a/apps/sim/lib/internal/athena/client.ts +++ b/apps/sim/lib/internal/athena/client.ts @@ -1,4 +1,5 @@ import { AthenaClient } from '@aws-sdk/client-athena' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' export interface AthenaConnectionConfig { region: string @@ -8,6 +9,7 @@ export interface AthenaConnectionConfig { export function createAthenaClient(config: AthenaConnectionConfig): AthenaClient { return new AthenaClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/box/client.test.ts b/apps/sim/lib/internal/box/client.test.ts index 290c27abc4f..80a6537945b 100644 --- a/apps/sim/lib/internal/box/client.test.ts +++ b/apps/sim/lib/internal/box/client.test.ts @@ -3,6 +3,13 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const mocks = vi.hoisted(() => ({ fetch: vi.fn() })) import { BoxClient, BoxUploadError } from '@/lib/internal/box/client' diff --git a/apps/sim/lib/internal/box/client.ts b/apps/sim/lib/internal/box/client.ts index be7cc2150f1..a7cfe8cfa7a 100644 --- a/apps/sim/lib/internal/box/client.ts +++ b/apps/sim/lib/internal/box/client.ts @@ -1,8 +1,11 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, } from '@/lib/core/utils/stream-limits' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + interface BoxUploadEntry { id?: string name?: string @@ -46,7 +49,7 @@ export class BoxClient { new Blob([new Uint8Array(buffer)], { type: 'application/octet-stream' }), fileName ) - const response = await fetch('https://upload.box.com/api/2.0/files/content', { + const response = await providerFetch('https://upload.box.com/api/2.0/files/content', { method: 'POST', headers: { Authorization: `Bearer ${this.accessToken}` }, body: formData, diff --git a/apps/sim/lib/internal/brex/client.test.ts b/apps/sim/lib/internal/brex/client.test.ts index cad850caf7b..918d56852fe 100644 --- a/apps/sim/lib/internal/brex/client.test.ts +++ b/apps/sim/lib/internal/brex/client.test.ts @@ -10,7 +10,9 @@ const mocks = vi.hoisted(() => ({ fetch: vi.fn(), })) -vi.mock('@/lib/core/security/input-validation.server', () => ({ +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), validateUrlWithDNS: mocks.validateUrl, secureFetchWithPinnedIP: mocks.pinnedFetch, })) @@ -38,6 +40,9 @@ describe('BrexReceiptClient', () => { expect(mocks.fetch).toHaveBeenCalledWith( 'https://api.brex.com/v1/expenses/card/expense%2Fid/receipt_upload', { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: DEFAULT_MAX_ERROR_BODY_BYTES, method: 'POST', headers: { Authorization: 'Bearer token', diff --git a/apps/sim/lib/internal/brex/client.ts b/apps/sim/lib/internal/brex/client.ts index 82f1b0fdca9..d21aef502b0 100644 --- a/apps/sim/lib/internal/brex/client.ts +++ b/apps/sim/lib/internal/brex/client.ts @@ -1,5 +1,6 @@ import { secureFetchWithPinnedIP, + secureFetchWithValidation, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { @@ -49,7 +50,10 @@ export class BrexReceiptClient { const endpoint = expenseId ? `${BREX_API_BASE}/v1/expenses/card/${encodeURIComponent(expenseId)}/receipt_upload` : `${BREX_API_BASE}/v1/expenses/card/receipt_match` - const response = await fetch(endpoint, { + const response = await secureFetchWithValidation(endpoint, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: DEFAULT_MAX_ERROR_BODY_BYTES, method: 'POST', headers: buildBrexHeaders(this.apiKey), body: JSON.stringify({ receipt_name: receiptName }), diff --git a/apps/sim/lib/internal/browser-use/operations/run-task.test.ts b/apps/sim/lib/internal/browser-use/operations/run-task.test.ts index 5974ffd4ad8..dc5c8d38491 100644 --- a/apps/sim/lib/internal/browser-use/operations/run-task.test.ts +++ b/apps/sim/lib/internal/browser-use/operations/run-task.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { executeRunTaskOperation } from '@/lib/internal/browser-use/operations/run-task' const mockFetch = vi.fn() diff --git a/apps/sim/lib/internal/browser-use/operations/run-task.ts b/apps/sim/lib/internal/browser-use/operations/run-task.ts index 4b9bf78b10a..45219a49b5c 100644 --- a/apps/sim/lib/internal/browser-use/operations/run-task.ts +++ b/apps/sim/lib/internal/browser-use/operations/run-task.ts @@ -3,6 +3,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { z } from 'zod' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { BrowserUseRunTaskParams, @@ -10,6 +11,8 @@ import type { BrowserUseTaskStep, } from '@/tools/browser_use/types' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('BrowserUseTool') const POLL_INTERVAL_MS = 5000 @@ -86,7 +89,7 @@ async function fetchBrowserUse( ): Promise { options.signal?.throwIfAborted() const hasBody = options.body !== undefined - const response = await fetch(`${API_BASE}${path}`, { + const response = await providerFetch(`${API_BASE}${path}`, { method: options.method ?? 'GET', headers: { ...(hasBody ? { 'Content-Type': 'application/json' } : {}), diff --git a/apps/sim/lib/internal/buffer/operations.test.ts b/apps/sim/lib/internal/buffer/operations.test.ts index d03661da628..47e0905d365 100644 --- a/apps/sim/lib/internal/buffer/operations.test.ts +++ b/apps/sim/lib/internal/buffer/operations.test.ts @@ -3,6 +3,13 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const mocks = vi.hoisted(() => ({ resolveFileInputToUrl: vi.fn(), })) diff --git a/apps/sim/lib/internal/buffer/operations.ts b/apps/sim/lib/internal/buffer/operations.ts index 644846f7179..4ceef9adbab 100644 --- a/apps/sim/lib/internal/buffer/operations.ts +++ b/apps/sim/lib/internal/buffer/operations.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { + createSsrfGuardedFetchWithDispatcher, secureFetchWithPinnedIP, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' @@ -18,6 +19,8 @@ import { parseBufferGraphQLResponse, } from '@/tools/buffer/types' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('BufferOperations') const VIDEO_EXTENSIONS = ['.mp4', '.mov', '.m4v', '.webm', '.avi'] const IMAGE_EXTENSIONS = ['.png', '.jpg', '.jpeg', '.gif', '.webp'] @@ -161,7 +164,7 @@ async function executePostMutation(args: { const { apiKey, mutation, input, context } = args let result: Record try { - const response = await fetch(BUFFER_API_URL, { + const response = await providerFetch(BUFFER_API_URL, { method: 'POST', headers: bufferHeaders(apiKey), body: JSON.stringify({ query: mutation, variables: { input } }), diff --git a/apps/sim/lib/internal/clickup/client.test.ts b/apps/sim/lib/internal/clickup/client.test.ts index e77484d265f..b071450be41 100644 --- a/apps/sim/lib/internal/clickup/client.test.ts +++ b/apps/sim/lib/internal/clickup/client.test.ts @@ -3,6 +3,13 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const mocks = vi.hoisted(() => ({ isPayloadSizeLimitError: vi.fn(), readResponseJsonWithLimit: vi.fn(), diff --git a/apps/sim/lib/internal/clickup/client.ts b/apps/sim/lib/internal/clickup/client.ts index 78da954d554..7cf27c3093a 100644 --- a/apps/sim/lib/internal/clickup/client.ts +++ b/apps/sim/lib/internal/clickup/client.ts @@ -1,4 +1,7 @@ -import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + createSsrfGuardedFetchWithDispatcher, + MAX_JSON_API_RESPONSE_BYTES, +} from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { ClickUpOperationError } from '@/lib/internal/clickup/errors' import { @@ -7,6 +10,8 @@ import { extractClickUpErrorMessage, } from '@/tools/clickup/shared' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + export async function uploadClickUpAttachment( accessToken: string, taskId: string, @@ -14,7 +19,7 @@ export async function uploadClickUpAttachment( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch( + const response = await providerFetch( `${CLICKUP_API_BASE_URL}/task/${encodeURIComponent(taskId)}/attachment`, { method: 'POST', diff --git a/apps/sim/lib/internal/cloudflare/operations/get-zone-settings.ts b/apps/sim/lib/internal/cloudflare/operations/get-zone-settings.ts index 5f4860882ed..2534143debc 100644 --- a/apps/sim/lib/internal/cloudflare/operations/get-zone-settings.ts +++ b/apps/sim/lib/internal/cloudflare/operations/get-zone-settings.ts @@ -1,4 +1,8 @@ import { getErrorMessage } from '@sim/utils/errors' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import { mapZoneSetting, zoneSettingUrl } from '@/tools/cloudflare/get_zone_settings' import type { @@ -31,7 +35,10 @@ export const executeGetZoneSettingsOperation: InternalToolOperationImplementatio const reads = await Promise.all( settingIds.map(async (settingId) => { try { - const response = await fetch(zoneSettingUrl(zoneId, settingId), { + const response = await secureFetchWithValidation(zoneSettingUrl(zoneId, settingId), { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, method: 'GET', headers, signal, diff --git a/apps/sim/lib/internal/cloudformation/client.ts b/apps/sim/lib/internal/cloudformation/client.ts index fb2eb8bca66..a94c1ebeb8e 100644 --- a/apps/sim/lib/internal/cloudformation/client.ts +++ b/apps/sim/lib/internal/cloudformation/client.ts @@ -4,6 +4,7 @@ import { type Parameter, type Tag, } from '@aws-sdk/client-cloudformation' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' export interface CloudFormationConnectionConfig { region: string @@ -15,6 +16,7 @@ export function createCloudFormationClient( config: CloudFormationConnectionConfig ): CloudFormationClient { return new CloudFormationClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/cloudtrail/client.ts b/apps/sim/lib/internal/cloudtrail/client.ts index 3d5b13d7bd9..a6c047331db 100644 --- a/apps/sim/lib/internal/cloudtrail/client.ts +++ b/apps/sim/lib/internal/cloudtrail/client.ts @@ -1,4 +1,5 @@ import { CloudTrailClient } from '@aws-sdk/client-cloudtrail' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' export interface CloudTrailConnectionConfig { region: string @@ -24,6 +25,7 @@ export function createCloudTrailClient( options: CreateCloudTrailClientOptions = {} ): CloudTrailClient { return new CloudTrailClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/cloudtrail/operations.ts b/apps/sim/lib/internal/cloudtrail/operations.ts index ad169ea7566..4929d853e76 100644 --- a/apps/sim/lib/internal/cloudtrail/operations.ts +++ b/apps/sim/lib/internal/cloudtrail/operations.ts @@ -230,7 +230,7 @@ export async function executeCloudtrailListTrails( ) { return withCloudTrailClient(input, async (client) => { const response = await client.send( - new ListTrailsCommand({ ...(input.nextToken ? { NextToken: input.nextToken } : {}) }), + new ListTrailsCommand(input.nextToken ? { NextToken: input.nextToken } : {}), { abortSignal: signal } ) return { diff --git a/apps/sim/lib/internal/cloudwatch/client.ts b/apps/sim/lib/internal/cloudwatch/client.ts index f198a8a25c2..06d9314ecc3 100644 --- a/apps/sim/lib/internal/cloudwatch/client.ts +++ b/apps/sim/lib/internal/cloudwatch/client.ts @@ -10,6 +10,7 @@ import { import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' interface AwsCredentials { region: string @@ -22,6 +23,7 @@ export function createCloudWatchClient( options?: { maxAttempts?: number } ): CloudWatchClient { return new CloudWatchClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, ...(options?.maxAttempts !== undefined && { maxAttempts: options.maxAttempts }), credentials: { @@ -33,6 +35,7 @@ export function createCloudWatchClient( export function createCloudWatchLogsClient(config: AwsCredentials): CloudWatchLogsClient { return new CloudWatchLogsClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/codepipeline/client.ts b/apps/sim/lib/internal/codepipeline/client.ts index 0061c45688d..ad9237d4763 100644 --- a/apps/sim/lib/internal/codepipeline/client.ts +++ b/apps/sim/lib/internal/codepipeline/client.ts @@ -1,4 +1,5 @@ import { CodePipelineClient } from '@aws-sdk/client-codepipeline' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' export interface CodePipelineConnectionConfig { region: string @@ -8,6 +9,7 @@ export interface CodePipelineConnectionConfig { export function createCodePipelineClient(config: CodePipelineConnectionConfig): CodePipelineClient { return new CodePipelineClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/confluence/client.test.ts b/apps/sim/lib/internal/confluence/client.test.ts index 324c227a55d..6bf8904dc5c 100644 --- a/apps/sim/lib/internal/confluence/client.test.ts +++ b/apps/sim/lib/internal/confluence/client.test.ts @@ -3,6 +3,13 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const { mockGetConfluenceCloudId } = vi.hoisted(() => ({ mockGetConfluenceCloudId: vi.fn(), })) diff --git a/apps/sim/lib/internal/confluence/client.ts b/apps/sim/lib/internal/confluence/client.ts index 603110ba36c..4390ef7be7f 100644 --- a/apps/sim/lib/internal/confluence/client.ts +++ b/apps/sim/lib/internal/confluence/client.ts @@ -1,5 +1,8 @@ import { validateJiraCloudId } from '@/lib/core/security/input-validation' -import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + createSsrfGuardedFetchWithDispatcher, + MAX_JSON_API_RESPONSE_BYTES, +} from '@/lib/core/security/input-validation.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -8,6 +11,8 @@ import { ConfluenceOperationError } from '@/lib/internal/confluence/errors' import { getConfluenceCloudId } from '@/tools/confluence/utils' import { parseAtlassianErrorMessage } from '@/tools/jira/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + export interface ConfluenceConnectionConfig { domain: string accessToken: string @@ -53,7 +58,7 @@ export class ConfluenceClient { async fetch(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { signal?.throwIfAborted() - return fetch(path, { + return providerFetch(path, { ...init, headers: { Accept: 'application/json', diff --git a/apps/sim/lib/internal/confluence/operations.test.ts b/apps/sim/lib/internal/confluence/operations.test.ts index 8a1c1d70b31..36bb00eccc8 100644 --- a/apps/sim/lib/internal/confluence/operations.test.ts +++ b/apps/sim/lib/internal/confluence/operations.test.ts @@ -3,6 +3,13 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const uploadMocks = vi.hoisted(() => ({ assertToolFileAccess: vi.fn(), downloadServableFileFromStorage: vi.fn(), diff --git a/apps/sim/lib/internal/crowdstrike/client.test.ts b/apps/sim/lib/internal/crowdstrike/client.test.ts index d48fc77b60c..e4062ecc171 100644 --- a/apps/sim/lib/internal/crowdstrike/client.test.ts +++ b/apps/sim/lib/internal/crowdstrike/client.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { CrowdStrikeAuthError, callCrowdStrike, diff --git a/apps/sim/lib/internal/crowdstrike/client.ts b/apps/sim/lib/internal/crowdstrike/client.ts index df44323b648..71ac5068946 100644 --- a/apps/sim/lib/internal/crowdstrike/client.ts +++ b/apps/sim/lib/internal/crowdstrike/client.ts @@ -1,8 +1,13 @@ import { isRecordLike } from '@sim/utils/object' -import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + createSsrfGuardedFetchWithDispatcher, + MAX_JSON_API_RESPONSE_BYTES, +} from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import type { CrowdStrikeBaseParams, CrowdStrikeCloud } from '@/tools/crowdstrike/types' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + export type JsonRecord = Record const CLOUD_BASE_URLS: Record = { @@ -192,7 +197,7 @@ export async function getAccessToken( ): Promise { signal?.throwIfAborted() const baseUrl = getCloudBaseUrl(params.cloud) - const response = await fetch(`${baseUrl}/oauth2/token`, { + const response = await providerFetch(`${baseUrl}/oauth2/token`, { method: 'POST', headers: { Accept: 'application/json', @@ -271,7 +276,7 @@ export async function callCrowdStrike( headers['Content-Type'] = 'application/json' } - const response = await fetch(buildUrl(baseUrl, options), { + const response = await providerFetch(buildUrl(baseUrl, options), { method: options.method, headers, body: options.body === undefined ? undefined : JSON.stringify(options.body), diff --git a/apps/sim/lib/internal/crowdstrike/operations.test.ts b/apps/sim/lib/internal/crowdstrike/operations.test.ts index 5a6672563a7..3e08683970d 100644 --- a/apps/sim/lib/internal/crowdstrike/operations.test.ts +++ b/apps/sim/lib/internal/crowdstrike/operations.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { chunkIdsByUrlBudget, executeCrowdStrikeOperation, diff --git a/apps/sim/lib/internal/cursor/operations.test.ts b/apps/sim/lib/internal/cursor/operations.test.ts index 9909d7a9bbd..ec5926c1263 100644 --- a/apps/sim/lib/internal/cursor/operations.test.ts +++ b/apps/sim/lib/internal/cursor/operations.test.ts @@ -8,7 +8,9 @@ const mocks = vi.hoisted(() => ({ validateUrlWithDNS: vi.fn(), })) -vi.mock('@/lib/core/security/input-validation.server', () => ({ +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, validateUrlWithDNS: mocks.validateUrlWithDNS, })) diff --git a/apps/sim/lib/internal/cursor/operations.ts b/apps/sim/lib/internal/cursor/operations.ts index 4ac9ba2cc81..20a044f4ac2 100644 --- a/apps/sim/lib/internal/cursor/operations.ts +++ b/apps/sim/lib/internal/cursor/operations.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { secureFetchWithPinnedIP, + secureFetchWithValidation, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { @@ -57,9 +58,12 @@ export async function downloadCursorArtifact( ): Promise { context.signal?.throwIfAborted() const authHeader = `Basic ${Buffer.from(`${input.apiKey}:`).toString('base64')}` - const artifactResponse = await fetch( + const artifactResponse = await secureFetchWithValidation( `https://api.cursor.com/v0/agents/${encodeURIComponent(input.agentId)}/artifacts/download?path=${encodeURIComponent(input.path)}`, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_CURSOR_METADATA_BYTES, method: 'GET', headers: { Authorization: authHeader }, signal: context.signal, diff --git a/apps/sim/lib/internal/datadog/operations/update-slo.ts b/apps/sim/lib/internal/datadog/operations/update-slo.ts index 074fb4c3f5e..92d0e9c64b1 100644 --- a/apps/sim/lib/internal/datadog/operations/update-slo.ts +++ b/apps/sim/lib/internal/datadog/operations/update-slo.ts @@ -1,3 +1,4 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { UpdateSloParams } from '@/tools/datadog/types' import { @@ -8,13 +9,15 @@ import { mergeSloUpdatePayload, } from '@/tools/datadog/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + export const executeUpdateSloOperation: InternalToolOperationImplementation< UpdateSloParams > = async (params, signal) => { const url = datadogApiUrl(params.site, `/api/v1/slo/${datadogPathSegment(params.sloId)}`) const headers = datadogHeaders(params) - const existingResponse = await fetch(url, { method: 'GET', headers, signal }) + const existingResponse = await providerFetch(url, { method: 'GET', headers, signal }) if (!existingResponse.ok) { return { success: false, @@ -33,7 +36,7 @@ export const executeUpdateSloOperation: InternalToolOperationImplementation< } } - const response = await fetch(url, { + const response = await providerFetch(url, { method: 'PUT', headers, body: JSON.stringify(mergeSloUpdatePayload(stored, params)), diff --git a/apps/sim/lib/internal/daytona/operations.ts b/apps/sim/lib/internal/daytona/operations.ts index 8682b958374..5030a555e6c 100644 --- a/apps/sim/lib/internal/daytona/operations.ts +++ b/apps/sim/lib/internal/daytona/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { DaytonaOperationError } from '@/lib/internal/daytona/errors' import { isDocNotReadyError } from '@/lib/uploads/utils/doc-not-ready' @@ -10,6 +11,8 @@ import { assertToolFileAccess } from '@/app/api/files/authorization' import type { DaytonaUploadFileParams, DaytonaUploadFileResponse } from '@/tools/daytona/types' import { daytonaToolboxUrl } from '@/tools/daytona/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('DaytonaOperations') const MAX_UPLOAD_SIZE_BYTES = 100 * 1024 * 1024 const MAX_DAYTONA_ERROR_BYTES = 256 * 1024 @@ -96,7 +99,7 @@ export async function uploadDaytonaFile( new Blob([new Uint8Array(fileBuffer)], { type: 'application/octet-stream' }), fileName ) - const response = await fetch( + const response = await providerFetch( daytonaToolboxUrl( input.sandboxId, `/files/upload-v2?path=${encodeURIComponent(destinationPath)}` diff --git a/apps/sim/lib/internal/discord/client.test.ts b/apps/sim/lib/internal/discord/client.test.ts index e0fecc2f939..6ce1a3d333a 100644 --- a/apps/sim/lib/internal/discord/client.test.ts +++ b/apps/sim/lib/internal/discord/client.test.ts @@ -3,6 +3,13 @@ */ import { describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const mocks = vi.hoisted(() => ({ readResponseJsonWithLimit: vi.fn() })) vi.mock('@/lib/core/utils/stream-limits', () => ({ diff --git a/apps/sim/lib/internal/discord/client.ts b/apps/sim/lib/internal/discord/client.ts index df86a4a1e15..fd12d8b6f78 100644 --- a/apps/sim/lib/internal/discord/client.ts +++ b/apps/sim/lib/internal/discord/client.ts @@ -1,8 +1,13 @@ import { isRecordLike } from '@sim/utils/object' -import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + createSsrfGuardedFetchWithDispatcher, + MAX_JSON_API_RESPONSE_BYTES, +} from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { DiscordOperationError } from '@/lib/internal/discord/errors' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + export async function sendDiscordMessage( botToken: string, channelId: string, @@ -11,15 +16,18 @@ export async function sendDiscordMessage( signal?: AbortSignal ): Promise> { signal?.throwIfAborted() - const response = await fetch(`https://discord.com/api/v10/channels/${channelId}/messages`, { - method: 'POST', - headers: { - Authorization: `Bot ${botToken}`, - ...(contentType === 'json' ? { 'Content-Type': 'application/json' } : {}), - }, - body, - signal, - }) + const response = await providerFetch( + `https://discord.com/api/v10/channels/${channelId}/messages`, + { + method: 'POST', + headers: { + Authorization: `Bot ${botToken}`, + ...(contentType === 'json' ? { 'Content-Type': 'application/json' } : {}), + }, + body, + signal, + } + ) let data: unknown try { data = await readResponseJsonWithLimit(response, { diff --git a/apps/sim/lib/internal/docusign/client.test.ts b/apps/sim/lib/internal/docusign/client.test.ts index 42f21599e8e..fae9a52c64c 100644 --- a/apps/sim/lib/internal/docusign/client.test.ts +++ b/apps/sim/lib/internal/docusign/client.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { DocuSignClient } from '@/lib/internal/docusign/client' function accountResponse(): Response { diff --git a/apps/sim/lib/internal/docusign/client.ts b/apps/sim/lib/internal/docusign/client.ts index 3e8894c5950..0933215d7cd 100644 --- a/apps/sim/lib/internal/docusign/client.ts +++ b/apps/sim/lib/internal/docusign/client.ts @@ -1,3 +1,4 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -7,6 +8,8 @@ import { import { DocuSignOperationError } from '@/lib/internal/docusign/errors' import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' +const contentFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'contentFetch' }).fetch + const MAX_DOCUSIGN_JSON_BYTES = 2 * 1024 * 1024 export const MAX_DOCUSIGN_DOCUMENT_BYTES = 25 * 1024 * 1024 const DOCUSIGN_FETCH_TIMEOUT_MS = 30_000 @@ -35,7 +38,7 @@ async function fetchDocusign( const abort = () => controller.abort(parentSignal?.reason ?? new Error('Request aborted')) parentSignal?.addEventListener('abort', abort, { once: true }) try { - return await fetch(input, { ...init, signal: controller.signal }) + return await contentFetch(input, { ...init, signal: controller.signal }) } finally { clearTimeout(timeout) parentSignal?.removeEventListener('abort', abort) diff --git a/apps/sim/lib/internal/dropbox/client.test.ts b/apps/sim/lib/internal/dropbox/client.test.ts index 3941ef9f6fc..9fd46ad078d 100644 --- a/apps/sim/lib/internal/dropbox/client.test.ts +++ b/apps/sim/lib/internal/dropbox/client.test.ts @@ -3,6 +3,11 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const mocks = vi.hoisted(() => ({ fetch: vi.fn() })) import { DropboxClient, DropboxUploadError } from '@/lib/internal/dropbox/client' diff --git a/apps/sim/lib/internal/dropbox/client.ts b/apps/sim/lib/internal/dropbox/client.ts index 5e0f66d6a0a..49ff6f58510 100644 --- a/apps/sim/lib/internal/dropbox/client.ts +++ b/apps/sim/lib/internal/dropbox/client.ts @@ -1,3 +1,4 @@ +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -35,21 +36,27 @@ export class DropboxClient { } ): Promise> { this.signal?.throwIfAborted() - const response = await fetch('https://content.dropboxapi.com/2/files/upload', { - method: 'POST', - headers: { - Authorization: `Bearer ${this.accessToken}`, - 'Content-Type': 'application/octet-stream', - 'Dropbox-API-Arg': httpHeaderSafeJson({ - path, - mode: options.mode || 'add', - autorename: options.autorename ?? false, - mute: options.mute ?? false, - }), - }, - body: new Uint8Array(buffer), - signal: this.signal, - }) + const response = await secureFetchWithValidation( + 'https://content.dropboxapi.com/2/files/upload', + { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + method: 'POST', + headers: { + Authorization: `Bearer ${this.accessToken}`, + 'Content-Type': 'application/octet-stream', + 'Dropbox-API-Arg': httpHeaderSafeJson({ + path, + mode: options.mode || 'add', + autorename: options.autorename ?? false, + mute: options.mute ?? false, + }), + }, + body: new Uint8Array(buffer), + signal: this.signal, + } + ) const data = await readResponseJsonWithLimit & DropboxErrorBody>( response, { diff --git a/apps/sim/lib/internal/dynamodb/client.ts b/apps/sim/lib/internal/dynamodb/client.ts index 76ee3ceef9c..9f9409a0690 100644 --- a/apps/sim/lib/internal/dynamodb/client.ts +++ b/apps/sim/lib/internal/dynamodb/client.ts @@ -8,10 +8,12 @@ import { ScanCommand, UpdateCommand, } from '@aws-sdk/lib-dynamodb' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' import type { DynamoDBConnectionConfig, DynamoDBTableSchema } from '@/tools/dynamodb/types' export function createDynamoDBClient(config: DynamoDBConnectionConfig): DynamoDBDocumentClient { const client = new DynamoDBClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, @@ -222,6 +224,7 @@ export async function deleteItem( */ export function createRawDynamoDBClient(config: DynamoDBConnectionConfig): DynamoDBClient { return new DynamoDBClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/elevenlabs/client.test.ts b/apps/sim/lib/internal/elevenlabs/client.test.ts index b55e22fc61b..644af77d778 100644 --- a/apps/sim/lib/internal/elevenlabs/client.test.ts +++ b/apps/sim/lib/internal/elevenlabs/client.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { generateElevenLabsAudio, MAX_ELEVENLABS_AUDIO_BYTES, diff --git a/apps/sim/lib/internal/elevenlabs/client.ts b/apps/sim/lib/internal/elevenlabs/client.ts index 9fc0e862db8..278a034fab6 100644 --- a/apps/sim/lib/internal/elevenlabs/client.ts +++ b/apps/sim/lib/internal/elevenlabs/client.ts @@ -2,6 +2,7 @@ import { createTimeoutAbortController, DEFAULT_EXECUTION_TIMEOUT_MS, } from '@/lib/core/execution-limits' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { consumeOrCancelBody, readResponseToBufferWithLimit } from '@/lib/core/utils/stream-limits' import { ElevenLabsOperationError } from '@/lib/internal/elevenlabs/errors' import type { @@ -10,6 +11,8 @@ import type { ElevenLabsSpeechToSpeechInput, } from '@/lib/internal/elevenlabs/schema' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const BASE_URL = 'https://api.elevenlabs.io/v1' export const MAX_ELEVENLABS_AUDIO_BYTES = 25 * 1024 * 1024 @@ -90,7 +93,7 @@ export async function generateElevenLabsAudio( const timeout = createTimeoutAbortController(DEFAULT_EXECUTION_TIMEOUT_MS, signal) try { const { url, init } = buildRequest(args) - const response = await fetch(url, { ...init, signal: timeout.signal }) + const response = await providerFetch(url, { ...init, signal: timeout.signal }) timeout.signal.throwIfAborted() if (!response.ok) { await consumeOrCancelBody(response) diff --git a/apps/sim/lib/internal/firecrawl/operations.test.ts b/apps/sim/lib/internal/firecrawl/operations.test.ts index ac004bfa4bb..5258ec96e85 100644 --- a/apps/sim/lib/internal/firecrawl/operations.test.ts +++ b/apps/sim/lib/internal/firecrawl/operations.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' import { RESOLVED_SECRET_PROVENANCE_FIELD, diff --git a/apps/sim/lib/internal/firecrawl/operations.ts b/apps/sim/lib/internal/firecrawl/operations.ts index 0a8c641fc93..079a8df1ecb 100644 --- a/apps/sim/lib/internal/firecrawl/operations.ts +++ b/apps/sim/lib/internal/firecrawl/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, isPayloadSizeLimitError, @@ -19,6 +20,8 @@ import { docNotReadyResponse } from '@/lib/uploads/utils/servable-file-response' import { assertToolFileAccess } from '@/app/api/files/authorization' import { hasFirecrawlParseModelInput } from '@/tools/firecrawl/model-input' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('FirecrawlParse') export interface FirecrawlOperationContext { @@ -85,7 +88,7 @@ export async function executeFirecrawlParse( formData.append('options', JSON.stringify(input.options)) } - const response = await fetch('https://api.firecrawl.dev/v2/parse', { + const response = await providerFetch('https://api.firecrawl.dev/v2/parse', { method: 'POST', headers: { Authorization: `Bearer ${input.apiKey}` }, body: formData, diff --git a/apps/sim/lib/internal/fireflies/operations.test.ts b/apps/sim/lib/internal/fireflies/operations.test.ts index e480b8d5e98..e2f3f082fea 100644 --- a/apps/sim/lib/internal/fireflies/operations.test.ts +++ b/apps/sim/lib/internal/fireflies/operations.test.ts @@ -2,6 +2,12 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + import { PRIVATE_MODEL_INPUT_PROVENANCE_HEADER } from '@/lib/execution/model-input-provenance' import { RESOLVED_SECRET_PROVENANCE_FIELD, diff --git a/apps/sim/lib/internal/fireflies/operations.ts b/apps/sim/lib/internal/fireflies/operations.ts index 10bbdc1e0fd..cc851053458 100644 --- a/apps/sim/lib/internal/fireflies/operations.ts +++ b/apps/sim/lib/internal/fireflies/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' import type { FirefliesUploadAudioInput } from '@/lib/internal/fireflies/schema' @@ -96,7 +97,10 @@ export async function executeFirefliesUploadAudio( if (body.clientReferenceId) input.client_reference_id = body.clientReferenceId if (body.attendees !== undefined) input.attendees = body.attendees - const response = await fetch(FIREFLIES_API_URL, { + const response = await secureFetchWithValidation(FIREFLIES_API_URL, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_FIREFLIES_RESPONSE_BYTES, method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/gmail/client.ts b/apps/sim/lib/internal/gmail/client.ts index f3a9bbf426b..9305586fc30 100644 --- a/apps/sim/lib/internal/gmail/client.ts +++ b/apps/sim/lib/internal/gmail/client.ts @@ -1,9 +1,12 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { type ReadResponseWithLimitOptions, readResponseJsonWithLimit, } from '@/lib/core/utils/stream-limits' import { GmailOperationError } from '@/lib/internal/gmail/errors' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' const GMAIL_METADATA_RESPONSE_MAX_BYTES = 1024 * 1024 const RESPONSE_LIMIT: ReadResponseWithLimitOptions = { @@ -36,7 +39,7 @@ export class GmailClient { async fetch(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { signal?.throwIfAborted() - return fetch(path, { + return providerFetch(path, { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/gmail/operations.test.ts b/apps/sim/lib/internal/gmail/operations.test.ts index bf4ec26feb5..da8107d57a2 100644 --- a/apps/sim/lib/internal/gmail/operations.test.ts +++ b/apps/sim/lib/internal/gmail/operations.test.ts @@ -3,6 +3,13 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const fileMocks = vi.hoisted(() => ({ assertToolFileAccess: vi.fn(), downloadServableFilesWithinBudget: vi.fn(), diff --git a/apps/sim/lib/internal/iam/client.ts b/apps/sim/lib/internal/iam/client.ts index 76a2a610692..9fd6859677f 100644 --- a/apps/sim/lib/internal/iam/client.ts +++ b/apps/sim/lib/internal/iam/client.ts @@ -38,10 +38,12 @@ import { SimulatePrincipalPolicyCommand, UpdateAccessKeyCommand, } from '@aws-sdk/client-iam' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' import type { IAMConnectionConfig, IAMSimulateContextEntry } from '@/tools/iam/types' export function createIAMClient(config: IAMConnectionConfig): IAMClient { return new IAMClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, @@ -316,9 +318,7 @@ export async function createAccessKey( userName?: string | null, signal?: AbortSignal ) { - const command = new CreateAccessKeyCommand({ - ...(userName ? { UserName: userName } : {}), - }) + const command = new CreateAccessKeyCommand(userName ? { UserName: userName } : {}) const response = await client.send(command, { abortSignal: signal }) const key = response.AccessKey diff --git a/apps/sim/lib/internal/identity-center/client.ts b/apps/sim/lib/internal/identity-center/client.ts index 02c5cbeef68..01c0a7bbc4e 100644 --- a/apps/sim/lib/internal/identity-center/client.ts +++ b/apps/sim/lib/internal/identity-center/client.ts @@ -27,6 +27,7 @@ import { SSOAdminClient, type TargetType, } from '@aws-sdk/client-sso-admin' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' import { AWS_FANOUT_CONCURRENCY, mapWithConcurrency, @@ -42,6 +43,7 @@ interface IdentityCenterConnectionConfig { export function createSSOAdminClient(config: IdentityCenterConnectionConfig): SSOAdminClient { return new SSOAdminClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, @@ -54,6 +56,7 @@ export function createIdentityStoreClient( config: IdentityCenterConnectionConfig ): IdentitystoreClient { return new IdentitystoreClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, @@ -68,6 +71,7 @@ export function createIdentityStoreClient( */ export function createOrganizationsClient(config: IdentityCenterConnectionConfig) { return new OrganizationsClient({ + requestHandler: createOutboundAwsHttpHandler(), region: resolveOrganizationsRegion(config.region), credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/image/operations.test.ts b/apps/sim/lib/internal/image/operations.test.ts index bf0d46ee125..a0de90f45c8 100644 --- a/apps/sim/lib/internal/image/operations.test.ts +++ b/apps/sim/lib/internal/image/operations.test.ts @@ -15,7 +15,9 @@ const mocks = vi.hoisted(() => ({ vi.stubGlobal('fetch', mocks.fetch) -vi.mock('@/lib/core/security/input-validation.server', () => ({ +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), validateUrlWithDNS: mocks.validateUrlWithDNS, secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, })) @@ -96,6 +98,34 @@ describe('image operations', () => { ]) }) + it('keeps the Gemini image key confined to its provider origin', async () => { + mocks.fetch.mockResolvedValueOnce( + Response.json({ + candidates: [ + { content: { parts: [{ inlineData: { data: 'cG5n', mimeType: 'image/png' } }] } }, + ], + }) + ) + + const response = await executeImageGeneration( + { provider: 'gemini', apiKey: 'gemini-key', prompt: 'Draw an image' }, + { userId: 'user-1', requestId: 'request-1' } + ) + + expect(response.status).toBe(200) + expect(mocks.fetch).toHaveBeenCalledWith( + expect.stringContaining('https://generativelanguage.googleapis.com/'), + expect.objectContaining({ + headers: { 'x-goog-api-key': 'gemini-key', 'Content-Type': 'application/json' }, + redirectPolicy: { + mode: 'standard', + sendCredentialsOnCrossOriginRedirect: false, + sensitiveHeaders: ['x-goog-api-key'], + }, + }) + ) + }) + it('cancels polling without resubmitting or storing an image', async () => { const controller = new AbortController() mocks.fetch.mockResolvedValueOnce( diff --git a/apps/sim/lib/internal/image/operations.ts b/apps/sim/lib/internal/image/operations.ts index a1af8fd4b66..629954805ae 100644 --- a/apps/sim/lib/internal/image/operations.ts +++ b/apps/sim/lib/internal/image/operations.ts @@ -5,6 +5,7 @@ import { isRecordLike } from '@sim/utils/object' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' import { secureFetchWithPinnedIP, + secureFetchWithValidation, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { @@ -438,15 +439,21 @@ async function generateWithOpenAI( requestBody.moderation = pickAllowed(body.moderation, OPENAI_MODERATION_LEVELS, 'auto') } - const openaiResponse = await fetch('https://api.openai.com/v1/images/generations', { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - signal, - }) + const openaiResponse = await secureFetchWithValidation( + 'https://api.openai.com/v1/images/generations', + { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_IMAGE_JSON_BYTES, + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + signal, + } + ) if (!openaiResponse.ok) { const error = await readResponseTextWithLimit(openaiResponse, { @@ -533,9 +540,16 @@ async function generateWithGemini( ...(Object.keys(imageConfig).length > 0 && { imageConfig }), } - const geminiResponse = await fetch( + const geminiResponse = await secureFetchWithValidation( `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent`, { + profile: 'configuredEndpoint', + redirectPolicy: { + mode: 'standard', + sendCredentialsOnCrossOriginRedirect: false, + sensitiveHeaders: ['x-goog-api-key'], + }, + maxResponseBytes: MAX_IMAGE_JSON_BYTES, method: 'POST', headers: { 'x-goog-api-key': apiKey, @@ -706,15 +720,21 @@ async function generateWithFalAI( requestBody.thinking_level = pickAllowed(body.thinkingLevel, ['minimal', 'high'], 'minimal') } - const createResponse = await fetch(`https://queue.fal.run/${modelConfig.endpoint}`, { - method: 'POST', - headers: { - Authorization: `Key ${apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(requestBody), - signal, - }) + const createResponse = await secureFetchWithValidation( + `https://queue.fal.run/${modelConfig.endpoint}`, + { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_IMAGE_JSON_BYTES, + method: 'POST', + headers: { + Authorization: `Key ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + signal, + } + ) if (!createResponse.ok) { const error = await readResponseTextWithLimit(createResponse, { diff --git a/apps/sim/lib/internal/instagram/execute-tool.test.ts b/apps/sim/lib/internal/instagram/execute-tool.test.ts index e568ce9c39b..b26013042de 100644 --- a/apps/sim/lib/internal/instagram/execute-tool.test.ts +++ b/apps/sim/lib/internal/instagram/execute-tool.test.ts @@ -3,6 +3,13 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const { mockDeleteFileMetadata, mockDeleteFiles, diff --git a/apps/sim/lib/internal/instagram/operations.ts b/apps/sim/lib/internal/instagram/operations.ts index 805c60e8453..96e19c3a6cc 100644 --- a/apps/sim/lib/internal/instagram/operations.ts +++ b/apps/sim/lib/internal/instagram/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { createMediaContainer, @@ -36,6 +37,8 @@ import { sanitizeFileName } from '@/executor/constants' import type { UserFile } from '@/executor/types' import { bearerHeaders, graphUrl, idString, readGraphError } from '@/tools/instagram/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('InstagramOperations') const MAX_GRAPH_METADATA_BYTES = 256 * 1024 const MAX_CAROUSEL_ITEMS = 10 @@ -123,7 +126,7 @@ async function fetchMediaMetadata({ fields: string signal?: AbortSignal }): Promise { - const response = await fetch(graphUrl(`/${encodeURIComponent(mediaId)}`, { fields }), { + const response = await providerFetch(graphUrl(`/${encodeURIComponent(mediaId)}`, { fields }), { headers: bearerHeaders(accessToken), signal, }) diff --git a/apps/sim/lib/internal/instagram/publishing.test.ts b/apps/sim/lib/internal/instagram/publishing.test.ts index e8d090dc97c..9d30e91a51e 100644 --- a/apps/sim/lib/internal/instagram/publishing.test.ts +++ b/apps/sim/lib/internal/instagram/publishing.test.ts @@ -4,6 +4,13 @@ import type { Logger } from '@sim/logger' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const { mockHasCloudStorage, mockResolveFileInputToUrl } = vi.hoisted(() => ({ mockHasCloudStorage: vi.fn(), mockResolveFileInputToUrl: vi.fn(), diff --git a/apps/sim/lib/internal/instagram/publishing.ts b/apps/sim/lib/internal/instagram/publishing.ts index 05f5b4c9a12..f17035c2866 100644 --- a/apps/sim/lib/internal/instagram/publishing.ts +++ b/apps/sim/lib/internal/instagram/publishing.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { hasCloudStorage } from '@/lib/uploads/core/storage-service' import { getFileExtension, @@ -17,6 +18,8 @@ import { readGraphJson, } from '@/tools/instagram/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + /** Covers Meta's poll-once-per-minute for ≤5 minutes while the container processes. */ export const INSTAGRAM_MEDIA_URL_TTL_SECONDS = 600 @@ -290,7 +293,7 @@ export async function resolveIgUserId( ): Promise { if (igUserId?.trim()) return igUserId.trim() - const response = await fetch(graphUrl('/me', { fields: 'user_id' }), { + const response = await providerFetch(graphUrl('/me', { fields: 'user_id' }), { headers: bearerHeaders(accessToken), signal, }) @@ -315,10 +318,13 @@ async function getContainerStatus( containerId: string, signal?: AbortSignal ): Promise<{ statusCode: ContainerStatusCode | null; status: string | null }> { - const response = await fetch(graphUrl(`/${containerId}`, { fields: 'status_code,status' }), { - headers: bearerHeaders(accessToken), - signal, - }) + const response = await providerFetch( + graphUrl(`/${containerId}`, { fields: 'status_code,status' }), + { + headers: bearerHeaders(accessToken), + signal, + } + ) if (!response.ok) { throw new Error(`Failed to get container status: ${await readGraphError(response)}`) } @@ -392,7 +398,7 @@ async function postGraphForm( if (value !== undefined && value !== null) form.set(key, String(value)) } - return fetch(graphUrl(path), { + return providerFetch(graphUrl(path), { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/internal/jira/client.test.ts b/apps/sim/lib/internal/jira/client.test.ts index c92412ca7b4..ec557b98130 100644 --- a/apps/sim/lib/internal/jira/client.test.ts +++ b/apps/sim/lib/internal/jira/client.test.ts @@ -3,6 +3,13 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const mocks = vi.hoisted(() => ({ getCloudId: vi.fn(), })) diff --git a/apps/sim/lib/internal/jira/client.ts b/apps/sim/lib/internal/jira/client.ts index 62243e95e08..3b35c297759 100644 --- a/apps/sim/lib/internal/jira/client.ts +++ b/apps/sim/lib/internal/jira/client.ts @@ -1,9 +1,14 @@ import { validateJiraCloudId } from '@/lib/core/security/input-validation' -import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + createSsrfGuardedFetchWithDispatcher, + MAX_JSON_API_RESPONSE_BYTES, +} from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { JiraOperationError } from '@/lib/internal/jira/errors' import { getJiraCloudId } from '@/tools/jira/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + interface JiraConnectionConfig { domain: string accessToken: string @@ -56,7 +61,7 @@ export class JiraClient { signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(url, { + const response = await providerFetch(url, { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/jsm/client.test.ts b/apps/sim/lib/internal/jsm/client.test.ts index 62b40ef807d..e932c2eff1a 100644 --- a/apps/sim/lib/internal/jsm/client.test.ts +++ b/apps/sim/lib/internal/jsm/client.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { afterEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { JsmClient } from '@/lib/internal/jsm/client' import type { JsmOperationError } from '@/lib/internal/jsm/errors' diff --git a/apps/sim/lib/internal/jsm/client.ts b/apps/sim/lib/internal/jsm/client.ts index 1dfb8df2665..5c19af13de6 100644 --- a/apps/sim/lib/internal/jsm/client.ts +++ b/apps/sim/lib/internal/jsm/client.ts @@ -1,8 +1,11 @@ import { validateJiraCloudId } from '@/lib/core/security/input-validation' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { JsmOperationError } from '@/lib/internal/jsm/errors' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' import { resolveAssetsContext } from '@/tools/jsm/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + export interface JsmConnectionConfig { domain: string accessToken: string @@ -74,7 +77,7 @@ export class JsmClient { async fetch(path: string, init: RequestInit = {}, signal?: AbortSignal): Promise { signal?.throwIfAborted() - return fetch(path, { + return providerFetch(path, { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/lambda/client.ts b/apps/sim/lib/internal/lambda/client.ts index 9c689061687..d2d220c26ce 100644 --- a/apps/sim/lib/internal/lambda/client.ts +++ b/apps/sim/lib/internal/lambda/client.ts @@ -8,6 +8,7 @@ import { type LayersListItem, type LayerVersionsListItem, } from '@aws-sdk/client-lambda' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' export interface LambdaConnectionConfig { region: string @@ -17,6 +18,7 @@ export interface LambdaConnectionConfig { export function createLambdaClient(config: LambdaConnectionConfig): LambdaClient { return new LambdaClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/latex/operations.test.ts b/apps/sim/lib/internal/latex/operations.test.ts index bdbad641579..b8606167b77 100644 --- a/apps/sim/lib/internal/latex/operations.test.ts +++ b/apps/sim/lib/internal/latex/operations.test.ts @@ -3,6 +3,13 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const mocks = vi.hoisted(() => ({ fetch: vi.fn(), uploadExecutionFile: vi.fn(), diff --git a/apps/sim/lib/internal/latex/operations.ts b/apps/sim/lib/internal/latex/operations.ts index 6baae21f1e3..c5d99e5e32f 100644 --- a/apps/sim/lib/internal/latex/operations.ts +++ b/apps/sim/lib/internal/latex/operations.ts @@ -1,4 +1,5 @@ import { truncate } from '@sim/utils/string' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit, readResponseToBufferWithLimit, @@ -9,6 +10,8 @@ import type { LatexCompileInput } from '@/lib/internal/latex/schema' import { StorageService } from '@/lib/uploads' import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const LATEX_COMPILE_URL = 'https://latex.ytotech.com/builds/sync' const MAX_PDF_BYTES = 25 * 1024 * 1024 const MAX_ERROR_JSON_BYTES = 4 * 1024 * 1024 @@ -80,7 +83,7 @@ export async function compileLatexDocument( const signal = context.signal ? AbortSignal.any([context.signal, timeoutSignal]) : timeoutSignal let response: Response try { - response = await fetch(LATEX_COMPILE_URL, { + response = await providerFetch(LATEX_COMPILE_URL, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ diff --git a/apps/sim/lib/internal/linq/client.ts b/apps/sim/lib/internal/linq/client.ts index 7ab0fd5e45f..2954d753c4a 100644 --- a/apps/sim/lib/internal/linq/client.ts +++ b/apps/sim/lib/internal/linq/client.ts @@ -2,6 +2,7 @@ import { isRecordLike } from '@sim/utils/object' import { MAX_JSON_API_RESPONSE_BYTES, secureFetchWithPinnedIP, + secureFetchWithValidation, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { @@ -38,7 +39,10 @@ export async function registerLinqAttachment( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(`${LINQ_API_BASE}/attachments`, { + const response = await secureFetchWithValidation(`${LINQ_API_BASE}/attachments`, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, method: 'POST', headers: linqHeaders(input.apiKey), body: JSON.stringify({ diff --git a/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.test.ts b/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.test.ts index 87a453e22f1..bfc01b3d8be 100644 --- a/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.test.ts +++ b/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.test.ts @@ -2,6 +2,12 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + import { executeAddUserAppRoleAssignmentOperation } from '@/lib/internal/microsoft-ad/operations/add-user-app-role-assignment' const INPUT = { diff --git a/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.ts b/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.ts index 73d3a03f79c..ff52270e598 100644 --- a/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.ts +++ b/apps/sim/lib/internal/microsoft-ad/operations/add-user-app-role-assignment.ts @@ -1,4 +1,8 @@ import { isRecordLike } from '@sim/utils/object' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import { mapAppRoleAssignment, @@ -16,9 +20,12 @@ export const executeAddUserAppRoleAssignmentOperation: InternalToolOperationImpl const { userId, resourceId, appRoleId } = readIdentifiers(params) const principalId = await resolveGraphUserObjectId(userId, params.accessToken, signal) - const response = await fetch( + const response = await secureFetchWithValidation( `https://graph.microsoft.com/v1.0/users/${encodeURIComponent(userId)}/appRoleAssignments`, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, method: 'POST', headers: { Authorization: `Bearer ${params.accessToken}`, diff --git a/apps/sim/lib/internal/microsoft-teams/client.ts b/apps/sim/lib/internal/microsoft-teams/client.ts index 1ac3399dcb4..b24940ecb33 100644 --- a/apps/sim/lib/internal/microsoft-teams/client.ts +++ b/apps/sim/lib/internal/microsoft-teams/client.ts @@ -1,8 +1,11 @@ import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { MicrosoftTeamsOperationError } from '@/lib/internal/microsoft-teams/errors' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const MICROSOFT_GRAPH_BASE_URL = 'https://graph.microsoft.com/v1.0' const MICROSOFT_GRAPH_RESPONSE_MAX_BYTES = 2 * 1024 * 1024 @@ -27,7 +30,7 @@ export class MicrosoftTeamsClient { signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(`${MICROSOFT_GRAPH_BASE_URL}${path}`, { + const response = await providerFetch(`${MICROSOFT_GRAPH_BASE_URL}${path}`, { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, diff --git a/apps/sim/lib/internal/microsoft-teams/operations.test.ts b/apps/sim/lib/internal/microsoft-teams/operations.test.ts index 927769c2429..7a39ade7a8d 100644 --- a/apps/sim/lib/internal/microsoft-teams/operations.test.ts +++ b/apps/sim/lib/internal/microsoft-teams/operations.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { isInternalToolFileResult } from '@/lib/internal/tool-operations/file-result' const mocks = vi.hoisted(() => ({ diff --git a/apps/sim/lib/internal/microsoft-teams/operations.ts b/apps/sim/lib/internal/microsoft-teams/operations.ts index 7cca5977186..72927418ef3 100644 --- a/apps/sim/lib/internal/microsoft-teams/operations.ts +++ b/apps/sim/lib/internal/microsoft-teams/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { MicrosoftTeamsClient, @@ -22,6 +23,8 @@ import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils. import { assertToolFileAccess } from '@/app/api/files/authorization' import type { UserFile } from '@/executor/types' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const MAX_GRAPH_RESPONSE_BYTES = 2 * 1024 * 1024 const MAX_TEAMS_FILE_SIZE = 4 * 1024 * 1024 const MENTION_PATTERN = /[^<]+<\/at>/i @@ -460,7 +463,7 @@ export async function deleteMicrosoftTeamsChatMessage( throw new MicrosoftTeamsOperationError('Chat ID and Message ID are required', 400) } - const meResponse = await fetch('https://graph.microsoft.com/v1.0/me', { + const meResponse = await providerFetch('https://graph.microsoft.com/v1.0/me', { headers: { Authorization: `Bearer ${input.accessToken}` }, signal: context.signal, }) @@ -472,7 +475,7 @@ export async function deleteMicrosoftTeamsChatMessage( ) } - const response = await fetch( + const response = await providerFetch( `https://graph.microsoft.com/v1.0/users/${encodeURIComponent(me.id)}/chats/${encodeURIComponent(chatId)}/messages/${encodeURIComponent(messageId)}/softDelete`, { method: 'POST', diff --git a/apps/sim/lib/internal/okta/operations/update-group.ts b/apps/sim/lib/internal/okta/operations/update-group.ts index dfcf829dbf5..37cbb353ab9 100644 --- a/apps/sim/lib/internal/okta/operations/update-group.ts +++ b/apps/sim/lib/internal/okta/operations/update-group.ts @@ -1,9 +1,12 @@ import { createLogger } from '@sim/logger' import { validateOktaDomain } from '@/lib/core/security/input-validation' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { OktaGroup, OktaUpdateGroupParams, OktaUpdateGroupResponse } from '@/tools/okta/types' import { mergeOktaGroupProfile, oktaHeaders, throwOktaError } from '@/tools/okta/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('OktaUpdateGroup') async function transformUpdateGroupResponse(response: Response): Promise { @@ -34,13 +37,13 @@ export const executeOktaUpdateGroupOperation: InternalToolOperationImplementatio const url = `https://${domain}/api/v1/groups/${encodeURIComponent(params.groupId.trim())}` const headers = oktaHeaders(params.apiKey) - const readResponse = await fetch(url, { headers, signal }) + const readResponse = await providerFetch(url, { headers, signal }) if (!readResponse.ok) { await throwOktaError(readResponse, logger, 'Failed to load group for update in Okta') } const existing: OktaGroup = await readResponse.json() - const writeResponse = await fetch(url, { + const writeResponse = await providerFetch(url, { method: 'PUT', headers, body: JSON.stringify({ profile: mergeOktaGroupProfile(existing.profile, params) }), diff --git a/apps/sim/lib/internal/onepassword/client.ts b/apps/sim/lib/internal/onepassword/client.ts index 1903bc34eec..8dc40c8b37c 100644 --- a/apps/sim/lib/internal/onepassword/client.ts +++ b/apps/sim/lib/internal/onepassword/client.ts @@ -11,6 +11,7 @@ import type { Website, } from '@1password/sdk' import { generateId } from '@sim/utils/id' +import { requireDirectOutboundTransport } from '@/lib/core/network/context.server' import { MAX_JSON_API_RESPONSE_BYTES, type SecureFetchResponse, @@ -246,6 +247,7 @@ export function resolveCredentials(params: CredentialParams): ResolvedCredential */ export async function createOnePasswordClient(serviceAccountToken: string, signal?: AbortSignal) { signal?.throwIfAborted() + await requireDirectOutboundTransport() const { createClient } = await import('@1password/sdk') signal?.throwIfAborted() const client = await createClient({ diff --git a/apps/sim/lib/internal/outlook/client.test.ts b/apps/sim/lib/internal/outlook/client.test.ts index c085bca9b04..214b87e2607 100644 --- a/apps/sim/lib/internal/outlook/client.test.ts +++ b/apps/sim/lib/internal/outlook/client.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { DEFAULT_MAX_ERROR_BODY_BYTES, PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { OutlookClient } from '@/lib/internal/outlook/client' import { OutlookOperationError } from '@/lib/internal/outlook/errors' diff --git a/apps/sim/lib/internal/outlook/client.ts b/apps/sim/lib/internal/outlook/client.ts index 2b7a2ac7e2b..7a3005dd64f 100644 --- a/apps/sim/lib/internal/outlook/client.ts +++ b/apps/sim/lib/internal/outlook/client.ts @@ -1,4 +1,5 @@ import { getErrorMessage } from '@sim/utils/errors' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -6,6 +7,8 @@ import { } from '@/lib/core/utils/stream-limits' import { OutlookOperationError } from '@/lib/internal/outlook/errors' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const MICROSOFT_GRAPH_BASE_URL = 'https://graph.microsoft.com/v1.0' const MICROSOFT_GRAPH_RESPONSE_MAX_BYTES = 10 * 1024 * 1024 @@ -41,7 +44,7 @@ export class OutlookClient { signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(this.url(path), { + const response = await providerFetch(this.url(path), { ...init, headers: { 'Content-Type': 'application/json', @@ -77,7 +80,7 @@ export class OutlookClient { signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(this.url(path), { + const response = await providerFetch(this.url(path), { ...init, headers: { Authorization: `Bearer ${this.accessToken}`, @@ -110,7 +113,7 @@ export class OutlookClient { signal?: AbortSignal ): Promise<{ buffer: Buffer; contentType: string | null }> { signal?.throwIfAborted() - const response = await fetch(this.url(path), { + const response = await providerFetch(this.url(path), { method: 'GET', headers: { Authorization: `Bearer ${this.accessToken}` }, signal, diff --git a/apps/sim/lib/internal/persona/operations.test.ts b/apps/sim/lib/internal/persona/operations.test.ts index c5eaf1ce5b9..3f6d08ddb9d 100644 --- a/apps/sim/lib/internal/persona/operations.test.ts +++ b/apps/sim/lib/internal/persona/operations.test.ts @@ -3,6 +3,11 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const mocks = vi.hoisted(() => ({ assertToolFileAccess: vi.fn(), downloadServableFileFromStorage: vi.fn(), diff --git a/apps/sim/lib/internal/persona/operations.ts b/apps/sim/lib/internal/persona/operations.ts index 3d8ec065e74..605288c0e41 100644 --- a/apps/sim/lib/internal/persona/operations.ts +++ b/apps/sim/lib/internal/persona/operations.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { PersonaOperationError } from '@/lib/internal/persona/errors' import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' @@ -46,7 +47,10 @@ export async function importPersonaAccounts( }) const buffer = resolved.buffer context.signal?.throwIfAborted() - const response = await fetch(`${PERSONA_API_BASE}/importer/accounts`, { + const response = await secureFetchWithValidation(`${PERSONA_API_BASE}/importer/accounts`, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_PERSONA_RESPONSE_BYTES, method: 'POST', headers: buildPersonaHeaders(input.apiKey), body: JSON.stringify({ diff --git a/apps/sim/lib/internal/quickbooks/operations.test.ts b/apps/sim/lib/internal/quickbooks/operations.test.ts index 27b9044ea6b..7c3c1619e09 100644 --- a/apps/sim/lib/internal/quickbooks/operations.test.ts +++ b/apps/sim/lib/internal/quickbooks/operations.test.ts @@ -16,9 +16,13 @@ vi.mock('@/lib/uploads/contexts/copilot', () => ({ vi.mock('@/lib/uploads/contexts/execution', () => ({ uploadExecutionFile: mocks.uploadExecutionFile, })) -vi.mock('@/lib/core/security/input-validation.server', () => ({ - createSsrfGuardedFetchWithDispatcher: () => ({ - fetch: mocks.guardedFetch, +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: ({ profile }: { profile: string }) => ({ + fetch: + profile === 'contentFetch' + ? mocks.guardedFetch + : (...args: Parameters) => fetch(...args), dispatcher: { close: mocks.closeDispatcher }, }), })) diff --git a/apps/sim/lib/internal/quickbooks/operations.ts b/apps/sim/lib/internal/quickbooks/operations.ts index 33707ff90a2..6cfe6e4874c 100644 --- a/apps/sim/lib/internal/quickbooks/operations.ts +++ b/apps/sim/lib/internal/quickbooks/operations.ts @@ -34,6 +34,8 @@ import { validateQuickBooksAttachmentFileType, } from '@/tools/quickbooks/documents_utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('QuickBooksInternalOperations') export interface QuickBooksOperationContext { @@ -93,7 +95,7 @@ async function downloadQuickBooksAttachment( body.quickBooksEnvironment ) const metadataSignal = quickBooksDocumentSignal(signal, QUICKBOOKS_DOCUMENT_METADATA_TIMEOUT_MS) - const downloadUrlResponse = await fetch(downloadUrl, { + const downloadUrlResponse = await providerFetch(downloadUrl, { method: 'GET', headers: { ...buildQuickBooksHeaders(body.accessToken), Accept: '*/*' }, signal: metadataSignal, @@ -183,7 +185,7 @@ async function downloadQuickBooksTransactionPdf( body.quickBooksEnvironment ) const transferSignal = quickBooksDocumentSignal(signal, QUICKBOOKS_DOCUMENT_TRANSFER_TIMEOUT_MS) - const response = await fetch(url, { + const response = await providerFetch(url, { method: 'GET', headers: { ...buildQuickBooksHeaders(body.accessToken), Accept: 'application/pdf' }, signal: transferSignal, @@ -221,7 +223,7 @@ export async function executeQuickBooksAddAttachment( const metadata = buildQuickBooksAttachableMetadata(data.targetType, data.targetId, { note: data.note!, }) - response = await fetch(url, { + response = await providerFetch(url, { method: 'POST', headers: { ...buildQuickBooksHeaders(data.accessToken), @@ -301,7 +303,7 @@ export async function executeQuickBooksAddAttachment( ), resolvedName ) - response = await fetch(url, { + response = await providerFetch(url, { method: 'POST', headers: buildQuickBooksHeaders(data.accessToken), body: formData, diff --git a/apps/sim/lib/internal/quickbooks/provider-operations.test.ts b/apps/sim/lib/internal/quickbooks/provider-operations.test.ts index 16c2fb1f4a4..7cfc3c325ea 100644 --- a/apps/sim/lib/internal/quickbooks/provider-operations.test.ts +++ b/apps/sim/lib/internal/quickbooks/provider-operations.test.ts @@ -3,6 +3,13 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + vi.mock('@/lib/core/config/env', () => ({ env: { QUICKBOOKS_ENV: 'production' }, })) diff --git a/apps/sim/lib/internal/quickbooks/provider-operations.ts b/apps/sim/lib/internal/quickbooks/provider-operations.ts index c6a2a644904..c99cbb25a07 100644 --- a/apps/sim/lib/internal/quickbooks/provider-operations.ts +++ b/apps/sim/lib/internal/quickbooks/provider-operations.ts @@ -1,4 +1,5 @@ import { filterUndefined } from '@sim/utils/object' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { buildQuickBooksCreateBillPaymentBody, buildQuickBooksUpdateBillBody, @@ -55,6 +56,8 @@ import { validateQuickBooksOptionalNumber, } from '@/tools/quickbooks/values' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + /** * Intuit constrains the BillPayment payment account by both classification * fields, not by `AccountType` alone. `BillPaymentCheck.BankAccountRef`: "The @@ -178,7 +181,7 @@ export async function executeQuickBooksCreateBillPaymentOperation( const paymentAccountId = params.paymentAccountId.trim() if (!paymentAccountId) throw new Error('paymentAccountId is required') - const accountResponse = await fetch( + const accountResponse = await providerFetch( buildQuickBooksEntityUrl(params, 'account', paymentAccountId), { method: 'GET', @@ -197,7 +200,7 @@ export async function executeQuickBooksCreateBillPaymentOperation( assertCompatiblePaymentAccount(account, params.paymentType, paymentAccountId) signal?.throwIfAborted() - const paymentResponse = await fetch( + const paymentResponse = await providerFetch( addQuickBooksRequestId(buildQuickBooksEntityUrl(params, 'billpayment'), params.requestId), { method: 'POST', @@ -273,7 +276,7 @@ export async function executeQuickBooksUpdateRefundReceiptOperation( params: QuickBooksUpdateRefundReceiptParams, signal?: AbortSignal ) { - const response = await fetch(buildQuickBooksEntityUrl(params, 'refundreceipt'), { + const response = await providerFetch(buildQuickBooksEntityUrl(params, 'refundreceipt'), { method: 'POST', headers: getQuickBooksToolHeaders(params.accessToken, 'application/json'), body: JSON.stringify(buildQuickBooksUpdateSalesDocumentBody(params)), @@ -301,7 +304,7 @@ export async function executeQuickBooksUpdateCustomerPaymentOperation( const syncToken = params.syncToken?.trim() if (!syncToken) throw new Error('syncToken is required') - const readResponse = await fetch(buildQuickBooksEntityUrl(params, 'payment', paymentId), { + const readResponse = await providerFetch(buildQuickBooksEntityUrl(params, 'payment', paymentId), { method: 'GET', headers: getQuickBooksToolHeaders(params.accessToken), signal, @@ -335,7 +338,7 @@ export async function executeQuickBooksUpdateCustomerPaymentOperation( paymentId, syncToken ) - const updateResponse = await fetch(buildQuickBooksEntityUrl(params, 'payment'), { + const updateResponse = await providerFetch(buildQuickBooksEntityUrl(params, 'payment'), { method: 'POST', headers: getQuickBooksToolHeaders(params.accessToken, 'application/json'), body: JSON.stringify(fullBody), diff --git a/apps/sim/lib/internal/quiver/client.test.ts b/apps/sim/lib/internal/quiver/client.test.ts index 08190bcc0ca..4fcd3257130 100644 --- a/apps/sim/lib/internal/quiver/client.test.ts +++ b/apps/sim/lib/internal/quiver/client.test.ts @@ -2,6 +2,12 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' import { requestQuiverSvg } from '@/lib/internal/quiver/client' @@ -26,6 +32,9 @@ describe('Quiver client', () => { ) ).resolves.toEqual({ data: [{ svg: '' }], id: 'generation-1' }) expect(fetchMock).toHaveBeenCalledWith('https://api.quiver.ai/v1/svgs/generations', { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/quiver/client.ts b/apps/sim/lib/internal/quiver/client.ts index acb7d512020..685f90e3505 100644 --- a/apps/sim/lib/internal/quiver/client.ts +++ b/apps/sim/lib/internal/quiver/client.ts @@ -1,5 +1,8 @@ import { createLogger } from '@sim/logger' -import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -19,7 +22,10 @@ export async function requestQuiverSvg( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch(`${QUIVER_API_BASE_URL}/${path}`, { + const response = await secureFetchWithValidation(`${QUIVER_API_BASE_URL}/${path}`, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/rds/client.ts b/apps/sim/lib/internal/rds/client.ts index 683e9a1aee4..3e80cd8ba93 100644 --- a/apps/sim/lib/internal/rds/client.ts +++ b/apps/sim/lib/internal/rds/client.ts @@ -5,10 +5,12 @@ import { RDSDataClient, type SqlParameter, } from '@aws-sdk/client-rds-data' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' import type { RdsConnectionConfig } from '@/tools/rds/types' export function createRdsClient(config: RdsConnectionConfig): RDSDataClient { return new RDSDataClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/resend/client.ts b/apps/sim/lib/internal/resend/client.ts index 3298db84a10..6dd6f428161 100644 --- a/apps/sim/lib/internal/resend/client.ts +++ b/apps/sim/lib/internal/resend/client.ts @@ -1,5 +1,8 @@ import { isRecordLike } from '@sim/utils/object' -import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { ResendOperationError } from '@/lib/internal/resend/errors' @@ -20,7 +23,10 @@ export async function sendResendEmail( signal?: AbortSignal ): Promise> { signal?.throwIfAborted() - const response = await fetch('https://api.resend.com/emails', { + const response = await secureFetchWithValidation('https://api.resend.com/emails', { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/internal/resend/resend.test.ts b/apps/sim/lib/internal/resend/resend.test.ts index ee6a209f487..a87e333a285 100644 --- a/apps/sim/lib/internal/resend/resend.test.ts +++ b/apps/sim/lib/internal/resend/resend.test.ts @@ -2,6 +2,12 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + import { executeResendSend } from '@/lib/internal/resend/operations' describe('Resend operation', () => { diff --git a/apps/sim/lib/internal/s3/client.ts b/apps/sim/lib/internal/s3/client.ts index 95d336d332a..576ede9ae1d 100644 --- a/apps/sim/lib/internal/s3/client.ts +++ b/apps/sim/lib/internal/s3/client.ts @@ -1,4 +1,5 @@ import { S3Client } from '@aws-sdk/client-s3' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' export interface S3ConnectionConfig { region: string @@ -8,6 +9,7 @@ export interface S3ConnectionConfig { export function createS3Client(config: S3ConnectionConfig): S3Client { return new S3Client({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/sailpoint/client.test.ts b/apps/sim/lib/internal/sailpoint/client.test.ts index 082c4068d4a..1d92404eb29 100644 --- a/apps/sim/lib/internal/sailpoint/client.test.ts +++ b/apps/sim/lib/internal/sailpoint/client.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { clearSailPointTokenStateForTests, getSailPointAccessToken, diff --git a/apps/sim/lib/internal/sailpoint/client.ts b/apps/sim/lib/internal/sailpoint/client.ts index 395492cea38..1c38672ce3e 100644 --- a/apps/sim/lib/internal/sailpoint/client.ts +++ b/apps/sim/lib/internal/sailpoint/client.ts @@ -2,13 +2,18 @@ import { createHash } from 'node:crypto' import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' -import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + createSsrfGuardedFetchWithDispatcher, + MAX_JSON_API_RESPONSE_BYTES, +} from '@/lib/core/security/input-validation.server' import { consumeOrCancelBody, DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, } from '@/lib/core/utils/stream-limits' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + export interface SailPointCredentials { clientId: string clientSecret: string @@ -156,7 +161,7 @@ async function exchangeAccessToken( while (true) { signal?.throwIfAborted() - const response = await fetch(tokenUrl, { + const response = await providerFetch(tokenUrl, { method: 'POST', headers: { Accept: 'application/json', @@ -258,7 +263,7 @@ export async function sailpointFetch( headers.set('Authorization', `Bearer ${token}`) if (!headers.has('Accept')) headers.set('Accept', 'application/json') - const response = await fetch(url, { + const response = await providerFetch(url, { ...init, cache: 'no-store', headers, diff --git a/apps/sim/lib/internal/sailpoint/execute-tool.test.ts b/apps/sim/lib/internal/sailpoint/execute-tool.test.ts index 5d3a8ac0478..0e5e020eb44 100644 --- a/apps/sim/lib/internal/sailpoint/execute-tool.test.ts +++ b/apps/sim/lib/internal/sailpoint/execute-tool.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' const fileMocks = vi.hoisted(() => ({ diff --git a/apps/sim/lib/internal/salesforce/operations/update-custom-field.test.ts b/apps/sim/lib/internal/salesforce/operations/update-custom-field.test.ts index e6d19cd9d48..778a1f8db37 100644 --- a/apps/sim/lib/internal/salesforce/operations/update-custom-field.test.ts +++ b/apps/sim/lib/internal/salesforce/operations/update-custom-field.test.ts @@ -2,6 +2,12 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + import { executeSalesforceUpdateCustomFieldOperation } from '@/lib/internal/salesforce/operations/update-custom-field' const PARAMS = { diff --git a/apps/sim/lib/internal/salesforce/operations/update-custom-field.ts b/apps/sim/lib/internal/salesforce/operations/update-custom-field.ts index 8bb290ef3ba..c4956538c74 100644 --- a/apps/sim/lib/internal/salesforce/operations/update-custom-field.ts +++ b/apps/sim/lib/internal/salesforce/operations/update-custom-field.ts @@ -1,5 +1,9 @@ import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' +import { + MAX_JSON_API_RESPONSE_BYTES, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { SalesforceUpdateCustomFieldParams, @@ -29,7 +33,13 @@ export const executeSalesforceUpdateCustomFieldOperation: InternalToolOperationI 'Content-Type': 'application/json', } - const readResponse = await fetch(url, { headers, signal }) + const readResponse = await secureFetchWithValidation(url, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + headers, + signal, + }) let existing: unknown try { existing = await readResponse.json() @@ -56,7 +66,10 @@ export const executeSalesforceUpdateCustomFieldOperation: InternalToolOperationI const metadata = mergeCustomFieldMetadata(existing.Metadata, params) - const patchResponse = await fetch(url, { + const patchResponse = await secureFetchWithValidation(url, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, method: 'PATCH', headers, body: JSON.stringify({ Metadata: metadata }), diff --git a/apps/sim/lib/internal/secrets-manager/client.ts b/apps/sim/lib/internal/secrets-manager/client.ts index 7e3808dc776..a2a76a17c92 100644 --- a/apps/sim/lib/internal/secrets-manager/client.ts +++ b/apps/sim/lib/internal/secrets-manager/client.ts @@ -12,6 +12,7 @@ import { UntagResourceCommand, UpdateSecretCommand, } from '@aws-sdk/client-secrets-manager' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' interface SecretsManagerConnectionConfig { region: string @@ -32,6 +33,7 @@ export function createSecretsManagerClient( config: SecretsManagerConnectionConfig ): SecretsManagerClient { return new SecretsManagerClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/sendgrid/client.ts b/apps/sim/lib/internal/sendgrid/client.ts index 75610b27c0b..26996c7add9 100644 --- a/apps/sim/lib/internal/sendgrid/client.ts +++ b/apps/sim/lib/internal/sendgrid/client.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { SendGridOperationError } from '@/lib/internal/sendgrid/errors' @@ -23,7 +24,10 @@ export async function sendSendGridMail( signal?: AbortSignal ): Promise { signal?.throwIfAborted() - const response = await fetch('https://api.sendgrid.com/v3/mail/send', { + const response = await secureFetchWithValidation('https://api.sendgrid.com/v3/mail/send', { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_SENDGRID_ERROR_BYTES, method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/internal/ses/client.ts b/apps/sim/lib/internal/ses/client.ts index 8ab4c75bdfa..52970ae309e 100644 --- a/apps/sim/lib/internal/ses/client.ts +++ b/apps/sim/lib/internal/ses/client.ts @@ -22,6 +22,7 @@ import { UpdateEmailTemplateCommand, } from '@aws-sdk/client-sesv2' import { z } from 'zod' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' import type { SESConnectionConfig } from '@/tools/ses/types' const SesBulkEmailDestinationSchema = z.object({ @@ -33,6 +34,7 @@ type SesBulkEmailDestination = z.infer export function createSESClient(config: SESConnectionConfig): SESv2Client { return new SESv2Client({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/slack/client.ts b/apps/sim/lib/internal/slack/client.ts index f40bb4b63b0..d06b1446cc2 100644 --- a/apps/sim/lib/internal/slack/client.ts +++ b/apps/sim/lib/internal/slack/client.ts @@ -1,3 +1,4 @@ +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' const MAX_SLACK_JSON_BYTES = 2 * 1024 * 1024 @@ -75,7 +76,10 @@ export async function requestSlackApi({ } const isForm = body instanceof URLSearchParams - const response = await fetch(url, { + const response = await secureFetchWithValidation(url.toString(), { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_SLACK_JSON_BYTES, method: httpMethod, headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/internal/slack/oauth.test.ts b/apps/sim/lib/internal/slack/oauth.test.ts index 9e4deeb48cf..68789ed2c72 100644 --- a/apps/sim/lib/internal/slack/oauth.test.ts +++ b/apps/sim/lib/internal/slack/oauth.test.ts @@ -1,5 +1,11 @@ /** @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + import { exchangeSlackBotAuthorization, revokeSlackBotAuthorization, @@ -35,8 +41,9 @@ describe('Slack bot OAuth exchange', () => { expect(request.headers.Authorization).toBe( `Basic ${Buffer.from('client:secret').toString('base64')}` ) - expect(request.body.get('redirect_uri')).toBe(input.redirectUri) - expect(request.body.get('code')).toBe('code') + const body = new URLSearchParams(request.body) + expect(body.get('redirect_uri')).toBe(input.redirectUri) + expect(body.get('code')).toBe('code') }) it.each([{ token_type: 'user' }, { ok: false, error: 'invalid_client_id' }])( 'rejects incompatible or unsuccessful grants: %j', diff --git a/apps/sim/lib/internal/slack/oauth.ts b/apps/sim/lib/internal/slack/oauth.ts index ced0697443a..c27387aa84c 100644 --- a/apps/sim/lib/internal/slack/oauth.ts +++ b/apps/sim/lib/internal/slack/oauth.ts @@ -1,6 +1,7 @@ import { Buffer } from 'node:buffer' import { z } from 'zod' import { OrchestrationError } from '@/lib/core/orchestration/types' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { requestSlackApi } from '@/lib/internal/slack/client' import { SLACK_SEARCH_SCOPES } from '@/lib/slack-search/constants' @@ -25,13 +26,16 @@ export async function exchangeSlackBotAuthorization(input: { code: string redirectUri: string }) { - const response = await fetch('https://slack.com/api/oauth.v2.access', { + const response = await secureFetchWithValidation('https://slack.com/api/oauth.v2.access', { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: 64 * 1024, method: 'POST', headers: { Authorization: `Basic ${Buffer.from(`${input.clientId}:${input.clientSecret}`).toString('base64')}`, 'Content-Type': 'application/x-www-form-urlencoded', }, - body: new URLSearchParams({ code: input.code, redirect_uri: input.redirectUri }), + body: new URLSearchParams({ code: input.code, redirect_uri: input.redirectUri }).toString(), signal: AbortSignal.timeout(10_000), }) const value = await readResponseJsonWithLimit(response, { diff --git a/apps/sim/lib/internal/slack/operations.test.ts b/apps/sim/lib/internal/slack/operations.test.ts index b4b76baf6be..f439b059909 100644 --- a/apps/sim/lib/internal/slack/operations.test.ts +++ b/apps/sim/lib/internal/slack/operations.test.ts @@ -19,9 +19,16 @@ vi.mock('@/lib/internal/slack/file-input', () => ({ forEachSlackAttachmentFile: mocks.resolveFiles, })) -vi.mock('@/lib/core/security/input-validation.server', () => ({ +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, - secureFetchWithValidation: mocks.secureFetchWithValidation, + secureFetchWithValidation: (url: string, options: RequestInit, paramName?: string) => + url.startsWith('https://slack.com/api/') + ? fetch(url, options) + : mocks.secureFetchWithValidation(url, options, paramName), validateUrlWithDNS: mocks.validateUrlWithDNS, })) diff --git a/apps/sim/lib/internal/slack/search-client.test.ts b/apps/sim/lib/internal/slack/search-client.test.ts index 864a82df65c..6df47fe79be 100644 --- a/apps/sim/lib/internal/slack/search-client.test.ts +++ b/apps/sim/lib/internal/slack/search-client.test.ts @@ -1,5 +1,11 @@ /** @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + import { getSlackSearchSender, verifySlackSearchBot } from '@/lib/internal/slack/search-client' import { SLACK_SEARCH_SCOPES } from '@/lib/slack-search/constants' diff --git a/apps/sim/lib/internal/sqs/client.ts b/apps/sim/lib/internal/sqs/client.ts index 9c762c0439d..80ec3182538 100644 --- a/apps/sim/lib/internal/sqs/client.ts +++ b/apps/sim/lib/internal/sqs/client.ts @@ -1,8 +1,10 @@ import { SQSClient } from '@aws-sdk/client-sqs' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' import type { SqsConnectionConfig } from '@/tools/sqs/types' export function createSqsClient(config: SqsConnectionConfig): SQSClient { return new SQSClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/square/operations.test.ts b/apps/sim/lib/internal/square/operations.test.ts index acef2374ce9..b4ba8d90fa0 100644 --- a/apps/sim/lib/internal/square/operations.test.ts +++ b/apps/sim/lib/internal/square/operations.test.ts @@ -3,6 +3,13 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const mocks = vi.hoisted(() => ({ assertToolFileAccess: vi.fn(), downloadFileFromStorage: vi.fn(), diff --git a/apps/sim/lib/internal/square/operations.ts b/apps/sim/lib/internal/square/operations.ts index 3fb8ffec558..5635458948e 100644 --- a/apps/sim/lib/internal/square/operations.ts +++ b/apps/sim/lib/internal/square/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -12,6 +13,8 @@ import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import { assertToolFileAccess } from '@/app/api/files/authorization' import { SQUARE_API_VERSION, SQUARE_BASE_URL } from '@/tools/square/types' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('SquareCatalogImage') const MAX_SQUARE_RESPONSE_BYTES = 10 * 1024 * 1024 @@ -71,7 +74,7 @@ export async function executeSquareCreateCatalogImage( input.fileName || userFile.name ) - const response = await fetch(`${SQUARE_BASE_URL}/v2/catalog/images`, { + const response = await providerFetch(`${SQUARE_BASE_URL}/v2/catalog/images`, { method: 'POST', headers: { Authorization: `Bearer ${input.accessToken}`, diff --git a/apps/sim/lib/internal/ssm/client.ts b/apps/sim/lib/internal/ssm/client.ts index 584708bad35..2c799faced2 100644 --- a/apps/sim/lib/internal/ssm/client.ts +++ b/apps/sim/lib/internal/ssm/client.ts @@ -63,6 +63,7 @@ import type { AwsSsmPutParameterBody } from '@/lib/api/contracts/tools/aws/ssm-p import type { AwsSsmSendCommandBody } from '@/lib/api/contracts/tools/aws/ssm-send-command' import type { AwsSsmStartAutomationExecutionBody } from '@/lib/api/contracts/tools/aws/ssm-start-automation-execution' import type { AwsSsmStopAutomationExecutionBody } from '@/lib/api/contracts/tools/aws/ssm-stop-automation-execution' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' interface SsmConnectionConfig { region: string @@ -72,6 +73,7 @@ interface SsmConnectionConfig { export function createSsmClient(config: SsmConnectionConfig): SSMClient { return new SSMClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, diff --git a/apps/sim/lib/internal/stagehand/operations.ts b/apps/sim/lib/internal/stagehand/operations.ts index 7cafab86303..ca3bc935b0c 100644 --- a/apps/sim/lib/internal/stagehand/operations.ts +++ b/apps/sim/lib/internal/stagehand/operations.ts @@ -1,7 +1,10 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' -import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' +import { + secureFetchWithValidation, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' import { isSensitiveKey, REDACTED_MARKER } from '@/lib/core/security/redaction' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { @@ -98,11 +101,21 @@ async function getLiveViewUrl(sessionId: string, signal?: AbortSignal): Promise< const browserbaseApiKey = getBrowserbaseApiKey() if (!browserbaseApiKey) return null try { - const response = await fetch(`https://api.browserbase.com/v1/sessions/${sessionId}/debug`, { - method: 'GET', - headers: { 'X-BB-API-Key': browserbaseApiKey }, - signal, - }) + const response = await secureFetchWithValidation( + `https://api.browserbase.com/v1/sessions/${sessionId}/debug`, + { + profile: 'configuredEndpoint', + redirectPolicy: { + mode: 'standard', + sendCredentialsOnCrossOriginRedirect: false, + sensitiveHeaders: ['x-bb-api-key'], + }, + maxResponseBytes: MAX_BROWSERBASE_DEBUG_RESPONSE_BYTES, + method: 'GET', + headers: { 'X-BB-API-Key': browserbaseApiKey }, + signal, + } + ) if (!response.ok) return null const data = await readResponseJsonWithLimit(response, { maxBytes: MAX_BROWSERBASE_DEBUG_RESPONSE_BYTES, diff --git a/apps/sim/lib/internal/sts/client.ts b/apps/sim/lib/internal/sts/client.ts index d5fa70efed9..833659551a5 100644 --- a/apps/sim/lib/internal/sts/client.ts +++ b/apps/sim/lib/internal/sts/client.ts @@ -9,10 +9,12 @@ import { STSClient, type Tag, } from '@aws-sdk/client-sts' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' import type { STSConnectionConfig } from '@/tools/sts/types' export function createSTSClient(config: STSConnectionConfig): STSClient { return new STSClient({ + requestHandler: createOutboundAwsHttpHandler(), region: config.region, credentials: { accessKeyId: config.accessKeyId, @@ -35,6 +37,7 @@ export function createSTSClient(config: STSConnectionConfig): STSClient { */ export function createUnauthenticatedSTSClient(region: string): STSClient { return new STSClient({ + requestHandler: createOutboundAwsHttpHandler(), region, credentials: { accessKeyId: 'anonymous', secretAccessKey: 'anonymous' }, }) diff --git a/apps/sim/lib/internal/stt/execute-tool.test.ts b/apps/sim/lib/internal/stt/execute-tool.test.ts index d451ce8164c..02cccb005d2 100644 --- a/apps/sim/lib/internal/stt/execute-tool.test.ts +++ b/apps/sim/lib/internal/stt/execute-tool.test.ts @@ -22,7 +22,10 @@ const { mockResolveInternalFileUrl: vi.fn(), })) -vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) vi.mock('@/lib/uploads/utils/file-utils', () => ({ extractStorageKey: vi.fn(() => 'storage-key'), isInternalFileUrl: mockIsInternalFileUrl, diff --git a/apps/sim/lib/internal/stt/operations.ts b/apps/sim/lib/internal/stt/operations.ts index 5785fa9beaa..7da729fb4ae 100644 --- a/apps/sim/lib/internal/stt/operations.ts +++ b/apps/sim/lib/internal/stt/operations.ts @@ -5,7 +5,10 @@ import { extractAudioFromVideo, isVideoFile } from '@/lib/audio/extractor' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { + createSsrfGuardedFetchWithDispatcher, + MAX_JSON_API_RESPONSE_BYTES, secureFetchWithPinnedIP, + secureFetchWithValidation, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' @@ -28,6 +31,8 @@ import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' import { assertToolFileAccess } from '@/app/api/files/authorization' import type { TranscriptSegment } from '@/tools/stt/types' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('SttOperations') const ELEVENLABS_STT_MODEL = 'scribe_v2' @@ -495,7 +500,7 @@ async function transcribeWithWhisper( } const endpoint = translate ? 'translations' : 'transcriptions' - const response = await fetch(`https://api.openai.com/v1/audio/${endpoint}`, { + const response = await providerFetch(`https://api.openai.com/v1/audio/${endpoint}`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -565,15 +570,21 @@ async function transcribeWithDeepgram( params.append('diarize', 'true') } - const response = await fetch(`https://api.deepgram.com/v1/listen?${params.toString()}`, { - method: 'POST', - headers: { - Authorization: `Token ${apiKey}`, - 'Content-Type': mimeType || 'audio/mpeg', - }, - body: new Uint8Array(audioBuffer), - signal, - }) + const response = await secureFetchWithValidation( + `https://api.deepgram.com/v1/listen?${params.toString()}`, + { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + method: 'POST', + headers: { + Authorization: `Token ${apiKey}`, + 'Content-Type': mimeType || 'audio/mpeg', + }, + body: new Uint8Array(audioBuffer), + signal, + } + ) if (!response.ok) { const error = (await response.json()) as DeepgramApiResponse @@ -648,7 +659,7 @@ async function transcribeWithElevenLabs( formData.append('timestamps_granularity', 'word') } - const response = await fetch('https://api.elevenlabs.io/v1/speech-to-text', { + const response = await providerFetch('https://api.elevenlabs.io/v1/speech-to-text', { method: 'POST', headers: { 'xi-api-key': apiKey, @@ -708,7 +719,10 @@ async function transcribeWithAssemblyAI( entities?: Record[] summary?: string }> { - const uploadResponse = await fetch('https://api.assemblyai.com/v2/upload', { + const uploadResponse = await secureFetchWithValidation('https://api.assemblyai.com/v2/upload', { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, method: 'POST', headers: { authorization: apiKey, @@ -767,15 +781,21 @@ async function transcribeWithAssemblyAI( transcriptRequest.summary_type = 'bullets' } - const transcriptResponse = await fetch('https://api.assemblyai.com/v2/transcript', { - method: 'POST', - headers: { - authorization: apiKey, - 'content-type': 'application/json', - }, - body: JSON.stringify(transcriptRequest), - signal, - }) + const transcriptResponse = await secureFetchWithValidation( + 'https://api.assemblyai.com/v2/transcript', + { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + method: 'POST', + headers: { + authorization: apiKey, + 'content-type': 'application/json', + }, + body: JSON.stringify(transcriptRequest), + signal, + } + ) if (!transcriptResponse.ok) { const error = (await transcriptResponse.json()) as AssemblyAiApiResponse @@ -791,12 +811,18 @@ async function transcribeWithAssemblyAI( const maxAttempts = Math.ceil(getMaxExecutionTimeout() / pollIntervalMs) while (attempts < maxAttempts) { - const statusResponse = await fetch(`https://api.assemblyai.com/v2/transcript/${id}`, { - headers: { - authorization: apiKey, - }, - signal, - }) + const statusResponse = await secureFetchWithValidation( + `https://api.assemblyai.com/v2/transcript/${id}`, + { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, + headers: { + authorization: apiKey, + }, + signal, + } + ) if (!statusResponse.ok) { const error = (await statusResponse.json()) as AssemblyAiApiResponse @@ -905,9 +931,12 @@ async function transcribeWithGemini( ], } - const response = await fetch( + const response = await secureFetchWithValidation( `https://generativelanguage.googleapis.com/v1beta/models/${modelName}:generateContent?key=${apiKey}`, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: MAX_JSON_API_RESPONSE_BYTES, method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/apps/sim/lib/internal/supabase/operations.test.ts b/apps/sim/lib/internal/supabase/operations.test.ts index dfec9986dfa..1b1da240482 100644 --- a/apps/sim/lib/internal/supabase/operations.test.ts +++ b/apps/sim/lib/internal/supabase/operations.test.ts @@ -3,6 +3,14 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const mocks = vi.hoisted(() => ({ assertToolFileAccess: vi.fn(), downloadServableFileFromStorage: vi.fn(), @@ -56,7 +64,15 @@ describe('executeSupabaseStorageUpload', () => { }) expect(fetch).toHaveBeenCalledWith( 'https://project1234.supabase.co/storage/v1/object/documents/folder/hello.txt', - expect.objectContaining({ method: 'POST' }) + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ apikey: 'service-key' }), + redirectPolicy: { + mode: 'standard', + sendCredentialsOnCrossOriginRedirect: false, + sensitiveHeaders: ['apikey'], + }, + }) ) }) diff --git a/apps/sim/lib/internal/supabase/operations.ts b/apps/sim/lib/internal/supabase/operations.ts index 33856d6ea81..56bb4dabc89 100644 --- a/apps/sim/lib/internal/supabase/operations.ts +++ b/apps/sim/lib/internal/supabase/operations.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { validateSupabaseProjectId } from '@/lib/core/security/input-validation' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { assertKnownSizeWithinLimit, isPayloadSizeLimitError, @@ -136,7 +137,14 @@ export async function executeSupabaseStorageUpload( } if (input.upsert) headers['x-upsert'] = 'true' - const response = await fetch(`${baseUrl}/${encodedBucket}/${encodedPath}`, { + const response = await secureFetchWithValidation(`${baseUrl}/${encodedBucket}/${encodedPath}`, { + profile: 'configuredEndpoint', + redirectPolicy: { + mode: 'standard', + sendCredentialsOnCrossOriginRedirect: false, + sensitiveHeaders: ['apikey'], + }, + maxResponseBytes: MAX_SUPABASE_RESPONSE_BYTES, method: 'POST', headers, body: new Uint8Array(upload.body), diff --git a/apps/sim/lib/internal/supabase/operations/storage-update-bucket.test.ts b/apps/sim/lib/internal/supabase/operations/storage-update-bucket.test.ts index a84af6dbe45..8b36213f299 100644 --- a/apps/sim/lib/internal/supabase/operations/storage-update-bucket.test.ts +++ b/apps/sim/lib/internal/supabase/operations/storage-update-bucket.test.ts @@ -2,6 +2,15 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { executeStorageUpdateBucketOperation } from '@/lib/internal/supabase/operations/storage-update-bucket' const INPUT = { diff --git a/apps/sim/lib/internal/supabase/operations/storage-update-bucket.ts b/apps/sim/lib/internal/supabase/operations/storage-update-bucket.ts index f2f6734d8d4..0728f6bae4e 100644 --- a/apps/sim/lib/internal/supabase/operations/storage-update-bucket.ts +++ b/apps/sim/lib/internal/supabase/operations/storage-update-bucket.ts @@ -1,5 +1,6 @@ import { getErrorMessage } from '@sim/utils/errors' import { filterUndefined } from '@sim/utils/object' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { InternalToolOperationImplementation } from '@/lib/internal/tool-operations/types' import type { SupabaseStorageUpdateBucketParams, @@ -7,6 +8,8 @@ import type { } from '@/tools/supabase/types' import { encodeStorageSegment, supabaseBaseUrl } from '@/tools/supabase/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + export const executeStorageUpdateBucketOperation: InternalToolOperationImplementation< SupabaseStorageUpdateBucketParams > = async ( @@ -43,7 +46,7 @@ export const executeStorageUpdateBucketOperation: InternalToolOperationImplement }) if (Object.keys(payload).length === 0) { - const currentResponse = await fetch(`${baseUrl}/storage/v1/bucket/${bucket}`, { + const currentResponse = await providerFetch(`${baseUrl}/storage/v1/bucket/${bucket}`, { method: 'GET', headers, redirect: 'error', @@ -65,7 +68,7 @@ export const executeStorageUpdateBucketOperation: InternalToolOperationImplement } } - const updateResponse = await fetch(`${baseUrl}/storage/v1/bucket/${bucket}`, { + const updateResponse = await providerFetch(`${baseUrl}/storage/v1/bucket/${bucket}`, { method: 'PUT', headers, body: JSON.stringify(payload), diff --git a/apps/sim/lib/internal/telegram/operations.test.ts b/apps/sim/lib/internal/telegram/operations.test.ts index 1ca7855cac9..5d03be77c98 100644 --- a/apps/sim/lib/internal/telegram/operations.test.ts +++ b/apps/sim/lib/internal/telegram/operations.test.ts @@ -3,6 +3,14 @@ */ import { assert, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { isInternalToolFileResult, type StoredToolFile, diff --git a/apps/sim/lib/internal/telegram/operations.ts b/apps/sim/lib/internal/telegram/operations.ts index b321ac86ee3..ba0135fa2ed 100644 --- a/apps/sim/lib/internal/telegram/operations.ts +++ b/apps/sim/lib/internal/telegram/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { TelegramOperationError } from '@/lib/internal/telegram/errors' import { @@ -13,6 +14,8 @@ import { assertToolFileAccess } from '@/app/api/files/authorization' import type { TelegramSendDocumentResponse } from '@/tools/telegram/types' import { convertMarkdownToHTML } from '@/tools/telegram/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('TelegramSendDocumentOperation') const MAX_TELEGRAM_DOCUMENT_BYTES = 50 * 1024 * 1024 const MAX_TELEGRAM_RESPONSE_BYTES = 2 * 1024 * 1024 @@ -100,7 +103,7 @@ export async function sendTelegramDocument( form.append('parse_mode', 'HTML') } - const response = await fetch( + const response = await providerFetch( `https://api.telegram.org/bot${encodeURIComponent(input.botToken)}/sendDocument`, { method: 'POST', body: form, signal: context.signal } ) diff --git a/apps/sim/lib/internal/textract/operations.ts b/apps/sim/lib/internal/textract/operations.ts index 7a021191c60..2e470df348a 100644 --- a/apps/sim/lib/internal/textract/operations.ts +++ b/apps/sim/lib/internal/textract/operations.ts @@ -21,6 +21,7 @@ import type { textractAnalyzeIdContract, textractParseContract, } from '@/lib/api/contracts/tools/media/document-parse' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance' import { parseS3Uri, resolveDocumentInput } from '@/lib/internal/textract/document-input' import { mapTextractSdkError, textractErrorResponse } from '@/lib/internal/textract/errors' @@ -83,6 +84,7 @@ function createTextractClient(input: { secretAccessKey: string }): TextractClient { return new TextractClient({ + requestHandler: createOutboundAwsHttpHandler(), region: input.region, credentials: { accessKeyId: input.accessKeyId, diff --git a/apps/sim/lib/internal/tiktok/operations.test.ts b/apps/sim/lib/internal/tiktok/operations.test.ts index 7d1afa786be..831068415c3 100644 --- a/apps/sim/lib/internal/tiktok/operations.test.ts +++ b/apps/sim/lib/internal/tiktok/operations.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' const mocks = vi.hoisted(() => ({ diff --git a/apps/sim/lib/internal/tiktok/operations.ts b/apps/sim/lib/internal/tiktok/operations.ts index ae2c255e2bc..ae47858f2ae 100644 --- a/apps/sim/lib/internal/tiktok/operations.ts +++ b/apps/sim/lib/internal/tiktok/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import type { TikTokUploadVideoDraftInput } from '@/lib/internal/tiktok/schema' import { @@ -18,6 +19,8 @@ import { assertToolFileAccess } from '@/app/api/files/authorization' import { tiktokPublishInitApiDataSchema } from '@/tools/tiktok/api-schemas' import { readTikTokApiResponse } from '@/tools/tiktok/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('TikTokUploadVideoDraft') const TIKTOK_VIDEO_MIME_TYPES = new Set(['video/mp4', 'video/quicktime', 'video/webm']) @@ -77,7 +80,7 @@ export async function executeTikTokUploadVideoDraft( if (videoSize === 0) return failureResponse('The video file is empty.', 400) const { chunkSize, totalChunkCount } = computeTikTokChunkPlan(videoSize) - const initResponse = await fetch( + const initResponse = await providerFetch( 'https://open.tiktokapis.com/v2/post/publish/inbox/video/init/', { method: 'POST', diff --git a/apps/sim/lib/internal/tiktok/upload.test.ts b/apps/sim/lib/internal/tiktok/upload.test.ts index d45050ce210..48e9c4e3911 100644 --- a/apps/sim/lib/internal/tiktok/upload.test.ts +++ b/apps/sim/lib/internal/tiktok/upload.test.ts @@ -3,6 +3,14 @@ */ import { Readable } from 'node:stream' import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' const { mockBackoffWithJitter, mockDownloadFileStream, mockHeadObject, mockParseRetryAfter } = diff --git a/apps/sim/lib/internal/tiktok/upload.ts b/apps/sim/lib/internal/tiktok/upload.ts index 7ec6d5687fc..af26c4dae50 100644 --- a/apps/sim/lib/internal/tiktok/upload.ts +++ b/apps/sim/lib/internal/tiktok/upload.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { assertKnownSizeWithinLimit, readResponseTextWithLimit, @@ -10,6 +11,8 @@ import { import type { StorageContext } from '@/lib/uploads/config' import { downloadFileStream, headObject } from '@/lib/uploads/core/storage-service' +const contentFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'contentFetch' }).fetch + const logger = createLogger('TikTokVideoUpload') /** TikTok accepts 5-64 MB chunks and allows the final chunk to absorb the remainder. */ @@ -235,7 +238,7 @@ async function uploadChunk(options: { for (let attempt = 1; attempt <= MAX_UPLOAD_ATTEMPTS; attempt++) { throwIfAborted(options.signal) - const response = await fetch(options.uploadUrl, { + const response = await contentFetch(options.uploadUrl, { method: 'PUT', headers: { 'Content-Type': options.mimeType, diff --git a/apps/sim/lib/internal/tts/client.test.ts b/apps/sim/lib/internal/tts/client.test.ts index d85af2ef383..5d747dc614f 100644 --- a/apps/sim/lib/internal/tts/client.test.ts +++ b/apps/sim/lib/internal/tts/client.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { MAX_TTS_AUDIO_BYTES, MAX_TTS_TEXT_BYTES, diff --git a/apps/sim/lib/internal/tts/client.ts b/apps/sim/lib/internal/tts/client.ts index c88a9b4b23c..26fa07d5e4a 100644 --- a/apps/sim/lib/internal/tts/client.ts +++ b/apps/sim/lib/internal/tts/client.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' import { validateAlphanumericId } from '@/lib/core/security/input-validation' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { assertKnownSizeWithinLimit, readResponseJsonWithLimit, @@ -21,6 +22,10 @@ import type { PlayHtTtsParams, } from '@/tools/tts/types' +const fetchProviderRequest = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}).fetch + const logger = createLogger('TtsClient') export const MAX_TTS_AUDIO_BYTES = 25 * 1024 * 1024 export const MAX_TTS_TEXT_BYTES = 10 * 1024 * 1024 @@ -51,14 +56,14 @@ async function providerFetch( timeoutMs?: number ): Promise { signal?.throwIfAborted() - if (!timeoutMs) return fetch(input, { ...init, signal }) + if (!timeoutMs) return fetchProviderRequest(input, { ...init, signal }) const controller = new AbortController() const timeout = setTimeout(() => controller.abort(new Error('TTS request timed out')), timeoutMs) const abort = () => controller.abort(signal?.reason ?? new Error('Request aborted')) signal?.addEventListener('abort', abort, { once: true }) try { - return await fetch(input, { ...init, signal: controller.signal }) + return await fetchProviderRequest(input, { ...init, signal: controller.signal }) } finally { clearTimeout(timeout) signal?.removeEventListener('abort', abort) diff --git a/apps/sim/lib/internal/uptimerobot/client.ts b/apps/sim/lib/internal/uptimerobot/client.ts index cfa44d61505..76093f3e3e3 100644 --- a/apps/sim/lib/internal/uptimerobot/client.ts +++ b/apps/sim/lib/internal/uptimerobot/client.ts @@ -1,10 +1,15 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + createSsrfGuardedFetchWithDispatcher, + MAX_JSON_API_RESPONSE_BYTES, +} from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' import { UptimeRobotOperationError } from '@/lib/internal/uptimerobot/errors' import { mapPsp, UPTIMEROBOT_API_BASE, type UptimeRobotPsp } from '@/tools/uptimerobot/types' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('UptimeRobotClient') function providerMessage(text: string, status: number): string { @@ -24,7 +29,7 @@ export async function requestUptimeRobotPsp(args: { }): Promise { const { apiKey, method, path, form, signal } = args signal?.throwIfAborted() - const response = await fetch(`${UPTIMEROBOT_API_BASE}${path}`, { + const response = await providerFetch(`${UPTIMEROBOT_API_BASE}${path}`, { method, headers: { Authorization: `Bearer ${apiKey}`, Accept: 'application/json' }, body: form, diff --git a/apps/sim/lib/internal/vanta/client.test.ts b/apps/sim/lib/internal/vanta/client.test.ts index e66394e5c88..c58edb3971a 100644 --- a/apps/sim/lib/internal/vanta/client.test.ts +++ b/apps/sim/lib/internal/vanta/client.test.ts @@ -2,6 +2,14 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { fetchVantaWithAuth } from '@/lib/internal/vanta/client' describe('Vanta provider client', () => { diff --git a/apps/sim/lib/internal/vanta/client.ts b/apps/sim/lib/internal/vanta/client.ts index 7cc9f479d7d..7040ec38601 100644 --- a/apps/sim/lib/internal/vanta/client.ts +++ b/apps/sim/lib/internal/vanta/client.ts @@ -1,9 +1,14 @@ import { LRUCache } from 'lru-cache' -import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + createSsrfGuardedFetchWithDispatcher, + MAX_JSON_API_RESPONSE_BYTES, +} from '@/lib/core/security/input-validation.server' import { consumeOrCancelBody, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { extractVantaError } from '@/lib/internal/vanta/normalizers' import type { VantaRegion } from '@/tools/vanta/types' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + export const VANTA_API_BASE_URLS: Record = { us: 'https://api.vanta.com', gov: 'https://api.vanta-gov.com', @@ -67,7 +72,7 @@ async function exchangeVantaToken( signal, AbortSignal.timeout(VANTA_TOKEN_EXCHANGE_TIMEOUT_MS), ]) - const response = await fetch(`${getVantaBaseUrl(params.region)}/oauth/token`, { + const response = await providerFetch(`${getVantaBaseUrl(params.region)}/oauth/token`, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/lib/internal/vanta/operations.test.ts b/apps/sim/lib/internal/vanta/operations.test.ts index df66cd149cc..44b60b3c326 100644 --- a/apps/sim/lib/internal/vanta/operations.test.ts +++ b/apps/sim/lib/internal/vanta/operations.test.ts @@ -3,6 +3,14 @@ */ import { assert, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + import { isInternalToolFileResult, type StoredToolFile, diff --git a/apps/sim/lib/internal/vanta/operations.ts b/apps/sim/lib/internal/vanta/operations.ts index c649d9b53af..6572cae1f9d 100644 --- a/apps/sim/lib/internal/vanta/operations.ts +++ b/apps/sim/lib/internal/vanta/operations.ts @@ -1,4 +1,7 @@ -import { MAX_JSON_API_RESPONSE_BYTES } from '@/lib/core/security/input-validation.server' +import { + createSsrfGuardedFetchWithDispatcher, + MAX_JSON_API_RESPONSE_BYTES, +} from '@/lib/core/security/input-validation.server' import { isPayloadSizeLimitError, readResponseJsonWithLimit, @@ -48,6 +51,8 @@ import { } from '@/lib/internal/vanta/normalizers' import type { VantaQueryBody } from '@/lib/internal/vanta/schema' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + interface VantaFileOperationContext { requestId: string signal?: AbortSignal @@ -427,7 +432,7 @@ export async function executeVantaQuery( scope, }, (accessToken) => - fetch(apiRequest.url, { + providerFetch(apiRequest.url, { method: apiRequest.method, headers: { Accept: 'application/json', @@ -487,7 +492,7 @@ export async function executeVantaUploadDocumentFile( ) if (input.description) formData.append('description', input.description) if (input.effectiveAtDate) formData.append('effectiveAtDate', input.effectiveAtDate) - return fetch(uploadUrl, { + return providerFetch(uploadUrl, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}` }, body: formData, @@ -529,7 +534,7 @@ export async function executeVantaDownloadDocumentFile( scope: VANTA_READ_SCOPE, }, (accessToken) => - fetch(mediaUrl, { + providerFetch(mediaUrl, { method: 'GET', headers: { Authorization: `Bearer ${accessToken}` }, cache: 'no-store', diff --git a/apps/sim/lib/internal/video/client.test.ts b/apps/sim/lib/internal/video/client.test.ts index 07698228b44..bd1dd00c4ae 100644 --- a/apps/sim/lib/internal/video/client.test.ts +++ b/apps/sim/lib/internal/video/client.test.ts @@ -3,6 +3,18 @@ */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +const { requestProfile } = vi.hoisted(() => ({ requestProfile: vi.fn() })) + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: ({ profile }: { profile: string }) => ({ + fetch: (...args: Parameters) => { + requestProfile(profile, String(args[0])) + return fetch(...args) + }, + }), +})) + vi.mock('@/lib/core/execution-limits', () => ({ getMaxExecutionTimeout: () => 5000 })) import { generateVideo } from '@/lib/internal/video/client' @@ -15,7 +27,10 @@ function jsonResponse(value: unknown, status = 200): Response { } describe('Video provider client', () => { - beforeEach(() => vi.useFakeTimers()) + beforeEach(() => { + vi.useFakeTimers() + requestProfile.mockClear() + }) afterEach(() => { vi.useRealTimers() @@ -46,6 +61,11 @@ describe('Video provider client', () => { await vi.advanceTimersByTimeAsync(5000) const result = await resultPromise + expect(requestProfile.mock.calls).toEqual([ + ['configuredEndpoint', 'https://api.dev.runwayml.com/v1/image_to_video'], + ['configuredEndpoint', 'https://api.dev.runwayml.com/v1/tasks/task-1'], + ['contentFetch', 'https://cdn.example/video.mp4'], + ]) expect(result).toMatchObject({ buffer: Buffer.from('video'), width: 1280, diff --git a/apps/sim/lib/internal/video/client.ts b/apps/sim/lib/internal/video/client.ts index 50a204bf37e..bd834acbc55 100644 --- a/apps/sim/lib/internal/video/client.ts +++ b/apps/sim/lib/internal/video/client.ts @@ -2,6 +2,7 @@ import { createLogger, type Logger } from '@sim/logger' import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { assertKnownSizeWithinLimit, DEFAULT_MAX_ERROR_BODY_BYTES, @@ -14,6 +15,9 @@ import { type FalAICostMetadata, getFalAICostMetadata } from '@/lib/tools/falai- import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' import type { UserFile } from '@/executor/types' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch +const contentFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'contentFetch' }).fetch + const logger = createLogger('VideoProviderClient') const MAX_VIDEO_OUTPUT_BYTES = 250 * 1024 * 1024 const MAX_VIDEO_REFERENCE_IMAGE_BYTES = 25 * 1024 * 1024 @@ -240,7 +244,7 @@ async function generateWithRunway( createPayload.promptImage = `data:${visualReference.type};base64,${refBase64}` } - const createResponse = await fetch('https://api.dev.runwayml.com/v1/image_to_video', { + const createResponse = await providerFetch('https://api.dev.runwayml.com/v1/image_to_video', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -267,7 +271,7 @@ async function generateWithRunway( while (attempts < maxAttempts) { await waitForProvider(context) - const statusResponse = await fetch(`https://api.dev.runwayml.com/v1/tasks/${taskId}`, { + const statusResponse = await providerFetch(`https://api.dev.runwayml.com/v1/tasks/${taskId}`, { headers: { Authorization: `Bearer ${apiKey}`, 'X-Runway-Version': '2024-11-06', @@ -294,7 +298,7 @@ async function generateWithRunway( throw new Error('No video URL in response') } - const videoResponse = await fetch(videoUrl, { signal }) + const videoResponse = await contentFetch(videoUrl, { signal }) if (!videoResponse.ok) { await readVideoErrorText(videoResponse, 'Runway video error response') throw new Error(`Failed to download video: ${videoResponse.status}`) @@ -355,7 +359,7 @@ async function generateWithVeo( }, } - const createResponse = await fetch( + const createResponse = await providerFetch( `https://generativelanguage.googleapis.com/v1beta/models/${modelName}:predictLongRunning`, { method: 'POST', @@ -384,7 +388,7 @@ async function generateWithVeo( while (attempts < maxAttempts) { await waitForProvider(context) - const statusResponse = await fetch( + const statusResponse = await providerFetch( `https://generativelanguage.googleapis.com/v1beta/${operationName}`, { headers: { @@ -419,7 +423,7 @@ async function generateWithVeo( throw new Error('No video URI in response') } - const videoResponse = await fetch(videoUri, { + const videoResponse = await contentFetch(videoUri, { headers: { 'x-goog-api-key': apiKey, }, @@ -482,15 +486,18 @@ async function generateWithLuma( createPayload.concepts = Array.isArray(cameraControl) ? cameraControl : [{ key: cameraControl }] } - const createResponse = await fetch('https://api.lumalabs.ai/dream-machine/v1/generations', { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(createPayload), - signal, - }) + const createResponse = await providerFetch( + 'https://api.lumalabs.ai/dream-machine/v1/generations', + { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(createPayload), + signal, + } + ) if (!createResponse.ok) { const error = await readVideoErrorText(createResponse, 'Luma create error response') @@ -508,7 +515,7 @@ async function generateWithLuma( while (attempts < maxAttempts) { await waitForProvider(context) - const statusResponse = await fetch( + const statusResponse = await providerFetch( `https://api.lumalabs.ai/dream-machine/v1/generations/${generationId}`, { headers: { @@ -537,7 +544,7 @@ async function generateWithLuma( throw new Error('No video URL in response') } - const videoResponse = await fetch(videoUrl, { signal }) + const videoResponse = await contentFetch(videoUrl, { signal }) if (!videoResponse.ok) { await readVideoErrorText(videoResponse, 'Luma video error response') throw new Error(`Failed to download video: ${videoResponse.status}`) @@ -589,7 +596,7 @@ async function generateWithMiniMax( const minimaxModel = model === 'hailuo-02' ? 'MiniMax-Hailuo-02' : 'MiniMax-Hailuo-2.3' - const createResponse = await fetch('https://api.minimax.io/v1/video_generation', { + const createResponse = await providerFetch('https://api.minimax.io/v1/video_generation', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, @@ -637,7 +644,7 @@ async function generateWithMiniMax( while (attempts < maxAttempts) { await waitForProvider(context) - const statusResponse = await fetch( + const statusResponse = await providerFetch( `https://api.minimax.io/v1/query/video_generation?task_id=${taskId}`, { headers: { @@ -676,7 +683,7 @@ async function generateWithMiniMax( throw new Error('No file_id in response') } - const fileResponse = await fetch( + const fileResponse = await providerFetch( `https://api.minimax.io/v1/files/retrieve?file_id=${fileId}`, { headers: { @@ -701,7 +708,7 @@ async function generateWithMiniMax( throw new Error('No download URL in file response') } - const videoResponse = await fetch(videoUrl, { signal }) + const videoResponse = await contentFetch(videoUrl, { signal }) if (!videoResponse.ok) { await readVideoErrorText(videoResponse, 'MiniMax video error response') throw new Error(`Failed to download video from URL: ${videoResponse.status}`) @@ -1060,7 +1067,7 @@ async function generateWithFalAI( requestBody.generate_audio = generateAudio } - const createResponse = await fetch(`https://queue.fal.run/${modelConfig.endpoint}`, { + const createResponse = await providerFetch(`https://queue.fal.run/${modelConfig.endpoint}`, { method: 'POST', headers: { Authorization: `Key ${apiKey}`, @@ -1100,7 +1107,7 @@ async function generateWithFalAI( while (attempts < maxAttempts) { await waitForProvider(context) - const statusResponse = await fetch(statusUrl, { + const statusResponse = await contentFetch(statusUrl, { headers: { Authorization: `Key ${apiKey}`, }, @@ -1125,7 +1132,7 @@ async function generateWithFalAI( logger.info(`[${requestId}] Fal.ai generation completed after ${attempts * 5}s`) - const resultResponse = await fetch( + const resultResponse = await contentFetch( getStringProperty(statusData, 'response_url') || responseUrl, { headers: { @@ -1153,7 +1160,7 @@ async function generateWithFalAI( throw new Error('No video URL in response') } - const videoResponse = await fetch(videoUrl, { signal }) + const videoResponse = await contentFetch(videoUrl, { signal }) if (!videoResponse.ok) { await readVideoErrorText(videoResponse, 'Fal.ai video error response') throw new Error(`Failed to download video: ${videoResponse.status}`) diff --git a/apps/sim/lib/internal/vision/client.test.ts b/apps/sim/lib/internal/vision/client.test.ts index d88a147695b..861bacf0fda 100644 --- a/apps/sim/lib/internal/vision/client.test.ts +++ b/apps/sim/lib/internal/vision/client.test.ts @@ -14,7 +14,11 @@ vi.mock('@google/genai', () => ({ models = { generateContent: mocks.generateContent } }, })) -vi.mock('@/lib/core/security/input-validation.server', () => ({ +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, })) diff --git a/apps/sim/lib/internal/vision/client.ts b/apps/sim/lib/internal/vision/client.ts index e3cdc7703bb..d7a0a826466 100644 --- a/apps/sim/lib/internal/vision/client.ts +++ b/apps/sim/lib/internal/vision/client.ts @@ -1,8 +1,10 @@ import { GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' import { isRecordLike } from '@sim/utils/object' +import { requireDirectOutboundTransport } from '@/lib/core/network/context.server' import type { EgressProfile } from '@/lib/core/security/egress/profiles' import { + createSsrfGuardedFetchWithDispatcher, MAX_JSON_API_RESPONSE_BYTES, secureFetchWithPinnedIP, } from '@/lib/core/security/input-validation.server' @@ -15,6 +17,8 @@ import { VisionOperationError } from '@/lib/internal/vision/errors' import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { convertUsageMetadata, extractTextContent } from '@/providers/google/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('VisionClient') const MAX_PROVIDER_ERROR_BYTES = 64 * 1024 @@ -121,6 +125,7 @@ async function analyzeWithGemini( signal?.throwIfAborted() const base64Payload = await fetchGeminiImage(input, signal) const { mediaType, base64Data } = parseDataImage(base64Payload) + await requireDirectOutboundTransport() const ai = new GoogleGenAI({ apiKey: input.apiKey }) const response = await ai.models.generateContent({ model: input.model, @@ -197,7 +202,7 @@ async function analyzeWithHttpProvider( } signal?.throwIfAborted() - const response = await fetch(apiUrl, { + const response = await providerFetch(apiUrl, { method: 'POST', headers, body: JSON.stringify(isClaude ? anthropicRequest(input) : openAiRequest(input)), diff --git a/apps/sim/lib/internal/whatsapp/operations.test.ts b/apps/sim/lib/internal/whatsapp/operations.test.ts index 6d9fe5abda7..74a89369060 100644 --- a/apps/sim/lib/internal/whatsapp/operations.test.ts +++ b/apps/sim/lib/internal/whatsapp/operations.test.ts @@ -16,7 +16,11 @@ vi.mock('@/lib/internal/whatsapp/client', () => ({ readWhatsAppGraphResponse: mocks.readGraph, })) -vi.mock('@/lib/core/security/input-validation.server', () => ({ +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), validateUrlWithDNS: mocks.validateUrl, secureFetchWithPinnedIP: mocks.secureFetch, })) diff --git a/apps/sim/lib/internal/whatsapp/operations.ts b/apps/sim/lib/internal/whatsapp/operations.ts index 2ddfef539de..3ee77dd6854 100644 --- a/apps/sim/lib/internal/whatsapp/operations.ts +++ b/apps/sim/lib/internal/whatsapp/operations.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { + createSsrfGuardedFetchWithDispatcher, secureFetchWithPinnedIP, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' @@ -35,6 +36,8 @@ import { WHATSAPP_MEDIA_MAX_BYTES, } from '@/tools/whatsapp/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('WhatsAppOperations') const DOWNLOAD_USER_AGENT = 'SimWhatsAppMedia/1.0' @@ -149,7 +152,7 @@ export async function executeWhatsAppSendMedia( caption: input.caption ?? undefined, filename, }) - const response = await fetch(buildMessagesUrl(input.phoneNumberId), { + const response = await providerFetch(buildMessagesUrl(input.phoneNumberId), { method: 'POST', headers: buildAuthHeaders(input.accessToken), body: JSON.stringify(messageBody), @@ -183,10 +186,13 @@ export async function executeWhatsAppGetMedia( context.signal?.throwIfAborted() const authorization = `Bearer ${input.accessToken.trim()}` try { - const metadataResponse = await fetch(buildMediaUrl(input.mediaId, input.phoneNumberId), { - headers: { Authorization: authorization }, - signal: context.signal, - }) + const metadataResponse = await providerFetch( + buildMediaUrl(input.mediaId, input.phoneNumberId), + { + headers: { Authorization: authorization }, + signal: context.signal, + } + ) const metadataBody = await readWhatsAppGraphResponse( metadataResponse, `WhatsApp media ${input.mediaId} metadata`, diff --git a/apps/sim/lib/internal/whatsapp/upload.test.ts b/apps/sim/lib/internal/whatsapp/upload.test.ts index ed23313574b..469675bbc4a 100644 --- a/apps/sim/lib/internal/whatsapp/upload.test.ts +++ b/apps/sim/lib/internal/whatsapp/upload.test.ts @@ -3,6 +3,13 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + }), +})) + const mocks = vi.hoisted(() => ({ processFile: vi.fn(), assertAccess: vi.fn(), diff --git a/apps/sim/lib/internal/whatsapp/upload.ts b/apps/sim/lib/internal/whatsapp/upload.ts index 8aa0d6452bb..ea389c3aeee 100644 --- a/apps/sim/lib/internal/whatsapp/upload.ts +++ b/apps/sim/lib/internal/whatsapp/upload.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readWhatsAppGraphResponse } from '@/lib/internal/whatsapp/client' import type { RawFileInput } from '@/lib/uploads/utils/file-utils' import { processSingleFileToUserFile } from '@/lib/uploads/utils/file-utils' @@ -11,6 +12,8 @@ import { whatsappMediaLimitFor, } from '@/tools/whatsapp/utils' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('WhatsAppMediaUpload') export interface UploadedWhatsAppMedia { @@ -93,7 +96,7 @@ export async function uploadWhatsAppMedia({ userFile.name ) - const response = await fetch(buildMediaUploadUrl(phoneNumberId), { + const response = await providerFetch(buildMediaUploadUrl(phoneNumberId), { method: 'POST', headers: { Authorization: `Bearer ${accessToken.trim()}` }, body: formData, diff --git a/apps/sim/lib/internal/wordpress/operations.ts b/apps/sim/lib/internal/wordpress/operations.ts index 33d5f1318f2..e7f35034f04 100644 --- a/apps/sim/lib/internal/wordpress/operations.ts +++ b/apps/sim/lib/internal/wordpress/operations.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -16,6 +17,8 @@ import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils. import { assertToolFileAccess } from '@/app/api/files/authorization' import type { WordPressUploadMediaResponse } from '@/tools/wordpress/types' +const providerFetch = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }).fetch + const logger = createLogger('WordPressOperations') const WORDPRESS_COM_API_BASE = 'https://public-api.wordpress.com/wp/v2/sites' const MAX_WORDPRESS_RESPONSE_BYTES = 2 * 1024 * 1024 @@ -96,7 +99,7 @@ export async function uploadWordPressMedia( if (input.altText) formData.append('alt_text', input.altText) if (input.description) formData.append('description', input.description) - const response = await fetch(`${WORDPRESS_COM_API_BASE}/${input.siteId}/media`, { + const response = await providerFetch(`${WORDPRESS_COM_API_BASE}/${input.siteId}/media`, { method: 'POST', headers: { Authorization: `Bearer ${input.accessToken}` }, body: formData, diff --git a/apps/sim/lib/internal/workday/client.test.ts b/apps/sim/lib/internal/workday/client.test.ts index 40b5f15d5e5..b9c0369cdd4 100644 --- a/apps/sim/lib/internal/workday/client.test.ts +++ b/apps/sim/lib/internal/workday/client.test.ts @@ -2,6 +2,12 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { buildServiceUrl, diff --git a/apps/sim/lib/internal/workday/client.ts b/apps/sim/lib/internal/workday/client.ts index faa15c6499e..55bf56c939f 100644 --- a/apps/sim/lib/internal/workday/client.ts +++ b/apps/sim/lib/internal/workday/client.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { validateWorkdayTenantUrl } from '@/lib/core/security/input-validation' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' const logger = createLogger('WorkdaySoapClient') @@ -561,7 +562,10 @@ async function callOperation( signal?.throwIfAborted() const envelope = buildEnvelope(operation, args, username, password) - const response = await fetch(endpoint, { + const response = await secureFetchWithValidation(endpoint, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + maxResponseBytes: WORKDAY_SOAP_RESPONSE_MAX_BYTES, method: 'POST', headers: { 'Content-Type': 'text/xml; charset=utf-8', diff --git a/apps/sim/lib/knowledge/access/confluence-site.test.ts b/apps/sim/lib/knowledge/access/confluence-site.test.ts index 543c63a168f..36f57365a4d 100644 --- a/apps/sim/lib/knowledge/access/confluence-site.test.ts +++ b/apps/sim/lib/knowledge/access/confluence-site.test.ts @@ -1,5 +1,11 @@ /** @vitest-environment node */ -import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { + dbChainMockFns, + inputValidationMock, + queueTableRows, + resetDbChainMock, + schemaMock, +} from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { CONFLUENCE_READ_ATTEMPT_TIMEOUT_MS, @@ -11,6 +17,11 @@ import { } from '@/lib/knowledge/access/confluence-site' import { MAX_KNOWLEDGE_ACCESS_CANDIDATES } from '@/lib/knowledge/access/types' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const mocks = vi.hoisted(() => ({ token: vi.fn(), decrypt: vi.fn(), fetch: vi.fn() })) vi.mock('@/lib/credentials/managed-oauth', () => ({ resolveManagedOAuthToken: mocks.token })) vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: mocks.decrypt })) @@ -82,8 +93,9 @@ describe('current Confluence site access', () => { 'https://api.atlassian.com/ex/confluence/cloud-1/wiki/rest/api/user/current', expect.objectContaining({ headers: { Authorization: 'Bearer alice-oauth-token', Accept: 'application/json' }, - cache: 'no-store', - redirect: 'error', + profile: 'configuredEndpoint', + maxRedirects: 0, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, signal: expect.any(AbortSignal), }) ) diff --git a/apps/sim/lib/knowledge/access/confluence-site.ts b/apps/sim/lib/knowledge/access/confluence-site.ts index afd80d523d5..cc467e813ff 100644 --- a/apps/sim/lib/knowledge/access/confluence-site.ts +++ b/apps/sim/lib/knowledge/access/confluence-site.ts @@ -7,6 +7,7 @@ import { normalizeAtlassianSiteUrl } from '@/lib/atlassian/discovery' import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { decryptSecret } from '@/lib/core/security/encryption' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { resolveManagedOAuthToken } from '@/lib/credentials/managed-oauth' import { confluenceSubjectToken } from '@/lib/knowledge/access/confluence-permissions' @@ -95,12 +96,13 @@ async function verifySite( signal: AbortSignal ): Promise { signal.throwIfAborted() - const response = await fetch( + const response = await secureFetchWithValidation( `https://api.atlassian.com/ex/confluence/${binding.cloudId}/wiki/rest/api/user/current`, { + profile: 'configuredEndpoint', + maxRedirects: 0, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, - redirect: 'error', - cache: 'no-store', signal, } ) diff --git a/apps/sim/lib/knowledge/access/github-installation.test.ts b/apps/sim/lib/knowledge/access/github-installation.test.ts index 1e14159bfc8..2f815877994 100644 --- a/apps/sim/lib/knowledge/access/github-installation.test.ts +++ b/apps/sim/lib/knowledge/access/github-installation.test.ts @@ -1,5 +1,11 @@ /** @vitest-environment node */ -import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { + dbChainMockFns, + inputValidationMock, + queueTableRows, + resetDbChainMock, + schemaMock, +} from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { GITHUB_READ_CONCURRENCY, @@ -10,6 +16,11 @@ import { } from '@/lib/knowledge/access/github-installation' import { MAX_KNOWLEDGE_ACCESS_CANDIDATES } from '@/lib/knowledge/access/types' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const mocks = vi.hoisted(() => ({ token: vi.fn(), installation: vi.fn(), @@ -94,8 +105,9 @@ describe('live GitHub installation reader access', () => { for (const [, init] of mocks.fetch.mock.calls) expect(init).toMatchObject({ headers: { Authorization: 'Bearer ghu_alice' }, - cache: 'no-store', - redirect: 'error', + profile: 'configuredEndpoint', + maxRedirects: 0, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, }) }) diff --git a/apps/sim/lib/knowledge/access/github-installation.ts b/apps/sim/lib/knowledge/access/github-installation.ts index 0e70615154c..d513bf77677 100644 --- a/apps/sim/lib/knowledge/access/github-installation.ts +++ b/apps/sim/lib/knowledge/access/github-installation.ts @@ -12,6 +12,7 @@ import { and, asc, eq, inArray, isNull, sql } from 'drizzle-orm' import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { decryptSecret } from '@/lib/core/security/encryption' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { resolveManagedOAuthToken } from '@/lib/credentials/managed-oauth' import { @@ -67,14 +68,15 @@ function positiveId(value: unknown): string | null { async function readGitHubJson(path: string, accessToken: string, signal: AbortSignal) { signal.throwIfAborted() - const response = await fetch(`https://api.github.com${path}`, { + const response = await secureFetchWithValidation(`https://api.github.com${path}`, { + profile: 'configuredEndpoint', + maxRedirects: 0, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28', }, - redirect: 'error', - cache: 'no-store', signal, }) if (!response.ok) { diff --git a/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts b/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts index e2b2cba69f3..125907a75e5 100644 --- a/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts +++ b/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts @@ -8,6 +8,7 @@ import { type WorkspaceUseCaseAuditEntry, } from '@/lib/core/application' import { authorizeOrganizationOperation } from '@/lib/core/application/organization-authorization' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { OrchestrationError, type OrchestrationRequestContext, @@ -230,11 +231,13 @@ export function defineAuthorizedKnowledgeUseCase< context: resolved.context, request, } - const result = await definition.execute(executionContext) - const resultContext = { ...executionContext, result } - recordOrganizationAudit(resultContext, resolved.context.organizationId) - await definition.afterSuccess?.(resultContext) - return result + return runWithOutboundOrganization(resolved.context.organizationId, async () => { + const result = await definition.execute(executionContext) + const resultContext = { ...executionContext, result } + recordOrganizationAudit(resultContext, resolved.context.organizationId) + await definition.afterSuccess?.(resultContext) + return result + }) }, } } diff --git a/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts b/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts index 8dd68d764c0..091ef0c3598 100644 --- a/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts +++ b/apps/sim/lib/knowledge/connectors/external-group-sync.test.ts @@ -5,6 +5,19 @@ import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@s import { beforeEach, describe, expect, it, vi } from 'vitest' import type { ConnectorDirectory } from '@/connectors/types' +const outbound = vi.hoisted(() => ({ enabled: false, workspace: vi.fn() })) +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: () => outbound.enabled, + resolveOutboundRoute: async (organizationId: string | null | undefined) => ({ organizationId }), +})) +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadActiveWorkspaceApplicationContext: outbound.workspace, +})) +beforeEach(() => { + outbound.enabled = false + outbound.workspace.mockReset() +}) + const { mockResolveTokenUserId, mockResolveToken, mockOpenDirectory, mockAvailability } = vi.hoisted(() => ({ mockResolveTokenUserId: vi.fn(), @@ -33,6 +46,10 @@ vi.mock('@/connectors/registry.server', () => ({ }, })) +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' import { refreshConnectorDirectory, refreshMirroredDirectory, @@ -193,10 +210,23 @@ describe('refreshConnectorDirectory', () => { * knowledge base owner, who is routinely a different member. */ it('resolves the token as the credential owner for an OAuth credential', async () => { + outbound.enabled = true + outbound.workspace.mockResolvedValue({ workspaceOrganizationId: 'current-org' }) queueTableRows(schemaMock.knowledgeConnector, [connectorRow()]) mockResolveTokenUserId.mockResolvedValue('credential-owner') + let tokenRoute: unknown + mockResolveToken.mockImplementationOnce(async () => { + tokenRoute = await resolveCurrentOutboundRoute() + return { accessToken: 'token', cloudId: 'cloud-1' } + }) - await expect(refreshConnectorDirectory('connector-1', 'req-1')).resolves.toBe('skipped') + await expect( + runWithOutboundOrganization('queued-org', () => + refreshConnectorDirectory('connector-1', 'req-1') + ) + ).resolves.toBe('skipped') + expect(tokenRoute).toEqual({ organizationId: 'current-org' }) + expect(outbound.workspace).toHaveBeenCalledExactlyOnceWith('ws-1') expect(mockResolveToken).toHaveBeenCalledWith( expect.objectContaining({ userId: 'credential-owner', accessMode: 'admin' }) ) @@ -220,6 +250,17 @@ describe('refreshConnectorDirectory', () => { { adminEmail: 'admin@corp.com' }, { cloudId: 'cloud-1' } ) + expect(outbound.workspace).not.toHaveBeenCalled() + }) + + it('skips a missing connector before resolving credentials or using inherited scope', async () => { + outbound.enabled = true + await expect( + runWithOutboundOrganization('queued-org', () => refreshConnectorDirectory('missing', 'req-1')) + ).resolves.toBe('skipped') + expect(outbound.workspace).not.toHaveBeenCalled() + expect(mockResolveToken).not.toHaveBeenCalled() + expect(mockOpenDirectory).not.toHaveBeenCalled() }) it('reports a connector whose credential no longer resolves rather than failing', async () => { diff --git a/apps/sim/lib/knowledge/connectors/external-group-sync.ts b/apps/sim/lib/knowledge/connectors/external-group-sync.ts index 230a5f92280..221c6802483 100644 --- a/apps/sim/lib/knowledge/connectors/external-group-sync.ts +++ b/apps/sim/lib/knowledge/connectors/external-group-sync.ts @@ -11,6 +11,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { chunkArray } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' import { and, eq, gt, inArray, isNull, lt, notInArray, or, sql } from 'drizzle-orm' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { resourceScopeColumns, resourceScopeFields, @@ -434,69 +435,77 @@ export async function refreshConnectorDirectory( return 'skipped' } - if ( - !( - await resolveKnowledgeAccessAvailability( - resourceScopeFields(resourceScopeFromOwner(connector)) - ) - ).sourceMirrored - ) { - return 'skipped' - } + return withResourceOutboundScope( + connector, + async (): Promise => { + if ( + !( + await resolveKnowledgeAccessAvailability( + resourceScopeFields(resourceScopeFromOwner(connector)) + ) + ).sourceMirrored + ) { + return 'skipped' + } - const connectorConfig = CONNECTOR_REGISTRY[connector.connectorType] - if (!connectorConfig?.openDirectory) return 'skipped' + const connectorConfig = CONNECTOR_REGISTRY[connector.connectorType] + if (!connectorConfig?.openDirectory) return 'skipped' - const credentialUserId = await resolveConnectorTokenUserId({ - credentialId: connector.credentialId, - ...resourceScopeFields(resourceScopeFromOwner(connector)), - fallbackUserId: connector.knowledgeBaseOwnerId, - }) - if (!credentialUserId) return 'unusable' + const credentialUserId = await resolveConnectorTokenUserId({ + credentialId: connector.credentialId, + ...resourceScopeFields(resourceScopeFromOwner(connector)), + fallbackUserId: connector.knowledgeBaseOwnerId, + }) + if (!credentialUserId) return 'unusable' - const sourceConfig = connector.sourceConfig as Record - const token = await resolveConnectorAccessToken({ - auth: connectorConfig.auth, - accessMode: 'admin', - connector, - userId: credentialUserId, - requestId, - sourceConfig, - }) - if (!token) return 'unusable' + const sourceConfig = connector.sourceConfig as Record + const token = await resolveConnectorAccessToken({ + auth: connectorConfig.auth, + accessMode: 'admin', + connector, + userId: credentialUserId, + requestId, + sourceConfig, + }) + if (!token) return 'unusable' - const recordError = async (lastSyncError: string | null) => { - await db - .update(knowledgeConnector) - .set({ lastSyncError, updatedAt: new Date() }) - .where( - and( - eq(knowledgeConnector.id, connector.id), - eq(knowledgeConnector.updatedAt, connector.updatedAt), - isNull(knowledgeConnector.syncLockToken), - isNull(knowledgeConnector.memberSyncLockToken), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - } - try { - const syncContext = syncContextForToken(token) - await connectorConfig.permissionConfig?.populateSyncContext(connector.id, syncContext) - const outcome = await refreshMirroredDirectory({ - ...resourceScopeFields(resourceScopeFromOwner(connector)), - connectorConfig, - sourceConfig, - syncContext, - accessToken: token.accessToken, - force: connector.lastSyncError?.startsWith(DIRECTORY_ERROR_PREFIX), - }) - if (outcome === 'refreshed' && connector.lastSyncError?.startsWith(DIRECTORY_ERROR_PREFIX)) { - await recordError(null) + const recordError = async (lastSyncError: string | null) => { + await db + .update(knowledgeConnector) + .set({ lastSyncError, updatedAt: new Date() }) + .where( + and( + eq(knowledgeConnector.id, connector.id), + eq(knowledgeConnector.updatedAt, connector.updatedAt), + isNull(knowledgeConnector.syncLockToken), + isNull(knowledgeConnector.memberSyncLockToken), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + } + try { + const syncContext = syncContextForToken(token) + await connectorConfig.permissionConfig?.populateSyncContext(connector.id, syncContext) + const outcome = await refreshMirroredDirectory({ + ...resourceScopeFields(resourceScopeFromOwner(connector)), + connectorConfig, + sourceConfig, + syncContext, + accessToken: token.accessToken, + force: connector.lastSyncError?.startsWith(DIRECTORY_ERROR_PREFIX), + }) + if ( + outcome === 'refreshed' && + connector.lastSyncError?.startsWith(DIRECTORY_ERROR_PREFIX) + ) { + await recordError(null) + } + return outcome + } catch (error) { + await recordError(getErrorMessage(error)) + throw error + } } - return outcome - } catch (error) { - await recordError(getErrorMessage(error)) - throw error - } + ) } diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.integration.test.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.integration.test.ts index 140e5c4e7d3..e0bb1e207d4 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.integration.test.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.integration.test.ts @@ -5,6 +5,15 @@ import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@s import { beforeEach, describe, expect, it, vi } from 'vitest' import type { ExternalDocument } from '@/connectors/types' +const outbound = vi.hoisted(() => ({ enabled: false, workspace: vi.fn() })) +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: () => outbound.enabled, + resolveOutboundRoute: async (organizationId: string | null | undefined) => ({ organizationId }), +})) +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadActiveWorkspaceApplicationContext: outbound.workspace, +})) + const mocks = vi.hoisted(() => ({ list: vi.fn(), get: vi.fn(), @@ -101,6 +110,10 @@ vi.mock('@/connectors/registry.server', () => ({ }, })) +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' import { CredentialGroupCredentialCursorNotFoundError, loadScopedAccountsCredentialListContext, @@ -304,14 +317,26 @@ describe('member engine with a dedicated content credential', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() + outbound.enabled = false + outbound.workspace.mockReset() dbChainMockFns.execute.mockImplementation(async () => [{ startedAt: new Date().toISOString() }]) }) it.each([undefined, 'organization'])( 'loads the account container within the canonical owner %s', async (organizationId) => { - const result = await arrange({ organizationId, contentFresh: true, noDueMembers: true })() + outbound.enabled = true + outbound.workspace.mockResolvedValue({ workspaceOrganizationId: 'current-org' }) + const run = arrange({ organizationId }) + let tokenRoute: unknown + mocks.token.mockImplementationOnce(async () => { + tokenRoute = await resolveCurrentOutboundRoute() + return { accessToken: 'service-token', cloudId: 'site' } + }) + const result = await runWithOutboundOrganization('queued-org', run) expect(result.error).toBeUndefined() + expect(tokenRoute).toEqual({ organizationId: organizationId ?? 'current-org' }) + expect(outbound.workspace).toHaveBeenCalledTimes(organizationId ? 0 : 1) expect(loadScopedAccountsCredentialListContext).toHaveBeenCalledWith( organizationId ? { kind: 'organization', organizationId } @@ -321,6 +346,16 @@ describe('member engine with a dedicated content credential', () => { } ) + it('refuses an archived workspace before claiming a member sync or contacting its provider', async () => { + outbound.enabled = true + outbound.workspace.mockResolvedValue(null) + const run = arrange() + await expect(run()).rejects.toMatchObject({ code: 'MISSING_SCOPE' }) + expect(dbChainMockFns.set).not.toHaveBeenCalled() + expect(mocks.token).not.toHaveBeenCalled() + expect(mocks.list).not.toHaveBeenCalled() + }) + it('invalidates authorization freshness and cursors before reusing a changed provider identity', async () => { const run = arrange({ changedIdentity: true, contentFresh: true, noDueMembers: true }) const result = await run() diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 186e56e8d72..3357161f6d8 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -21,6 +21,7 @@ import { assertBillingAttributionSnapshot, type BillingAttributionSnapshot, } from '@/lib/billing/core/billing-attribution' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { @@ -1713,544 +1714,547 @@ export async function executeMemberSync( userId: kbRow.userId, } - const runId = generateId() - const connector = await acquireMemberSyncLock(connectorId, runId, options.dispatchToken) - if (!connector) { - const [current] = await db - .select({ - status: knowledgeConnector.status, - memberSyncStatus: knowledgeConnector.memberSyncStatus, - memberSyncLockToken: knowledgeConnector.memberSyncLockToken, - syncLockToken: knowledgeConnector.syncLockToken, - }) - .from(knowledgeConnector) - .where(eq(knowledgeConnector.id, connectorId)) - .limit(1) - if ( - current?.memberSyncStatus === 'disabled' || - current?.syncLockToken || - (current && !MEMBER_LOCKABLE_CONNECTOR_STATUSES.some((status) => status === current.status)) - ) { - logger.info('Connector is not accepting member syncs, skipping', { - connectorId, - status: current.status, - }) - return skipped(result, 'connector_not_syncable') - } - if (options.dispatchToken && current?.memberSyncLockToken !== options.dispatchToken) { - logger.info('Member sync superseded by a newer dispatch, skipping', { connectorId }) - return skipped(result, 'dispatch_superseded') - } - logger.info('Member sync already in progress, skipping', { connectorId }) - return skipped(result, 'sync_in_progress') - } - - const runStartedAt = new Date() - const run: MemberSyncRun = { - connectorId, - knowledgeBaseId: connector.knowledgeBaseId, - ...resourceScopeFields(resourceScopeFromOwner(kbRow)), - runId, - runStartedAt, - deadlineAt: runStartedAt.getTime() + MEMBER_SYNC_SOFT_BUDGET_SECONDS * 1000, - result, - lease: createMemberSyncLease(connectorId, runId), - } - await insertMemberSyncLog(runId, connectorId, runStartedAt) - - try { - /** - * Where the feature is off — flag, plan, or a flag read that could not - * reach its source — nothing changes: readers already see no member-scoped - * document, and the run waits for the next schedule to look again. - */ - if (!(await isKnowledgeMemberAccessAvailable(run))) { - await deferMemberSync(run, connector.syncIntervalMinutes) - return { - ...skipped(result, 'connector_not_syncable'), - error: 'Per-member access is not available for this workspace', + return withResourceOutboundScope(kbOwner, async (): Promise => { + const runId = generateId() + const connector = await acquireMemberSyncLock(connectorId, runId, options.dispatchToken) + if (!connector) { + const [current] = await db + .select({ + status: knowledgeConnector.status, + memberSyncStatus: knowledgeConnector.memberSyncStatus, + memberSyncLockToken: knowledgeConnector.memberSyncLockToken, + syncLockToken: knowledgeConnector.syncLockToken, + }) + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connectorId)) + .limit(1) + if ( + current?.memberSyncStatus === 'disabled' || + current?.syncLockToken || + (current && !MEMBER_LOCKABLE_CONNECTOR_STATUSES.some((status) => status === current.status)) + ) { + logger.info('Connector is not accepting member syncs, skipping', { + connectorId, + status: current.status, + }) + return skipped(result, 'connector_not_syncable') } - } - if (!connector.credentialGroupId || !connector.credentialGroupOptionId) { - await disableMemberSync(run, 'Connector is no longer attached to a Credential Group option') - return { - ...skipped(result, 'connector_not_syncable'), - error: 'Connector is no longer attached to a Credential Group option', + if (options.dispatchToken && current?.memberSyncLockToken !== options.dispatchToken) { + logger.info('Member sync superseded by a newer dispatch, skipping', { connectorId }) + return skipped(result, 'dispatch_superseded') } + logger.info('Member sync already in progress, skipping', { connectorId }) + return skipped(result, 'sync_in_progress') } - if (!connectorConfig.permissionScopedListing || connectorConfig.auth.mode !== 'oauth') { - throw new Error(`Connector ${connectorConfig.id} cannot sync per member`) - } - if (connector.credentialId && !connectorConfig.supportsSeparateContentCredential) { - throw new Error(`${connectorConfig.name} does not support a separate content credential`) - } - const binding = { - credentialGroupId: connector.credentialGroupId, - credentialGroupOptionId: connector.credentialGroupOptionId, - } - const sourceConfig = connector.sourceConfig as Record - if (connector.accessRewritePending && !(await finishPendingAccessRewrite(run))) { - /** The rewrite is not done, so nothing is listed yet; the next run picks it up at once. */ - result.membersRemaining = true - const landed = await completeMemberSync(run, connector.syncIntervalMinutes) - if (!landed) return skipped(result, 'sync_superseded') - logger.info('Member sync spent its budget hiding documents after a mode switch', { - connectorId, - runId, - }) - return result + const runStartedAt = new Date() + const run: MemberSyncRun = { + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + ...resourceScopeFields(resourceScopeFromOwner(kbRow)), + runId, + runStartedAt, + deadlineAt: runStartedAt.getTime() + MEMBER_SYNC_SOFT_BUDGET_SECONDS * 1000, + result, + lease: createMemberSyncLease(connectorId, runId), } + await insertMemberSyncLog(runId, connectorId, runStartedAt) - const contentDue = - Boolean(connector.listingCheckpoint) || - options.forceContentRefresh || - connector.accessRewritePending || - !connector.lastSyncAt || - connector.syncIntervalMinutes <= 0 || - runStartedAt.getTime() - connector.lastSyncAt.getTime() >= - connector.syncIntervalMinutes * 60_000 - if (connector.credentialId && options.forceContentRefresh) { - /** An interrupted explicit crawl stays due when its continuation no longer carries the force flag. */ - await withMemberLease(run, (tx) => - tx - .update(knowledgeConnector) - .set({ lastSyncAt: null, updatedAt: new Date() }) - .where(stillHoldsMemberSyncLock(connectorId, runId)) - ) - } - const serviceContent = connector.credentialId - ? contentDue - ? await syncDedicatedMemberContent({ - run, - connector, - connectorConfig, - sourceConfig, - kbOwner, - billingAttribution, - }) - : { complete: true } - : undefined - /** - * Anyone who joined the workspace since the last run is invited now, so - * membership grows on its own; the invitation is the only thing they need. - */ - const invited = run.workspaceId - ? await inviteWorkspaceMembersToCredentialGroup({ - workspaceId: run.workspaceId, - credentialGroupId: connector.credentialGroupId, - beforeBatch: run.lease.beatIfDue, - deadlineAt: run.deadlineAt, - }).catch((error) => { - logger.warn('Failed to invite new workspace members during a member run', { - connectorId, - error: getErrorMessage(error), + try { + /** + * Where the feature is off — flag, plan, or a flag read that could not + * reach its source — nothing changes: readers already see no member-scoped + * document, and the run waits for the next schedule to look again. + */ + if (!(await isKnowledgeMemberAccessAvailable(run))) { + await deferMemberSync(run, connector.syncIntervalMinutes) + return { + ...skipped(result, 'connector_not_syncable'), + error: 'Per-member access is not available for this workspace', + } + } + if (!connector.credentialGroupId || !connector.credentialGroupOptionId) { + await disableMemberSync(run, 'Connector is no longer attached to a Credential Group option') + return { + ...skipped(result, 'connector_not_syncable'), + error: 'Connector is no longer attached to a Credential Group option', + } + } + if (!connectorConfig.permissionScopedListing || connectorConfig.auth.mode !== 'oauth') { + throw new Error(`Connector ${connectorConfig.id} cannot sync per member`) + } + if (connector.credentialId && !connectorConfig.supportsSeparateContentCredential) { + throw new Error(`${connectorConfig.name} does not support a separate content credential`) + } + const binding = { + credentialGroupId: connector.credentialGroupId, + credentialGroupOptionId: connector.credentialGroupOptionId, + } + const sourceConfig = connector.sourceConfig as Record + + if (connector.accessRewritePending && !(await finishPendingAccessRewrite(run))) { + /** The rewrite is not done, so nothing is listed yet; the next run picks it up at once. */ + result.membersRemaining = true + const landed = await completeMemberSync(run, connector.syncIntervalMinutes) + if (!landed) return skipped(result, 'sync_superseded') + logger.info('Member sync spent its budget hiding documents after a mode switch', { + connectorId, + runId, + }) + return result + } + + const contentDue = + Boolean(connector.listingCheckpoint) || + options.forceContentRefresh || + connector.accessRewritePending || + !connector.lastSyncAt || + connector.syncIntervalMinutes <= 0 || + runStartedAt.getTime() - connector.lastSyncAt.getTime() >= + connector.syncIntervalMinutes * 60_000 + if (connector.credentialId && options.forceContentRefresh) { + /** An interrupted explicit crawl stays due when its continuation no longer carries the force flag. */ + await withMemberLease(run, (tx) => + tx + .update(knowledgeConnector) + .set({ lastSyncAt: null, updatedAt: new Date() }) + .where(stillHoldsMemberSyncLock(connectorId, runId)) + ) + } + const serviceContent = connector.credentialId + ? contentDue + ? await syncDedicatedMemberContent({ + run, + connector, + connectorConfig, + sourceConfig, + kbOwner, + billingAttribution, + }) + : { complete: true } + : undefined + /** + * Anyone who joined the workspace since the last run is invited now, so + * membership grows on its own; the invitation is the only thing they need. + */ + const invited = run.workspaceId + ? await inviteWorkspaceMembersToCredentialGroup({ + workspaceId: run.workspaceId, + credentialGroupId: connector.credentialGroupId, + beforeBatch: run.lease.beatIfDue, + deadlineAt: run.deadlineAt, + }).catch((error) => { + logger.warn('Failed to invite new workspace members during a member run', { + connectorId, + error: getErrorMessage(error), + }) + return null }) - return null + : null + if (invited && invited.invited > 0) { + logger.info('Invited new workspace members to the connector credential group', { + connectorId, + ...invited, }) - : null - if (invited && invited.invited > 0) { - logger.info('Invited new workspace members to the connector credential group', { - connectorId, - ...invited, - }) - } - if ( - !(await reconcileMembership( - run, - binding, - connector.directoryCheckpoint, - Boolean(options.forceContentRefresh) - )) - ) { - result.membersRemaining = true - if (!(await completeMemberSync(run, connector.syncIntervalMinutes))) - return skipped(result, 'sync_superseded') - return result - } + } + if ( + !(await reconcileMembership( + run, + binding, + connector.directoryCheckpoint, + Boolean(options.forceContentRefresh) + )) + ) { + result.membersRemaining = true + if (!(await completeMemberSync(run, connector.syncIntervalMinutes))) + return skipped(result, 'sync_superseded') + return result + } - const credentialIdByMemberId = new Map() - const tokens = createMemberTokenCache({ - run, - connectorConfig, - credentialIdByMemberId, - sourceConfig, - }) + const credentialIdByMemberId = new Map() + const tokens = createMemberTokenCache({ + run, + connectorConfig, + credentialIdByMemberId, + sourceConfig, + }) - while (Date.now() < run.deadlineAt) { - const member = await claimNextMember(run) - if (!member) break - result.membersClaimed += 1 - credentialIdByMemberId.clear() - credentialIdByMemberId.set(member.id, member.credentialId) - const syncContext: Record = { - syncRunId: runId, - memberId: member.id, - ...PER_MEMBER_LISTING_CONTEXT, - } - let contentFailures = false - const processPage = async ( - documents: ExternalDocument[], - checkpoint: ListingCheckpoint, - durableCheckpoint = true - ) => { - const externalIds = documents.map((item) => item.externalId) - const observeAttempted = async (attempted: ExternalDocument[]) => { - if (attempted.length === 0) return - const documentIds = [ - ...( - await loadDocumentIdsByExternalId( - connectorId, - attempted.map((item) => item.externalId) + while (Date.now() < run.deadlineAt) { + const member = await claimNextMember(run) + if (!member) break + result.membersClaimed += 1 + credentialIdByMemberId.clear() + credentialIdByMemberId.set(member.id, member.credentialId) + const syncContext: Record = { + syncRunId: runId, + memberId: member.id, + ...PER_MEMBER_LISTING_CONTEXT, + } + let contentFailures = false + const processPage = async ( + documents: ExternalDocument[], + checkpoint: ListingCheckpoint, + durableCheckpoint = true + ) => { + const externalIds = documents.map((item) => item.externalId) + const observeAttempted = async (attempted: ExternalDocument[]) => { + if (attempted.length === 0) return + const documentIds = [ + ...( + await loadDocumentIdsByExternalId( + connectorId, + attempted.map((item) => item.externalId) + ) + ).values(), + ] + await withMemberLease(run, async (tx) => { + result.observationsAdded += await recordMemberObservations( + tx, + member.id, + documentIds, + checkpoint.generationId ) - ).values(), - ] - await withMemberLease(run, async (tx) => { - result.observationsAdded += await recordMemberObservations( - tx, - member.id, - documentIds, - checkpoint.generationId - ) - await materializeDocumentAcls(connectorId, documentIds, tx) - if (durableCheckpoint && checkpoint.contentFailures) { - await tx - .update(knowledgeConnectorMember) - .set({ listingCheckpoint: checkpoint }) - .where(eq(knowledgeConnectorMember.id, member.id)) - } - if (!serviceContent) { - for (let offset = 0; offset < documentIds.length; offset += 500) { + await materializeDocumentAcls(connectorId, documentIds, tx) + if (durableCheckpoint && checkpoint.contentFailures) { await tx - .update(document) - .set({ sourceSeenAt: run.runStartedAt }) - .where( - and( - eq(document.connectorId, connectorId), - inArray(document.id, documentIds.slice(offset, offset + 500)) + .update(knowledgeConnectorMember) + .set({ listingCheckpoint: checkpoint }) + .where(eq(knowledgeConnectorMember.id, member.id)) + } + if (!serviceContent) { + for (let offset = 0; offset < documentIds.length; offset += 500) { + await tx + .update(document) + .set({ sourceSeenAt: run.runStartedAt }) + .where( + and( + eq(document.connectorId, connectorId), + inArray(document.id, documentIds.slice(offset, offset + 500)) + ) ) - ) + } } - } - }) - result.docsListed += attempted.length - } - if (!serviceContent) { - const corpus = await loadPageCorpus(connectorId, externalIds) - const pageState = createSyncRunState(result) - let rejectedCredentialError: Error | undefined - /** Commit sibling observations and failures before capacity pressure can end this page. */ - const persistAttempted = async (attempted: ExternalDocument[]) => { - if (rejectedCredentialError) throw rejectedCredentialError - if (attempted.some((item) => pageState.failedExternalIds.has(item.externalId))) { - await persistSourceDocumentFailures({ - knowledgeBaseId: connector.knowledgeBaseId, - connectorId, - connectorType: connector.connectorType, - documents: attempted, - failedExternalIds: pageState.failedExternalIds, - sourceFailures: pageState.sourceFailures, - priorByExternalId: corpus.priorByExternalId, - sourceConfig, - access: 'members', - lease: run.lease, - }) - checkpoint.contentFailures = true - contentFailures = true - result.listingIncomplete = true - } - await observeAttempted(attempted) + }) + result.docsListed += attempted.length } - const pendingOps = classifyListing({ - externalDocs: documents.filter((item) => { - const alreadyRead = corpus.priorByExternalId.get(item.externalId)?.sourceSeenAt - if ( - alreadyRead && - alreadyRead >= run.runStartedAt && - corpus.priorByExternalId.get(item.externalId)?.contentHash !== null - ) { - result.docsUnchanged += 1 - return false + if (!serviceContent) { + const corpus = await loadPageCorpus(connectorId, externalIds) + const pageState = createSyncRunState(result) + let rejectedCredentialError: Error | undefined + /** Commit sibling observations and failures before capacity pressure can end this page. */ + const persistAttempted = async (attempted: ExternalDocument[]) => { + if (rejectedCredentialError) throw rejectedCredentialError + if (attempted.some((item) => pageState.failedExternalIds.has(item.externalId))) { + await persistSourceDocumentFailures({ + knowledgeBaseId: connector.knowledgeBaseId, + connectorId, + connectorType: connector.connectorType, + documents: attempted, + failedExternalIds: pageState.failedExternalIds, + sourceFailures: pageState.sourceFailures, + priorByExternalId: corpus.priorByExternalId, + sourceConfig, + access: 'members', + lease: run.lease, + }) + checkpoint.contentFailures = true + contentFailures = true + result.listingIncomplete = true } - return true - }), - corpus, - forceRehydrate: false, - state: pageState, - }) - const pendingIds = new Set(pendingOps.map((op) => op.extDoc.externalId)) - await persistAttempted(documents.filter((item) => !pendingIds.has(item.externalId))) - const finished = await processDocOps({ - connectorId, - connector, - sourceConfig, - kbOwner, - billingAttribution, - pendingOps, - corpus, - forceRehydrate: false, - state: pageState, - hydration: { - concurrency: connectorConfig.contentConcurrency, - getDocument: async (externalId) => { - if (rejectedCredentialError) throw rejectedCredentialError - try { - return await connectorConfig.getDocument( - await tokens.get(member.id), - sourceConfig, - externalId, - syncContext - ) - } catch (error) { - if (connectorConfig.isCredentialInvalidError?.(error) === true) { - rejectedCredentialError = toError(error) - if (await tokens.reject(member.id)) - await recordMemberFailure( - run, - member, - error, - connector.syncIntervalMinutes, - true - ) - } - throw error + await observeAttempted(attempted) + } + const pendingOps = classifyListing({ + externalDocs: documents.filter((item) => { + const alreadyRead = corpus.priorByExternalId.get(item.externalId)?.sourceSeenAt + if ( + alreadyRead && + alreadyRead >= run.runStartedAt && + corpus.priorByExternalId.get(item.externalId)?.contentHash !== null + ) { + result.docsUnchanged += 1 + return false } + return true + }), + corpus, + forceRehydrate: false, + state: pageState, + }) + const pendingIds = new Set(pendingOps.map((op) => op.extDoc.externalId)) + await persistAttempted(documents.filter((item) => !pendingIds.has(item.externalId))) + const finished = await processDocOps({ + connectorId, + connector, + sourceConfig, + kbOwner, + billingAttribution, + pendingOps, + corpus, + forceRehydrate: false, + state: pageState, + hydration: { + concurrency: connectorConfig.contentConcurrency, + getDocument: async (externalId) => { + if (rejectedCredentialError) throw rejectedCredentialError + try { + return await connectorConfig.getDocument( + await tokens.get(member.id), + sourceConfig, + externalId, + syncContext + ) + } catch (error) { + if (connectorConfig.isCredentialInvalidError?.(error) === true) { + rejectedCredentialError = toError(error) + if (await tokens.reject(member.id)) + await recordMemberFailure( + run, + member, + error, + connector.syncIntervalMinutes, + true + ) + } + throw error + } + }, }, - }, - lease: run.lease, - documentAccess: 'members', - deadlineAt: durableCheckpoint ? run.deadlineAt : undefined, - onBatchComplete: async (attempted) => { - result.docsHydratedOnce += attempted.filter((item) => item.contentDeferred).length - await persistAttempted(attempted) - }, - }) - if (rejectedCredentialError) throw rejectedCredentialError - if (!finished) return false - } else { - await observeAttempted(documents) + lease: run.lease, + documentAccess: 'members', + deadlineAt: durableCheckpoint ? run.deadlineAt : undefined, + onBatchComplete: async (attempted) => { + result.docsHydratedOnce += attempted.filter((item) => item.contentDeferred).length + await persistAttempted(attempted) + }, + }) + if (rejectedCredentialError) throw rejectedCredentialError + if (!finished) return false + } else { + await observeAttempted(documents) + } } - } - const listed = await listForMember({ - run, - member, - connectorConfig, - sourceConfig, - tokens, - syncContext, - syncIntervalMinutes: connector.syncIntervalMinutes, - forceFull: Boolean( - serviceContent && - (result.docsAdded > 0 || - (connector.lastSyncAt && - (!member.memberSyncedThrough || member.memberSyncedThrough < connector.lastSyncAt))) - ), - processPage, - }) - if (listed.kind === 'failed') continue - if (listed.checkpoint?.contentFailures) result.listingIncomplete = true - if (listed.documents.length > 0) { - await processPage( - listed.documents, - beginListingCheckpoint({ - fingerprint: listingFingerprint({ connectorId, memberId: member.id }), - generationId: runId, - startedAt: listed.startedAt, - }), - false + const listed = await listForMember({ + run, + member, + connectorConfig, + sourceConfig, + tokens, + syncContext, + syncIntervalMinutes: connector.syncIntervalMinutes, + forceFull: Boolean( + serviceContent && + (result.docsAdded > 0 || + (connector.lastSyncAt && + (!member.memberSyncedThrough || + member.memberSyncedThrough < connector.lastSyncAt))) + ), + processPage, + }) + if (listed.kind === 'failed') continue + if (listed.checkpoint?.contentFailures) result.listingIncomplete = true + if (listed.documents.length > 0) { + await processPage( + listed.documents, + beginListingCheckpoint({ + fingerprint: listingFingerprint({ connectorId, memberId: member.id }), + generationId: runId, + startedAt: listed.startedAt, + }), + false + ) + } + const listedCount = listed.checkpoint?.listedCount ?? listed.documents.length + const suspect = + listed.mode === 'full' && + !listed.authoritative && + listed.complete && + classifySuspectListing(listedCount, member.lastListedCount ?? 0) !== null + const outcome: MemberListingOutcome = { + member, + mode: listed.mode, + listingStartedAt: listed.startedAt, + seenExternalIds: new Set(listed.documents.map((doc) => doc.externalId)), + removedExternalIds: listed.removedExternalIds, + listedCount, + complete: listed.complete, + resumable: listed.resumable, + suspect, + contentFailures: contentFailures || Boolean(listed.checkpoint?.contentFailures), + changeCursor: suspect ? undefined : listed.changeCursor, + checkpoint: listed.checkpoint, + observationRunId: listed.observationRunId, + } + const relevantIds = [...outcome.seenExternalIds, ...outcome.removedExternalIds] + const affected = await applyMemberListing( + run, + outcome, + await loadDocumentIdsByExternalId(connectorId, relevantIds), + connector.syncIntervalMinutes ) + await withMemberLease(run, (tx) => materializeDocumentAcls(connectorId, affected, tx)) } - const listedCount = listed.checkpoint?.listedCount ?? listed.documents.length - const suspect = - listed.mode === 'full' && - !listed.authoritative && - listed.complete && - classifySuspectListing(listedCount, member.lastListedCount ?? 0) !== null - const outcome: MemberListingOutcome = { - member, - mode: listed.mode, - listingStartedAt: listed.startedAt, - seenExternalIds: new Set(listed.documents.map((doc) => doc.externalId)), - removedExternalIds: listed.removedExternalIds, - listedCount, - complete: listed.complete, - resumable: listed.resumable, - suspect, - contentFailures: contentFailures || Boolean(listed.checkpoint?.contentFailures), - changeCursor: suspect ? undefined : listed.changeCursor, - checkpoint: listed.checkpoint, - observationRunId: listed.observationRunId, - } - const relevantIds = [...outcome.seenExternalIds, ...outcome.removedExternalIds] - const affected = await applyMemberListing( - run, - outcome, - await loadDocumentIdsByExternalId(connectorId, relevantIds), - connector.syncIntervalMinutes - ) - await withMemberLease(run, (tx) => materializeDocumentAcls(connectorId, affected, tx)) - } - /** A service-owned corpus outlives its last observer; only the content pass removes it. */ - if (!serviceContent) { - /** - * Nobody has completed a listing yet — a connector that just entered - * members mode, waiting for its first member to connect — so an - * unobserved document says nothing about access and must not be - * tombstoned, let alone purged a week later. - */ - const [listed] = await db - .select({ count: sql`count(*)::int` }) - .from(knowledgeConnectorMember) - .where( - and( - eq(knowledgeConnectorMember.connectorId, connectorId), - sql`${knowledgeConnectorMember.lastCompleteListingAt} IS NOT NULL` + /** A service-owned corpus outlives its last observer; only the content pass removes it. */ + if (!serviceContent) { + /** + * Nobody has completed a listing yet — a connector that just entered + * members mode, waiting for its first member to connect — so an + * unobserved document says nothing about access and must not be + * tombstoned, let alone purged a week later. + */ + const [listed] = await db + .select({ count: sql`count(*)::int` }) + .from(knowledgeConnectorMember) + .where( + and( + eq(knowledgeConnectorMember.connectorId, connectorId), + sql`${knowledgeConnectorMember.lastCompleteListingAt} IS NOT NULL` + ) ) - ) - const lifecycle = await applyMemberDocumentLifecycle({ + const lifecycle = await applyMemberDocumentLifecycle({ + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + runId, + lease: run.lease, + withLease: (fn) => withMemberLease(run, fn), + deadlineAt: run.deadlineAt, + allowRemoval: (listed?.count ?? 0) > 0, + }) + result.docsTombstoned = lifecycle.tombstoned + result.docsResurrected = lifecycle.resurrected + result.docsPurged = lifecycle.purged + result.docsDeleted = lifecycle.purged + result.membersRemaining = !lifecycle.finished + } + + await sweepStuckDocuments({ connectorId, knowledgeBaseId: connector.knowledgeBaseId, - runId, + syncStartedAt: runStartedAt, + retryCutoff: new Date(Date.now() - RETRY_WINDOW_DAYS * 24 * 60 * 60 * 1000), + billingAttribution, + result, lease: run.lease, - withLease: (fn) => withMemberLease(run, fn), - deadlineAt: run.deadlineAt, - allowRemoval: (listed?.count ?? 0) > 0, }) - result.docsTombstoned = lifecycle.tombstoned - result.docsResurrected = lifecycle.resurrected - result.docsPurged = lifecycle.purged - result.docsDeleted = lifecycle.purged - result.membersRemaining = !lifecycle.finished - } - await sweepStuckDocuments({ - connectorId, - knowledgeBaseId: connector.knowledgeBaseId, - syncStartedAt: runStartedAt, - retryCutoff: new Date(Date.now() - RETRY_WINDOW_DAYS * 24 * 60 * 60 * 1000), - billingAttribution, - result, - lease: run.lease, - }) - - result.membersRemaining = - result.membersRemaining || - serviceContent?.complete === false || - (await countDueMembers(run, binding)) > 0 - const landed = await completeMemberSync(run, connector.syncIntervalMinutes) - if (!landed) { - logger.warn( - 'Member sync result discarded — connector was reclaimed while this run was executing', - { - connectorId, - runId, - } - ) - return skipped(result, 'sync_superseded') - } - logger.info('Member sync completed', { connectorId, runId, ...result }) - return result - } catch (error) { - if (error instanceof SyncLockLostException) { - logger.warn('Member sync abandoned — lock was reclaimed while this run was executing', { - connectorId, - runId, - }) - return skipped(result, 'sync_superseded') - } - if (error instanceof ConnectorDeletedException) { - logger.info('Connector deleted during member sync', { connectorId }) - await failMemberSyncLog(runId, result, 'Connector deleted during sync').catch((logError) => - logger.error('Failed to record member sync failure', { - connectorId, - error: getErrorMessage(logError), - }) - ) - return skipped(result, 'connector_deleted_during_sync') - } - if (error instanceof MemberBindingGoneError) { - try { - await disableMemberSync(run, error.message) - } catch (disableError) { - if (!(disableError instanceof SyncLockLostException)) throw disableError - logger.warn('Member sync abandoned — lock was reclaimed before it could be disabled', { + result.membersRemaining = + result.membersRemaining || + serviceContent?.complete === false || + (await countDueMembers(run, binding)) > 0 + const landed = await completeMemberSync(run, connector.syncIntervalMinutes) + if (!landed) { + logger.warn( + 'Member sync result discarded — connector was reclaimed while this run was executing', + { + connectorId, + runId, + } + ) + return skipped(result, 'sync_superseded') + } + logger.info('Member sync completed', { connectorId, runId, ...result }) + return result + } catch (error) { + if (error instanceof SyncLockLostException) { + logger.warn('Member sync abandoned — lock was reclaimed while this run was executing', { connectorId, runId, }) return skipped(result, 'sync_superseded') } - return { ...skipped(result, 'connector_not_syncable'), error: error.message } - } + if (error instanceof ConnectorDeletedException) { + logger.info('Connector deleted during member sync', { connectorId }) + await failMemberSyncLog(runId, result, 'Connector deleted during sync').catch((logError) => + logger.error('Failed to record member sync failure', { + connectorId, + error: getErrorMessage(logError), + }) + ) + return skipped(result, 'connector_deleted_during_sync') + } + if (error instanceof MemberBindingGoneError) { + try { + await disableMemberSync(run, error.message) + } catch (disableError) { + if (!(disableError instanceof SyncLockLostException)) throw disableError + logger.warn('Member sync abandoned — lock was reclaimed before it could be disabled', { + connectorId, + runId, + }) + return skipped(result, 'sync_superseded') + } + return { ...skipped(result, 'connector_not_syncable'), error: error.message } + } - if (getConnectorSyncDeferral(error)) { - try { - result.deferred = await deferConnectorSync({ - connectorId, - knowledgeBaseId: connector.knowledgeBaseId, - runId, - lease: run.lease, - kind: 'member', - result, - error, - }) - result.listingIncomplete = true - logger.info('Member source sync deferred', { connectorId, ...result.deferred }) - return result - } catch (persistenceError) { - logger.error('Failed to persist member source deferral', { - connectorId, - error: - getConnectorFailureDiagnostic(persistenceError)?.message ?? - toError(persistenceError).message, - }) - result.error = 'Could not persist the member sync retry after provider deferral' - return result + if (getConnectorSyncDeferral(error)) { + try { + result.deferred = await deferConnectorSync({ + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + runId, + lease: run.lease, + kind: 'member', + result, + error, + }) + result.listingIncomplete = true + logger.info('Member source sync deferred', { connectorId, ...result.deferred }) + return result + } catch (persistenceError) { + logger.error('Failed to persist member source deferral', { + connectorId, + error: + getConnectorFailureDiagnostic(persistenceError)?.message ?? + toError(persistenceError).message, + }) + result.error = 'Could not persist the member sync retry after provider deferral' + return result + } } - } - const diagnostic = getConnectorFailureDiagnostic(error) - const errorMessage = diagnostic?.message ?? toError(error).message - const retryAfterMs = getRetryAfterMs(error) - logger.error('Member sync failed', { connectorId, runId, error: errorMessage, diagnostic }) - try { - await failMemberSyncLog(runId, result, errorMessage) - const failureUpdate = - error instanceof ConnectorSyncCapacityError - ? { - memberSyncStatus: 'error' as const, - lastMemberSyncError: errorMessage, - nextMemberSyncAt: null, - memberSyncConsecutiveFailures: connector.memberSyncConsecutiveFailures, - memberSyncLockToken: null, - memberSyncLockLeaseAt: null, - updatedAt: new Date(), - } - : buildMemberSyncFailureUpdate( - new Date(), - connector.memberSyncConsecutiveFailures, - errorMessage, - retryAfterMs - ) - const written = await db - .update(knowledgeConnector) - .set(failureUpdate) - .where(stillHoldsMemberSyncLock(connectorId, runId)) - .returning({ id: knowledgeConnector.id }) - if (written.length === 0) { - logger.warn('Member sync failure discarded — connector was reclaimed', { + const diagnostic = getConnectorFailureDiagnostic(error) + const errorMessage = diagnostic?.message ?? toError(error).message + const retryAfterMs = getRetryAfterMs(error) + logger.error('Member sync failed', { connectorId, runId, error: errorMessage, diagnostic }) + try { + await failMemberSyncLog(runId, result, errorMessage) + const failureUpdate = + error instanceof ConnectorSyncCapacityError + ? { + memberSyncStatus: 'error' as const, + lastMemberSyncError: errorMessage, + nextMemberSyncAt: null, + memberSyncConsecutiveFailures: connector.memberSyncConsecutiveFailures, + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: new Date(), + } + : buildMemberSyncFailureUpdate( + new Date(), + connector.memberSyncConsecutiveFailures, + errorMessage, + retryAfterMs + ) + const written = await db + .update(knowledgeConnector) + .set(failureUpdate) + .where(stillHoldsMemberSyncLock(connectorId, runId)) + .returning({ id: knowledgeConnector.id }) + if (written.length === 0) { + logger.warn('Member sync failure discarded — connector was reclaimed', { + connectorId, + runId, + }) + } + } catch (recoveryError) { + logger.error('Failed to record member sync failure', { connectorId, - runId, + error: + getConnectorFailureDiagnostic(recoveryError)?.message ?? toError(recoveryError).message, }) } - } catch (recoveryError) { - logger.error('Failed to record member sync failure', { - connectorId, - error: - getConnectorFailureDiagnostic(recoveryError)?.message ?? toError(recoveryError).message, - }) + result.error = errorMessage + return result } - result.error = errorMessage - return result - } + }) } diff --git a/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts b/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts index de2d738b106..a2b4dd5e323 100644 --- a/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-content-pass.test.ts @@ -1,6 +1,7 @@ /** @vitest-environment node */ import { dbChainMockFns, + inputValidationMock, queueTableRows, resetDbChainMock as resetDatabaseMock, schemaMock, @@ -61,6 +62,7 @@ vi.mock('@/lib/knowledge/documents/storage-cleanup', () => ({ isKnowledgeBaseOwnedStorageKey: (key: string) => key.startsWith('kb/'), })) vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: {} })) +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) interface StoredPage { id: string diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index dc11c0d278d..af438b779b4 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -14,6 +14,10 @@ import { } from '@sim/testing' import { generateShortId } from '@sim/utils/id' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' import * as connectorTokens from '@/lib/knowledge/connectors/access-token' import { executeSync, isConnectorRunnableStatus } from '@/lib/knowledge/connectors/sync-engine' import { @@ -34,6 +38,18 @@ function resetDbChainMock() { } vi.mock('drizzle-orm', () => drizzleOrmMock) +const outbound = vi.hoisted(() => ({ enabled: false, workspace: vi.fn() })) +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: () => outbound.enabled, + resolveOutboundRoute: async (organizationId: string | null | undefined) => ({ organizationId }), +})) +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadActiveWorkspaceApplicationContext: outbound.workspace, +})) +beforeEach(() => { + outbound.enabled = false + outbound.workspace.mockReset() +}) const { mockProcessDocumentsWithQueue, mockUploadFile } = vi.hoisted(() => ({ mockProcessDocumentsWithQueue: vi.fn(), mockUploadFile: vi.fn(), @@ -2701,6 +2717,55 @@ describe('executeSync heartbeats during the listing phase', () => { dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-1', accessMode: 'workspace' }]) } + it.each([false, true])( + 'resolves provider credentials with routing enabled=%s', + async (enabled) => { + outbound.enabled = enabled + outbound.workspace.mockResolvedValue({ workspaceOrganizationId: 'current-org' }) + primeSyncUpToListing() + dbChainMockFns.returning.mockReset().mockResolvedValueOnce([CONNECTOR]) + let tokenRoute: unknown + const resolveToken = vi + .spyOn(connectorTokens, 'resolveConnectorAccessToken') + .mockImplementationOnce(async () => { + tokenRoute = await resolveCurrentOutboundRoute() + throw new Error('Stop after scoped token resolution') + }) + try { + await runWithOutboundOrganization('queued-org', () => + executeSync('c-1', { billingAttribution: { workspaceId: 'ws-1' } as never }) + ) + expect(tokenRoute).toEqual({ organizationId: enabled ? 'current-org' : 'queued-org' }) + expect(outbound.workspace).toHaveBeenCalledTimes(enabled ? 1 : 0) + if (enabled) expect(outbound.workspace).toHaveBeenCalledWith('ws-1') + } finally { + resolveToken.mockRestore() + } + } + ) + + it('refuses an archived workspace before claiming the connector or resolving tokens', async () => { + outbound.enabled = true + outbound.workspace.mockResolvedValue(null) + primeSyncUpToListing() + await expect( + executeSync('c-1', { billingAttribution: { workspaceId: 'ws-1' } as never }) + ).rejects.toMatchObject({ code: 'MISSING_SCOPE' }) + expect(dbChainMockFns.set).not.toHaveBeenCalled() + expect(mockListDocuments).not.toHaveBeenCalled() + }) + + it('skips a missing connector without consuming inherited outbound scope or taking a lock', async () => { + outbound.enabled = true + const result = await runWithOutboundOrganization('queued-org', () => + executeSync('missing', { billingAttribution: { workspaceId: 'ws-1' } as never }) + ) + expect(result.skipReason).toBe('connector_unavailable') + expect(outbound.workspace).not.toHaveBeenCalled() + expect(dbChainMockFns.set).not.toHaveBeenCalled() + expect(mockListDocuments).not.toHaveBeenCalled() + }) + it.each(['workspace', 'admin'] as const)( 'uses the locked source mode %s when resolving its token', async (accessMode) => { diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index d1e9ef65b85..78bec1bf562 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -15,6 +15,7 @@ import { assertBillingAttributionSnapshot, type BillingAttributionSnapshot, } from '@/lib/billing/core/billing-attribution' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { EMPTY_ACL } from '@/lib/knowledge/access/tokens' import { @@ -805,543 +806,551 @@ export async function executeSync( ) } assertBillingAttributionOwner(billingAttribution, kbOwner) - /** - * Identifies this run for the terminal writes. Generated before the CAS and - * written by it, so ownership is established atomically with the lock — and - * reused as the sync-log row id, which makes the connector row point at the - * run that holds it. - */ - const syncLogId = generateId() - - const lockResult = await db - .update(knowledgeConnector) - .set(buildSyncLockAcquisition(syncLogId, new Date())) - .where( - and( - inArray(knowledgeConnector.accessMode, CONTENT_ENGINE_ACCESS_MODES), - eq(knowledgeConnector.id, connectorId), - inArray(knowledgeConnector.status, LOCKABLE_CONNECTOR_STATUSES), - /** - * Proves this run is consuming the queue entry that was made for it. - * - * A task delayed past the lease is reclaimed and replaced, and the - * status check alone would let that stale task take the replacement's - * entry — running superseded options (a plain sync where the user had - * just asked for a full resync) while the replacement is turned away as - * `sync_in_progress`. Matching the token is the same discipline - * {@link holdsSyncLockToken} already applies to the `syncing` phase, - * extended to the phase before it. - */ - ...(options.dispatchToken - ? [eq(knowledgeConnector.syncLockToken, options.dispatchToken)] - : []), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .returning() - - if (lockResult.length === 0) { + return withResourceOutboundScope(kbOwner, async (): Promise => { /** - * Distinguishes the two ways the CAS can find no row. Costs one read on a - * path that already decided not to work, and the alternative is reporting a - * connector someone paused as a concurrency conflict. + * Identifies this run for the terminal writes. Generated before the CAS and + * written by it, so ownership is established atomically with the lock — and + * reused as the sync-log row id, which makes the connector row point at the + * run that holds it. */ - const [current] = await db - .select({ - status: knowledgeConnector.status, - syncLockToken: knowledgeConnector.syncLockToken, - }) - .from(knowledgeConnector) - .where(eq(knowledgeConnector.id, connectorId)) - .limit(1) + const syncLogId = generateId() - /** - * Status is checked before ownership because pausing a queued connector - * releases its token, so a mismatch is the *symptom* there and the status is - * the actual reason. Testing ownership first would report every - * pause-while-queued — the common case — as a superseded dispatch, losing - * the distinction this branch exists to draw. - */ - if (current?.status === 'paused' || current?.status === 'disabled') { - logger.info('Connector is not accepting syncs, skipping', { - connectorId, - status: current.status, - }) - return { ...result, skipReason: 'connector_not_syncable' } - } + const lockResult = await db + .update(knowledgeConnector) + .set(buildSyncLockAcquisition(syncLogId, new Date())) + .where( + and( + inArray(knowledgeConnector.accessMode, CONTENT_ENGINE_ACCESS_MODES), + eq(knowledgeConnector.id, connectorId), + inArray(knowledgeConnector.status, LOCKABLE_CONNECTOR_STATUSES), + /** + * Proves this run is consuming the queue entry that was made for it. + * + * A task delayed past the lease is reclaimed and replaced, and the + * status check alone would let that stale task take the replacement's + * entry — running superseded options (a plain sync where the user had + * just asked for a full resync) while the replacement is turned away as + * `sync_in_progress`. Matching the token is the same discipline + * {@link holdsSyncLockToken} already applies to the `syncing` phase, + * extended to the phase before it. + */ + ...(options.dispatchToken + ? [eq(knowledgeConnector.syncLockToken, options.dispatchToken)] + : []), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning() - if (options.dispatchToken && current?.syncLockToken !== options.dispatchToken) { - logger.info('Sync superseded by a newer dispatch, skipping', { connectorId }) - return { ...result, skipReason: 'dispatch_superseded' } - } + if (lockResult.length === 0) { + /** + * Distinguishes the two ways the CAS can find no row. Costs one read on a + * path that already decided not to work, and the alternative is reporting a + * connector someone paused as a concurrency conflict. + */ + const [current] = await db + .select({ + status: knowledgeConnector.status, + syncLockToken: knowledgeConnector.syncLockToken, + }) + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connectorId)) + .limit(1) - logger.info('Sync already in progress, skipping', { connectorId }) - return { ...result, skipReason: 'sync_in_progress' } - } + /** + * Status is checked before ownership because pausing a queued connector + * releases its token, so a mismatch is the *symptom* there and the status is + * the actual reason. Testing ownership first would report every + * pause-while-queued — the common case — as a superseded dispatch, losing + * the distinction this branch exists to draw. + */ + if (current?.status === 'paused' || current?.status === 'disabled') { + logger.info('Connector is not accepting syncs, skipping', { + connectorId, + status: current.status, + }) + return { ...result, skipReason: 'connector_not_syncable' } + } - /** - * The row returned by the lock is the authoritative sync snapshot. A source update - * committed before the lock is included here; one attempted after it sees `syncing` - * and conflicts instead of letting this worker process stale configuration. - */ - const connector = lockResult[0] - /** The lock CAS only takes a content-engine row; this is the type's word for the same fact. */ - if (!isContentEngineAccessMode(connector.accessMode)) { - throw new Error(`Connector ${connectorId} left the content engine's modes while locked`) - } - const accessMode = connector.accessMode - const mirrored = mirrorsSourceAcls(connector.accessMode) - const sourceConfig = connector.sourceConfig as Record - const syncStartedAt = new Date() - const lease = createContentSyncLease(connectorId, syncLogId) - await db.insert(knowledgeConnectorSyncLog).values({ - id: syncLogId, - connectorId, - status: 'started', - startedAt: syncStartedAt, - }) + if (options.dispatchToken && current?.syncLockToken !== options.dispatchToken) { + logger.info('Sync superseded by a newer dispatch, skipping', { connectorId }) + return { ...result, skipReason: 'dispatch_superseded' } + } + + logger.info('Sync already in progress, skipping', { connectorId }) + return { ...result, skipReason: 'sync_in_progress' } + } - try { /** - * OAuth credentials are workspace-scoped and shared, so the member who authorized - * one is often not the knowledge base owner. Resolve the credential's own account - * owner — token reads are scoped to `account.userId`, so passing the KB owner - * resolves no token at all. Resolved once here rather than inside - * `resolveAccessToken` so per-page refreshes don't repeat the lookup. + * The row returned by the lock is the authoritative sync snapshot. A source update + * committed before the lock is included here; one attempted after it sees `syncing` + * and conflicts instead of letting this worker process stale configuration. */ - const credentialUserId = await resolveConnectorTokenUserId({ - credentialId: connector.credentialId, - ...resourceScopeFields(resourceScopeFromOwner(kbOwner)), - fallbackUserId: userId, - }) - if (!credentialUserId) { - throw new Error( - `Credential ${connector.credentialId} is not usable from workspace ${kbOwner.workspaceId} — reconnect the credential` - ) + const connector = lockResult[0] + /** The lock CAS only takes a content-engine row; this is the type's word for the same fact. */ + if (!isContentEngineAccessMode(connector.accessMode)) { + throw new Error(`Connector ${connectorId} left the content engine's modes while locked`) } + const accessMode = connector.accessMode + const mirrored = mirrorsSourceAcls(connector.accessMode) + const sourceConfig = connector.sourceConfig as Record + const syncStartedAt = new Date() + const lease = createContentSyncLease(connectorId, syncLogId) + await db.insert(knowledgeConnectorSyncLog).values({ + id: syncLogId, + connectorId, + status: 'started', + startedAt: syncStartedAt, + }) - let credentialToken = await resolveAccessToken( - connector, - connectorConfig, - credentialUserId, - sourceConfig, - accessMode - ) - /** Re-resolves the token for every OAuth call after the first, so a long run outlives a short-lived token. */ - const refreshOAuthToken = async (): Promise => { - if (connectorConfig.auth.mode === 'oauth') { - credentialToken = await resolveAccessToken( - connector, - connectorConfig, - credentialUserId, - sourceConfig, - accessMode + try { + /** + * OAuth credentials are workspace-scoped and shared, so the member who authorized + * one is often not the knowledge base owner. Resolve the credential's own account + * owner — token reads are scoped to `account.userId`, so passing the KB owner + * resolves no token at all. Resolved once here rather than inside + * `resolveAccessToken` so per-page refreshes don't repeat the lookup. + */ + const credentialUserId = await resolveConnectorTokenUserId({ + credentialId: connector.credentialId, + ...resourceScopeFields(resourceScopeFromOwner(kbOwner)), + fallbackUserId: userId, + }) + if (!credentialUserId) { + throw new Error( + `Credential ${connector.credentialId} is not usable from workspace ${kbOwner.workspaceId} — reconnect the credential` ) } - } - /** - * A credential that already knows its cloud id seeds the same `syncContext` - * slot the connector would otherwise memoise it into. Confluence discovers - * it by calling `accessible-resources` with a bearer token; an Atlassian - * service account holds an API token that cannot make that call, so for it - * the seed is the only source. Connectors need no service-account branch. - */ - const syncContext: Record = { - syncRunId: generateId(), - ...syncContextForToken(credentialToken), - /** Tells a connector to carry permissions with its listing; without it, none are read. */ - ...(mirrored ? { mirrorsSourceAcls: true } : {}), - } - if (mirrored) - await connectorConfig.permissionConfig?.populateSyncContext(connectorId, syncContext) + let credentialToken = await resolveAccessToken( + connector, + connectorConfig, + credentialUserId, + sourceConfig, + accessMode + ) + /** Re-resolves the token for every OAuth call after the first, so a long run outlives a short-lived token. */ + const refreshOAuthToken = async (): Promise => { + if (connectorConfig.auth.mode === 'oauth') { + credentialToken = await resolveAccessToken( + connector, + connectorConfig, + credentialUserId, + sourceConfig, + accessMode + ) + } + } + + /** + * A credential that already knows its cloud id seeds the same `syncContext` + * slot the connector would otherwise memoise it into. Confluence discovers + * it by calling `accessible-resources` with a bearer token; an Atlassian + * service account holds an API token that cannot make that call, so for it + * the seed is the only source. Connectors need no service-account branch. + */ + const syncContext: Record = { + syncRunId: generateId(), + ...syncContextForToken(credentialToken), + /** Tells a connector to carry permissions with its listing; without it, none are read. */ + ...(mirrored ? { mirrorsSourceAcls: true } : {}), + } + if (mirrored) + await connectorConfig.permissionConfig?.populateSyncContext(connectorId, syncContext) - // Shared cutoff for both the tombstone-retry bound below and the stuck-document - // retry near the end of this sync — same RETRY_WINDOW_DAYS window, one computation. - const retryCutoff = new Date(Date.now() - RETRY_WINDOW_DAYS * 24 * 60 * 60 * 1000) + // Shared cutoff for both the tombstone-retry bound below and the stuck-document + // retry near the end of this sync — same RETRY_WINDOW_DAYS window, one computation. + const retryCutoff = new Date(Date.now() - RETRY_WINDOW_DAYS * 24 * 60 * 60 * 1000) - /** - * Bounded to the same retry window as the stuck-document retry below: a - * document whose refresh keeps failing every sync (e.g. permanently - * oversized) would otherwise be a tombstone that never resolves, forcing a - * full listing — and its listing-time overhead — for this connector - * forever. Past the window, this connector stops forcing full syncs on its - * account; the document itself is unaffected and stays tombstoned either way. - * - * Known accepted trade-off: once past the window, a still-tombstoned - * document that's unchanged-but-genuinely-present at the source can only - * be resurrected by a full listing — and nothing here forces one anymore. - * On a connector that never runs a full sync again (persistent incremental - * syncMode, no manual full resync), that document stays correctly - * invisible (excluded everywhere by `isNull(deletedAt)`, so no - * search/billing/listing leakage) but unresolved indefinitely. This is - * deliberately not "fixed" by hard-deleting it after the window expires — - * that would delete a document we have no positive evidence is actually - * gone, reintroducing the exact risk this whole design exists to avoid. - */ - const hasTombstonedDocs = await db - .select({ id: document.id }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - isNull(document.archivedAt), - or( - and(isNotNull(document.deletedAt), gt(document.deletedAt, retryCutoff)), - isNull(document.contentHash) + /** + * Bounded to the same retry window as the stuck-document retry below: a + * document whose refresh keeps failing every sync (e.g. permanently + * oversized) would otherwise be a tombstone that never resolves, forcing a + * full listing — and its listing-time overhead — for this connector + * forever. Past the window, this connector stops forcing full syncs on its + * account; the document itself is unaffected and stays tombstoned either way. + * + * Known accepted trade-off: once past the window, a still-tombstoned + * document that's unchanged-but-genuinely-present at the source can only + * be resurrected by a full listing — and nothing here forces one anymore. + * On a connector that never runs a full sync again (persistent incremental + * syncMode, no manual full resync), that document stays correctly + * invisible (excluded everywhere by `isNull(deletedAt)`, so no + * search/billing/listing leakage) but unresolved indefinitely. This is + * deliberately not "fixed" by hard-deleting it after the window expires — + * that would delete a document we have no positive evidence is actually + * gone, reintroducing the exact risk this whole design exists to avoid. + */ + const hasTombstonedDocs = await db + .select({ id: document.id }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + isNull(document.archivedAt), + or( + and(isNotNull(document.deletedAt), gt(document.deletedAt, retryCutoff)), + isNull(document.contentHash) + ) ) ) - ) - .limit(1) - .then((rows) => rows.length > 0) - - /** - * Determine if this sync should be incremental. A `rehydrate` request forces a - * full listing too: re-hydration must see *every* document (a container page can - * be unchanged itself yet transclude a page that changed), and an incremental - * listing would omit those unchanged containers, so they'd never be re-fetched. - */ - const isIncremental = - !mirrored && - shouldRunIncrementalSync( - connectorConfig.supportsIncrementalSync, - connector.syncMode, - options?.fullSync, - options?.rehydrate, - hasTombstonedDocs, - connector.lastSyncAt - ) - const lastSyncAt = - isIncremental && connector.lastSyncAt ? new Date(connector.lastSyncAt) : undefined - - /** - * Re-hydrate and re-index connectors whose rendered content can drift without a - * hash change (transclusions) — see `ConnectorMeta.rehydrateOnFullSync`. Driven - * by the dedicated `rehydrate` request (the "Full resync" action) or implied by a - * true `fullSync`. It forces a full listing (above) and re-indexes unchanged - * deferred docs, but — unlike `fullSync` — it does NOT bypass any - * deletion-reconciliation safety guard. Incremental syncs of other connectors - * stay hash-gated. - */ - const forceRehydrate = Boolean( - (options?.rehydrate || options?.fullSync) && connectorConfig.rehydrateOnFullSync - ) + .limit(1) + .then((rows) => rows.length > 0) - let directoryRefreshed: Promise = Promise.resolve(undefined) - if (mirrored) { /** - * A switch into this mode hides every document before it flips, and one - * whose rewrite outgrew its request budget leaves the rest for the next - * run. It has to be finished *before this run lists anything*: the - * documents it did not reach are still readable by the whole workspace, - * and the completion write below clears the flag on the strength of this - * pass having left none under the mode the connector came from. The - * workspace-mode equivalent runs at completion instead, because restoring - * is safe to do last; hiding is not. + * Determine if this sync should be incremental. A `rehydrate` request forces a + * full listing too: re-hydration must see *every* document (a container page can + * be unchanged itself yet transclude a page that changed), and an incremental + * listing would omit those unchanged containers, so they'd never be re-fetched. */ - if (connector.accessRewritePending) { - await rewriteConnectorAcls(connectorId, EMPTY_ACL, { - beforeBatch: lease.beatIfDue, - lease, - }) - } + const isIncremental = + !mirrored && + shouldRunIncrementalSync( + connectorConfig.supportsIncrementalSync, + connector.syncMode, + options?.fullSync, + options?.rehydrate, + hasTombstonedDocs, + connector.lastSyncAt + ) + const lastSyncAt = + isIncremental && connector.lastSyncAt ? new Date(connector.lastSyncAt) : undefined + /** - * Started before the listing and awaited before the ACLs are written: a - * group grant this crawl writes must never point at membership nobody - * has read, and the scheduler's refresh is a cadence, not a guarantee. - * Observe failures immediately while allowing content ingestion to finish. - * The terminal sync write below still reports directory failures. + * Re-hydrate and re-index connectors whose rendered content can drift without a + * hash change (transclusions) — see `ConnectorMeta.rehydrateOnFullSync`. Driven + * by the dedicated `rehydrate` request (the "Full resync" action) or implied by a + * true `fullSync`. It forces a full listing (above) and re-indexes unchanged + * deferred docs, but — unlike `fullSync` — it does NOT bypass any + * deletion-reconciliation safety guard. Incremental syncs of other connectors + * stay hash-gated. */ - directoryRefreshed = refreshMirroredDirectory({ - ...resourceScopeFields(resourceScopeFromOwner(kbOwner)), + const forceRehydrate = Boolean( + (options?.rehydrate || options?.fullSync) && connectorConfig.rehydrateOnFullSync + ) + + let directoryRefreshed: Promise = Promise.resolve(undefined) + if (mirrored) { + /** + * A switch into this mode hides every document before it flips, and one + * whose rewrite outgrew its request budget leaves the rest for the next + * run. It has to be finished *before this run lists anything*: the + * documents it did not reach are still readable by the whole workspace, + * and the completion write below clears the flag on the strength of this + * pass having left none under the mode the connector came from. The + * workspace-mode equivalent runs at completion instead, because restoring + * is safe to do last; hiding is not. + */ + if (connector.accessRewritePending) { + await rewriteConnectorAcls(connectorId, EMPTY_ACL, { + beforeBatch: lease.beatIfDue, + lease, + }) + } + /** + * Started before the listing and awaited before the ACLs are written: a + * group grant this crawl writes must never point at membership nobody + * has read, and the scheduler's refresh is a cadence, not a guarantee. + * Observe failures immediately while allowing content ingestion to finish. + * The terminal sync write below still reports directory failures. + */ + directoryRefreshed = refreshMirroredDirectory({ + ...resourceScopeFields(resourceScopeFromOwner(kbOwner)), + connectorConfig, + sourceConfig, + syncContext, + accessToken: credentialToken.accessToken, + force: + Boolean(options.fullSync) || + connector.consecutiveFailures > 0 || + connector.lastSyncError?.startsWith(DIRECTORY_ERROR_PREFIX), + }).then(() => undefined, toError) + } + + const contentPass = await runConnectorContentPass({ + connectorId, + connector, connectorConfig, sourceConfig, syncContext, - accessToken: credentialToken.accessToken, - force: - Boolean(options.fullSync) || - connector.consecutiveFailures > 0 || - connector.lastSyncError?.startsWith(DIRECTORY_ERROR_PREFIX), - }).then(() => undefined, toError) - } - - const contentPass = await runConnectorContentPass({ - connectorId, - connector, - connectorConfig, - sourceConfig, - syncContext, - lastSyncAt, - kbOwner, - billingAttribution, - result, - forceRehydrate, - getAccessToken: async (pageNum) => { - if (pageNum > 0) await refreshOAuthToken() - return credentialToken.accessToken - }, - hydration: { - concurrency: connectorConfig.contentConcurrency, - beforeHydration: refreshOAuthToken, - getDocument: (externalId) => - connectorConfig.getDocument( - credentialToken.accessToken, - sourceConfig, - externalId, - syncContext - ), - }, - lease, - documentAccess: connector.accessMode, - runId: syncLogId, - leaseKind: 'content', - fingerprint: listingFingerprint({ - connectorType: connector.connectorType, - credentialId: connector.credentialId, - encryptedApiKey: connector.encryptedApiKey, - sourceConfig, - accessMode: connector.accessMode, - }), - fullSync: options.fullSync, - deadlineAt: syncStartedAt.getTime() + (CONNECTOR_SYNC_MAX_DURATION_SECONDS - 300) * 1000, - onPage: mirrored - ? async (externalDocs, generationStartedAt) => { - await directoryRefreshed - await applySourceMirroredAcls({ - connectorId, - connectorConfig, + lastSyncAt, + kbOwner, + billingAttribution, + result, + forceRehydrate, + getAccessToken: async (pageNum) => { + if (pageNum > 0) await refreshOAuthToken() + return credentialToken.accessToken + }, + hydration: { + concurrency: connectorConfig.contentConcurrency, + beforeHydration: refreshOAuthToken, + getDocument: (externalId) => + connectorConfig.getDocument( + credentialToken.accessToken, sourceConfig, - syncContext, - accessToken: credentialToken.accessToken, - externalDocs, - generationStartedAt, - ownedExternalIds: [], - lease, - }) - } - : undefined, - }) + externalId, + syncContext + ), + }, + lease, + documentAccess: connector.accessMode, + runId: syncLogId, + leaseKind: 'content', + fingerprint: listingFingerprint({ + connectorType: connector.connectorType, + credentialId: connector.credentialId, + encryptedApiKey: connector.encryptedApiKey, + sourceConfig, + accessMode: connector.accessMode, + }), + fullSync: options.fullSync, + deadlineAt: syncStartedAt.getTime() + (CONNECTOR_SYNC_MAX_DURATION_SECONDS - 300) * 1000, + onPage: mirrored + ? async (externalDocs, generationStartedAt) => { + await directoryRefreshed + await applySourceMirroredAcls({ + connectorId, + connectorConfig, + sourceConfig, + syncContext, + accessToken: credentialToken.accessToken, + externalDocs, + generationStartedAt, + ownedExternalIds: [], + lease, + }) + } + : undefined, + }) - result.listingIncomplete = isContentPassIncomplete(contentPass) - const reconciliationHoldNotice = contentPass.holdNotice - const directoryError = await directoryRefreshed - if (directoryError) throw directoryError + result.listingIncomplete = isContentPassIncomplete(contentPass) + const reconciliationHoldNotice = contentPass.holdNotice + const directoryError = await directoryRefreshed + if (directoryError) throw directoryError - const postBatchPresence = await checkSyncTargetPresence(connectorId, connector.knowledgeBaseId) - if (postBatchPresence.connectorDeleted) { - throw new ConnectorDeletedException(connectorId) - } - if (postBatchPresence.knowledgeBaseDeleted) { - throw new Error(`Knowledge base ${connector.knowledgeBaseId} was deleted during sync`) - } - - await sweepStuckDocuments({ - connectorId, - knowledgeBaseId: connector.knowledgeBaseId, - syncStartedAt, - retryCutoff, - billingAttribution, - result, - lease, - }) + const postBatchPresence = await checkSyncTargetPresence( + connectorId, + connector.knowledgeBaseId + ) + if (postBatchPresence.connectorDeleted) { + throw new ConnectorDeletedException(connectorId) + } + if (postBatchPresence.knowledgeBaseDeleted) { + throw new Error(`Knowledge base ${connector.knowledgeBaseId} was deleted during sync`) + } - const completionLanded = await completeSuccessfulSync( - connectorId, - connector.knowledgeBaseId, - syncLogId, - effectiveConnectorSyncIntervalMinutes(connector.accessMode, connector.syncIntervalMinutes), - result, - reconciliationHoldNotice, - contentPass - ) + await sweepStuckDocuments({ + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + syncStartedAt, + retryCutoff, + billingAttribution, + result, + lease, + }) - if (!completionLanded) { - logger.warn('Sync result discarded — connector was reclaimed while this run was executing', { + const completionLanded = await completeSuccessfulSync( connectorId, + connector.knowledgeBaseId, syncLogId, - ...result, - }) - return markSyncSuperseded(result) - } + effectiveConnectorSyncIntervalMinutes(connector.accessMode, connector.syncIntervalMinutes), + result, + reconciliationHoldNotice, + contentPass + ) - logger.info('Sync completed', { connectorId, ...result }) - return result - } catch (error) { - let connectorDeleted = error instanceof ConnectorDeletedException - if (error instanceof SyncLockLostException) { - /** A checkpoint can discover an archive before the next batch's presence check. */ - const [ownedArchive] = await db - .select({ - archivedAt: knowledgeConnector.archivedAt, - deletedAt: knowledgeConnector.deletedAt, - }) - .from(knowledgeConnector) - .where( - and( - holdsSyncLockToken(connectorId, syncLogId), - or(isNotNull(knowledgeConnector.archivedAt), isNotNull(knowledgeConnector.deletedAt)) - ) + if (!completionLanded) { + logger.warn( + 'Sync result discarded — connector was reclaimed while this run was executing', + { + connectorId, + syncLogId, + ...result, + } ) - .limit(1) - connectorDeleted = Boolean(ownedArchive?.archivedAt || ownedArchive?.deletedAt) - if (!connectorDeleted) { - /** A replacement-owned connector must receive no writes from this run. */ - logger.warn('Sync abandoned — lock was reclaimed while this run was executing', { - connectorId, - syncLogId, - ...result, - }) return markSyncSuperseded(result) } - } - - if (connectorDeleted) { - logger.info('Connector deleted during sync, cleaning up', { connectorId }) - try { - await releaseSyncLockOnDeletedConnector(connectorId, syncLogId) + logger.info('Sync completed', { connectorId, ...result }) + return result + } catch (error) { + let connectorDeleted = error instanceof ConnectorDeletedException + if (error instanceof SyncLockLostException) { + /** A checkpoint can discover an archive before the next batch's presence check. */ + const [ownedArchive] = await db + .select({ + archivedAt: knowledgeConnector.archivedAt, + deletedAt: knowledgeConnector.deletedAt, + }) + .from(knowledgeConnector) + .where( + and( + holdsSyncLockToken(connectorId, syncLogId), + or(isNotNull(knowledgeConnector.archivedAt), isNotNull(knowledgeConnector.deletedAt)) + ) + ) + .limit(1) + connectorDeleted = Boolean(ownedArchive?.archivedAt || ownedArchive?.deletedAt) + if (!connectorDeleted) { + /** A replacement-owned connector must receive no writes from this run. */ + logger.warn('Sync abandoned — lock was reclaimed while this run was executing', { + connectorId, + syncLogId, + ...result, + }) + return markSyncSuperseded(result) + } + } - /** - * Includes pending-removal tombstones. Page IDs so deleting a connector - * with a legacy corpus above the sync admission cap cannot materialize - * the entire corpus in the cleanup worker. - */ - let afterDocumentId: string | undefined - while (true) { - const connectorDocs = await db - .select({ id: document.id }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - isNull(document.archivedAt), - afterDocumentId ? gt(document.id, afterDocumentId) : undefined + if (connectorDeleted) { + logger.info('Connector deleted during sync, cleaning up', { connectorId }) + + try { + await releaseSyncLockOnDeletedConnector(connectorId, syncLogId) + + /** + * Includes pending-removal tombstones. Page IDs so deleting a connector + * with a legacy corpus above the sync admission cap cannot materialize + * the entire corpus in the cleanup worker. + */ + let afterDocumentId: string | undefined + while (true) { + const connectorDocs = await db + .select({ id: document.id }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + isNull(document.archivedAt), + afterDocumentId ? gt(document.id, afterDocumentId) : undefined + ) ) + .orderBy(asc(document.id)) + .limit(CONNECTOR_DELETION_CLEANUP_BATCH_SIZE) + if (connectorDocs.length === 0) break + + await hardDeleteDocuments( + connectorDocs.map((doc) => doc.id), + syncLogId, + connectorId ) - .orderBy(asc(document.id)) - .limit(CONNECTOR_DELETION_CLEANUP_BATCH_SIZE) - if (connectorDocs.length === 0) break + afterDocumentId = connectorDocs.at(-1)?.id + if (connectorDocs.length < CONNECTOR_DELETION_CLEANUP_BATCH_SIZE) break + } - await hardDeleteDocuments( - connectorDocs.map((doc) => doc.id), - syncLogId, - connectorId - ) - afterDocumentId = connectorDocs.at(-1)?.id - if (connectorDocs.length < CONNECTOR_DELETION_CLEANUP_BATCH_SIZE) break + await completeSyncLog(syncLogId, 'failed', result, { + errorMessage: 'Connector deleted during sync', + }) + } catch (cleanupError) { + logger.error('Failed to clean up after connector deletion', { + connectorId, + error: toError(cleanupError).message, + }) } - await completeSyncLog(syncLogId, 'failed', result, { - errorMessage: 'Connector deleted during sync', - }) - } catch (cleanupError) { - logger.error('Failed to clean up after connector deletion', { - connectorId, - error: toError(cleanupError).message, - }) + result.skipReason = 'connector_deleted_during_sync' + return result } - result.skipReason = 'connector_deleted_during_sync' - return result - } - - if (getConnectorSyncDeferral(error)) { - try { - result.deferred = await deferConnectorSync({ - connectorId, - knowledgeBaseId: connector.knowledgeBaseId, - runId: syncLogId, - lease, - kind: 'content', - result, - error, - }) - result.listingIncomplete = true - logger.info('Connector source sync deferred', { - connectorId, - ...result.deferred, - docsAdvanced: - result.docsAdded + result.docsUpdated + result.docsUnchanged + result.docsSkipped, - }) - return result - } catch (persistenceError) { - logger.error('Failed to persist connector source deferral', { - connectorId, - error: - getConnectorFailureDiagnostic(persistenceError)?.message ?? - toError(persistenceError).message, - }) - result.error = 'Could not persist the connector retry after provider deferral' - return result + if (getConnectorSyncDeferral(error)) { + try { + result.deferred = await deferConnectorSync({ + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + runId: syncLogId, + lease, + kind: 'content', + result, + error, + }) + result.listingIncomplete = true + logger.info('Connector source sync deferred', { + connectorId, + ...result.deferred, + docsAdvanced: + result.docsAdded + result.docsUpdated + result.docsUnchanged + result.docsSkipped, + }) + return result + } catch (persistenceError) { + logger.error('Failed to persist connector source deferral', { + connectorId, + error: + getConnectorFailureDiagnostic(persistenceError)?.message ?? + toError(persistenceError).message, + }) + result.error = 'Could not persist the connector retry after provider deferral' + return result + } } - } - const diagnostic = getConnectorFailureDiagnostic(error) - const errorMessage = diagnostic?.message ?? toError(error).message - const retryAfterMs = getRetryAfterMs(error) - const rateLimited = isRateLimitError(error) - logger.error('Sync failed', { - connectorId, - diagnostic, - error: errorMessage, - ...(retryAfterMs === undefined ? {} : { retryAfterMs }), - }) + const diagnostic = getConnectorFailureDiagnostic(error) + const errorMessage = diagnostic?.message ?? toError(error).message + const retryAfterMs = getRetryAfterMs(error) + const rateLimited = isRateLimitError(error) + logger.error('Sync failed', { + connectorId, + diagnostic, + error: errorMessage, + ...(retryAfterMs === undefined ? {} : { retryAfterMs }), + }) - try { - await completeSyncLog(syncLogId, 'failed', result, { errorMessage }) - - const failureUpdate = - error instanceof ConnectorSyncCapacityError - ? buildSyncCapacityUpdate(new Date(), connector.consecutiveFailures, errorMessage) - : rateLimited - ? buildSyncRateLimitUpdate( - new Date(), - connector.consecutiveFailures, - errorMessage, - retryAfterMs - ) - : buildSyncFailureUpdate( - new Date(), - connector.consecutiveFailures, - errorMessage, - retryAfterMs - ) + try { + await completeSyncLog(syncLogId, 'failed', result, { errorMessage }) + + const failureUpdate = + error instanceof ConnectorSyncCapacityError + ? buildSyncCapacityUpdate(new Date(), connector.consecutiveFailures, errorMessage) + : rateLimited + ? buildSyncRateLimitUpdate( + new Date(), + connector.consecutiveFailures, + errorMessage, + retryAfterMs + ) + : buildSyncFailureUpdate( + new Date(), + connector.consecutiveFailures, + errorMessage, + retryAfterMs + ) + + if (failureUpdate.status === 'disabled') { + logger.warn('Connector disabled after repeated failures', { + connectorId, + consecutiveFailures: failureUpdate.consecutiveFailures, + }) + } + + const failureWriteLanded = await writeTerminalConnectorState( + connectorId, + syncLogId, + failureUpdate + ) - if (failureUpdate.status === 'disabled') { - logger.warn('Connector disabled after repeated failures', { + /** + * Deliberately does NOT get {@link markSyncSuperseded}. `result.error` + * is set to the real failure cause below, so replacing it with lifecycle + * control flow would destroy the diagnostic. The supersession is carried + * by this log line instead. + */ + if (!failureWriteLanded) { + logger.warn( + 'Sync failure discarded — connector was reclaimed while this run was executing', + { connectorId, syncLogId, error: errorMessage } + ) + } + } catch (recoveryError) { + logger.error('Failed to record sync failure', { connectorId, - consecutiveFailures: failureUpdate.consecutiveFailures, + error: + getConnectorFailureDiagnostic(recoveryError)?.message ?? toError(recoveryError).message, }) } - const failureWriteLanded = await writeTerminalConnectorState( - connectorId, - syncLogId, - failureUpdate - ) - - /** - * Deliberately does NOT get {@link markSyncSuperseded}. `result.error` - * is set to the real failure cause below, so replacing it with lifecycle - * control flow would destroy the diagnostic. The supersession is carried - * by this log line instead. - */ - if (!failureWriteLanded) { - logger.warn( - 'Sync failure discarded — connector was reclaimed while this run was executing', - { connectorId, syncLogId, error: errorMessage } - ) - } - } catch (recoveryError) { - logger.error('Failed to record sync failure', { - connectorId, - error: - getConnectorFailureDiagnostic(recoveryError)?.message ?? toError(recoveryError).message, - }) + result.error = errorMessage + return result } - - result.error = errorMessage - return result - } + }) } diff --git a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts index 24b77c321fa..bb3fa3bf105 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts @@ -13,6 +13,17 @@ import { } from '@sim/testing' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +const outbound = vi.hoisted(() => ({ enabled: false, workspace: vi.fn() })) + +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: () => outbound.enabled, + resolveOutboundRoute: async (organizationId: string | null | undefined) => ({ organizationId }), +})) + +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadActiveWorkspaceApplicationContext: outbound.workspace, +})) + const { mockCheckAttributedUsageLimits, mockBatchTrigger, @@ -72,6 +83,10 @@ import { markInsideTriggerRun, resetInsideTriggerRunForTests, } from '@/lib/core/config/trigger-runtime' +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { BYOK_EMBEDDING_CREDENTIAL_REJECTION_MESSAGE, @@ -93,6 +108,8 @@ import { MAX_PROCESSING_ATTEMPTS } from '@/lib/knowledge/documents/types' const mockEmbeddingCapacity = vi.fn() beforeEach(() => { + outbound.enabled = false + outbound.workspace.mockReset() resetIngestionUsageGateCache() vi.spyOn(billingAttribution, 'checkAttributedUsageLimits').mockImplementation( mockCheckAttributedUsageLimits @@ -258,19 +275,34 @@ describe('knowledge document processing source', () => { }) it('uses the persisted document source instead of stale queued source fields', async () => { - await processDocumentAsync( - 'knowledge-base-1', - 'document-1', - { - filename: 'stale.pdf', - fileUrl: 'https://example.com/stale.pdf', - fileSize: 1, - mimeType: 'text/plain', - }, - {}, - BILLING_ATTRIBUTION + outbound.enabled = true + outbound.workspace.mockResolvedValue({ workspaceOrganizationId: 'current-org' }) + let providerRoute: unknown + mockProcessDocument.mockImplementationOnce(async () => { + providerRoute = await resolveCurrentOutboundRoute() + return { + chunks: [], + metadata: { chunkCount: 0, tokenCount: 0, characterCount: 0 }, + } + }) + + await runWithOutboundOrganization('queued-org', () => + processDocumentAsync( + 'knowledge-base-1', + 'document-1', + { + filename: 'stale.pdf', + fileUrl: 'https://example.com/stale.pdf', + fileSize: 1, + mimeType: 'text/plain', + }, + {}, + BILLING_ATTRIBUTION + ) ) + expect(providerRoute).toEqual({ organizationId: 'current-org' }) + expect(outbound.workspace).toHaveBeenCalledExactlyOnceWith('workspace-1') expect(mockGetFileMetadataByKeys).toHaveBeenCalledWith( [PERSISTED_KEY], 'workspace', diff --git a/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts b/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts index d57161fbc33..be01e3baa2b 100644 --- a/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processor-secret-provenance.test.ts @@ -1,10 +1,16 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { interruptibleSleep } from '@sim/utils/helpers' import { PDFDocument } from 'pdf-lib' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const { mockDownloadFileFromUrl, mockGenerateInternalToken, diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts index a0691b43220..f8afbdf4941 100644 --- a/apps/sim/lib/knowledge/documents/document-processor.ts +++ b/apps/sim/lib/knowledge/documents/document-processor.ts @@ -23,6 +23,7 @@ import { waitForProviderAdmission, } from '@/lib/core/rate-limiter/provider-admission' import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, isPayloadSizeLimitError, @@ -621,9 +622,12 @@ async function makeOCRRequest( const requestSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal try { - const response = await fetch(endpoint, { + const response = await secureFetchWithValidation(endpoint, { + profile: 'configuredEndpoint', + maxRedirects: 20, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, method: 'POST', - headers, + headers: Object.fromEntries(new Headers(headers)), body: typeof body === 'string' ? body : JSON.stringify(body), signal: requestSignal, }) @@ -679,7 +683,7 @@ async function makeOCRRequest( return new Response(responseText, { status: response.status, - headers: response.headers, + headers: [...response.headers], }) } catch (error) { signal?.throwIfAborted() diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts index 723bf3bc6f3..1f6c6f08a53 100644 --- a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts +++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts @@ -6,8 +6,14 @@ * the routing: the text layer is tried first, and OCR is reached only when it is * missing or unreadable. */ +import { inputValidationMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const { mockParseBuffer, mockDownload, mockToken, mockBaseUrl, mockExecuteMistralParse } = vi.hoisted(() => ({ mockParseBuffer: vi.fn(), diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 65eb4b14f95..188a5c0308d 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -53,6 +53,7 @@ import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' import { env, envNumber } from '@/lib/core/config/env' import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/env-flags' import { isInsideTriggerRun } from '@/lib/core/config/trigger-runtime' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' @@ -1540,482 +1541,487 @@ export async function processDocumentAsync( const ctx = contextRows[0] processingFilename = ctx.filename - const persistedDocData = { - filename: ctx.filename, - fileUrl: ctx.fileUrl, - fileSize: ctx.fileSize, - mimeType: ctx.mimeType, - } + await withResourceOutboundScope(ctx, async () => { + const persistedDocData = { + filename: ctx.filename, + fileUrl: ctx.fileUrl, + fileSize: ctx.fileSize, + mimeType: ctx.mimeType, + } - /** - * Claiming is guarded by both completion status and queue generation. - * - * Without a status predicate this write was reachable for a finished - * document — a late or duplicate dispatch would flip `completed` back to - * `processing`, discard the pass that had already indexed and billed, and - * index it a second time. `pending`, `failed`, and `processing` remain - * claimable so a Trigger retry can recover if an earlier attempt threw - * before persisting its failure. Queued workers also match the exact stamp - * carried in their payload. A retry or recovery sweep re-stamps the row, so - * an older delayed quota continuation becomes a harmless no-op instead of - * stealing the newer pass. - */ - /** - * Queue acceptance can precede the parent's pending-state write. The published - * successor may adopt that exact processing generation; stamping its token - * fences the parent's delayed write and refunds admission at most once. - */ - const predecessor = - attemptContext?.processingQueueToken && attemptContext.processingPredecessorToken - ? and( - eq(document.processingStatus, 'processing'), - eq(document.processingQueueToken, attemptContext.processingPredecessorToken) + /** + * Claiming is guarded by both completion status and queue generation. + * + * Without a status predicate this write was reachable for a finished + * document — a late or duplicate dispatch would flip `completed` back to + * `processing`, discard the pass that had already indexed and billed, and + * index it a second time. `pending`, `failed`, and `processing` remain + * claimable so a Trigger retry can recover if an earlier attempt threw + * before persisting its failure. Queued workers also match the exact stamp + * carried in their payload. A retry or recovery sweep re-stamps the row, so + * an older delayed quota continuation becomes a harmless no-op instead of + * stealing the newer pass. + */ + /** + * Queue acceptance can precede the parent's pending-state write. The published + * successor may adopt that exact processing generation; stamping its token + * fences the parent's delayed write and refunds admission at most once. + */ + const predecessor = + attemptContext?.processingQueueToken && attemptContext.processingPredecessorToken + ? and( + eq(document.processingStatus, 'processing'), + eq(document.processingQueueToken, attemptContext.processingPredecessorToken) + ) + : undefined + const claimed = await db + .update(document) + .set({ + processingStatus: 'processing', + processingStartedAt, + processingDeferredUntil: null, + processingCompletedAt: null, + processingError: null, + ...(attemptContext?.processingQueueToken + ? { processingQueueToken: attemptContext.processingQueueToken } + : {}), + ...(predecessor && attemptContext?.refundPredecessorAdmission + ? { + processingAttempts: sql`CASE WHEN ${document.processingQueueToken} = ${attemptContext.processingPredecessorToken} THEN GREATEST(${document.processingAttempts} - 1, 0) ELSE ${document.processingAttempts} END`, + } + : {}), + }) + .where( + and( + eq(document.id, documentId), + inArray(document.processingStatus, ['pending', 'processing', 'failed']), + not(skippedDocumentCondition()), + ...(predecessor + ? [or(and(...queueGenerationConditions(attemptContext)), predecessor)] + : queueGenerationConditions(attemptContext)), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt) ) - : undefined - const claimed = await db - .update(document) - .set({ - processingStatus: 'processing', - processingStartedAt, - processingDeferredUntil: null, - processingCompletedAt: null, - processingError: null, - ...(attemptContext?.processingQueueToken - ? { processingQueueToken: attemptContext.processingQueueToken } - : {}), - ...(predecessor && attemptContext?.refundPredecessorAdmission - ? { - processingAttempts: sql`CASE WHEN ${document.processingQueueToken} = ${attemptContext.processingPredecessorToken} THEN GREATEST(${document.processingAttempts} - 1, 0) ELSE ${document.processingAttempts} END`, - } - : {}), - }) - .where( - and( - eq(document.id, documentId), - inArray(document.processingStatus, ['pending', 'processing', 'failed']), - not(skippedDocumentCondition()), - ...(predecessor - ? [or(and(...queueGenerationConditions(attemptContext)), predecessor)] - : queueGenerationConditions(attemptContext)), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) ) - ) - .returning({ id: document.id }) + .returning({ id: document.id }) - if (claimed.length === 0) { - logger.info( - `[${documentId}] Skipping document processing: superseded, already active, completed, archived, or deleted` - ) - return - } + if (claimed.length === 0) { + logger.info( + `[${documentId}] Skipping document processing: superseded, already active, completed, archived, or deleted` + ) + return + } - attemptContext?.onClaimed?.() - - logger.info(`[${documentId}] Status updated to 'processing', starting document processor`) - - const rawConfig = ctx.chunkingConfig as { - maxSize?: number - minSize?: number - overlap?: number - strategy?: ChunkingStrategy - strategyOptions?: StrategyOptions - } | null - const kbConfig = { - maxSize: rawConfig?.maxSize ?? 1024, - minSize: rawConfig?.minSize ?? 100, - overlap: rawConfig?.overlap ?? 200, - } + attemptContext?.onClaimed?.() + + logger.info(`[${documentId}] Status updated to 'processing', starting document processor`) + + const rawConfig = ctx.chunkingConfig as { + maxSize?: number + minSize?: number + overlap?: number + strategy?: ChunkingStrategy + strategyOptions?: StrategyOptions + } | null + const kbConfig = { + maxSize: rawConfig?.maxSize ?? 1024, + minSize: rawConfig?.minSize ?? 100, + overlap: rawConfig?.overlap ?? 200, + } - const kbEmbedding: KbEmbeddingTarget = { - model: ctx.embeddingModel, - dimensions: toKbEmbeddingDimensions(ctx.embeddingDimension), - } - const kbEmbeddingModel = kbEmbedding.model - const queuedBillingContext = hasDocumentProcessingBillingScope(providedBillingContext) - ? assertDocumentProcessingBillingContext(providedBillingContext) - : undefined - const restoredBillingAttribution = - queuedBillingContext && queuedBillingContext.billingScope !== 'non-workspace' - ? queuedBillingContext.billingAttribution - : providedBillingContext && !queuedBillingContext - ? assertBillingAttributionSnapshot(providedBillingContext) - : undefined - if (queuedBillingContext?.billingScope === 'non-workspace') { - throw new Error('Document processing billing scope does not match knowledge base ownership') - } - if (!restoredBillingAttribution) { - throw new Error('Billing attribution is required for queued document processing') - } - const billingAttribution = restoredBillingAttribution - assertBillingAttributionOwner(billingAttribution, ctx) - const documentActorUserId = billingAttribution.actorUserId - - const usageGate = await checkIngestionUsageLimits(billingAttribution) - if (usageGate.isExceeded) { - logger.warn(`[${documentId}] Usage limit reached — skipping document indexing`) - throw new UsageLimitDocumentProcessingError( - usageGate.message ?? 'Usage limit exceeded. Please upgrade your plan to continue.' - ) - } - let billableEmbeddingTokens = 0 - let embeddingModelName = kbEmbeddingModel - let embeddingPricingId = kbEmbeddingModel + const kbEmbedding: KbEmbeddingTarget = { + model: ctx.embeddingModel, + dimensions: toKbEmbeddingDimensions(ctx.embeddingDimension), + } + const kbEmbeddingModel = kbEmbedding.model + const queuedBillingContext = hasDocumentProcessingBillingScope(providedBillingContext) + ? assertDocumentProcessingBillingContext(providedBillingContext) + : undefined + const restoredBillingAttribution = + queuedBillingContext && queuedBillingContext.billingScope !== 'non-workspace' + ? queuedBillingContext.billingAttribution + : providedBillingContext && !queuedBillingContext + ? assertBillingAttributionSnapshot(providedBillingContext) + : undefined + if (queuedBillingContext?.billingScope === 'non-workspace') { + throw new Error('Document processing billing scope does not match knowledge base ownership') + } + if (!restoredBillingAttribution) { + throw new Error('Billing attribution is required for queued document processing') + } + const billingAttribution = restoredBillingAttribution + assertBillingAttributionOwner(billingAttribution, ctx) + const documentActorUserId = billingAttribution.actorUserId + + const usageGate = await checkIngestionUsageLimits(billingAttribution) + if (usageGate.isExceeded) { + logger.warn(`[${documentId}] Usage limit reached — skipping document indexing`) + throw new UsageLimitDocumentProcessingError( + usageGate.message ?? 'Usage limit exceeded. Please upgrade your plan to continue.' + ) + } + let billableEmbeddingTokens = 0 + let embeddingModelName = kbEmbeddingModel + let embeddingPricingId = kbEmbeddingModel - const currentSourceFileProvenance = await loadCurrentSourceFileSecretProvenance({ - fileUrl: persistedDocData.fileUrl, - workspaceId: ctx.workspaceId, - }) - const documentSecretContext = await loadKnowledgeDocumentSecretRegistry( - documentId, - { - userId: documentActorUserId, - ...(ctx.workspaceId ? { workspaceId: ctx.workspaceId } : {}), - }, - currentSourceFileProvenance - ) + const currentSourceFileProvenance = await loadCurrentSourceFileSecretProvenance({ + fileUrl: persistedDocData.fileUrl, + workspaceId: ctx.workspaceId, + }) + const documentSecretContext = await loadKnowledgeDocumentSecretRegistry( + documentId, + { + userId: documentActorUserId, + ...(ctx.workspaceId ? { workspaceId: ctx.workspaceId } : {}), + }, + currentSourceFileProvenance + ) - let processingCommitted = false - const processingDeadlineAt = Math.min( - startTime + TIMEOUTS.OVERALL_PROCESSING - 15_000, - (attemptContext?.deadlineAt ?? Number.POSITIVE_INFINITY) - 15_000 - ) - await withTimeout( - (signal) => - runWithKnowledgeModelInputProvenance( - documentSecretContext.registry, - async () => { - await assertKnowledgeEmbeddingCapacity({ - ...kbEmbedding, - workspaceId: ctx.workspaceId, - signal, - }) - const processed = await processDocument( - persistedDocData.fileUrl, - persistedDocData.filename, - persistedDocData.mimeType, - kbConfig.maxSize, - kbConfig.overlap, - kbConfig.minSize, - { - ...sourceFileAccessFor(ctx.connectorId, documentActorUserId), + let processingCommitted = false + const processingDeadlineAt = Math.min( + startTime + TIMEOUTS.OVERALL_PROCESSING - 15_000, + (attemptContext?.deadlineAt ?? Number.POSITIVE_INFINITY) - 15_000 + ) + await withTimeout( + (signal) => + runWithKnowledgeModelInputProvenance( + documentSecretContext.registry, + async () => { + await assertKnowledgeEmbeddingCapacity({ + ...kbEmbedding, + workspaceId: ctx.workspaceId, signal, - processingDeadlineAt, - ...(indexingPassId - ? { ocrCheckpoint: { knowledgeBaseId, documentId, indexingPassId } } - : {}), - }, - ctx.workspaceId, - rawConfig?.strategy, - rawConfig?.strategyOptions - ) + }) + const processed = await processDocument( + persistedDocData.fileUrl, + persistedDocData.filename, + persistedDocData.mimeType, + kbConfig.maxSize, + kbConfig.overlap, + kbConfig.minSize, + { + ...sourceFileAccessFor(ctx.connectorId, documentActorUserId), + signal, + processingDeadlineAt, + ...(indexingPassId + ? { ocrCheckpoint: { knowledgeBaseId, documentId, indexingPassId } } + : {}), + }, + ctx.workspaceId, + rawConfig?.strategy, + rawConfig?.strategyOptions + ) - signal.throwIfAborted() - assertDocumentChunkCountWithinLimit(processed.chunks.length) + signal.throwIfAborted() + assertDocumentChunkCountWithinLimit(processed.chunks.length) - const now = new Date() + const now = new Date() - logger.info( - `[${documentId}] Document parsed successfully, generating embeddings for ${processed.chunks.length} chunks` - ) + logger.info( + `[${documentId}] Document parsed successfully, generating embeddings for ${processed.chunks.length} chunks` + ) - const chunkTexts = processed.chunks.map((chunk) => chunk.text) - const embeddingModelInfo = getEmbeddingModelInfo(kbEmbeddingModel) - const chunkTokenCounts: number[] = [] - for (let chunkIndex = 0; chunkIndex < chunkTexts.length; chunkIndex++) { - const tokenCount = estimateTokenCount( - chunkTexts[chunkIndex], - embeddingModelInfo.tokenizerProvider - ).count - chunkTokenCounts.push(tokenCount) - if (tokenCount > embeddingModelInfo.maxInputTokens) { - throw new PermanentDocumentProcessingError( - 'document_complexity_limit', - `Chunk ${chunkIndex + 1} contains ${tokenCount.toLocaleString()} estimated tokens, exceeding the ${embeddingModelInfo.maxInputTokens.toLocaleString()}-token limit for ${kbEmbeddingModel}. Reduce the knowledge-base chunk size and retry.` - ) + const chunkTexts = processed.chunks.map((chunk) => chunk.text) + const embeddingModelInfo = getEmbeddingModelInfo(kbEmbeddingModel) + const chunkTokenCounts: number[] = [] + for (let chunkIndex = 0; chunkIndex < chunkTexts.length; chunkIndex++) { + const tokenCount = estimateTokenCount( + chunkTexts[chunkIndex], + embeddingModelInfo.tokenizerProvider + ).count + chunkTokenCounts.push(tokenCount) + if (tokenCount > embeddingModelInfo.maxInputTokens) { + throw new PermanentDocumentProcessingError( + 'document_complexity_limit', + `Chunk ${chunkIndex + 1} contains ${tokenCount.toLocaleString()} estimated tokens, exceeding the ${embeddingModelInfo.maxInputTokens.toLocaleString()}-token limit for ${kbEmbeddingModel}. Reduce the knowledge-base chunk size and retry.` + ) + } } - } - const embeddings: number[][] = [] - const embeddingSourceHash = indexingPassId - ? sha256Hex(JSON.stringify(chunkTexts.map((text) => sha256Hex(text)))) - : undefined + const embeddings: number[][] = [] + const embeddingSourceHash = indexingPassId + ? sha256Hex(JSON.stringify(chunkTexts.map((text) => sha256Hex(text)))) + : undefined - if (chunkTexts.length > 0) { - const batchSize = LARGE_DOC_CONFIG.MAX_EMBEDDING_BATCH - const totalBatches = Math.ceil(chunkTexts.length / batchSize) + if (chunkTexts.length > 0) { + const batchSize = LARGE_DOC_CONFIG.MAX_EMBEDDING_BATCH + const totalBatches = Math.ceil(chunkTexts.length / batchSize) - logger.info(`[${documentId}] Generating embeddings in ${totalBatches} batches`) + logger.info(`[${documentId}] Generating embeddings in ${totalBatches} batches`) - for (let i = 0; i < chunkTexts.length; i += batchSize) { - signal.throwIfAborted() - const batch = chunkTexts.slice(i, i + batchSize) - const batchNum = Math.floor(i / batchSize) + 1 + for (let i = 0; i < chunkTexts.length; i += batchSize) { + signal.throwIfAborted() + const batch = chunkTexts.slice(i, i + batchSize) + const batchNum = Math.floor(i / batchSize) + 1 - logger.info( - `[${documentId}] Processing embedding batch ${batchNum}/${totalBatches}` - ) - const { - embeddings: batchEmbeddings, - billableTokens: batchBillableTokens, - modelName, - pricingId, - } = await generateEmbeddings( - batch, - kbEmbedding, - ctx.workspaceId, - signal, - indexingPassId && embeddingSourceHash - ? createEmbeddingCheckpoints({ - knowledgeBaseId, - documentId, - indexingPassId, - sourceHash: embeddingSourceHash, - batchOffset: i, - deadlineAt: processingDeadlineAt, - }) - : undefined - ) - for (const emb of batchEmbeddings) { - embeddings.push(emb) - } - billableEmbeddingTokens += batchBillableTokens - if (i === 0) { - embeddingModelName = modelName - embeddingPricingId = pricingId + logger.info( + `[${documentId}] Processing embedding batch ${batchNum}/${totalBatches}` + ) + const { + embeddings: batchEmbeddings, + billableTokens: batchBillableTokens, + modelName, + pricingId, + } = await generateEmbeddings( + batch, + kbEmbedding, + ctx.workspaceId, + signal, + indexingPassId && embeddingSourceHash + ? createEmbeddingCheckpoints({ + knowledgeBaseId, + documentId, + indexingPassId, + sourceHash: embeddingSourceHash, + batchOffset: i, + deadlineAt: processingDeadlineAt, + }) + : undefined + ) + for (const emb of batchEmbeddings) { + embeddings.push(emb) + } + billableEmbeddingTokens += batchBillableTokens + if (i === 0) { + embeddingModelName = modelName + embeddingPricingId = pricingId + } } } - } - if (embeddings.length !== processed.chunks.length) { - throw new Error( - `Embedding generation returned ${embeddings.length} vectors for ${processed.chunks.length} chunks` + if (embeddings.length !== processed.chunks.length) { + throw new Error( + `Embedding generation returned ${embeddings.length} vectors for ${processed.chunks.length} chunks` + ) + } + const documentTags = ctx + + logger.info( + `[${documentId}] Embeddings generated, creating embedding records with tags` ) - } - const documentTags = ctx - logger.info( - `[${documentId}] Embeddings generated, creating embedding records with tags` - ) + const chunkProvenances = processed.chunks.map((chunk) => + documentSecretContext.tracked + ? documentSecretContext.registry + ? durableSecretProvenanceFromRegistry( + documentSecretContext.registry, + chunk.text + ) + : EXACT_EMPTY_DURABLE_SECRET_PROVENANCE + : undefined + ) + const embeddingRecords = processed.chunks.map((chunk, chunkIndex) => ({ + id: generateId(), + knowledgeBaseId, + documentId, + chunkIndex, + chunkHash: sha256Hex(chunk.text), + content: chunk.text, + secretProvenanceVersion: chunkProvenances[chunkIndex] ? 1 : null, + contentLength: chunk.text.length, + tokenCount: chunkTokenCounts[chunkIndex], + ...embeddingVectorValues(kbEmbedding.dimensions, embeddings[chunkIndex]), + embeddingModel: kbEmbeddingModel, + startOffset: chunk.metadata.startIndex, + endOffset: chunk.metadata.endIndex, + tag1: documentTags.tag1, + tag2: documentTags.tag2, + tag3: documentTags.tag3, + tag4: documentTags.tag4, + tag5: documentTags.tag5, + tag6: documentTags.tag6, + tag7: documentTags.tag7, + number1: documentTags.number1, + number2: documentTags.number2, + number3: documentTags.number3, + number4: documentTags.number4, + number5: documentTags.number5, + date1: documentTags.date1, + date2: documentTags.date2, + boolean1: documentTags.boolean1, + boolean2: documentTags.boolean2, + boolean3: documentTags.boolean3, + createdAt: now, + updatedAt: now, + })) - const chunkProvenances = processed.chunks.map((chunk) => - documentSecretContext.tracked - ? documentSecretContext.registry - ? durableSecretProvenanceFromRegistry(documentSecretContext.registry, chunk.text) - : EXACT_EMPTY_DURABLE_SECRET_PROVENANCE - : undefined - ) - const embeddingRecords = processed.chunks.map((chunk, chunkIndex) => ({ - id: generateId(), - knowledgeBaseId, - documentId, - chunkIndex, - chunkHash: sha256Hex(chunk.text), - content: chunk.text, - secretProvenanceVersion: chunkProvenances[chunkIndex] ? 1 : null, - contentLength: chunk.text.length, - tokenCount: chunkTokenCounts[chunkIndex], - ...embeddingVectorValues(kbEmbedding.dimensions, embeddings[chunkIndex]), - embeddingModel: kbEmbeddingModel, - startOffset: chunk.metadata.startIndex, - endOffset: chunk.metadata.endIndex, - tag1: documentTags.tag1, - tag2: documentTags.tag2, - tag3: documentTags.tag3, - tag4: documentTags.tag4, - tag5: documentTags.tag5, - tag6: documentTags.tag6, - tag7: documentTags.tag7, - number1: documentTags.number1, - number2: documentTags.number2, - number3: documentTags.number3, - number4: documentTags.number4, - number5: documentTags.number5, - date1: documentTags.date1, - date2: documentTags.date2, - boolean1: documentTags.boolean1, - boolean2: documentTags.boolean2, - boolean3: documentTags.boolean3, - createdAt: now, - updatedAt: now, - })) - - signal.throwIfAborted() - processingCommitted = await db.transaction(async (tx) => { signal.throwIfAborted() - const activeDocument = await tx - .select({ id: document.id }) - .from(document) - .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id)) - .where( - and( - eq(document.id, documentId), - eq(document.processingStatus, 'processing'), - eq(document.processingStartedAt, processingStartedAt), - ...queueGenerationConditions(attemptContext), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - isNull(knowledgeBase.deletedAt) + processingCommitted = await db.transaction(async (tx) => { + signal.throwIfAborted() + const activeDocument = await tx + .select({ id: document.id }) + .from(document) + .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id)) + .where( + and( + eq(document.id, documentId), + eq(document.processingStatus, 'processing'), + eq(document.processingStartedAt, processingStartedAt), + ...queueGenerationConditions(attemptContext), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + isNull(knowledgeBase.deletedAt) + ) ) - ) - .for('update', { of: document }) - .limit(1) - - if (activeDocument.length === 0) { - return false - } - - if (embeddingRecords.length > 0) { - await tx.delete(embedding).where(eq(embedding.documentId, documentId)) + .for('update', { of: document }) + .limit(1) - const insertBatchSize = LARGE_DOC_CONFIG.MAX_CHUNKS_PER_BATCH - const batches: (typeof embeddingRecords)[] = [] - for (let i = 0; i < embeddingRecords.length; i += insertBatchSize) { - batches.push(embeddingRecords.slice(i, i + insertBatchSize)) + if (activeDocument.length === 0) { + return false } - logger.info(`[${documentId}] Inserting ${embeddingRecords.length} embeddings`) - for (const batch of batches) { - signal.throwIfAborted() - await tx.insert(embedding).values(batch) - } - const provenanceRecords = embeddingRecords.flatMap((record, index) => { - const provenance = chunkProvenances[index] - if (!provenance) return [] - return [ - { - embeddingId: record.id, - contentHash: record.chunkHash, - status: provenance.status, - entries: provenance.status === 'exact' ? [...provenance.entries] : [], - updatedAt: now, - }, - ] - }) - for (let i = 0; i < provenanceRecords.length; i += insertBatchSize) { - signal.throwIfAborted() - await tx - .insert(embeddingSecretProvenance) - .values(provenanceRecords.slice(i, i + insertBatchSize)) + if (embeddingRecords.length > 0) { + await tx.delete(embedding).where(eq(embedding.documentId, documentId)) + + const insertBatchSize = LARGE_DOC_CONFIG.MAX_CHUNKS_PER_BATCH + const batches: (typeof embeddingRecords)[] = [] + for (let i = 0; i < embeddingRecords.length; i += insertBatchSize) { + batches.push(embeddingRecords.slice(i, i + insertBatchSize)) + } + + logger.info(`[${documentId}] Inserting ${embeddingRecords.length} embeddings`) + for (const batch of batches) { + signal.throwIfAborted() + await tx.insert(embedding).values(batch) + } + const provenanceRecords = embeddingRecords.flatMap((record, index) => { + const provenance = chunkProvenances[index] + if (!provenance) return [] + return [ + { + embeddingId: record.id, + contentHash: record.chunkHash, + status: provenance.status, + entries: provenance.status === 'exact' ? [...provenance.entries] : [], + updatedAt: now, + }, + ] + }) + for (let i = 0; i < provenanceRecords.length; i += insertBatchSize) { + signal.throwIfAborted() + await tx + .insert(embeddingSecretProvenance) + .values(provenanceRecords.slice(i, i + insertBatchSize)) + } } - } - signal.throwIfAborted() - await tx - .update(document) - .set({ - chunkCount: processed.metadata.chunkCount, - tokenCount: processed.metadata.tokenCount, - characterCount: processed.metadata.characterCount, - processingStatus: 'completed', - processingCompletedAt: now, - processingError: null, - /** A completed pass restores the retry allowance for a future failure. */ - processingAttempts: 0, - processingQueueToken: null, - processingQueuedAt: null, - processingDeferredUntil: null, - }) - .where( - and( - eq(document.id, documentId), - eq(document.processingStatus, 'processing'), - eq(document.processingStartedAt, processingStartedAt), - ...queueGenerationConditions(attemptContext), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) + signal.throwIfAborted() + await tx + .update(document) + .set({ + chunkCount: processed.metadata.chunkCount, + tokenCount: processed.metadata.tokenCount, + characterCount: processed.metadata.characterCount, + processingStatus: 'completed', + processingCompletedAt: now, + processingError: null, + /** A completed pass restores the retry allowance for a future failure. */ + processingAttempts: 0, + processingQueueToken: null, + processingQueuedAt: null, + processingDeferredUntil: null, + }) + .where( + and( + eq(document.id, documentId), + eq(document.processingStatus, 'processing'), + eq(document.processingStartedAt, processingStartedAt), + ...queueGenerationConditions(attemptContext), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt) + ) ) - ) - signal.throwIfAborted() - return true - }) - }, - { - opaqueInputSafe: - documentSecretContext.provenance.status === 'exact' && - documentSecretContext.provenance.entries.length === 0, - } - ), - Math.max(1, processingDeadlineAt - Date.now()), - 'Document processing', - attemptContext?.signal - ) + signal.throwIfAborted() + return true + }) + }, + { + opaqueInputSafe: + documentSecretContext.provenance.status === 'exact' && + documentSecretContext.provenance.entries.length === 0, + } + ), + Math.max(1, processingDeadlineAt - Date.now()), + 'Document processing', + attemptContext?.signal + ) - if (!processingCommitted) { - logger.info(`[${documentId}] Discarded output from an obsolete processing attempt`) - return - } + if (!processingCommitted) { + logger.info(`[${documentId}] Discarded output from an obsolete processing attempt`) + return + } - const processingTime = Date.now() - startTime - logger.info(`[${documentId}] Successfully processed document in ${processingTime}ms`) + const processingTime = Date.now() - startTime + logger.info(`[${documentId}] Successfully processed document in ${processingTime}ms`) - if (billableEmbeddingTokens > 0) { - try { - const costMultiplier = getCostMultiplier() - const { total: cost } = calculateCost( - embeddingPricingId, - billableEmbeddingTokens, - 0, - false, - costMultiplier - ) - if (cost > 0) { - /** - * Dedup identity for this embedding charge. `usage_log.event_key` is - * derived from `sourceReference` and guarded by a permanent unique - * index — usage_log rows are never pruned, there is no retention job - * — so the granularity has to separate two cases for all time: - * - * - A retry of the same pass must collapse. `knowledge-process-document` - * runs up to `KB_CONFIG_MAX_ATTEMPTS` attempts and the stale-document - * sweep can re-dispatch on top of that, so any per-attempt component - * (a `Date.now()` stamp, `processingStartedAt`) bills one indexing - * pass several times over. - * - A genuinely new pass must not collapse. A content change, a - * rehydrate, or a user-triggered reprocess pays a real embedding - * bill, and keying on `documentId` alone would suppress that charge - * permanently. - * - * `indexingPassId` is exactly that discriminator. Without one, the - * resolved pricing id is the safest fallback: it still collapses - * attempts and still re-bills a knowledge base whose embedding model - * changed. Token counts are deliberately left out — OCR-backed parsing - * is not bit-stable across attempts, so they would break the dedup - * they appear to sharpen. - */ - const usageSourceReference = [ - 'knowledge-document', - documentId, - indexingPassId ?? `model:${embeddingPricingId}`, - ].join(':') - await recordUsage({ - userId: documentActorUserId, - workspaceId: ctx.workspaceId ?? undefined, - ...toBillingContext(billingAttribution), - entries: [ - { - category: 'model', - source: 'knowledge-base', - description: embeddingModelName, - cost, - sourceReference: usageSourceReference, - metadata: { inputTokens: billableEmbeddingTokens, outputTokens: 0 }, - }, - ], - }) - await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) - } else { - logger.warn( - `[${documentId}] Embedding model "${embeddingModelName}" has no pricing entry — billing skipped`, - { billableEmbeddingTokens, embeddingModelName } + if (billableEmbeddingTokens > 0) { + try { + const costMultiplier = getCostMultiplier() + const { total: cost } = calculateCost( + embeddingPricingId, + billableEmbeddingTokens, + 0, + false, + costMultiplier ) + if (cost > 0) { + /** + * Dedup identity for this embedding charge. `usage_log.event_key` is + * derived from `sourceReference` and guarded by a permanent unique + * index — usage_log rows are never pruned, there is no retention job + * — so the granularity has to separate two cases for all time: + * + * - A retry of the same pass must collapse. `knowledge-process-document` + * runs up to `KB_CONFIG_MAX_ATTEMPTS` attempts and the stale-document + * sweep can re-dispatch on top of that, so any per-attempt component + * (a `Date.now()` stamp, `processingStartedAt`) bills one indexing + * pass several times over. + * - A genuinely new pass must not collapse. A content change, a + * rehydrate, or a user-triggered reprocess pays a real embedding + * bill, and keying on `documentId` alone would suppress that charge + * permanently. + * + * `indexingPassId` is exactly that discriminator. Without one, the + * resolved pricing id is the safest fallback: it still collapses + * attempts and still re-bills a knowledge base whose embedding model + * changed. Token counts are deliberately left out — OCR-backed parsing + * is not bit-stable across attempts, so they would break the dedup + * they appear to sharpen. + */ + const usageSourceReference = [ + 'knowledge-document', + documentId, + indexingPassId ?? `model:${embeddingPricingId}`, + ].join(':') + await recordUsage({ + userId: documentActorUserId, + workspaceId: ctx.workspaceId ?? undefined, + ...toBillingContext(billingAttribution), + entries: [ + { + category: 'model', + source: 'knowledge-base', + description: embeddingModelName, + cost, + sourceReference: usageSourceReference, + metadata: { inputTokens: billableEmbeddingTokens, outputTokens: 0 }, + }, + ], + }) + await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) + } else { + logger.warn( + `[${documentId}] Embedding model "${embeddingModelName}" has no pricing entry — billing skipped`, + { billableEmbeddingTokens, embeddingModelName } + ) + } + } catch (billingError) { + logger.error(`[${documentId}] Failed to record embedding usage`, { error: billingError }) } - } catch (billingError) { - logger.error(`[${documentId}] Failed to record embedding usage`, { error: billingError }) } - } + }) } catch (error) { const processingTime = Date.now() - startTime const embeddingQuotaExhausted = isEmbeddingQuotaExhaustion(error) diff --git a/apps/sim/lib/knowledge/documents/utils.test.ts b/apps/sim/lib/knowledge/documents/utils.test.ts index 36e742545a1..ee0858a5cc8 100644 --- a/apps/sim/lib/knowledge/documents/utils.test.ts +++ b/apps/sim/lib/knowledge/documents/utils.test.ts @@ -10,8 +10,13 @@ const { mockSecureFetchWithValidation } = vi.hoisted(() => ({ mockSecureFetchWithValidation: vi.fn(), })) -vi.mock('@/lib/core/security/input-validation.server', () => ({ +vi.mock('@/lib/core/security/input-validation.server', async (importOriginal) => ({ + ...(await importOriginal()), secureFetchWithValidation: mockSecureFetchWithValidation, + createSsrfGuardedFetchWithDispatcher: () => ({ + fetch: (...args: Parameters) => fetch(...args), + dispatcher: { close: vi.fn(), destroy: vi.fn() }, + }), })) import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' diff --git a/apps/sim/lib/knowledge/documents/utils.ts b/apps/sim/lib/knowledge/documents/utils.ts index ce8a6fbb321..a89de4c96ae 100644 --- a/apps/sim/lib/knowledge/documents/utils.ts +++ b/apps/sim/lib/knowledge/documents/utils.ts @@ -4,6 +4,7 @@ import { interruptibleSleep } from '@sim/utils/helpers' import { randomFloat } from '@sim/utils/random' import { parseRetryAfter } from '@sim/utils/retry' import { truncate } from '@sim/utils/string' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { redactSensitiveValues } from '@/lib/core/security/redaction' import { DEFAULT_MAX_ERROR_BODY_BYTES, @@ -604,6 +605,9 @@ export const VALIDATE_RETRY_OPTIONS: RetryOptions = { maxDelayMs: 10000, } +/** Untrusted retry URLs stay public; configured services supply their profile-aware fetcher. */ +let retryTransport: ReturnType | undefined + /** * Bounds requests and response bodies within one retry budget. */ @@ -612,6 +616,9 @@ export async function fetchWithRetry( options: RequestInit = {}, retryOptions: RetryOptions = {} ): Promise { + const fetcher = + retryOptions.fetcher ?? + (retryTransport ??= createSsrfGuardedFetchWithDispatcher({ profile: 'contentFetch' })).fetch const callerSignal = options.signal ? retryOptions.signal ? AbortSignal.any([options.signal, retryOptions.signal]) @@ -625,7 +632,7 @@ export async function fetchWithRetry( signal, AbortSignal.timeout(Math.max(0, Math.ceil(deadlineAt - Date.now()))), ]) - const response = await (retryOptions.fetcher ?? fetch)(url, { + const response = await fetcher(url, { ...options, signal: requestSignal, }) diff --git a/apps/sim/lib/knowledge/reranker.test.ts b/apps/sim/lib/knowledge/reranker.test.ts index 75ea03141a2..e078e031401 100644 --- a/apps/sim/lib/knowledge/reranker.test.ts +++ b/apps/sim/lib/knowledge/reranker.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { setupGlobalFetchMock } from '@sim/testing/mocks' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { @@ -8,6 +9,11 @@ import type { TokenBucketReservation, } from '@/lib/core/rate-limiter/storage/adapter' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const admission = vi.hoisted(() => ({ consume: vi.fn(), setCooldown: vi.fn(), diff --git a/apps/sim/lib/knowledge/reranker.ts b/apps/sim/lib/knowledge/reranker.ts index 56440caa2dc..db22ce59cde 100644 --- a/apps/sim/lib/knowledge/reranker.ts +++ b/apps/sim/lib/knowledge/reranker.ts @@ -10,6 +10,7 @@ import { recordProviderCooldown, waitForProviderAdmission, } from '@/lib/core/rate-limiter/provider-admission' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseJsonWithLimit, @@ -26,6 +27,8 @@ import { } from '@/lib/knowledge/model-input-provenance' import { isSupportedRerankerModel } from '@/lib/knowledge/reranker-models' +let providerTransport: ReturnType | undefined + const logger = createLogger('Reranker') const RERANK_OPERATION_TIMEOUT_MS = 30_000 @@ -159,7 +162,9 @@ export async function rerank( maxWaitMs: Math.max(0, deadlineAt - Date.now()), }) attempt += 1 - const res = await fetch('https://api.cohere.com/v2/rerank', { + const res = await (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch('https://api.cohere.com/v2/rerank', { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, diff --git a/apps/sim/lib/media/falai.ts b/apps/sim/lib/media/falai.ts index db4dc0a6152..066b262594a 100644 --- a/apps/sim/lib/media/falai.ts +++ b/apps/sim/lib/media/falai.ts @@ -4,6 +4,7 @@ import { isRecordLike } from '@sim/utils/object' import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' import { secureFetchWithPinnedIP, + secureFetchWithValidation, validateUrlWithDNS, } from '@/lib/core/security/input-validation.server' import { @@ -85,7 +86,10 @@ export async function runFalQueue( input: Record, apiKey: string ): Promise { - const createResponse = await fetch(`https://queue.fal.run/${endpoint}`, { + const createResponse = await secureFetchWithValidation(`https://queue.fal.run/${endpoint}`, { + profile: 'configuredEndpoint', + maxRedirects: 20, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, method: 'POST', headers: { Authorization: `Key ${apiKey}`, 'Content-Type': 'application/json' }, body: JSON.stringify(input), @@ -116,7 +120,12 @@ export async function runFalQueue( for (let attempt = 0; attempt < maxAttempts; attempt++) { await sleep(POLL_INTERVAL_MS) - const statusResponse = await fetch(statusUrl, { headers: { Authorization: `Key ${apiKey}` } }) + const statusResponse = await secureFetchWithValidation(statusUrl, { + profile: 'contentFetch', + maxRedirects: 20, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + headers: { Authorization: `Key ${apiKey}` }, + }) if (!statusResponse.ok) { const body = await readResponseTextWithLimit(statusResponse, { maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, @@ -138,9 +147,15 @@ export async function runFalQueue( if (statusData.error) { throw new Error(`Fal.ai generation failed: ${falErrorMessage(statusData.error)}`) } - const resultResponse = await fetch(getStringProp(statusData, 'response_url') || responseUrl, { - headers: { Authorization: `Key ${apiKey}` }, - }) + const resultResponse = await secureFetchWithValidation( + getStringProp(statusData, 'response_url') || responseUrl, + { + profile: 'contentFetch', + maxRedirects: 20, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + headers: { Authorization: `Key ${apiKey}` }, + } + ) if (!resultResponse.ok) { const body = await readResponseTextWithLimit(resultResponse, { maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, diff --git a/apps/sim/lib/messaging/email/providers/gmail.test.ts b/apps/sim/lib/messaging/email/providers/gmail.test.ts index 8147ab1e350..255cdb3bcfd 100644 --- a/apps/sim/lib/messaging/email/providers/gmail.test.ts +++ b/apps/sim/lib/messaging/email/providers/gmail.test.ts @@ -3,7 +3,7 @@ * * @vitest-environment node */ -import { resetEnvMock, setEnv } from '@sim/testing' +import { inputValidationMock, resetEnvMock, setEnv } from '@sim/testing' import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' afterAll(resetEnvMock) @@ -31,6 +31,11 @@ vi.mock('google-auth-library', () => ({ import { createGmailProvider } from '@/lib/messaging/email/providers/gmail' import type { ProcessedEmailData } from '@/lib/messaging/email/types' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const VALID_CREDENTIALS = JSON.stringify({ client_email: 'mailer@my-project.iam.gserviceaccount.com', private_key: '-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----\n', diff --git a/apps/sim/lib/messaging/email/providers/gmail.ts b/apps/sim/lib/messaging/email/providers/gmail.ts index 281b05dda19..a5240121b5a 100644 --- a/apps/sim/lib/messaging/email/providers/gmail.ts +++ b/apps/sim/lib/messaging/email/providers/gmail.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { JWT } from 'google-auth-library' import MailComposer from 'nodemailer/lib/mail-composer' import { env } from '@/lib/core/config/env' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import type { MailProvider, ProcessedEmailData, SendEmailResult } from '@/lib/messaging/email/types' const logger = createLogger('GmailMailProvider') @@ -114,14 +115,15 @@ export function createGmailProvider(): MailProvider | null { ) } - const response = await fetch(GMAIL_SEND_ENDPOINT, { + const response = await secureFetchWithValidation(GMAIL_SEND_ENDPOINT, { + profile: 'configuredEndpoint', + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, method: 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'message/rfc822', }, - // Buffer is a valid BodyInit at runtime; undici's types only admit ArrayBufferView - body: raw as BodyInit, + body: raw, }) if (!response.ok) { diff --git a/apps/sim/lib/oauth/connection-network.test.ts b/apps/sim/lib/oauth/connection-network.test.ts new file mode 100644 index 00000000000..60288efff55 --- /dev/null +++ b/apps/sim/lib/oauth/connection-network.test.ts @@ -0,0 +1,151 @@ +/** @vitest-environment node */ +import { resetEnvMock, setEnv } from '@sim/testing' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const { findAccount, updateAccount, processDraft } = vi.hoisted(() => ({ + findAccount: vi.fn(), + updateAccount: vi.fn(() => ({ where: vi.fn().mockResolvedValue(undefined) })), + processDraft: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ + db: { + query: { account: { findFirst: findAccount } }, + update: () => ({ set: updateAccount }), + }, +})) +vi.mock('@/lib/credentials/draft-processor', () => ({ processCredentialDraft: processDraft })) +vi.mock('@/lib/oauth/credential-service', () => ({ safeAccountInsert: vi.fn() })) + +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' +import { createGitHubRepositoriesProvider } from '@/lib/oauth/github-repositories' +import { exchangeMondayAuthorizationCode } from '@/lib/oauth/monday' +import { revokeQuickBooksToken } from '@/lib/oauth/quickbooks' +import { completeShopifyOAuthConnection } from '@/lib/oauth/shopify' + +function json(value: unknown) { + return new Response(JSON.stringify(value), { status: 200 }) +} + +describe('OAuth account lifecycle networking', () => { + beforeAll(async () => { + setEnv({ + OUTBOUND_ROUTING_SOURCE: 'env', + OUTBOUND_ROUTING_CONFIG: JSON.stringify({ + schemaVersion: 1, + revision: 'test', + defaultRoute: { kind: 'direct' }, + organizations: {}, + }), + }) + await expect(resolveCurrentOutboundRoute()).rejects.toThrow('MISSING_SCOPE') + }) + + beforeEach(() => { + vi.clearAllMocks() + }) + + afterEach(() => vi.unstubAllGlobals()) + afterAll(resetEnvMock) + + it('completes GitHub code exchange and identity verification without a resource scope', async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + json({ + access_token: 'ghu_access', + refresh_token: 'ghr_refresh', + expires_in: 28800, + refresh_token_expires_in: 15897600, + token_type: 'bearer', + scope: '', + }) + ) + .mockResolvedValueOnce(json({ id: 1234, login: 'octocat', type: 'User', name: 'Octocat' })) + .mockResolvedValueOnce(json([{ email: 'user@example.com', primary: true, verified: true }])) + vi.stubGlobal('fetch', fetchMock) + const provider = createGitHubRepositoriesProvider({ + clientId: 'app-client', + clientSecret: 'app-secret', + redirectURI: 'https://sim.example/callback', + }) + const tokens = await provider.getToken!({ + code: 'code', + codeVerifier: 'verifier', + redirectURI: 'https://sim.example/callback', + }) + await expect(provider.getUserInfo!(tokens)).resolves.toMatchObject({ + email: 'user@example.com', + emailVerified: true, + }) + expect(fetchMock).toHaveBeenCalledTimes(3) + }) + + it('completes Monday code exchange without a resource scope', async () => { + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + json({ + access_token: 'monday-access-token', + refresh_token: 'monday-refresh-token', + token_type: 'Bearer', + scope: 'me:read', + expires_in: 3600, + }) + ) + ) + await expect( + exchangeMondayAuthorizationCode({ + clientId: 'client-id', + clientSecret: 'client-secret', + code: 'code', + codeVerifier: 'verifier', + redirectUri: 'https://sim.example/callback', + }) + ).resolves.toMatchObject({ + accessToken: 'monday-access-token', + refreshToken: 'monday-refresh-token', + }) + }) + + it('revokes a user-owned QuickBooks account without inventing a resource scope', async () => { + const fetchMock = vi.fn().mockResolvedValue(json({})) + vi.stubGlobal('fetch', fetchMock) + await expect( + revokeQuickBooksToken('refresh-token', { + clientId: 'client-id', + clientSecret: 'client-secret', + environment: 'sandbox', + webhookVerifierToken: 'webhook-verifier', + }) + ).resolves.toBeUndefined() + expect(fetchMock).toHaveBeenCalledWith( + 'https://developer.api.intuit.com/v2/oauth2/tokens/revoke', + expect.objectContaining({ method: 'POST', body: JSON.stringify({ token: 'refresh-token' }) }) + ) + }) + + it('validates Shopify and completes its exact draft without a resource scope', async () => { + const signal = new AbortController().signal + const fetchMock = vi.fn().mockResolvedValue(json({ shop: { id: 42 } })) + vi.stubGlobal('fetch', fetchMock) + findAccount.mockResolvedValue({ id: 'existing-account' }) + await completeShopifyOAuthConnection({ + accessToken: 'shopify-access-token', + shopDomain: 'example.myshopify.com', + userId: 'user-1', + draftId: 'draft-from-state', + signal, + }) + expect(fetchMock).toHaveBeenCalledWith( + expect.stringContaining('example.myshopify.com/admin/api/'), + expect.objectContaining({ signal }) + ) + expect(processDraft).toHaveBeenCalledWith({ + draftId: 'draft-from-state', + userId: 'user-1', + providerId: 'shopify', + accountId: 'existing-account', + }) + }) +}) diff --git a/apps/sim/lib/oauth/credential-service.test.ts b/apps/sim/lib/oauth/credential-service.test.ts index d28450cba11..f78ce246d6e 100644 --- a/apps/sim/lib/oauth/credential-service.test.ts +++ b/apps/sim/lib/oauth/credential-service.test.ts @@ -3,7 +3,7 @@ */ import { generateKeyPairSync, verify } from 'node:crypto' import { account, credential } from '@sim/db/schema' -import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { dbChainMockFns, inputValidationMock, queueTableRows, resetDbChainMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -51,8 +51,7 @@ vi.mock('@/lib/oauth/microsoft', () => ({ PROACTIVE_REFRESH_THRESHOLD_DAYS: 7, })) -vi.mock('@/lib/oauth/oauth', () => ({ - OAUTH_PROVIDERS: {}, +vi.mock('@/lib/oauth/refresh-token.server', () => ({ refreshOAuthToken: mocks.refreshOAuthToken, })) @@ -83,6 +82,8 @@ import { } from '@/lib/oauth/credential-service' import { GOOGLE_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/oauth/types' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const RAW_CREDENTIAL_ID = 'credential-raw-secret-id' const RAW_ACCOUNT_ID = 'account-raw-secret-id' const RAW_USER_ID = 'user-raw-secret-id' diff --git a/apps/sim/lib/oauth/credential-service.ts b/apps/sim/lib/oauth/credential-service.ts index 33181593f2d..8d8d90e4889 100644 --- a/apps/sim/lib/oauth/credential-service.ts +++ b/apps/sim/lib/oauth/credential-service.ts @@ -8,6 +8,7 @@ import { withLeaderLock } from '@/lib/concurrency/leader-lock' import { coalesceLocally } from '@/lib/concurrency/singleflight' import { env } from '@/lib/core/config/env' import { decryptSecret } from '@/lib/core/security/encryption' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { isClientCredentialAccountProviderId } from '@/lib/credentials/client-credential-accounts/descriptors' import { getClientCredentialAccountMinter, @@ -35,9 +36,9 @@ import { isMicrosoftProvider, PROACTIVE_REFRESH_THRESHOLD_DAYS, } from '@/lib/oauth/microsoft' -import { refreshOAuthToken } from '@/lib/oauth/oauth' import { decryptQuickBooksOAuthClientConfig } from '@/lib/oauth/quickbooks-client-config' import { getOAuthRefreshCoordinationIdentity } from '@/lib/oauth/refresh-coordination' +import { refreshOAuthToken } from '@/lib/oauth/refresh-token.server' import { extractSlackTeamId, fanOutSlackTokenChain, @@ -61,6 +62,10 @@ import { slackBotCredentialVersion, } from '@/lib/slack-search/app-configuration' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('OAuthCredentialService') export interface CredentialTokenResolutionOptions { @@ -279,7 +284,7 @@ export async function getServiceAccountToken( const jwt = `${signingInput}.${signature}` - const response = await fetch(tokenUri, { + const response = await providerFetch(tokenUri, { method: 'POST', redirect: 'error', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, diff --git a/apps/sim/lib/oauth/github-installation-credential.test.ts b/apps/sim/lib/oauth/github-installation-credential.test.ts index 29657563997..ec6e1f0c888 100644 --- a/apps/sim/lib/oauth/github-installation-credential.test.ts +++ b/apps/sim/lib/oauth/github-installation-credential.test.ts @@ -1,6 +1,6 @@ /** @vitest-environment node */ import { credential } from '@sim/db/schema' -import { queueTableRows, resetDbChainMock } from '@sim/testing' +import { inputValidationMock, queueTableRows, resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -13,11 +13,14 @@ vi.mock('@/lib/oauth/github-installation', () => ({ parseGitHubInstallationBinding: mocks.parseBinding, resolveGitHubInstallationAccessToken: mocks.resolveToken, })) -vi.mock('@/lib/oauth/oauth', () => ({ OAUTH_PROVIDERS: {}, refreshOAuthToken: vi.fn() })) +vi.mock('@/lib/oauth/oauth', () => ({ OAUTH_PROVIDERS: {} })) +vi.mock('@/lib/oauth/refresh-token.server', () => ({ refreshOAuthToken: vi.fn() })) import { resolveServiceAccountToken } from '@/lib/oauth/credential-service' import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const row = { type: 'service_account', providerId: GITHUB_INSTALLATION_PROVIDER_ID, diff --git a/apps/sim/lib/oauth/github-installation.test.ts b/apps/sim/lib/oauth/github-installation.test.ts index c49835e6514..01b5a019d74 100644 --- a/apps/sim/lib/oauth/github-installation.test.ts +++ b/apps/sim/lib/oauth/github-installation.test.ts @@ -1,6 +1,6 @@ /** @vitest-environment node */ import { generateKeyPairSync, verify } from 'node:crypto' -import { resetEnvMock, setEnv } from '@sim/testing' +import { inputValidationMock, resetEnvMock, setEnv } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { assertGitHubInstallationActive, @@ -15,6 +15,8 @@ import { } from '@/lib/oauth/github-installation' import type { GitHubInstallationBinding } from '@/lib/oauth/github-installation-types' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const { privateKey, publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }) const privateKeyPem = privateKey.export({ type: 'pkcs8', format: 'pem' }).toString() const user = { id: 9, type: 'User' } diff --git a/apps/sim/lib/oauth/github-installation.ts b/apps/sim/lib/oauth/github-installation.ts index 270ba839567..8f7cf002181 100644 --- a/apps/sim/lib/oauth/github-installation.ts +++ b/apps/sim/lib/oauth/github-installation.ts @@ -1,6 +1,7 @@ import { createHash, createPrivateKey, createSign } from 'node:crypto' import { z } from 'zod' import { env } from '@/lib/core/config/env' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import type { GitHubInstallationBinding, @@ -9,6 +10,10 @@ import type { } from '@/lib/oauth/github-installation-types' import { parseGitHubRepository } from '@/lib/oauth/github-repository' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const API_URL = 'https://api.github.com' const PAGE_SIZE = 100 const MAX_PAGES = 10 @@ -158,7 +163,7 @@ async function request( operation: GitHubInstallationOperation = 'installation' ): Promise { const requestSignal = AbortSignal.any([signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)]) - const response = await fetch(`${API_URL}${path}`, { + const response = await providerFetch(`${API_URL}${path}`, { method: body === undefined ? 'GET' : 'POST', headers: { Accept: 'application/vnd.github+json', diff --git a/apps/sim/lib/oauth/github-repositories.test.ts b/apps/sim/lib/oauth/github-repositories.test.ts index c2820879cc1..4b81ccf4b19 100644 --- a/apps/sim/lib/oauth/github-repositories.test.ts +++ b/apps/sim/lib/oauth/github-repositories.test.ts @@ -1,5 +1,7 @@ /** @vitest-environment node */ -import { resetEnvMock, setEnv } from '@sim/testing' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + +import { inputValidationMock, resetEnvMock, setEnv } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { getManagedOAuthConnectorPolicy } from '@/lib/auth/connectors/managed-oauth' import { buildConnectorProviders } from '@/lib/auth/connectors/providers' @@ -9,7 +11,7 @@ import { verifyGitHubRepositoriesIdentity, } from '@/lib/oauth/github-repositories' import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' -import { refreshOAuthToken } from '@/lib/oauth/oauth' +import { refreshOAuthToken } from '@/lib/oauth/refresh-token.server' const tokenResponse = { access_token: 'ghu_access', diff --git a/apps/sim/lib/oauth/oauth.test.ts b/apps/sim/lib/oauth/oauth.test.ts index b4ee51d2a42..017c6be0cfe 100644 --- a/apps/sim/lib/oauth/oauth.test.ts +++ b/apps/sim/lib/oauth/oauth.test.ts @@ -1,4 +1,4 @@ -import { createMockFetch, resetEnvMock, setEnv } from '@sim/testing' +import { createMockFetch, inputValidationMock, resetEnvMock, setEnv } from '@sim/testing' import { getOAuth2Tokens } from 'better-auth/oauth2' import { genericOAuth } from 'better-auth/plugins' import { getTestInstance } from 'better-auth/test' @@ -76,10 +76,12 @@ import { getPerRequestOAuthLinkScopes, getSlackApprovalGatedScopes, OAUTH_PROVIDERS, - refreshOAuthToken, } from '@/lib/oauth' +import { refreshOAuthToken } from '@/lib/oauth/refresh-token.server' import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + /** Compares real icon components by identity; the global `@/components/icons` stub in vitest.setup.ts would make that vacuous. */ vi.unmock('@/components/icons') diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index 1d6891b9d6c..57ff8cabd61 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -1,5 +1,3 @@ -import { createLogger } from '@sim/logger' -import { toError } from '@sim/utils/errors' import { AirtableIcon, AsanaIcon, @@ -67,37 +65,13 @@ import { ZohoDeskIcon, ZoomIcon, } from '@/components/icons' -import { env } from '@/lib/core/config/env' -import { - type OAuthClientCapabilityField, - type OAuthClientCapabilityId, - requireOAuthClientCapability, -} from '@/lib/core/config/env-capabilities' import { isSlackExtendedScopesEnabled } from '@/lib/core/config/env-flags' -import { redactExactSensitiveValues } from '@/lib/core/security/redaction' -import { - DEFAULT_MAX_ERROR_BODY_BYTES, - readResponseTextWithLimit, -} from '@/lib/core/utils/stream-limits' -import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' -import { - GITHUB_TOKEN_URL, - parseGitHubRepositoriesTokenResponse, -} from '@/lib/oauth/github-repositories' -import { parseInstagramLongLivedToken } from '@/lib/oauth/instagram' -import { MONDAY_OAUTH_TOKEN_URL, resolveMondayAccessTokenExpiresAt } from '@/lib/oauth/monday' -import type { QuickBooksOAuthClientConfig } from '@/lib/oauth/quickbooks-client-config' -import { QUICKBOOKS_TOKEN_URL } from '@/lib/oauth/quickbooks-constants' import { SALESFORCE_ADDITIONAL_PROVIDER_IDS, - SALESFORCE_LOGIN_HOSTS, SALESFORCE_PROVIDER_ID_LABELS, } from '@/lib/oauth/salesforce' -import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' -import type { OAuthProviderConfig } from './types' - -const logger = createLogger('OAuth') +import type { OAuthProviderConfig } from '@/lib/oauth/types' /** * Slack scopes requested only where the app is approved for them, gated by @@ -1522,967 +1496,3 @@ export const OAUTH_PROVIDERS: Record = { defaultService: 'spotify', }, } - -interface ProviderAuthConfig { - tokenEndpoint: string - clientId: string - clientSecret: string - useBasicAuth: boolean - additionalHeaders?: Record - supportsRefreshTokenRotation?: boolean - /** - * If true, the refresh token is sent in the Authorization header as Bearer token - * instead of in the request body. Used by Cal.com. - */ - refreshTokenInAuthHeader?: boolean - /** - * If true, the token endpoint expects a JSON body with Content-Type: application/json - * instead of the default application/x-www-form-urlencoded. Used by Notion. - */ - useJsonBody?: boolean - /** - * Token refresh strategy. `instagram_long_lived` uses Meta's GET - * `refresh_access_token?grant_type=ig_refresh_token` flow instead of a - * standard OAuth refresh_token POST. - */ - refreshStrategy?: 'standard' | 'instagram_long_lived' - /** - * Body param name to use for the client identifier instead of the standard `client_id`. - * TikTok requires `client_key` instead. - */ - clientIdParamName?: string -} - -function getConfiguredClientCredentials( - providerId: TCapabilityId, - clientIdField: NoInfer>, - clientSecretField?: NoInfer> -): Pick { - const { values } = requireOAuthClientCapability(providerId, env) - return { - clientId: values[clientIdField], - clientSecret: clientSecretField ? values[clientSecretField] : '', - } -} - -/** - * Get OAuth provider configuration for token refresh - */ -function getProviderAuthConfig( - provider: string, - clientOverride?: Pick -): ProviderAuthConfig { - if (clientOverride && provider !== 'quickbooks') { - throw new Error(`OAuth client override is not supported for provider ${provider}`) - } - switch (provider) { - case 'google': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'google', - 'GOOGLE_CLIENT_ID', - 'GOOGLE_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://oauth2.googleapis.com/token', - clientId, - clientSecret, - useBasicAuth: false, - } - } - case 'x': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'x', - 'X_CLIENT_ID', - 'X_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.x.com/2/oauth2/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'tiktok': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'tiktok', - 'TIKTOK_CLIENT_ID', - 'TIKTOK_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://open.tiktokapis.com/v2/oauth/token/', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - // TikTok requires `client_key` in the token request body instead of `client_id`. - clientIdParamName: 'client_key', - } - } - case 'confluence': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'confluence', - 'CONFLUENCE_CLIENT_ID', - 'CONFLUENCE_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://auth.atlassian.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'jira': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'jira', - 'JIRA_CLIENT_ID', - 'JIRA_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://auth.atlassian.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'calcom': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'calcom', - 'CALCOM_CLIENT_ID' - ) - return { - tokenEndpoint: 'https://app.cal.com/api/auth/oauth/refreshToken', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - // Cal.com requires refresh token in Authorization header, not body - refreshTokenInAuthHeader: true, - } - } - case 'airtable': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'airtable', - 'AIRTABLE_CLIENT_ID', - 'AIRTABLE_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://airtable.com/oauth2/v1/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'bitbucket': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'bitbucket', - 'BITBUCKET_CLIENT_ID', - 'BITBUCKET_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://bitbucket.org/site/oauth2/access_token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'github-repositories': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'github-repositories', - 'GITHUB_APP_CLIENT_ID', - 'GITHUB_APP_CLIENT_SECRET' - ) - return { - tokenEndpoint: GITHUB_TOKEN_URL, - clientId, - clientSecret, - useBasicAuth: false, - additionalHeaders: { Accept: 'application/json' }, - supportsRefreshTokenRotation: true, - } - } - case 'notion': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'notion', - 'NOTION_CLIENT_ID', - 'NOTION_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.notion.com/v1/oauth/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - useJsonBody: true, - } - } - case 'microsoft': - case 'outlook': - case 'onedrive': - case 'sharepoint': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'microsoft', - 'MICROSOFT_CLIENT_ID', - 'MICROSOFT_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://login.microsoftonline.com/common/oauth2/v2.0/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'clickup': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'clickup', - 'CLICKUP_CLIENT_ID', - 'CLICKUP_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.clickup.com/api/v2/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'linear': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'linear', - 'LINEAR_CLIENT_ID', - 'LINEAR_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.linear.app/oauth/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'attio': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'attio', - 'ATTIO_CLIENT_ID', - 'ATTIO_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://app.attio.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - } - } - case 'box': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'box', - 'BOX_CLIENT_ID', - 'BOX_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.box.com/oauth2/token', - clientId, - clientSecret, - useBasicAuth: false, - // Box refresh tokens are single-use: "the Refresh Token is invalidated and a - // new Refresh Token is returned" and "A Refresh Token is valid for 60 days and - // can be used to obtain a new Access Token and Refresh Token only once." - // (developer.box.com/guides/authentication/tokens/refresh). Without rotation the - // new token is discarded and the credential dies on the second refresh. - supportsRefreshTokenRotation: true, - } - } - case 'docusign': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'docusign', - 'DOCUSIGN_CLIENT_ID', - 'DOCUSIGN_CLIENT_SECRET' - ) - return { - tokenEndpoint: getDocusignOAuthUrl('/oauth/token'), - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'dropbox': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'dropbox', - 'DROPBOX_CLIENT_ID', - 'DROPBOX_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.dropboxapi.com/oauth2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'slack': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'slack', - 'SLACK_CLIENT_ID', - 'SLACK_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://slack.com/api/oauth.v2.access', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'reddit': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'reddit', - 'REDDIT_CLIENT_ID', - 'REDDIT_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://www.reddit.com/api/v1/access_token', - clientId, - clientSecret, - useBasicAuth: true, - additionalHeaders: { - 'User-Agent': REDDIT_USER_AGENT, - }, - } - } - case 'wealthbox': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'wealthbox', - 'WEALTHBOX_CLIENT_ID', - 'WEALTHBOX_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://app.crmworkspace.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'webflow': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'webflow', - 'WEBFLOW_CLIENT_ID', - 'WEBFLOW_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.webflow.com/oauth/access_token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'asana': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'asana', - 'ASANA_CLIENT_ID', - 'ASANA_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://app.asana.com/-/oauth_token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'pipedrive': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'pipedrive', - 'PIPEDRIVE_CLIENT_ID', - 'PIPEDRIVE_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://oauth.pipedrive.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'quickbooks': { - if (!clientOverride) { - throw new Error('QuickBooks OAuth client configuration is missing') - } - return { - tokenEndpoint: QUICKBOOKS_TOKEN_URL, - clientId: clientOverride.clientId, - clientSecret: clientOverride.clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'hubspot': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'hubspot', - 'HUBSPOT_CLIENT_ID', - 'HUBSPOT_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://api.hubapi.com/oauth/v1/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'linkedin': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'linkedin', - 'LINKEDIN_CLIENT_ID', - 'LINKEDIN_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://www.linkedin.com/oauth/v2/accessToken', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'instagram': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'instagram', - 'INSTAGRAM_CLIENT_ID', - 'INSTAGRAM_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://graph.instagram.com/refresh_access_token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - refreshStrategy: 'instagram_long_lived', - } - } - case 'salesforce': - case 'salesforce-sandbox': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'salesforce', - 'SALESFORCE_CLIENT_ID', - 'SALESFORCE_CLIENT_SECRET' - ) - // A refresh token is only redeemable at the authorization server that - // issued it: a sandbox token posted to login.salesforce.com fails with - // `invalid_grant`. One Connected App's consumer key is valid at both - // hosts, so only the endpoint differs. - return { - tokenEndpoint: `https://${SALESFORCE_LOGIN_HOSTS[provider]}/services/oauth2/token`, - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: true, - } - } - case 'shopify': { - // Shopify access tokens don't expire and don't support refresh tokens - // This configuration is provided for completeness but won't be used for token refresh - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'shopify', - 'SHOPIFY_CLIENT_ID', - 'SHOPIFY_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://accounts.shopify.com/oauth/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'zoom': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'zoom', - 'ZOOM_CLIENT_ID', - 'ZOOM_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://zoom.us/oauth/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: true, - } - } - case 'wordpress': { - // WordPress.com does NOT support refresh tokens - // Users will need to re-authorize when tokens expire (~2 weeks) - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'wordpress', - 'WORDPRESS_CLIENT_ID', - 'WORDPRESS_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://public-api.wordpress.com/oauth2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'spotify': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'spotify', - 'SPOTIFY_CLIENT_ID', - 'SPOTIFY_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://accounts.spotify.com/api/token', - clientId, - clientSecret, - useBasicAuth: true, - supportsRefreshTokenRotation: false, - } - } - case 'monday': { - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'monday', - 'MONDAY_CLIENT_ID', - 'MONDAY_CLIENT_SECRET' - ) - return { - tokenEndpoint: MONDAY_OAUTH_TOKEN_URL, - clientId, - clientSecret, - useBasicAuth: false, - useJsonBody: true, - supportsRefreshTokenRotation: true, - } - } - case 'manageengine-sdp': { - // ServiceDesk Plus Cloud authenticates through Zoho, so the grant is the - // same one Zoho Desk uses and shares its client credentials: scopes are - // chosen per authorization request, not per API-console client, so one - // registered client serves both products. - // - // Rotation stays off for the same reason as zoho-desk below - Zoho's - // refresh_token grant returns a new access token but no new refresh token. - // accounts.zoho.com is correct because the authorize and code-exchange - // legs in lib/auth/connectors/providers.ts are pinned to the US accounts - // server, so every refresh token in the system is US-issued. Data - // residency for API calls is honored separately, via the block's data - // center selector. - // Keyed on the `zoho-desk` capability, which is what - // `resolveOAuthClientCapabilityId('manageengine-sdp')` aliases to — the - // capability names the env pair, not the product. - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'zoho-desk', - 'ZOHO_CLIENT_ID', - 'ZOHO_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - case 'zoho-desk': { - // Zoho's refresh_token grant returns a new access token but no new refresh - // token, so rotation stays off (the existing refresh token is preserved). - // The refresh must target the accounts server of the data center that issued - // the token - "if location=eu, you will need to make access token request to - // https://accounts.zoho.eu" (zoho.com/accounts/protocol/oauth/multi-dc.html). - // accounts.zoho.com is correct here because the authorize and code-exchange - // legs in lib/auth/connectors/providers.ts are also pinned to the US accounts - // server, so every refresh token in the system is US-issued. Making refresh - // DC-aware requires making the grant DC-aware first (read the `accounts-server` - // callback param) and threading the credential's persisted `__zoho_domain__` - // marker into refreshOAuthToken, which today only receives the token string. - // Data residency for API calls is already honored via that persisted Desk base. - const { clientId, clientSecret } = getConfiguredClientCredentials( - 'zoho-desk', - 'ZOHO_CLIENT_ID', - 'ZOHO_CLIENT_SECRET' - ) - return { - tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', - clientId, - clientSecret, - useBasicAuth: false, - supportsRefreshTokenRotation: false, - } - } - default: - throw new Error(`Unsupported provider: ${provider}`) - } -} - -/** - * Build the authentication request headers and body for OAuth token refresh - */ -function buildAuthRequest( - config: ProviderAuthConfig, - refreshToken: string -): { headers: Record; bodyParams: Record; useJsonBody?: boolean } { - const headers: Record = { - 'Content-Type': config.useJsonBody ? 'application/json' : 'application/x-www-form-urlencoded', - ...config.additionalHeaders, - } - - const bodyParams: Record = { - grant_type: 'refresh_token', - } - - // Handle refresh token placement - if (config.refreshTokenInAuthHeader) { - // Cal.com style: refresh token in Authorization header as Bearer token - headers.Authorization = `Bearer ${refreshToken}` - } else { - // Standard OAuth: refresh token in request body - bodyParams.refresh_token = refreshToken - } - - if (config.useBasicAuth) { - // Use Basic Authentication - credentials in Authorization header only - const basicAuth = Buffer.from(`${config.clientId}:${config.clientSecret}`).toString('base64') - headers.Authorization = `Basic ${basicAuth}` - } else { - // Use body credentials - include client credentials in request body - bodyParams[config.clientIdParamName || 'client_id'] = config.clientId - if (config.clientSecret) { - bodyParams.client_secret = config.clientSecret - } - } - - return { headers, bodyParams, useJsonBody: config.useJsonBody } -} - -/** - * Resolves the key {@link getProviderAuthConfig} is switched on for a stored - * credential's provider id. - * - * Normally that is the base provider, because every service in a family - * refreshes against the same endpoint with the same client. A provider id - * listed in a service's `additionalProviderIds` is the exception: it names a - * *different* authorization server for the same service, so it must reach - * `getProviderAuthConfig` intact — collapsing it to the base would silently - * refresh a sandbox token against the production endpoint. - */ -function getBaseProviderForService(providerId: string): string { - if (providerId in OAUTH_PROVIDERS) { - return providerId - } - - for (const [baseProvider, config] of Object.entries(OAUTH_PROVIDERS)) { - for (const service of Object.values(config.services)) { - if (service.providerId === providerId) { - return baseProvider - } - if (service.additionalProviderIds?.includes(providerId)) { - return providerId - } - } - } - - throw new Error(`Unknown OAuth provider: ${providerId}`) -} - -export interface RefreshTokenSuccess { - ok: true - accessToken: string - expiresIn: number - refreshToken: string - refreshTokenExpiresIn?: number -} - -export interface RefreshTokenFailure { - ok: false - errorCode?: string - message?: string -} - -export type RefreshTokenResult = RefreshTokenSuccess | RefreshTokenFailure - -function extractErrorCode(value: unknown): string | undefined { - if (value && typeof value === 'object' && 'error' in value) { - const error = (value as { error: unknown }).error - if (typeof error === 'string') return error - if (error && typeof error === 'object' && 'code' in error) { - const code = (error as { code: unknown }).code - if (typeof code === 'string' || typeof code === 'number') return String(code) - } - } - return undefined -} - -function safeOAuthErrorCode(value: unknown, secrets: string[]): string | undefined { - const errorCode = extractErrorCode(value) - if (!errorCode) return undefined - const safeCode = redactExactSensitiveValues(errorCode, secrets).trim().toLowerCase() - return /^[a-z0-9][a-z0-9._:-]{0,127}$/.test(safeCode) ? safeCode : undefined -} - -/** - * Hard deadline on the token-endpoint exchange. This function does not coalesce - * on its own; its sole production caller (`performCoalescedRefresh` in the OAuth - * utils) shares one in-flight refresh across concurrent callers for a credential. - * Without this bound a hung endpoint would wedge every joiner on that key until - * the undici socket defaults (~5 min) gave up. - */ -const TOKEN_REFRESH_TIMEOUT_MS = 15_000 - -function parseOAuthResponse(responseText: string): unknown { - try { - return JSON.parse(responseText) - } catch { - return responseText - } -} - -function oauthResponseRecord(value: unknown): Record | undefined { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as Record) - : undefined -} - -const OAUTH_RESPONSE_OMITTED = '[token endpoint response omitted]' - -async function refreshInstagramLongLivedToken( - config: ProviderAuthConfig, - longLivedToken: string, - providerId: string -): Promise { - const url = new URL(config.tokenEndpoint) - url.searchParams.set('grant_type', 'ig_refresh_token') - url.searchParams.set('access_token', longLivedToken) - - const response = await fetch(url.toString(), { - method: 'GET', - redirect: 'error', - signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS), - }) - - const responseText = await readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'Instagram token refresh response', - }) - const responseData = parseOAuthResponse(responseText) - - if (!response.ok) { - const exactSecrets = [longLivedToken, config.clientSecret ?? ''] - const errorCode = safeOAuthErrorCode(responseData, exactSecrets) - logger.error('Instagram long-lived token refresh failed:', { - status: response.status, - error: OAUTH_RESPONSE_OMITTED, - errorCode, - providerId, - tokenEndpoint: config.tokenEndpoint, - }) - return { - ok: false, - errorCode, - message: `Failed to refresh token: ${response.status} ${OAUTH_RESPONSE_OMITTED}`, - } - } - - const payload = parseInstagramLongLivedToken(responseData) - if (!payload) { - logger.warn('Invalid Instagram refresh response', { providerId }) - return { ok: false, message: 'Invalid Instagram token refresh response' } - } - - logger.info('Instagram long-lived token refreshed successfully', { - expiresIn: payload.expires_in, - providerId, - }) - - // Instagram returns a new long-lived token; store it as both access and refresh. - return { - ok: true, - accessToken: payload.access_token, - expiresIn: payload.expires_in, - refreshToken: payload.access_token, - } -} - -export async function refreshOAuthToken( - providerId: string, - refreshToken: string, - clientOverride?: Pick -): Promise { - const exactSecrets = [refreshToken] - try { - const provider = getBaseProviderForService(providerId) - - const config = getProviderAuthConfig(provider, clientOverride) - if (config.clientSecret) exactSecrets.push(config.clientSecret) - - if (config.refreshStrategy === 'instagram_long_lived') { - return await refreshInstagramLongLivedToken(config, refreshToken, providerId) - } - - const { headers, bodyParams, useJsonBody } = buildAuthRequest(config, refreshToken) - - const response = await fetch(config.tokenEndpoint, { - method: 'POST', - headers, - body: useJsonBody ? JSON.stringify(bodyParams) : new URLSearchParams(bodyParams).toString(), - redirect: 'error', - signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS), - }) - - const responseText = await readResponseTextWithLimit(response, { - maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, - label: 'OAuth token refresh response', - }) - const responseData = parseOAuthResponse(responseText) - - if (!response.ok) { - const errorCode = safeOAuthErrorCode(responseData, exactSecrets) - - logger.error('Token refresh failed:', { - status: response.status, - error: OAUTH_RESPONSE_OMITTED, - errorCode, - providerId, - tokenEndpoint: config.tokenEndpoint, - hasClientId: !!config.clientId, - hasClientSecret: !!config.clientSecret, - hasRefreshToken: !!refreshToken, - }) - return { - ok: false, - errorCode, - message: `Failed to refresh token: ${response.status} ${OAUTH_RESPONSE_OMITTED}`, - } - } - - const data = oauthResponseRecord(responseData) - if (!data) { - logger.warn('Invalid OAuth token refresh response', { providerId }) - return { ok: false, message: 'Invalid OAuth token refresh response' } - } - - if ( - data.ok === false || - (provider === 'github-repositories' && typeof data.error === 'string') - ) { - const errorCode = safeOAuthErrorCode(data, exactSecrets) - logger.error('Token refresh failed:', { - status: response.status, - error: OAUTH_RESPONSE_OMITTED, - errorCode, - providerId, - tokenEndpoint: config.tokenEndpoint, - hasClientId: !!config.clientId, - hasClientSecret: !!config.clientSecret, - hasRefreshToken: !!refreshToken, - }) - return { - ok: false, - errorCode, - message: `Failed to refresh token: ${OAUTH_RESPONSE_OMITTED}`, - } - } - - if (provider === 'github-repositories') { - const tokens = parseGitHubRepositoriesTokenResponse(data) - return { - ok: true, - accessToken: tokens.access_token, - refreshToken: tokens.refresh_token, - expiresIn: tokens.expires_in, - refreshTokenExpiresIn: tokens.refresh_token_expires_in, - } - } - - const accessToken = - typeof data.access_token === 'string' && data.access_token.length > 0 - ? data.access_token - : undefined - - let newRefreshToken: string | undefined - if ( - config.supportsRefreshTokenRotation && - typeof data.refresh_token === 'string' && - data.refresh_token.length > 0 - ) { - newRefreshToken = data.refresh_token - logger.info(`Received new refresh token from ${provider}`) - } - if (provider === 'monday' && !newRefreshToken) { - logger.warn('Monday token refresh response omitted its rotating refresh token') - return { ok: false, message: 'Invalid Monday token refresh response' } - } - if (provider === 'quickbooks' && !newRefreshToken) { - logger.warn('QuickBooks token refresh response omitted its rotating refresh token') - return { ok: false, message: 'Invalid QuickBooks token refresh response' } - } - - const rawExpiresIn = data.expires_in ?? data.expiresIn - const parsedExpiresIn = - typeof rawExpiresIn === 'number' || typeof rawExpiresIn === 'string' - ? Number(rawExpiresIn) - : Number.NaN - const responseExpiresIn = - Number.isFinite(parsedExpiresIn) && parsedExpiresIn > 0 ? parsedExpiresIn : undefined - const expiresIn = - provider === 'monday' && accessToken - ? Math.max( - 1, - Math.ceil( - (resolveMondayAccessTokenExpiresAt(accessToken, responseExpiresIn).getTime() - - Date.now()) / - 1000 - ) - ) - : (responseExpiresIn ?? 3600) - - const rawRefreshTokenExpiresIn = data.x_refresh_token_expires_in - const parsedRefreshTokenExpiresIn = - typeof rawRefreshTokenExpiresIn === 'number' || typeof rawRefreshTokenExpiresIn === 'string' - ? Number(rawRefreshTokenExpiresIn) - : Number.NaN - const refreshTokenExpiresIn = - provider === 'quickbooks' && - Number.isSafeInteger(parsedRefreshTokenExpiresIn) && - parsedRefreshTokenExpiresIn > 0 - ? parsedRefreshTokenExpiresIn - : undefined - - if (!accessToken) { - // Log only the shape, never `data` itself - on a partial success it can - // carry live tokens. - logger.warn('No access token found in refresh response', { - providerId, - responseKeys: Object.keys(data ?? {}), - }) - return { ok: false, message: 'No access token in refresh response' } - } - - logger.info('Token refreshed successfully with expiration', { - expiresIn, - hasNewRefreshToken: !!newRefreshToken, - provider, - }) - - return { - ok: true, - accessToken, - expiresIn, - refreshToken: newRefreshToken ?? refreshToken, - ...(refreshTokenExpiresIn ? { refreshTokenExpiresIn } : {}), - } - } catch (error) { - const normalized = toError(error) - const message = - normalized.name === 'PayloadSizeLimitError' || normalized.message.startsWith('OAuth client ') - ? normalized.message - : 'Token refresh failed' - logger.error('Error refreshing token', { errorType: normalized.name }) - return { ok: false, message } - } -} diff --git a/apps/sim/lib/oauth/quickbooks.test.ts b/apps/sim/lib/oauth/quickbooks.test.ts index c415201976e..469e9f0bbc4 100644 --- a/apps/sim/lib/oauth/quickbooks.test.ts +++ b/apps/sim/lib/oauth/quickbooks.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch } = vi.hoisted(() => ({ mockFetch: vi.fn() })) @@ -20,6 +21,8 @@ import { } from '@/lib/oauth/quickbooks' import { deriveQuickBooksWebhookAppKey } from '@/lib/oauth/quickbooks-client-config' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + describe('QuickBooks account identity', () => { it('round-trips an opaque OpenID subject without narrowing its valid punctuation', () => { const accountId = createQuickBooksAccountId('123456789', 'issuer:subject', CLIENT_CONFIG) diff --git a/apps/sim/lib/oauth/quickbooks.ts b/apps/sim/lib/oauth/quickbooks.ts index 2f4f9b6d840..901e15498f1 100644 --- a/apps/sim/lib/oauth/quickbooks.ts +++ b/apps/sim/lib/oauth/quickbooks.ts @@ -1,3 +1,4 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, @@ -19,6 +20,10 @@ import { type QuickBooksEnvironment, } from '@/tools/quickbooks/client' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const QUICKBOOKS_ACCOUNT_PREFIX = 'quickbooks:v2:' const QUICKBOOKS_REVOCATION_URL = 'https://developer.api.intuit.com/v2/oauth2/tokens/revoke' const QUICKBOOKS_MAX_REVOCATION_ERROR_BYTES = 64 * 1024 @@ -183,7 +188,7 @@ export async function fetchQuickBooksConnectionProfile( clientConfig: Pick ): Promise { const realmId = normalizeQuickBooksRealmId(callbackRealmId) - const response = await fetch(getQuickBooksUserInfoUrl(clientConfig.environment), { + const response = await providerFetch(getQuickBooksUserInfoUrl(clientConfig.environment), { headers: buildQuickBooksHeaders(accessToken), signal: AbortSignal.timeout(QUICKBOOKS_OAUTH_REQUEST_TIMEOUT_MS), }) @@ -254,7 +259,7 @@ export async function exchangeQuickBooksAuthorizationCode(params: { signal?: AbortSignal }): Promise { const clientConfig = normalizeQuickBooksOAuthClientConfig(params.clientConfig) - const response = await fetch(QUICKBOOKS_TOKEN_URL, { + const response = await providerFetch(QUICKBOOKS_TOKEN_URL, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/lib/oauth/refresh-token.server.ts b/apps/sim/lib/oauth/refresh-token.server.ts new file mode 100644 index 00000000000..ebe82069658 --- /dev/null +++ b/apps/sim/lib/oauth/refresh-token.server.ts @@ -0,0 +1,992 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { env } from '@/lib/core/config/env' +import { + type OAuthClientCapabilityField, + type OAuthClientCapabilityId, + requireOAuthClientCapability, +} from '@/lib/core/config/env-capabilities' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' +import { redactExactSensitiveValues } from '@/lib/core/security/redaction' +import { + DEFAULT_MAX_ERROR_BODY_BYTES, + readResponseTextWithLimit, +} from '@/lib/core/utils/stream-limits' +import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' +import { + GITHUB_TOKEN_URL, + parseGitHubRepositoriesTokenResponse, +} from '@/lib/oauth/github-repositories' +import { parseInstagramLongLivedToken } from '@/lib/oauth/instagram' +import { MONDAY_OAUTH_TOKEN_URL, resolveMondayAccessTokenExpiresAt } from '@/lib/oauth/monday' +import { OAUTH_PROVIDERS } from '@/lib/oauth/oauth' +import type { QuickBooksOAuthClientConfig } from '@/lib/oauth/quickbooks-client-config' +import { QUICKBOOKS_TOKEN_URL } from '@/lib/oauth/quickbooks-constants' +import { SALESFORCE_LOGIN_HOSTS } from '@/lib/oauth/salesforce' +import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' + +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + +const logger = createLogger('OAuth') + +interface ProviderAuthConfig { + tokenEndpoint: string + clientId: string + clientSecret: string + useBasicAuth: boolean + additionalHeaders?: Record + supportsRefreshTokenRotation?: boolean + /** + * If true, the refresh token is sent in the Authorization header as Bearer token + * instead of in the request body. Used by Cal.com. + */ + refreshTokenInAuthHeader?: boolean + /** + * If true, the token endpoint expects a JSON body with Content-Type: application/json + * instead of the default application/x-www-form-urlencoded. Used by Notion. + */ + useJsonBody?: boolean + /** + * Token refresh strategy. `instagram_long_lived` uses Meta's GET + * `refresh_access_token?grant_type=ig_refresh_token` flow instead of a + * standard OAuth refresh_token POST. + */ + refreshStrategy?: 'standard' | 'instagram_long_lived' + /** + * Body param name to use for the client identifier instead of the standard `client_id`. + * TikTok requires `client_key` instead. + */ + clientIdParamName?: string +} + +function getConfiguredClientCredentials( + providerId: TCapabilityId, + clientIdField: NoInfer>, + clientSecretField?: NoInfer> +): Pick { + const { values } = requireOAuthClientCapability(providerId, env) + return { + clientId: values[clientIdField], + clientSecret: clientSecretField ? values[clientSecretField] : '', + } +} + +/** + * Get OAuth provider configuration for token refresh + */ +function getProviderAuthConfig( + provider: string, + clientOverride?: Pick +): ProviderAuthConfig { + if (clientOverride && provider !== 'quickbooks') { + throw new Error(`OAuth client override is not supported for provider ${provider}`) + } + switch (provider) { + case 'google': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'google', + 'GOOGLE_CLIENT_ID', + 'GOOGLE_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://oauth2.googleapis.com/token', + clientId, + clientSecret, + useBasicAuth: false, + } + } + case 'x': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'x', + 'X_CLIENT_ID', + 'X_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.x.com/2/oauth2/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'tiktok': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'tiktok', + 'TIKTOK_CLIENT_ID', + 'TIKTOK_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://open.tiktokapis.com/v2/oauth/token/', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + // TikTok requires `client_key` in the token request body instead of `client_id`. + clientIdParamName: 'client_key', + } + } + case 'confluence': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'confluence', + 'CONFLUENCE_CLIENT_ID', + 'CONFLUENCE_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://auth.atlassian.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'jira': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'jira', + 'JIRA_CLIENT_ID', + 'JIRA_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://auth.atlassian.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'calcom': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'calcom', + 'CALCOM_CLIENT_ID' + ) + return { + tokenEndpoint: 'https://app.cal.com/api/auth/oauth/refreshToken', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + // Cal.com requires refresh token in Authorization header, not body + refreshTokenInAuthHeader: true, + } + } + case 'airtable': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'airtable', + 'AIRTABLE_CLIENT_ID', + 'AIRTABLE_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://airtable.com/oauth2/v1/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'bitbucket': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'bitbucket', + 'BITBUCKET_CLIENT_ID', + 'BITBUCKET_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://bitbucket.org/site/oauth2/access_token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'github-repositories': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'github-repositories', + 'GITHUB_APP_CLIENT_ID', + 'GITHUB_APP_CLIENT_SECRET' + ) + return { + tokenEndpoint: GITHUB_TOKEN_URL, + clientId, + clientSecret, + useBasicAuth: false, + additionalHeaders: { Accept: 'application/json' }, + supportsRefreshTokenRotation: true, + } + } + case 'notion': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'notion', + 'NOTION_CLIENT_ID', + 'NOTION_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.notion.com/v1/oauth/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + useJsonBody: true, + } + } + case 'microsoft': + case 'outlook': + case 'onedrive': + case 'sharepoint': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'microsoft', + 'MICROSOFT_CLIENT_ID', + 'MICROSOFT_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://login.microsoftonline.com/common/oauth2/v2.0/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'clickup': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'clickup', + 'CLICKUP_CLIENT_ID', + 'CLICKUP_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.clickup.com/api/v2/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'linear': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'linear', + 'LINEAR_CLIENT_ID', + 'LINEAR_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.linear.app/oauth/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'attio': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'attio', + 'ATTIO_CLIENT_ID', + 'ATTIO_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://app.attio.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + } + } + case 'box': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'box', + 'BOX_CLIENT_ID', + 'BOX_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.box.com/oauth2/token', + clientId, + clientSecret, + useBasicAuth: false, + // Box refresh tokens are single-use: "the Refresh Token is invalidated and a + // new Refresh Token is returned" and "A Refresh Token is valid for 60 days and + // can be used to obtain a new Access Token and Refresh Token only once." + // (developer.box.com/guides/authentication/tokens/refresh). Without rotation the + // new token is discarded and the credential dies on the second refresh. + supportsRefreshTokenRotation: true, + } + } + case 'docusign': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'docusign', + 'DOCUSIGN_CLIENT_ID', + 'DOCUSIGN_CLIENT_SECRET' + ) + return { + tokenEndpoint: getDocusignOAuthUrl('/oauth/token'), + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'dropbox': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'dropbox', + 'DROPBOX_CLIENT_ID', + 'DROPBOX_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.dropboxapi.com/oauth2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'slack': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'slack', + 'SLACK_CLIENT_ID', + 'SLACK_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://slack.com/api/oauth.v2.access', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'reddit': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'reddit', + 'REDDIT_CLIENT_ID', + 'REDDIT_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://www.reddit.com/api/v1/access_token', + clientId, + clientSecret, + useBasicAuth: true, + additionalHeaders: { + 'User-Agent': REDDIT_USER_AGENT, + }, + } + } + case 'wealthbox': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'wealthbox', + 'WEALTHBOX_CLIENT_ID', + 'WEALTHBOX_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://app.crmworkspace.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'webflow': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'webflow', + 'WEBFLOW_CLIENT_ID', + 'WEBFLOW_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.webflow.com/oauth/access_token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'asana': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'asana', + 'ASANA_CLIENT_ID', + 'ASANA_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://app.asana.com/-/oauth_token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'pipedrive': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'pipedrive', + 'PIPEDRIVE_CLIENT_ID', + 'PIPEDRIVE_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://oauth.pipedrive.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'quickbooks': { + if (!clientOverride) { + throw new Error('QuickBooks OAuth client configuration is missing') + } + return { + tokenEndpoint: QUICKBOOKS_TOKEN_URL, + clientId: clientOverride.clientId, + clientSecret: clientOverride.clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'hubspot': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'hubspot', + 'HUBSPOT_CLIENT_ID', + 'HUBSPOT_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://api.hubapi.com/oauth/v1/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'linkedin': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'linkedin', + 'LINKEDIN_CLIENT_ID', + 'LINKEDIN_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://www.linkedin.com/oauth/v2/accessToken', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'instagram': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'instagram', + 'INSTAGRAM_CLIENT_ID', + 'INSTAGRAM_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://graph.instagram.com/refresh_access_token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + refreshStrategy: 'instagram_long_lived', + } + } + case 'salesforce': + case 'salesforce-sandbox': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'salesforce', + 'SALESFORCE_CLIENT_ID', + 'SALESFORCE_CLIENT_SECRET' + ) + // A refresh token is only redeemable at the authorization server that + // issued it: a sandbox token posted to login.salesforce.com fails with + // `invalid_grant`. One Connected App's consumer key is valid at both + // hosts, so only the endpoint differs. + return { + tokenEndpoint: `https://${SALESFORCE_LOGIN_HOSTS[provider]}/services/oauth2/token`, + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: true, + } + } + case 'shopify': { + // Shopify access tokens don't expire and don't support refresh tokens + // This configuration is provided for completeness but won't be used for token refresh + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'shopify', + 'SHOPIFY_CLIENT_ID', + 'SHOPIFY_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://accounts.shopify.com/oauth/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'zoom': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'zoom', + 'ZOOM_CLIENT_ID', + 'ZOOM_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://zoom.us/oauth/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: true, + } + } + case 'wordpress': { + // WordPress.com does NOT support refresh tokens + // Users will need to re-authorize when tokens expire (~2 weeks) + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'wordpress', + 'WORDPRESS_CLIENT_ID', + 'WORDPRESS_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://public-api.wordpress.com/oauth2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'spotify': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'spotify', + 'SPOTIFY_CLIENT_ID', + 'SPOTIFY_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://accounts.spotify.com/api/token', + clientId, + clientSecret, + useBasicAuth: true, + supportsRefreshTokenRotation: false, + } + } + case 'monday': { + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'monday', + 'MONDAY_CLIENT_ID', + 'MONDAY_CLIENT_SECRET' + ) + return { + tokenEndpoint: MONDAY_OAUTH_TOKEN_URL, + clientId, + clientSecret, + useBasicAuth: false, + useJsonBody: true, + supportsRefreshTokenRotation: true, + } + } + case 'manageengine-sdp': { + // ServiceDesk Plus Cloud authenticates through Zoho, so the grant is the + // same one Zoho Desk uses and shares its client credentials: scopes are + // chosen per authorization request, not per API-console client, so one + // registered client serves both products. + // + // Rotation stays off for the same reason as zoho-desk below - Zoho's + // refresh_token grant returns a new access token but no new refresh token. + // accounts.zoho.com is correct because the authorize and code-exchange + // legs in lib/auth/connectors/providers.ts are pinned to the US accounts + // server, so every refresh token in the system is US-issued. Data + // residency for API calls is honored separately, via the block's data + // center selector. + // Keyed on the `zoho-desk` capability, which is what + // `resolveOAuthClientCapabilityId('manageengine-sdp')` aliases to — the + // capability names the env pair, not the product. + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'zoho-desk', + 'ZOHO_CLIENT_ID', + 'ZOHO_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + case 'zoho-desk': { + // Zoho's refresh_token grant returns a new access token but no new refresh + // token, so rotation stays off (the existing refresh token is preserved). + // The refresh must target the accounts server of the data center that issued + // the token - "if location=eu, you will need to make access token request to + // https://accounts.zoho.eu" (zoho.com/accounts/protocol/oauth/multi-dc.html). + // accounts.zoho.com is correct here because the authorize and code-exchange + // legs in lib/auth/connectors/providers.ts are also pinned to the US accounts + // server, so every refresh token in the system is US-issued. Making refresh + // DC-aware requires making the grant DC-aware first (read the `accounts-server` + // callback param) and threading the credential's persisted `__zoho_domain__` + // marker into refreshOAuthToken, which today only receives the token string. + // Data residency for API calls is already honored via that persisted Desk base. + const { clientId, clientSecret } = getConfiguredClientCredentials( + 'zoho-desk', + 'ZOHO_CLIENT_ID', + 'ZOHO_CLIENT_SECRET' + ) + return { + tokenEndpoint: 'https://accounts.zoho.com/oauth/v2/token', + clientId, + clientSecret, + useBasicAuth: false, + supportsRefreshTokenRotation: false, + } + } + default: + throw new Error(`Unsupported provider: ${provider}`) + } +} + +/** + * Build the authentication request headers and body for OAuth token refresh + */ +function buildAuthRequest( + config: ProviderAuthConfig, + refreshToken: string +): { headers: Record; bodyParams: Record; useJsonBody?: boolean } { + const headers: Record = { + 'Content-Type': config.useJsonBody ? 'application/json' : 'application/x-www-form-urlencoded', + ...config.additionalHeaders, + } + + const bodyParams: Record = { + grant_type: 'refresh_token', + } + + if (config.refreshTokenInAuthHeader) { + // Cal.com style: refresh token in Authorization header as Bearer token + headers.Authorization = `Bearer ${refreshToken}` + } else { + bodyParams.refresh_token = refreshToken + } + + if (config.useBasicAuth) { + const basicAuth = Buffer.from(`${config.clientId}:${config.clientSecret}`).toString('base64') + headers.Authorization = `Basic ${basicAuth}` + } else { + bodyParams[config.clientIdParamName || 'client_id'] = config.clientId + if (config.clientSecret) { + bodyParams.client_secret = config.clientSecret + } + } + + return { headers, bodyParams, useJsonBody: config.useJsonBody } +} + +/** + * Resolves the key {@link getProviderAuthConfig} is switched on for a stored + * credential's provider id. + * + * Normally that is the base provider, because every service in a family + * refreshes against the same endpoint with the same client. A provider id + * listed in a service's `additionalProviderIds` is the exception: it names a + * *different* authorization server for the same service, so it must reach + * `getProviderAuthConfig` intact — collapsing it to the base would silently + * refresh a sandbox token against the production endpoint. + */ +function getBaseProviderForService(providerId: string): string { + if (providerId in OAUTH_PROVIDERS) { + return providerId + } + + for (const [baseProvider, config] of Object.entries(OAUTH_PROVIDERS)) { + for (const service of Object.values(config.services)) { + if (service.providerId === providerId) { + return baseProvider + } + if (service.additionalProviderIds?.includes(providerId)) { + return providerId + } + } + } + + throw new Error(`Unknown OAuth provider: ${providerId}`) +} + +export interface RefreshTokenSuccess { + ok: true + accessToken: string + expiresIn: number + refreshToken: string + refreshTokenExpiresIn?: number +} + +export interface RefreshTokenFailure { + ok: false + errorCode?: string + message?: string +} + +export type RefreshTokenResult = RefreshTokenSuccess | RefreshTokenFailure + +function extractErrorCode(value: unknown): string | undefined { + if (value && typeof value === 'object' && 'error' in value) { + const error = (value as { error: unknown }).error + if (typeof error === 'string') return error + if (error && typeof error === 'object' && 'code' in error) { + const code = (error as { code: unknown }).code + if (typeof code === 'string' || typeof code === 'number') return String(code) + } + } + return undefined +} + +function safeOAuthErrorCode(value: unknown, secrets: string[]): string | undefined { + const errorCode = extractErrorCode(value) + if (!errorCode) return undefined + const safeCode = redactExactSensitiveValues(errorCode, secrets).trim().toLowerCase() + return /^[a-z0-9][a-z0-9._:-]{0,127}$/.test(safeCode) ? safeCode : undefined +} + +/** + * Hard deadline on the token-endpoint exchange. This function does not coalesce + * on its own; its sole production caller (`performCoalescedRefresh` in the OAuth + * utils) shares one in-flight refresh across concurrent callers for a credential. + * Without this bound a hung endpoint would wedge every joiner on that key until + * the undici socket defaults (~5 min) gave up. + */ +const TOKEN_REFRESH_TIMEOUT_MS = 15_000 + +function parseOAuthResponse(responseText: string): unknown { + try { + return JSON.parse(responseText) + } catch { + return responseText + } +} + +function oauthResponseRecord(value: unknown): Record | undefined { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined +} + +const OAUTH_RESPONSE_OMITTED = '[token endpoint response omitted]' + +async function refreshInstagramLongLivedToken( + config: ProviderAuthConfig, + longLivedToken: string, + providerId: string +): Promise { + const url = new URL(config.tokenEndpoint) + url.searchParams.set('grant_type', 'ig_refresh_token') + url.searchParams.set('access_token', longLivedToken) + + const response = await providerFetch(url.toString(), { + method: 'GET', + redirect: 'error', + signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS), + }) + + const responseText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'Instagram token refresh response', + }) + const responseData = parseOAuthResponse(responseText) + + if (!response.ok) { + const exactSecrets = [longLivedToken, config.clientSecret ?? ''] + const errorCode = safeOAuthErrorCode(responseData, exactSecrets) + logger.error('Instagram long-lived token refresh failed:', { + status: response.status, + error: OAUTH_RESPONSE_OMITTED, + errorCode, + providerId, + tokenEndpoint: config.tokenEndpoint, + }) + return { + ok: false, + errorCode, + message: `Failed to refresh token: ${response.status} ${OAUTH_RESPONSE_OMITTED}`, + } + } + + const payload = parseInstagramLongLivedToken(responseData) + if (!payload) { + logger.warn('Invalid Instagram refresh response', { providerId }) + return { ok: false, message: 'Invalid Instagram token refresh response' } + } + + logger.info('Instagram long-lived token refreshed successfully', { + expiresIn: payload.expires_in, + providerId, + }) + + // Instagram returns a new long-lived token; store it as both access and refresh. + return { + ok: true, + accessToken: payload.access_token, + expiresIn: payload.expires_in, + refreshToken: payload.access_token, + } +} + +export async function refreshOAuthToken( + providerId: string, + refreshToken: string, + clientOverride?: Pick +): Promise { + const exactSecrets = [refreshToken] + try { + const provider = getBaseProviderForService(providerId) + + const config = getProviderAuthConfig(provider, clientOverride) + if (config.clientSecret) exactSecrets.push(config.clientSecret) + + if (config.refreshStrategy === 'instagram_long_lived') { + return await refreshInstagramLongLivedToken(config, refreshToken, providerId) + } + + const { headers, bodyParams, useJsonBody } = buildAuthRequest(config, refreshToken) + + const response = await providerFetch(config.tokenEndpoint, { + method: 'POST', + headers, + body: useJsonBody ? JSON.stringify(bodyParams) : new URLSearchParams(bodyParams).toString(), + redirect: 'error', + signal: AbortSignal.timeout(TOKEN_REFRESH_TIMEOUT_MS), + }) + + const responseText = await readResponseTextWithLimit(response, { + maxBytes: DEFAULT_MAX_ERROR_BODY_BYTES, + label: 'OAuth token refresh response', + }) + const responseData = parseOAuthResponse(responseText) + + if (!response.ok) { + const errorCode = safeOAuthErrorCode(responseData, exactSecrets) + + logger.error('Token refresh failed:', { + status: response.status, + error: OAUTH_RESPONSE_OMITTED, + errorCode, + providerId, + tokenEndpoint: config.tokenEndpoint, + hasClientId: !!config.clientId, + hasClientSecret: !!config.clientSecret, + hasRefreshToken: !!refreshToken, + }) + return { + ok: false, + errorCode, + message: `Failed to refresh token: ${response.status} ${OAUTH_RESPONSE_OMITTED}`, + } + } + + const data = oauthResponseRecord(responseData) + if (!data) { + logger.warn('Invalid OAuth token refresh response', { providerId }) + return { ok: false, message: 'Invalid OAuth token refresh response' } + } + + if ( + data.ok === false || + (provider === 'github-repositories' && typeof data.error === 'string') + ) { + const errorCode = safeOAuthErrorCode(data, exactSecrets) + logger.error('Token refresh failed:', { + status: response.status, + error: OAUTH_RESPONSE_OMITTED, + errorCode, + providerId, + tokenEndpoint: config.tokenEndpoint, + hasClientId: !!config.clientId, + hasClientSecret: !!config.clientSecret, + hasRefreshToken: !!refreshToken, + }) + return { + ok: false, + errorCode, + message: `Failed to refresh token: ${OAUTH_RESPONSE_OMITTED}`, + } + } + + if (provider === 'github-repositories') { + const tokens = parseGitHubRepositoriesTokenResponse(data) + return { + ok: true, + accessToken: tokens.access_token, + refreshToken: tokens.refresh_token, + expiresIn: tokens.expires_in, + refreshTokenExpiresIn: tokens.refresh_token_expires_in, + } + } + + const accessToken = + typeof data.access_token === 'string' && data.access_token.length > 0 + ? data.access_token + : undefined + + let newRefreshToken: string | undefined + if ( + config.supportsRefreshTokenRotation && + typeof data.refresh_token === 'string' && + data.refresh_token.length > 0 + ) { + newRefreshToken = data.refresh_token + logger.info(`Received new refresh token from ${provider}`) + } + if (provider === 'monday' && !newRefreshToken) { + logger.warn('Monday token refresh response omitted its rotating refresh token') + return { ok: false, message: 'Invalid Monday token refresh response' } + } + if (provider === 'quickbooks' && !newRefreshToken) { + logger.warn('QuickBooks token refresh response omitted its rotating refresh token') + return { ok: false, message: 'Invalid QuickBooks token refresh response' } + } + + const rawExpiresIn = data.expires_in ?? data.expiresIn + const parsedExpiresIn = + typeof rawExpiresIn === 'number' || typeof rawExpiresIn === 'string' + ? Number(rawExpiresIn) + : Number.NaN + const responseExpiresIn = + Number.isFinite(parsedExpiresIn) && parsedExpiresIn > 0 ? parsedExpiresIn : undefined + const expiresIn = + provider === 'monday' && accessToken + ? Math.max( + 1, + Math.ceil( + (resolveMondayAccessTokenExpiresAt(accessToken, responseExpiresIn).getTime() - + Date.now()) / + 1000 + ) + ) + : (responseExpiresIn ?? 3600) + + const rawRefreshTokenExpiresIn = data.x_refresh_token_expires_in + const parsedRefreshTokenExpiresIn = + typeof rawRefreshTokenExpiresIn === 'number' || typeof rawRefreshTokenExpiresIn === 'string' + ? Number(rawRefreshTokenExpiresIn) + : Number.NaN + const refreshTokenExpiresIn = + provider === 'quickbooks' && + Number.isSafeInteger(parsedRefreshTokenExpiresIn) && + parsedRefreshTokenExpiresIn > 0 + ? parsedRefreshTokenExpiresIn + : undefined + + if (!accessToken) { + // Log only the shape, never `data` itself - on a partial success it can + // carry live tokens. + logger.warn('No access token found in refresh response', { + providerId, + responseKeys: Object.keys(data ?? {}), + }) + return { ok: false, message: 'No access token in refresh response' } + } + + logger.info('Token refreshed successfully with expiration', { + expiresIn, + hasNewRefreshToken: !!newRefreshToken, + provider, + }) + + return { + ok: true, + accessToken, + expiresIn, + refreshToken: newRefreshToken ?? refreshToken, + ...(refreshTokenExpiresIn ? { refreshTokenExpiresIn } : {}), + } + } catch (error) { + const normalized = toError(error) + const message = + normalized.name === 'PayloadSizeLimitError' || normalized.message.startsWith('OAuth client ') + ? normalized.message + : 'Token refresh failed' + logger.error('Error refreshing token', { errorType: normalized.name }) + return { ok: false, message } + } +} diff --git a/apps/sim/lib/selectors/application/execute-selector.test.ts b/apps/sim/lib/selectors/application/execute-selector.test.ts index e0f2a9e0625..3beb4bd0ce7 100644 --- a/apps/sim/lib/selectors/application/execute-selector.test.ts +++ b/apps/sim/lib/selectors/application/execute-selector.test.ts @@ -22,6 +22,13 @@ const mocks = vi.hoisted(() => ({ sanitize: vi.fn(), authorizePersonalSearch: vi.fn(), requireOrganizationMembership: vi.fn(), + routingEnabled: vi.fn(() => false), + resolveRoute: vi.fn(async (organizationId: string | null | undefined) => ({ organizationId })), +})) + +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: mocks.routingEnabled, + resolveOutboundRoute: mocks.resolveRoute, })) vi.mock('@/lib/knowledge/application/personal-search-account', () => ({ @@ -77,6 +84,10 @@ const mockResolvePermissionGroupConfig = permissionGroupScopeMockFns.mockResolvePermissionGroupConfig import { selectorScopeSchema } from '@/lib/api/contracts/selectors/execute' +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' import { executeSelector } from '@/lib/selectors/application/execute-selector' import { getSelectorManifestEntry } from '@/lib/selectors/manifest' @@ -106,6 +117,7 @@ function execute(inputOverrides: Record = {}) { describe('executeSelector', () => { beforeEach(() => { vi.clearAllMocks() + mocks.routingEnabled.mockReturnValue(false) mocks.events.length = 0 mocks.resolveScope.mockImplementation(async () => { mocks.events.push('canonical-scope') @@ -200,6 +212,48 @@ describe('executeSelector', () => { expect(mocks.executeAttachment).not.toHaveBeenCalled() }) + it.each([false, true])( + 'uses canonical organization routing for discovery with personal setup %s', + async (personalSetup) => { + mocks.routingEnabled.mockReturnValue(true) + const organizationScope = { kind: 'organization' as const, organizationId: 'org-1' } + const signal = new AbortController().signal + mocks.resolveScope.mockResolvedValueOnce({ + organizationId: 'org-1', + workspaceId: undefined, + selectorKey: 'jira.projectKeys', + selectorManifest: getSelectorManifestEntry('jira.projectKeys'), + selectorScope: organizationScope, + }) + mocks.resolveReferences.mockResolvedValueOnce({ + context: { oauthCredential: 'credential-1', domain: 'example.atlassian.net' }, + request: { kind: 'list' }, + references: new Map(), + }) + mocks.executeAttachment.mockImplementationOnce(async (args: ExecuteServerSelectorArgs) => { + expect(await resolveCurrentOutboundRoute()).toEqual({ organizationId: 'org-1' }) + expect(args.signal).toBe(signal) + return { kind: 'list', items: [] } + }) + + await runWithOutboundOrganization('caller-org', async () => { + await expect( + execute({ + selectorKey: 'jira.projectKeys', + scope: organizationScope, + context: { oauthCredential: 'credential-1', domain: 'example.atlassian.net' }, + signal, + ...(personalSetup ? { personalSearchSetup: 'jira' } : {}), + }) + ).resolves.toEqual({ kind: 'list', items: [] }) + expect(await resolveCurrentOutboundRoute()).toEqual({ organizationId: 'caller-org' }) + }) + expect(mocks.executeAttachment).toHaveBeenCalledOnce() + expect(mocks.authorizePersonalSearch).toHaveBeenCalledTimes(personalSetup ? 1 : 0) + expect(mocks.requireOrganizationMembership).toHaveBeenCalledTimes(personalSetup ? 0 : 1) + } + ) + it('rejects a personal setup marker outside its approved provider selector and organization scope', async () => { await expect(execute({ personalSearchSetup: 'jira' })).rejects.toBeInstanceOf( SelectorContextUnavailableError diff --git a/apps/sim/lib/selectors/application/execute-selector.ts b/apps/sim/lib/selectors/application/execute-selector.ts index 32ca55fb3b5..ca802667784 100644 --- a/apps/sim/lib/selectors/application/execute-selector.ts +++ b/apps/sim/lib/selectors/application/execute-selector.ts @@ -7,6 +7,7 @@ import { import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' import type { OperationUseCase } from '@/lib/core/application/operation' import { requireOrganizationMembership } from '@/lib/core/application/organization-authorization' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { authorizePersonalSearchSetup } from '@/lib/knowledge/application/personal-search-account' import { type CredentialAuditRequest, recordCredentialAccess } from '@/lib/oauth/token-resolution' @@ -371,6 +372,7 @@ export const executeSelector: OperationUseCase< scope: args.input.scope, }) validateAuthorizedInput(args.input, context) - return executeAuthorizedSelector({ principal: args.principal, input: args.input, context }) + const executionArgs = { principal: args.principal, input: args.input, context } + return withResourceOutboundScope(context, () => executeAuthorizedSelector(executionArgs)) }, } diff --git a/apps/sim/lib/selectors/server/providers/bigquery.test.ts b/apps/sim/lib/selectors/server/providers/bigquery.test.ts index 87840705600..a03bbc2e72b 100644 --- a/apps/sim/lib/selectors/server/providers/bigquery.test.ts +++ b/apps/sim/lib/selectors/server/providers/bigquery.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/selectors/server/providers/bitbucket.test.ts b/apps/sim/lib/selectors/server/providers/bitbucket.test.ts index 0f90de7d37d..8c698c00b76 100644 --- a/apps/sim/lib/selectors/server/providers/bitbucket.test.ts +++ b/apps/sim/lib/selectors/server/providers/bitbucket.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/selectors/server/providers/confluence.test.ts b/apps/sim/lib/selectors/server/providers/confluence.test.ts index 5faa48c5e68..ecf209566b9 100644 --- a/apps/sim/lib/selectors/server/providers/confluence.test.ts +++ b/apps/sim/lib/selectors/server/providers/confluence.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveCredentialBundle, mockResolveCloudId } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/selectors/server/providers/google.test.ts b/apps/sim/lib/selectors/server/providers/google.test.ts index 0f737ec42fa..6013a85f71d 100644 --- a/apps/sim/lib/selectors/server/providers/google.test.ts +++ b/apps/sim/lib/selectors/server/providers/google.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/selectors/server/providers/harmonic.test.ts b/apps/sim/lib/selectors/server/providers/harmonic.test.ts index 3a7a3793d37..730fc9a0b38 100644 --- a/apps/sim/lib/selectors/server/providers/harmonic.test.ts +++ b/apps/sim/lib/selectors/server/providers/harmonic.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveCredentialBundle } = vi.hoisted(() => ({ @@ -16,6 +17,8 @@ import { createSelectorProtectedValues } from '@/lib/selectors/server/protected- import { harmonicSelectorAttachments } from '@/lib/selectors/server/providers/harmonic' import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + function detailArgs(id: string): ExecuteServerSelectorArgs { return { selectorKey: 'harmonic.savedSearches', diff --git a/apps/sim/lib/selectors/server/providers/harmonic.ts b/apps/sim/lib/selectors/server/providers/harmonic.ts index 52f4f04eeed..39afa93e50c 100644 --- a/apps/sim/lib/selectors/server/providers/harmonic.ts +++ b/apps/sim/lib/selectors/server/providers/harmonic.ts @@ -1,5 +1,6 @@ import { isPlainRecord } from '@sim/utils/object' import { z } from 'zod' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import type { ServerSelectorKey } from '@/lib/selectors/manifest' import { SelectorOptionsUnavailableError } from '@/lib/selectors/server/errors' @@ -12,6 +13,10 @@ import { type ServerSelectorAttachmentMap, } from '@/lib/selectors/server/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + type HarmonicSelectorKey = Extract const HARMONIC_URL = 'https://api.harmonic.ai/savedSearches' @@ -105,7 +110,7 @@ async function listSavedSearches( let response: Response try { - response = await fetch(HARMONIC_URL, { + response = await providerFetch(HARMONIC_URL, { headers: { Accept: 'application/json', apikey: accessToken }, redirect: 'error', signal, diff --git a/apps/sim/lib/selectors/server/providers/hubspot.test.ts b/apps/sim/lib/selectors/server/providers/hubspot.test.ts index c421f7189b6..cb671e1e48a 100644 --- a/apps/sim/lib/selectors/server/providers/hubspot.test.ts +++ b/apps/sim/lib/selectors/server/providers/hubspot.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/selectors/server/providers/jira.test.ts b/apps/sim/lib/selectors/server/providers/jira.test.ts index a3807abda7a..4d2ec7e8785 100644 --- a/apps/sim/lib/selectors/server/providers/jira.test.ts +++ b/apps/sim/lib/selectors/server/providers/jira.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveSelectorAtlassianCloudId, mockResolveSelectorCredentialBundle } = diff --git a/apps/sim/lib/selectors/server/providers/jsm.test.ts b/apps/sim/lib/selectors/server/providers/jsm.test.ts index b8503ca8bef..623d0b98181 100644 --- a/apps/sim/lib/selectors/server/providers/jsm.test.ts +++ b/apps/sim/lib/selectors/server/providers/jsm.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveJsmAuth, mockResolveCloudId } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/selectors/server/providers/microsoft.test.ts b/apps/sim/lib/selectors/server/providers/microsoft.test.ts index c408a88ef33..53b4170fc11 100644 --- a/apps/sim/lib/selectors/server/providers/microsoft.test.ts +++ b/apps/sim/lib/selectors/server/providers/microsoft.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/selectors/server/providers/monday.test.ts b/apps/sim/lib/selectors/server/providers/monday.test.ts index 19ac53d0e76..b394a1e6032 100644 --- a/apps/sim/lib/selectors/server/providers/monday.test.ts +++ b/apps/sim/lib/selectors/server/providers/monday.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/selectors/server/providers/notion.test.ts b/apps/sim/lib/selectors/server/providers/notion.test.ts index 55b955fb5db..15a918cc866 100644 --- a/apps/sim/lib/selectors/server/providers/notion.test.ts +++ b/apps/sim/lib/selectors/server/providers/notion.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/selectors/server/providers/pipedrive.test.ts b/apps/sim/lib/selectors/server/providers/pipedrive.test.ts index d09256e27c4..7861e5ca9eb 100644 --- a/apps/sim/lib/selectors/server/providers/pipedrive.test.ts +++ b/apps/sim/lib/selectors/server/providers/pipedrive.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveSelectorCredentialBundle } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/selectors/server/providers/provider-http.test.ts b/apps/sim/lib/selectors/server/providers/provider-http.test.ts index df8de2820e6..f41906eeb1a 100644 --- a/apps/sim/lib/selectors/server/providers/provider-http.test.ts +++ b/apps/sim/lib/selectors/server/providers/provider-http.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' import { SelectorConnectionUnavailableError, @@ -12,6 +13,8 @@ import { RetryableProviderNetworkError, } from '@/lib/selectors/server/providers/provider-http' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const mockFetch = vi.fn() function openBody(onCancel: () => void): ReadableStream { diff --git a/apps/sim/lib/selectors/server/providers/provider-http.ts b/apps/sim/lib/selectors/server/providers/provider-http.ts index 00394622b00..6eb383c7847 100644 --- a/apps/sim/lib/selectors/server/providers/provider-http.ts +++ b/apps/sim/lib/selectors/server/providers/provider-http.ts @@ -1,9 +1,14 @@ import { parseRetryAfter } from '@sim/utils/retry' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { SelectorConnectionUnavailableError, SelectorOptionsUnavailableError, } from '@/lib/selectors/server/errors' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const PROVIDER_TIMEOUT_MS = 30_000 const MAX_PROVIDER_RESPONSE_BYTES = 16 * 1024 * 1024 @@ -83,7 +88,7 @@ export async function fetchProviderJsonWithStatus( const timeoutSignal = AbortSignal.timeout(PROVIDER_TIMEOUT_MS) const signal = init?.signal ? AbortSignal.any([init.signal, timeoutSignal]) : timeoutSignal try { - response = await fetch(input, { ...init, signal, redirect: init?.redirect ?? 'error' }) + response = await providerFetch(input, { ...init, signal, redirect: init?.redirect ?? 'error' }) } catch (error) { if (init?.signal?.aborted) throw error if (options.passthroughNetworkErrors) throw new RetryableProviderNetworkError() diff --git a/apps/sim/lib/selectors/server/providers/sharepoint.test.ts b/apps/sim/lib/selectors/server/providers/sharepoint.test.ts index 72327b52785..f02eb4c7c82 100644 --- a/apps/sim/lib/selectors/server/providers/sharepoint.test.ts +++ b/apps/sim/lib/selectors/server/providers/sharepoint.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/selectors/server/providers/snowflake.test.ts b/apps/sim/lib/selectors/server/providers/snowflake.test.ts index b3d9fb08df3..0e1fc2fcedd 100644 --- a/apps/sim/lib/selectors/server/providers/snowflake.test.ts +++ b/apps/sim/lib/selectors/server/providers/snowflake.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveCredentialBundle } = vi.hoisted(() => ({ @@ -16,6 +17,8 @@ import { createSelectorProtectedValues } from '@/lib/selectors/server/protected- import { snowflakeSelectorAttachments } from '@/lib/selectors/server/providers/snowflake' import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const STATEMENT_HANDLE = '019c06a4-0000-df4f-0000-00100006589e' function jsonResponse(body: unknown, status = 200): Response { diff --git a/apps/sim/lib/selectors/server/providers/snowflake.ts b/apps/sim/lib/selectors/server/providers/snowflake.ts index 3708f047c1e..81084465f13 100644 --- a/apps/sim/lib/selectors/server/providers/snowflake.ts +++ b/apps/sim/lib/selectors/server/providers/snowflake.ts @@ -1,3 +1,4 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { MAX_SELECTOR_OPTIONS } from '@/lib/selectors/limits' import type { ServerSelectorKey } from '@/lib/selectors/manifest' import { @@ -21,6 +22,10 @@ import { readSnowflakeResult, } from '@/tools/snowflake/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + type SnowflakeSelectorKey = Extract type SnowflakeScopeLevel = 'account' | 'database' | 'schema' @@ -89,7 +94,7 @@ function requireStatementHandle(value: string): string { } async function fetchSnowflakeResponse(url: string, init: RequestInit): Promise { - const response = await fetch(url, { ...init, redirect: 'error' }) + const response = await providerFetch(url, { ...init, redirect: 'error' }) if (!response.ok) { await response.body?.cancel().catch(() => undefined) throw selectorProviderStatusError(response.status) diff --git a/apps/sim/lib/selectors/server/providers/webflow.test.ts b/apps/sim/lib/selectors/server/providers/webflow.test.ts index 2b5cff89f2d..68abb81da7b 100644 --- a/apps/sim/lib/selectors/server/providers/webflow.test.ts +++ b/apps/sim/lib/selectors/server/providers/webflow.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/selectors/server/providers/zoho-desk.test.ts b/apps/sim/lib/selectors/server/providers/zoho-desk.test.ts index 6b6fda70dd9..26cf8ed6278 100644 --- a/apps/sim/lib/selectors/server/providers/zoho-desk.test.ts +++ b/apps/sim/lib/selectors/server/providers/zoho-desk.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockResolveSelectorCredentialBundle, mockSecureFetchWithValidation } = vi.hoisted(() => ({ @@ -9,6 +10,7 @@ const { mockResolveSelectorCredentialBundle, mockSecureFetchWithValidation } = v })) vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, secureFetchWithValidation: mockSecureFetchWithValidation, })) diff --git a/apps/sim/lib/selectors/server/providers/zoom.test.ts b/apps/sim/lib/selectors/server/providers/zoom.test.ts index f2a274816eb..81939f838b6 100644 --- a/apps/sim/lib/selectors/server/providers/zoom.test.ts +++ b/apps/sim/lib/selectors/server/providers/zoom.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetch, mockResolveSelectorOAuthAccessToken } = vi.hoisted(() => ({ diff --git a/apps/sim/lib/table/application/folders.test.ts b/apps/sim/lib/table/application/folders.test.ts index 30f0b60af42..56af8ea6f6a 100644 --- a/apps/sim/lib/table/application/folders.test.ts +++ b/apps/sim/lib/table/application/folders.test.ts @@ -64,6 +64,8 @@ describe('listTableFoldersUseCase', () => { vi.clearAllMocks() mocks.resolveWorkspaceContext.mockResolvedValue({ workspaceId: 'ws-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, billedAccountUserId: 'owner-1', }) mocks.resolvePermission.mockResolvedValue('admin') diff --git a/apps/sim/lib/webhooks/polling/gmail.ts b/apps/sim/lib/webhooks/polling/gmail.ts index b2deb341b52..311107715d2 100644 --- a/apps/sim/lib/webhooks/polling/gmail.ts +++ b/apps/sim/lib/webhooks/polling/gmail.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getProviderConfig, type PollingProviderHandler, @@ -16,6 +17,10 @@ import { processPolledWebhookEvent } from '@/lib/webhooks/processor' import type { GmailAttachment } from '@/tools/gmail/types' import { downloadAttachments, extractAttachmentInfo } from '@/tools/gmail/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + interface GmailWebhookConfig { labelIds: string[] labelFilterBehavior: 'INCLUDE' | 'EXCLUDE' @@ -161,7 +166,7 @@ async function fetchNewEmails( historyUrl += `&pageToken=${pageToken}` } - const historyResponse = await fetch(historyUrl, { + const historyResponse = await providerFetch(historyUrl, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -321,7 +326,7 @@ async function searchEmails( const query = `${baseQuery}${timeConstraint}` const searchUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages?q=${encodeURIComponent(query)}&maxResults=${config.maxEmailsPerPoll || 25}` - const searchResponse = await fetch(searchUrl, { + const searchResponse = await providerFetch(searchUrl, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -383,7 +388,7 @@ async function getGmailProfileHistoryId( logger: Logger ): Promise { try { - const response = await fetch('https://gmail.googleapis.com/gmail/v1/users/me/profile', { + const response = await providerFetch('https://gmail.googleapis.com/gmail/v1/users/me/profile', { headers: { Authorization: `Bearer ${accessToken}` }, }) if (!response.ok) { @@ -403,7 +408,7 @@ async function getGmailProfileHistoryId( async function getEmailDetails(accessToken: string, messageId: string): Promise { const messageUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages/${messageId}?format=full` - const messageResponse = await fetch(messageUrl, { + const messageResponse = await providerFetch(messageUrl, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -573,7 +578,7 @@ async function markEmailAsRead(accessToken: string, messageId: string, logger: L const modifyUrl = `https://gmail.googleapis.com/gmail/v1/users/me/messages/${messageId}/modify` try { - const response = await fetch(modifyUrl, { + const response = await providerFetch(modifyUrl, { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/lib/webhooks/polling/google-calendar.ts b/apps/sim/lib/webhooks/polling/google-calendar.ts index 94a3c120e8c..9d036418a0c 100644 --- a/apps/sim/lib/webhooks/polling/google-calendar.ts +++ b/apps/sim/lib/webhooks/polling/google-calendar.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical' import { getProviderConfig, @@ -15,6 +16,10 @@ import { } from '@/lib/webhooks/polling/utils' import { processPolledWebhookEvent } from '@/lib/webhooks/processor' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const CALENDAR_API_BASE = 'https://www.googleapis.com/calendar/v3' const MAX_EVENTS_PER_POLL = 50 const MAX_PAGES = 10 @@ -209,7 +214,7 @@ async function fetchChangedEvents( const encodedCalendarId = encodeURIComponent(calendarId) const url = `${CALENDAR_API_BASE}/calendars/${encodedCalendarId}/events?${params.toString()}` - const response = await fetch(url, { + const response = await providerFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) diff --git a/apps/sim/lib/webhooks/polling/google-drive.ts b/apps/sim/lib/webhooks/polling/google-drive.ts index aaf321c14e8..b7ff0b9f8e7 100644 --- a/apps/sim/lib/webhooks/polling/google-drive.ts +++ b/apps/sim/lib/webhooks/polling/google-drive.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical' import { getProviderConfig, @@ -15,6 +16,10 @@ import { } from '@/lib/webhooks/polling/utils' import { processPolledWebhookEvent } from '@/lib/webhooks/processor' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const MAX_FILES_PER_POLL = 50 const MAX_KNOWN_FILE_IDS = 1000 const MAX_PAGES = 10 @@ -218,7 +223,7 @@ async function getStartPageToken( } const url = `${DRIVE_API_BASE}/changes/startPageToken?${params.toString()}` - const response = await fetch(url, { + const response = await providerFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -267,7 +272,7 @@ async function fetchChanges( } const url = `${DRIVE_API_BASE}/changes?${params.toString()}` - const response = await fetch(url, { + const response = await providerFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) diff --git a/apps/sim/lib/webhooks/polling/google-sheets.ts b/apps/sim/lib/webhooks/polling/google-sheets.ts index 72147225469..5b0b4a16444 100644 --- a/apps/sim/lib/webhooks/polling/google-sheets.ts +++ b/apps/sim/lib/webhooks/polling/google-sheets.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readCanonicalTriggerValue } from '@/lib/webhooks/polling/canonical' import { getProviderConfig, @@ -15,6 +16,10 @@ import { } from '@/lib/webhooks/polling/utils' import { processPolledWebhookEvent } from '@/lib/webhooks/processor' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const MAX_ROWS_PER_POLL = 100 /** Maximum number of leading rows to scan when auto-detecting the header row. */ @@ -269,7 +274,7 @@ async function getDriveFileModifiedTime( logger: Logger ): Promise { try { - const response = await fetch( + const response = await providerFetch( `https://www.googleapis.com/drive/v3/files/${fileId}?fields=modifiedTime`, { headers: { Authorization: `Bearer ${accessToken}` } } ) @@ -308,7 +313,7 @@ async function fetchSheetState( }) const url = `https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}/values/${encodedSheet}!A:Z?${params.toString()}` - const response = await fetch(url, { + const response = await providerFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) @@ -362,7 +367,7 @@ async function fetchRowRange( }) const url = `https://sheets.googleapis.com/v4/spreadsheets/${spreadsheetId}/values/${encodedSheet}!${startRow}:${endRow}?${params.toString()}` - const response = await fetch(url, { + const response = await providerFetch(url, { headers: { Authorization: `Bearer ${accessToken}` }, }) diff --git a/apps/sim/lib/webhooks/polling/hubspot.test.ts b/apps/sim/lib/webhooks/polling/hubspot.test.ts index 286fedc98a0..a7653004082 100644 --- a/apps/sim/lib/webhooks/polling/hubspot.test.ts +++ b/apps/sim/lib/webhooks/polling/hubspot.test.ts @@ -1,9 +1,12 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { describe, expect, it } from 'vitest' import { buildUserFilters } from '@/lib/webhooks/polling/hubspot' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + describe('buildUserFilters', () => { it('translates pipeline/stage/owner shortcuts into EQ filters', () => { const filters = buildUserFilters({ diff --git a/apps/sim/lib/webhooks/polling/hubspot.ts b/apps/sim/lib/webhooks/polling/hubspot.ts index a854f28511b..5a471f0c3f8 100644 --- a/apps/sim/lib/webhooks/polling/hubspot.ts +++ b/apps/sim/lib/webhooks/polling/hubspot.ts @@ -1,6 +1,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { pollingIdempotency } from '@/lib/core/idempotency/service' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { getProviderConfig, type PollingProviderHandler, @@ -14,6 +15,10 @@ import { } from '@/lib/webhooks/polling/utils' import { processPolledWebhookEvent } from '@/lib/webhooks/processor' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + type HubSpotBuiltInObjectType = 'contact' | 'company' | 'deal' | 'ticket' type HubSpotEventType = 'created' | 'updated' | 'property_changed' @@ -682,7 +687,7 @@ async function fetchHubSpotChanges(args: FetchArgs): Promise '') diff --git a/apps/sim/lib/webhooks/polling/orchestrator.test.ts b/apps/sim/lib/webhooks/polling/orchestrator.test.ts new file mode 100644 index 00000000000..a5b5ff6d483 --- /dev/null +++ b/apps/sim/lib/webhooks/polling/orchestrator.test.ts @@ -0,0 +1,97 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + enabled: vi.fn(() => true), + workspace: vi.fn(), + route: vi.fn(async (organizationId: string | null | undefined) => ({ organizationId })), + webhooks: vi.fn(), + poll: vi.fn(), +})) + +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: mocks.enabled, + resolveOutboundRoute: mocks.route, +})) +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadActiveWorkspaceApplicationContext: mocks.workspace, +})) +vi.mock('@/lib/webhooks/polling/registry', () => ({ + getPollingHandler: () => ({ provider: 'gmail', label: 'Gmail', pollWebhook: mocks.poll }), +})) +vi.mock('@/lib/webhooks/polling/utils', async (importOriginal) => ({ + ...(await importOriginal()), + fetchActiveWebhooks: mocks.webhooks, +})) + +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' +import { pollProvider } from '@/lib/webhooks/polling/orchestrator' +import type { PollWebhookContext } from '@/lib/webhooks/polling/types' + +function entry(workspaceId: string | null) { + return { + webhook: { id: `webhook-${workspaceId}` }, + workflow: { id: `workflow-${workspaceId}`, workspaceId }, + } +} + +describe('polling outbound scope', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.enabled.mockReturnValue(true) + }) + + it('isolates concurrent organizations and personal workspaces and restores the caller', async () => { + const organizations = new Map([ + ['workspace-a', 'org-a'], + ['workspace-b', 'org-b'], + ['workspace-personal', null], + ]) + mocks.webhooks.mockResolvedValue([...organizations.keys()].map(entry)) + mocks.workspace.mockImplementation(async (workspaceId: string) => ({ + workspaceOrganizationId: organizations.get(workspaceId), + })) + let arrived = 0 + let release = () => {} + const allStarted = new Promise((resolve) => { + release = resolve + }) + mocks.poll.mockImplementation(async ({ workflowData }: PollWebhookContext) => { + arrived++ + if (arrived === organizations.size) release() + await allStarted + expect(await resolveCurrentOutboundRoute()).toEqual({ + organizationId: organizations.get(workflowData.workspaceId!), + }) + return 'success' + }) + + await runWithOutboundOrganization('caller-org', async () => { + expect(await pollProvider('gmail')).toEqual({ total: 3, successful: 3, failed: 0 }) + expect(await resolveCurrentOutboundRoute()).toEqual({ organizationId: 'caller-org' }) + }) + expect(mocks.workspace).toHaveBeenCalledTimes(3) + expect(mocks.poll).toHaveBeenCalledTimes(3) + }) + + it('does not contact the provider for a missing or archived workspace', async () => { + mocks.webhooks.mockResolvedValue([entry('removed-workspace')]) + mocks.workspace.mockResolvedValue(null) + + expect(await pollProvider('gmail')).toEqual({ total: 1, successful: 0, failed: 1 }) + expect(mocks.poll).not.toHaveBeenCalled() + }) + + it('preserves legacy polling without ownership reads when routing is unconfigured', async () => { + mocks.enabled.mockReturnValue(false) + mocks.webhooks.mockResolvedValue([entry(null)]) + mocks.poll.mockResolvedValue('success') + + expect(await pollProvider('gmail')).toEqual({ total: 1, successful: 1, failed: 0 }) + expect(mocks.workspace).not.toHaveBeenCalled() + expect(mocks.poll).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/webhooks/polling/orchestrator.ts b/apps/sim/lib/webhooks/polling/orchestrator.ts index 6a7f553eb7d..d6cf75822b5 100644 --- a/apps/sim/lib/webhooks/polling/orchestrator.ts +++ b/apps/sim/lib/webhooks/polling/orchestrator.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { generateShortId } from '@sim/utils/id' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { getPollingHandler } from '@/lib/webhooks/polling/registry' import type { PollSummary } from '@/lib/webhooks/polling/types' import { fetchActiveWebhooks, runWithConcurrency } from '@/lib/webhooks/polling/utils' @@ -26,12 +27,14 @@ export async function pollProvider(providerName: string): Promise { activeWebhooks, async (entry) => { const requestId = generateShortId() - return handler.pollWebhook({ - webhookData: entry.webhook, - workflowData: entry.workflow, - requestId, - logger, - }) + return withResourceOutboundScope({ workspaceId: entry.workflow.workspaceId }, () => + handler.pollWebhook({ + webhookData: entry.webhook, + workflowData: entry.workflow, + requestId, + logger, + }) + ) }, logger ) diff --git a/apps/sim/lib/workflows/executor/execution-core.ts b/apps/sim/lib/workflows/executor/execution-core.ts index a465c6b48e4..09fbb27c2d7 100644 --- a/apps/sim/lib/workflows/executor/execution-core.ts +++ b/apps/sim/lib/workflows/executor/execution-core.ts @@ -19,6 +19,8 @@ import { getTimeoutErrorMessage, isTimeoutAbortReason, } from '@/lib/core/execution-limits' +import { isOutboundRoutingEnabled } from '@/lib/core/network/config.server' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { withDatabaseReadRetry } from '@/lib/db/read-retry' import { getExecutionEnvironment } from '@/lib/environment/utils' import { clearExecutionCancellation } from '@/lib/execution/cancellation' @@ -29,6 +31,7 @@ import type { LoggingSession } from '@/lib/logs/execution/logging-session' import { redactLargeValueRefsInValue } from '@/lib/logs/execution/pii-large-values' import { redactObjectStrings } from '@/lib/logs/execution/pii-redaction' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' +import { resolveActiveWorkflowApplicationContext } from '@/lib/workflows/application/context' import { waitForChildRuns } from '@/lib/workflows/custom-blocks/child-execution' import { getCustomBlockRowsForWorkspace } from '@/lib/workflows/custom-blocks/operations' import { resolveStartBlockRunIdentity } from '@/lib/workflows/executor/start-run-identity' @@ -397,7 +400,13 @@ export async function executeWorkflowCore( label: 'getCustomBlockRowsForWorkspace', }) : [] - return withCustomBlockOverlay(rows, () => executeWorkflowCoreImpl(options)) + const execute = () => withCustomBlockOverlay(rows, () => executeWorkflowCoreImpl(options)) + if (!isOutboundRoutingEnabled()) return execute() + const context = await resolveActiveWorkflowApplicationContext({ + workflowId: options.snapshot.metadata.workflowId, + assertedWorkspaceId: workspaceId, + }) + return runWithOutboundOrganization(context.workspaceOrganizationId, execute) } async function executeWorkflowCoreImpl( diff --git a/apps/sim/lib/workspaces/admin-move-source-impact.ts b/apps/sim/lib/workspaces/admin-move-source-impact.ts index 687f89ec9ac..8a770c24d6e 100644 --- a/apps/sim/lib/workspaces/admin-move-source-impact.ts +++ b/apps/sim/lib/workspaces/admin-move-source-impact.ts @@ -65,7 +65,7 @@ const ENTERPRISE_GATED_SECTION_LABELS: Record | undefined + const logger = createLogger('AnthropicProvider') export const anthropicProvider: ProviderConfig = { @@ -24,7 +27,16 @@ export const anthropicProvider: ProviderConfig = { providerLabel: 'Anthropic', createClient: (apiKey) => { const cacheKey = `anthropic::${apiKey}` - return getCachedProviderClient(cacheKey, () => new Anthropic({ apiKey })) + return getCachedProviderClient( + cacheKey, + () => + new Anthropic({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, + apiKey, + }) + ) }, logger, }) diff --git a/apps/sim/providers/azure-anthropic/index.test.ts b/apps/sim/providers/azure-anthropic/index.test.ts index 1de56f6ed15..51556652a51 100644 --- a/apps/sim/providers/azure-anthropic/index.test.ts +++ b/apps/sim/providers/azure-anthropic/index.test.ts @@ -10,6 +10,8 @@ const { anthropicArgs, mockValidate, mockCreatePinnedFetch, + guardedFetchFn, + mockCreateGuardedFetch, mockExecuteAnthropic, sentinelFetch, } = vi.hoisted(() => { @@ -20,7 +22,10 @@ const { anthropicArgs.push(opts) } } + const guardedFetchFn = vi.fn() return { + guardedFetchFn, + mockCreateGuardedFetch: vi.fn(() => ({ fetch: guardedFetchFn })), mockAnthropic: MockAnthropic, anthropicArgs, mockValidate: vi.fn(), @@ -32,6 +37,7 @@ const { vi.mock('@anthropic-ai/sdk', () => ({ default: mockAnthropic })) vi.mock('@/lib/core/security/input-validation.server', () => ({ + createSsrfGuardedFetchWithDispatcher: mockCreateGuardedFetch, validateUrlWithDNS: mockValidate, createPinnedFetch: mockCreatePinnedFetch, })) @@ -95,7 +101,8 @@ describe('azureAnthropicProvider — SSRF pinning', () => { expect(mockValidate).not.toHaveBeenCalled() expect(mockCreatePinnedFetch).not.toHaveBeenCalled() - expect(buildClientOptions()).not.toHaveProperty('fetch') + expect(buildClientOptions().fetch).toBe(guardedFetchFn) + expect(mockCreateGuardedFetch).toHaveBeenCalledWith({ profile: 'configuredEndpoint' }) }) it('keeps the registry model in core and resolves a separate Azure wire model', async () => { diff --git a/apps/sim/providers/azure-anthropic/index.ts b/apps/sim/providers/azure-anthropic/index.ts index dff9f1be274..52e8fcc2a66 100644 --- a/apps/sim/providers/azure-anthropic/index.ts +++ b/apps/sim/providers/azure-anthropic/index.ts @@ -1,13 +1,19 @@ import Anthropic from '@anthropic-ai/sdk' import { createLogger } from '@sim/logger' import { env } from '@/lib/core/config/env' -import { createPinnedFetch, validateUrlWithDNS } from '@/lib/core/security/input-validation.server' +import { + createPinnedFetch, + createSsrfGuardedFetchWithDispatcher, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { executeAnthropicProviderRequest } from '@/providers/anthropic/core' import { getCachedProviderClient } from '@/providers/client-cache' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import type { ProviderConfig, ProviderRequest, ProviderResponse } from '@/providers/types' +let providerTransport: ReturnType | undefined + const logger = createLogger('AzureAnthropicProvider') export const azureAnthropicProvider: ProviderConfig = { @@ -77,9 +83,13 @@ export const azureAnthropicProvider: ProviderConfig = { cacheKey, () => new Anthropic({ + fetch: + pinnedFetch ?? + (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, baseURL, apiKey, - ...(pinnedFetch ? { fetch: pinnedFetch } : {}), defaultHeaders: { 'anthropic-version': anthropicVersion, }, diff --git a/apps/sim/providers/azure-openai/index.test.ts b/apps/sim/providers/azure-openai/index.test.ts index 48c7431cb36..02885ce2efd 100644 --- a/apps/sim/providers/azure-openai/index.test.ts +++ b/apps/sim/providers/azure-openai/index.test.ts @@ -12,6 +12,8 @@ const { mockChatCreate, mockValidate, mockCreatePinnedFetch, + guardedFetchFn, + mockCreateGuardedFetch, mockExecuteResponses, sentinelFetch, mockIsChatCompletionsEndpoint, @@ -28,7 +30,10 @@ const { azureOpenAIArgs.push(opts) } } + const guardedFetchFn = vi.fn() return { + guardedFetchFn, + mockCreateGuardedFetch: vi.fn(() => ({ fetch: guardedFetchFn })), mockAzureOpenAI: MockAzureOpenAI, azureOpenAIArgs, mockChatCreate, @@ -46,6 +51,7 @@ const { vi.mock('openai', () => ({ AzureOpenAI: mockAzureOpenAI })) vi.mock('@/providers', () => ({ MAX_TOOL_ITERATIONS: 20 })) vi.mock('@/lib/core/security/input-validation.server', () => ({ + createSsrfGuardedFetchWithDispatcher: mockCreateGuardedFetch, validateUrlWithDNS: mockValidate, createPinnedFetch: mockCreatePinnedFetch, })) @@ -153,14 +159,15 @@ describe('azureOpenAIProvider — SSRF pinning', () => { expect(responsesConfig().fetch).toBe(sentinelFetch) }) - it('passes no custom fetch when the endpoint comes from trusted server env', async () => { + it('uses the configured-endpoint transport when the endpoint comes from server env', async () => { setEnv({ AZURE_OPENAI_ENDPOINT: 'https://trusted.openai.azure.com' }) await azureOpenAIProvider.executeRequest(request({ azureEndpoint: undefined })) expect(mockValidate).not.toHaveBeenCalled() expect(mockCreatePinnedFetch).not.toHaveBeenCalled() - expect(responsesConfig().fetch).toBeUndefined() + expect(responsesConfig().fetch).toBe(guardedFetchFn) + expect(mockCreateGuardedFetch).toHaveBeenCalledWith({ profile: 'configuredEndpoint' }) }) it('throws and never reaches the Responses core when validation blocks the endpoint', async () => { @@ -198,7 +205,7 @@ describe('azureOpenAIProvider — SSRF pinning', () => { expect(azureOpenAIArgs[0]).toMatchObject({ fetch: sentinelFetch }) }) - it('constructs the AzureOpenAI client without a custom fetch for a trusted env endpoint', async () => { + it('uses the shared outbound transport for a trusted env endpoint', async () => { mockIsChatCompletionsEndpoint.mockReturnValue(true) setEnv({ AZURE_OPENAI_ENDPOINT: @@ -212,7 +219,7 @@ describe('azureOpenAIProvider — SSRF pinning', () => { await azureOpenAIProvider.executeRequest(request({ azureEndpoint: undefined })) expect(mockCreatePinnedFetch).not.toHaveBeenCalled() - expect(azureOpenAIArgs[0]).not.toHaveProperty('fetch') + expect(azureOpenAIArgs[0].fetch).toBe(guardedFetchFn) }) it('projects the settled tool-loop answer without a final streaming request', async () => { diff --git a/apps/sim/providers/azure-openai/index.ts b/apps/sim/providers/azure-openai/index.ts index 72251763d99..84944232c96 100644 --- a/apps/sim/providers/azure-openai/index.ts +++ b/apps/sim/providers/azure-openai/index.ts @@ -14,7 +14,11 @@ import type { } from 'openai/resources/chat/completions' import type { ReasoningEffort } from 'openai/resources/shared' import { env } from '@/lib/core/config/env' -import { createPinnedFetch, validateUrlWithDNS } from '@/lib/core/security/input-validation.server' +import { + createPinnedFetch, + createSsrfGuardedFetchWithDispatcher, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { prepareProviderAttachments } from '@/providers/attachments' @@ -51,6 +55,8 @@ import { sumToolCosts, } from '@/providers/utils' +let providerTransport: ReturnType | undefined + /** `verbosity` narrowed from `string` to a literal union in openai v5. */ type ChatCompletionVerbosity = NonNullable @@ -81,10 +87,14 @@ async function executeChatCompletionsRequest( }) const azureOpenAI = new AzureOpenAI({ + fetch: + pinnedFetch ?? + (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, apiKey: request.apiKey!, apiVersion: azureApiVersion, endpoint: azureEndpoint, - ...(pinnedFetch ? { fetch: pinnedFetch } : {}), }) const allMessages: ChatCompletionMessageParam[] = [] @@ -749,7 +759,11 @@ export const azureOpenAIProvider: ProviderConfig = { 'api-key': apiKey, }, logger, - fetch: pinnedFetch, + fetch: + pinnedFetch ?? + (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, } ) } @@ -774,7 +788,11 @@ export const azureOpenAIProvider: ProviderConfig = { 'api-key': apiKey, }, logger, - fetch: pinnedFetch, + fetch: + pinnedFetch ?? + (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, } ) }, diff --git a/apps/sim/providers/baseten/index.ts b/apps/sim/providers/baseten/index.ts index 5b00cecfcf0..ec8f06b111d 100644 --- a/apps/sim/providers/baseten/index.ts +++ b/apps/sim/providers/baseten/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -38,6 +39,8 @@ import { sumToolCosts, } from '@/providers/utils' +let providerTransport: ReturnType | undefined + const logger = createLogger('BasetenProvider') /** @@ -87,6 +90,9 @@ export const basetenProvider: ProviderConfig = { } const client = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey: request.apiKey, baseURL: 'https://inference.baseten.co/v1', diff --git a/apps/sim/providers/bedrock/index.test.ts b/apps/sim/providers/bedrock/index.test.ts index bf4b2fdecdd..4471f152a6d 100644 --- a/apps/sim/providers/bedrock/index.test.ts +++ b/apps/sim/providers/bedrock/index.test.ts @@ -108,6 +108,7 @@ describe('bedrockProvider credential handling', () => { }) expect(BedrockRuntimeClient).toHaveBeenCalledWith({ + requestHandler: expect.objectContaining({ handle: expect.any(Function) }), region: 'us-east-1', credentials: { accessKeyId: 'AKIAIOSFODNN7EXAMPLE', @@ -120,6 +121,7 @@ describe('bedrockProvider credential handling', () => { await bedrockProvider.executeRequest(baseRequest) expect(BedrockRuntimeClient).toHaveBeenCalledWith({ + requestHandler: expect.objectContaining({ handle: expect.any(Function) }), region: 'us-east-1', }) }) @@ -131,6 +133,7 @@ describe('bedrockProvider credential handling', () => { }) expect(BedrockRuntimeClient).toHaveBeenCalledWith({ + requestHandler: expect.objectContaining({ handle: expect.any(Function) }), region: 'eu-west-1', }) }) diff --git a/apps/sim/providers/bedrock/index.ts b/apps/sim/providers/bedrock/index.ts index 28b9b112d27..01158801178 100644 --- a/apps/sim/providers/bedrock/index.ts +++ b/apps/sim/providers/bedrock/index.ts @@ -17,6 +17,7 @@ import { import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' +import { createOutboundAwsHttpHandler } from '@/lib/core/network/aws-handler.server' import { validateAwsRegion } from '@/lib/core/security/input-validation' import type { IterationToolCall, NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' @@ -164,7 +165,11 @@ export const bedrockProvider: ProviderConfig = { : 'default-chain' const client = getCachedProviderClient( `bedrock::${region}::${credentialKey}`, - () => new BedrockRuntimeClient(clientConfig) + () => + new BedrockRuntimeClient({ + ...clientConfig, + requestHandler: createOutboundAwsHttpHandler(), + }) ) const messages: BedrockMessage[] = [] diff --git a/apps/sim/providers/deepseek/index.ts b/apps/sim/providers/deepseek/index.ts index 2897d8e9ab0..64072b31bd2 100644 --- a/apps/sim/providers/deepseek/index.ts +++ b/apps/sim/providers/deepseek/index.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -29,6 +30,8 @@ import { trackForcedToolUsage, } from '@/providers/utils' +let providerTransport: ReturnType | undefined + const logger = createLogger('DeepseekProvider') export const deepseekProvider: ProviderConfig = { @@ -51,6 +54,9 @@ export const deepseekProvider: ProviderConfig = { try { const deepseek = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey: request.apiKey, baseURL: 'https://api.deepseek.com', diff --git a/apps/sim/providers/file-attachments.server.test.ts b/apps/sim/providers/file-attachments.server.test.ts index f88a10bf5f3..783432af09c 100644 --- a/apps/sim/providers/file-attachments.server.test.ts +++ b/apps/sim/providers/file-attachments.server.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' import { @@ -16,6 +17,11 @@ import { import { runWithProviderRuntimeContext } from '@/providers/runtime-context' import type { ProviderRequest } from '@/providers/types' +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + const { mockDownloadServableFileFromStorage, mockGeneratePresignedDownloadUrl, diff --git a/apps/sim/providers/file-attachments.server.ts b/apps/sim/providers/file-attachments.server.ts index 39fcbf9267e..42925153b34 100644 --- a/apps/sim/providers/file-attachments.server.ts +++ b/apps/sim/providers/file-attachments.server.ts @@ -2,6 +2,8 @@ import { FileState, GoogleGenAI } from '@google/genai' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' +import { requireDirectOutboundTransport } from '@/lib/core/network/context.server' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { StorageService } from '@/lib/uploads' import { resolveTrustedFileContext } from '@/lib/uploads/utils/file-utils' import { downloadServableFileFromStorage } from '@/lib/uploads/utils/file-utils.server' @@ -18,6 +20,8 @@ import { } from '@/providers/attachments' import type { Message, ProviderId, ProviderRequest } from '@/providers/types' +let providerTransport: ReturnType | undefined + const logger = createLogger('ProviderFileAttachments') const OPENAI_FILES_ENDPOINT = 'https://api.openai.com/v1/files' @@ -138,6 +142,7 @@ export async function uploadLargeFilesToProvider( if (groups.length === 0) return const maxBytes = getProviderAttachmentMaxBytes(providerId) + if (providerId === 'google') await requireDirectOutboundTransport() const ai = providerId === 'google' ? new GoogleGenAI({ apiKey: request.apiKey }) : null for (const group of groups) { @@ -232,7 +237,9 @@ async function uploadOpenAIFile( form.append('expires_after[seconds]', String(OPENAI_FILE_EXPIRY_SECONDS)) form.append('file', blob, file.name) - const response = await fetch(OPENAI_FILES_ENDPOINT, { + const response = await (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch(OPENAI_FILES_ENDPOINT, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}` }, body: form, diff --git a/apps/sim/providers/fireworks/index.ts b/apps/sim/providers/fireworks/index.ts index 5ebb31606cd..2d1e3237a13 100644 --- a/apps/sim/providers/fireworks/index.ts +++ b/apps/sim/providers/fireworks/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -39,6 +40,8 @@ import { sumToolCosts, } from '@/providers/utils' +let providerTransport: ReturnType | undefined + const logger = createLogger('FireworksProvider') /** @@ -88,6 +91,9 @@ export const fireworksProvider: ProviderConfig = { } const client = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey: request.apiKey, baseURL: 'https://api.fireworks.ai/inference/v1', diff --git a/apps/sim/providers/index.ts b/apps/sim/providers/index.ts index 01a64387c99..14acaf2e4d9 100644 --- a/apps/sim/providers/index.ts +++ b/apps/sim/providers/index.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { getApiKeyWithBYOK } from '@/lib/api-key/byok' import { env, envNumber } from '@/lib/core/config/env' +import { requireDirectOutboundTransport } from '@/lib/core/network/context.server' import { filterModelSafeWorkspaceFileAttachments } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' import type { StreamingExecution } from '@/executor/types' import { @@ -265,6 +266,7 @@ export async function executeProviderRequest( } const response = await runWithProviderRuntimeContext(requestRuntimeContext, async () => { + if (providerId === 'google' || providerId === 'vertex') await requireDirectOutboundTransport() await attachLargeFileRemoteUrls(modelSafeRequest, providerId) await uploadLargeFilesToProvider(modelSafeRequest, providerId) return provider.executeRequest(modelSafeRequest) diff --git a/apps/sim/providers/kimi/index.ts b/apps/sim/providers/kimi/index.ts index 0b25f27b5fe..5b7e80d0d41 100644 --- a/apps/sim/providers/kimi/index.ts +++ b/apps/sim/providers/kimi/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -37,6 +38,8 @@ import { trackForcedToolUsage, } from '@/providers/utils' +let providerTransport: ReturnType | undefined + const logger = createLogger('KimiProvider') const KIMI_BASE_URL = 'https://api.moonshot.ai/v1' @@ -101,6 +104,9 @@ export const kimiProvider: ProviderConfig = { try { const kimi = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey: request.apiKey, baseURL: KIMI_BASE_URL, diff --git a/apps/sim/providers/litellm/index.ts b/apps/sim/providers/litellm/index.ts index 2bdcb7e4a66..ec8f5bdb993 100644 --- a/apps/sim/providers/litellm/index.ts +++ b/apps/sim/providers/litellm/index.ts @@ -4,6 +4,10 @@ import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' import { env } from '@/lib/core/config/env' +import { + createSsrfGuardedFetchWithDispatcher, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -36,6 +40,8 @@ import { } from '@/providers/utils' import { useProvidersStore } from '@/stores/providers' +let providerTransport: ReturnType | undefined + const logger = createLogger('LiteLLMProvider') const LITELLM_VERSION = '1.0.0' @@ -68,7 +74,12 @@ export const litellmProvider: ProviderConfig = { headers.Authorization = `Bearer ${env.LITELLM_API_KEY}` } - const response = await fetch(`${baseUrl}/v1/models`, { headers }) + const response = await secureFetchWithValidation(`${baseUrl}/v1/models`, { + profile: 'selfHostedService', + maxRedirects: 20, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + headers, + }) if (!response.ok) { await response.text().catch(() => {}) useProvidersStore.getState().setProviderModels('litellm', []) @@ -111,6 +122,9 @@ export const litellmProvider: ProviderConfig = { const apiKey = request.apiKey || env.LITELLM_API_KEY || 'empty' const litellm = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'selfHostedService', + })).fetch, ...openAICompatTransport(), apiKey, baseURL: `${baseUrl}/v1`, diff --git a/apps/sim/providers/meta/index.ts b/apps/sim/providers/meta/index.ts index e342b2d026b..4dc5d0969e2 100644 --- a/apps/sim/providers/meta/index.ts +++ b/apps/sim/providers/meta/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -30,6 +31,8 @@ import { sumToolCosts, } from '@/providers/utils' +let providerTransport: ReturnType | undefined + const logger = createLogger('MetaProvider') const META_BASE_URL = 'https://api.meta.ai/v1' @@ -54,6 +57,9 @@ export const metaProvider: ProviderConfig = { try { const meta = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey: request.apiKey, baseURL: META_BASE_URL, diff --git a/apps/sim/providers/mistral/index.ts b/apps/sim/providers/mistral/index.ts index bd88f7d371c..547f727f498 100644 --- a/apps/sim/providers/mistral/index.ts +++ b/apps/sim/providers/mistral/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -31,6 +32,8 @@ import { trackForcedToolUsage, } from '@/providers/utils' +let providerTransport: ReturnType | undefined + const logger = createLogger('MistralProvider') /** @@ -62,6 +65,9 @@ export const mistralProvider: ProviderConfig = { } const mistral = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey: request.apiKey, baseURL: 'https://api.mistral.ai/v1', diff --git a/apps/sim/providers/nvidia/index.ts b/apps/sim/providers/nvidia/index.ts index 5973ba2af54..87d727aa785 100644 --- a/apps/sim/providers/nvidia/index.ts +++ b/apps/sim/providers/nvidia/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -32,6 +33,8 @@ import { trackForcedToolUsage, } from '@/providers/utils' +let providerTransport: ReturnType | undefined + const logger = createLogger('NvidiaProvider') const NVIDIA_BASE_URL = 'https://integrate.api.nvidia.com/v1' @@ -61,6 +64,9 @@ export const nvidiaProvider: ProviderConfig = { try { const nvidia = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey: request.apiKey, baseURL: NVIDIA_BASE_URL, diff --git a/apps/sim/providers/ollama-cloud/index.ts b/apps/sim/providers/ollama-cloud/index.ts index 11d7bb4372b..c82e42f8568 100644 --- a/apps/sim/providers/ollama-cloud/index.ts +++ b/apps/sim/providers/ollama-cloud/index.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import OpenAI from 'openai' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { getProviderDefaultModel, getProviderModels } from '@/providers/models' import { executeOllamaProviderRequest } from '@/providers/ollama/core' @@ -7,6 +8,8 @@ import { createReadableStreamFromOllamaCloudStream } from '@/providers/ollama-cl import { openAICompatTransport } from '@/providers/transport' import type { ProviderConfig, ProviderRequest, ProviderResponse } from '@/providers/types' +let providerTransport: ReturnType | undefined + const logger = createLogger('OllamaCloudProvider') /** Ollama Cloud OpenAI-compatible endpoint. BYOK only — Sim never hosts a key or bills usage. */ @@ -37,6 +40,9 @@ export const ollamaCloudProvider: ProviderConfig = { providerLabel: 'Ollama Cloud', createClient: () => new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey, baseURL: OLLAMA_CLOUD_BASE_URL, diff --git a/apps/sim/providers/ollama/index.ts b/apps/sim/providers/ollama/index.ts index cd0e16f8bd7..eea97c8c09e 100644 --- a/apps/sim/providers/ollama/index.ts +++ b/apps/sim/providers/ollama/index.ts @@ -1,6 +1,10 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import OpenAI from 'openai' +import { + createSsrfGuardedFetchWithDispatcher, + secureFetchWithValidation, +} from '@/lib/core/security/input-validation.server' import { getOllamaUrl } from '@/lib/core/utils/urls' import type { StreamingExecution } from '@/executor/types' import { executeOllamaProviderRequest } from '@/providers/ollama/core' @@ -10,6 +14,8 @@ import { openAICompatTransport } from '@/providers/transport' import type { ProviderConfig, ProviderRequest, ProviderResponse } from '@/providers/types' import { useProvidersStore } from '@/stores/providers' +let providerTransport: ReturnType | undefined + const logger = createLogger('OllamaProvider') const OLLAMA_HOST = getOllamaUrl() @@ -28,7 +34,11 @@ export const ollamaProvider: ProviderConfig = { } try { - const response = await fetch(`${OLLAMA_HOST}/api/tags`) + const response = await secureFetchWithValidation(`${OLLAMA_HOST}/api/tags`, { + profile: 'selfHostedService', + maxRedirects: 20, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + }) if (!response.ok) { await response.text().catch(() => {}) useProvidersStore.getState().setProviderModels('ollama', []) @@ -53,6 +63,9 @@ export const ollamaProvider: ProviderConfig = { providerLabel: 'Ollama', createClient: () => new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'selfHostedService', + })).fetch, ...openAICompatTransport(), apiKey: 'empty', baseURL: `${OLLAMA_HOST}/v1`, diff --git a/apps/sim/providers/openai/core.ts b/apps/sim/providers/openai/core.ts index 27c5903ee52..29adf823cea 100644 --- a/apps/sim/providers/openai/core.ts +++ b/apps/sim/providers/openai/core.ts @@ -4,6 +4,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { truncate } from '@sim/utils/string' import type OpenAI from 'openai' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { createOpenAIResponsesStreamingToolLoopStream } from '@/providers/openai/streaming-tool-loop' @@ -42,6 +43,8 @@ import { toResponsesToolChoice, } from './utils' +let providerTransport: ReturnType | undefined + /** * Rejects a `/v1/responses` body reporting a generation that did not succeed — the * endpoint answers HTTP 200 for both `status: 'failed'` and `status: 'incomplete'`. @@ -122,7 +125,7 @@ export interface ResponsesProviderConfig { /** * Optional fetch implementation. Used to pin the connection to a pre-validated * IP (DNS-rebinding/SSRF protection) when the endpoint is user-supplied. - * Defaults to the global fetch. + * Defaults to the shared guarded transport for configured provider endpoints. */ fetch?: typeof fetch } @@ -135,7 +138,10 @@ export async function executeResponsesProviderRequest( config: ResponsesProviderConfig ): Promise { const { logger } = config - const fetchImpl = config.fetch ?? fetch + const fetchImpl = + config.fetch ?? + (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' })) + .fetch logger.info(`Preparing ${config.providerLabel} request`, { model: request.model, diff --git a/apps/sim/providers/openrouter/index.ts b/apps/sim/providers/openrouter/index.ts index fbfeddd6193..88b8d002370 100644 --- a/apps/sim/providers/openrouter/index.ts +++ b/apps/sim/providers/openrouter/index.ts @@ -6,6 +6,7 @@ import type { ChatCompletionCreateParamsStreaming, ChatCompletionMessage, } from 'openai/resources/chat/completions' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -45,6 +46,8 @@ import { sumToolCosts, } from '@/providers/utils' +let providerTransport: ReturnType | undefined + const logger = createLogger('OpenRouterProvider') type OpenRouterAssistantMessage = ChatCompletionMessage & { @@ -102,6 +105,9 @@ export const openRouterProvider: ProviderConfig = { } const client = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey: request.apiKey, baseURL: 'https://openrouter.ai/api/v1', diff --git a/apps/sim/providers/openrouter/utils.ts b/apps/sim/providers/openrouter/utils.ts index 2f8a7850fc9..a86beb437e9 100644 --- a/apps/sim/providers/openrouter/utils.ts +++ b/apps/sim/providers/openrouter/utils.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' import type { CompletionUsage } from 'openai/resources/completions' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { createOpenAICompatibleAgentEventStream } from '@/providers/openai-compat/stream-events' import type { AgentStreamEvent } from '@/providers/stream-events' import { checkForForcedToolUsageOpenAI } from '@/providers/utils' @@ -24,7 +25,10 @@ const CACHE_TTL_MS = 5 * 60 * 1000 // 5 minutes async function fetchModelCapabilities(): Promise> { try { - const response = await fetch('https://openrouter.ai/api/v1/models', { + const response = await secureFetchWithValidation('https://openrouter.ai/api/v1/models', { + profile: 'configuredEndpoint', + maxRedirects: 20, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, headers: { 'Content-Type': 'application/json' }, }) @@ -36,10 +40,10 @@ async function fetchModelCapabilities(): Promise> return new Map() } - const data = await response.json() + const data = (await response.json()) as { data?: OpenRouterModelData[] } const capabilities = new Map() - for (const model of (data.data ?? []) as OpenRouterModelData[]) { + for (const model of data.data ?? []) { const supportedParams = model.supported_parameters ?? [] capabilities.set(model.id, { supportsStructuredOutputs: supportedParams.includes('structured_outputs'), diff --git a/apps/sim/providers/sakana/index.ts b/apps/sim/providers/sakana/index.ts index 33c9861a626..a461ea24489 100644 --- a/apps/sim/providers/sakana/index.ts +++ b/apps/sim/providers/sakana/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -31,6 +32,8 @@ import { trackForcedToolUsage, } from '@/providers/utils' +let providerTransport: ReturnType | undefined + const logger = createLogger('SakanaProvider') const SAKANA_BASE_URL = 'https://api.sakana.ai/v1' @@ -55,6 +58,9 @@ export const sakanaProvider: ProviderConfig = { try { const sakana = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey: request.apiKey, baseURL: SAKANA_BASE_URL, diff --git a/apps/sim/providers/together/index.ts b/apps/sim/providers/together/index.ts index 27d0f7ed769..78b343d8b95 100644 --- a/apps/sim/providers/together/index.ts +++ b/apps/sim/providers/together/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -38,6 +39,8 @@ import { sumToolCosts, } from '@/providers/utils' +let providerTransport: ReturnType | undefined + const logger = createLogger('TogetherProvider') /** @@ -87,6 +90,9 @@ export const togetherProvider: ProviderConfig = { } const client = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey: request.apiKey, baseURL: 'https://api.together.ai/v1', diff --git a/apps/sim/providers/vllm/index.test.ts b/apps/sim/providers/vllm/index.test.ts index 1057bb89e5e..33c04eb1541 100644 --- a/apps/sim/providers/vllm/index.test.ts +++ b/apps/sim/providers/vllm/index.test.ts @@ -14,6 +14,8 @@ const { mockCreateStream, mockValidateUrlWithDNS, mockCreatePinnedFetch, + guardedFetchFn, + mockCreateGuardedFetch, pinnedFetchFn, } = vi.hoisted(() => { const openAIArgs: Array> = [] @@ -25,7 +27,10 @@ const { openAIArgs.push(opts) } } + const guardedFetchFn = vi.fn() return { + guardedFetchFn, + mockCreateGuardedFetch: vi.fn(() => ({ fetch: guardedFetchFn })), mockCreate, openAIArgs, mockOpenAI: MockOpenAI, @@ -41,6 +46,7 @@ const { vi.mock('openai', () => ({ default: mockOpenAI })) vi.mock('@/lib/core/security/input-validation.server', () => ({ + createSsrfGuardedFetchWithDispatcher: mockCreateGuardedFetch, validateUrlWithDNS: mockValidateUrlWithDNS, createPinnedFetch: mockCreatePinnedFetch, })) @@ -151,7 +157,7 @@ describe('vllmProvider', () => { }) describe('endpoint SSRF protection', () => { - it('does not validate or pin when no endpoint is supplied (uses env base URL)', async () => { + it('uses the self-hosted transport when the endpoint comes from server env', async () => { mockCreate.mockResolvedValueOnce(chatResponse('hi')) await vllmProvider.executeRequest({ @@ -162,7 +168,8 @@ describe('vllmProvider', () => { expect(mockValidateUrlWithDNS).not.toHaveBeenCalled() expect(mockCreatePinnedFetch).not.toHaveBeenCalled() expect(openAIArgs[0].baseURL).toBe('http://localhost:8000/v1') - expect(openAIArgs[0].fetch).toBeUndefined() + expect(openAIArgs[0].fetch).toBe(guardedFetchFn) + expect(mockCreateGuardedFetch).toHaveBeenCalledWith({ profile: 'selfHostedService' }) }) it('does not duplicate an existing /v1 API prefix', async () => { diff --git a/apps/sim/providers/vllm/index.ts b/apps/sim/providers/vllm/index.ts index 3bad1cba441..d53c6077e7d 100644 --- a/apps/sim/providers/vllm/index.ts +++ b/apps/sim/providers/vllm/index.ts @@ -4,7 +4,12 @@ import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' import { env } from '@/lib/core/config/env' -import { createPinnedFetch, validateUrlWithDNS } from '@/lib/core/security/input-validation.server' +import { + createPinnedFetch, + createSsrfGuardedFetchWithDispatcher, + secureFetchWithValidation, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -37,6 +42,8 @@ import { import { checkForForcedToolUsage, createReadableStreamFromVLLMStream } from '@/providers/vllm/utils' import { useProvidersStore } from '@/stores/providers' +let providerTransport: ReturnType | undefined + const logger = createLogger('VLLMProvider') const VLLM_VERSION = '1.0.0' @@ -70,7 +77,12 @@ export const vllmProvider: ProviderConfig = { headers.Authorization = `Bearer ${env.VLLM_API_KEY}` } - const response = await fetch(`${apiBaseUrl}/models`, { headers }) + const response = await secureFetchWithValidation(`${apiBaseUrl}/models`, { + profile: 'selfHostedService', + maxRedirects: 20, + redirectPolicy: { mode: 'standard', sendCredentialsOnCrossOriginRedirect: false }, + headers, + }) if (!response.ok) { await response.text().catch(() => {}) useProvidersStore.getState().setProviderModels('vllm', []) @@ -148,10 +160,14 @@ export const vllmProvider: ProviderConfig = { `vllm::${apiKey}::${apiBaseUrl}::${pinnedIP ?? 'no-pin'}`, () => new OpenAI({ + fetch: + pinnedFetch ?? + (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'selfHostedService', + })).fetch, ...openAICompatTransport(), apiKey, baseURL: apiBaseUrl, - ...(pinnedFetch ? { fetch: pinnedFetch } : {}), }) ) diff --git a/apps/sim/providers/xai/index.ts b/apps/sim/providers/xai/index.ts index 0f2fff891df..a59ca21d541 100644 --- a/apps/sim/providers/xai/index.ts +++ b/apps/sim/providers/xai/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -36,6 +37,8 @@ import { createResponseFormatPayload, } from '@/providers/xai/utils' +let providerTransport: ReturnType | undefined + const logger = createLogger('XAIProvider') /** @@ -68,6 +71,9 @@ export const xAIProvider: ProviderConfig = { } const xai = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey: request.apiKey, baseURL: 'https://api.x.ai/v1', diff --git a/apps/sim/providers/zai/index.ts b/apps/sim/providers/zai/index.ts index bce2deb33af..28415e342bb 100644 --- a/apps/sim/providers/zai/index.ts +++ b/apps/sim/providers/zai/index.ts @@ -3,6 +3,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import OpenAI from 'openai' import type { ChatCompletionChunk } from 'openai/resources/chat/completions' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { StreamingExecution } from '@/executor/types' import { MAX_TOOL_ITERATIONS } from '@/providers' import { formatMessagesForProvider } from '@/providers/attachments' @@ -31,6 +32,8 @@ import { } from '@/providers/utils' import { createReadableStreamFromZaiStream } from '@/providers/zai/utils' +let providerTransport: ReturnType | undefined + const logger = createLogger('ZaiProvider') const ZAI_BASE_URL = 'https://api.z.ai/api/paas/v4' @@ -84,6 +87,9 @@ export const zaiProvider: ProviderConfig = { try { const zai = new OpenAI({ + fetch: (providerTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + })).fetch, ...openAICompatTransport(), apiKey: request.apiKey, baseURL: ZAI_BASE_URL, diff --git a/apps/sim/tools/apify/run_actor_async.ts b/apps/sim/tools/apify/run_actor_async.ts index bfd78999919..16fb14310d3 100644 --- a/apps/sim/tools/apify/run_actor_async.ts +++ b/apps/sim/tools/apify/run_actor_async.ts @@ -1,8 +1,13 @@ import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { ApifyRun, RunActorParams, RunActorResult } from '@/tools/apify/types' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const POLL_INTERVAL_MS = 5000 const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS @@ -145,7 +150,7 @@ export const apifyRunActorAsyncTool: ToolConfig await sleep(POLL_INTERVAL_MS) elapsedTime += POLL_INTERVAL_MS - const statusResponse = await fetch(`https://api.apify.com/v2/actor-runs/${runId}`, { + const statusResponse = await providerFetch(`https://api.apify.com/v2/actor-runs/${runId}`, { headers: { Authorization: `Bearer ${params.apiKey}`, }, @@ -170,7 +175,7 @@ export const apifyRunActorAsyncTool: ToolConfig ) { if (run.status === 'SUCCEEDED' && run.defaultDatasetId) { const limit = Math.max(1, Math.min(params.itemLimit || 100, 250000)) - const itemsResponse = await fetch( + const itemsResponse = await providerFetch( `https://api.apify.com/v2/datasets/${run.defaultDatasetId}/items?limit=${limit}`, { headers: { diff --git a/apps/sim/tools/azure_devops/azure-devops.test.ts b/apps/sim/tools/azure_devops/azure-devops.test.ts index b1c126f22fb..4cae586248f 100644 --- a/apps/sim/tools/azure_devops/azure-devops.test.ts +++ b/apps/sim/tools/azure_devops/azure-devops.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, describe, expect, it, vi } from 'vitest' import { hasToolId } from '@/tools/tool-ids' import { isAzureDevOpsEventMatch } from '@/triggers/azure_devops/utils' diff --git a/apps/sim/tools/azure_devops/get_work_items_batch.ts b/apps/sim/tools/azure_devops/get_work_items_batch.ts index 3b5200d2af3..15222d1c7e8 100644 --- a/apps/sim/tools/azure_devops/get_work_items_batch.ts +++ b/apps/sim/tools/azure_devops/get_work_items_batch.ts @@ -1,3 +1,4 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { AzureDevOpsWorkItem, GetWorkItemsBatchParams, @@ -7,6 +8,10 @@ import type { AzureDevOpsRawWorkItem } from '@/tools/azure_devops/utils' import { formatWorkItem, mapWorkItem } from '@/tools/azure_devops/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + export const getWorkItemsBatchTool: ToolConfig = { id: 'azure_devops_get_work_items_batch', @@ -94,7 +99,7 @@ export const getWorkItemsBatchTool: ToolConfig = { id: 'azure_devops_query_work_items', name: 'Azure DevOps Query Work Items', @@ -83,7 +88,7 @@ export const queryWorkItemsTool: ToolConfig inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { executeCbinsightsChatOperation } from '@/lib/internal/cbinsights/operations/chat' import { executeCbinsightsGetCommercialMaturityHistoryOperation } from '@/lib/internal/cbinsights/operations/get-commercial-maturity-history' diff --git a/apps/sim/tools/cbinsights/utils.ts b/apps/sim/tools/cbinsights/utils.ts index db88c45bb25..2af234b1e38 100644 --- a/apps/sim/tools/cbinsights/utils.ts +++ b/apps/sim/tools/cbinsights/utils.ts @@ -1,5 +1,6 @@ import { getErrorMessage } from '@sim/utils/errors' import { LRUCache } from 'lru-cache' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -12,6 +13,10 @@ import type { } from '@/tools/cbinsights/types' import type { ToolResponse } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + /** CB Insights API v2 origin, as declared by the published Swagger document. */ export const CBINSIGHTS_API_BASE = 'https://api.cbinsights.com' @@ -124,7 +129,7 @@ async function authorize( clientSecret: string, signal?: AbortSignal ): Promise { - const response = await fetch(`${CBINSIGHTS_API_BASE}/v2/authorize`, { + const response = await providerFetch(`${CBINSIGHTS_API_BASE}/v2/authorize`, { method: 'POST', headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, body: JSON.stringify({ clientId, clientSecret }), @@ -202,7 +207,7 @@ export async function cbInsightsRequest( const effectiveSignal = signal ? AbortSignal.any([signal, timeout]) : timeout const send = async (token: string) => - fetch(`${CBINSIGHTS_API_BASE}${spec.path}`, { + providerFetch(`${CBINSIGHTS_API_BASE}${spec.path}`, { method: 'POST', headers: { Authorization: `Bearer ${token}`, diff --git a/apps/sim/tools/cloudflare/cloudflare.test.ts b/apps/sim/tools/cloudflare/cloudflare.test.ts index 01e65b0a60d..25a3a4b6b46 100644 --- a/apps/sim/tools/cloudflare/cloudflare.test.ts +++ b/apps/sim/tools/cloudflare/cloudflare.test.ts @@ -10,6 +10,13 @@ * These tests assert the seeded default reaching each tool for operations whose * control is deliberately not last, so re-introducing a collision goes red. */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + import { afterEach, describe, expect, it, vi } from 'vitest' import { executeGetZoneSettingsOperation } from '@/lib/internal/cloudflare/operations/get-zone-settings' import { CloudflareBlock } from '@/blocks/blocks/cloudflare' diff --git a/apps/sim/tools/datadog/datadog.test.ts b/apps/sim/tools/datadog/datadog.test.ts index 302cb2645db..f9d3b29e8ef 100644 --- a/apps/sim/tools/datadog/datadog.test.ts +++ b/apps/sim/tools/datadog/datadog.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { beforeEach, describe, expect, it, vi } from 'vitest' import { executeUpdateSloOperation } from '@/lib/internal/datadog/operations/update-slo' import * as datadogTools from '@/tools/datadog' diff --git a/apps/sim/tools/dropbox/download.ts b/apps/sim/tools/dropbox/download.ts index dbd57e8894c..db7f762d806 100644 --- a/apps/sim/tools/dropbox/download.ts +++ b/apps/sim/tools/dropbox/download.ts @@ -1,4 +1,5 @@ import { omit } from '@sim/utils/object' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { httpHeaderSafeJson } from '@/lib/core/utils/validation' import type { DropboxDownloadParams, @@ -7,6 +8,10 @@ import type { } from '@/tools/dropbox/types' import type { ToolConfig, ToolFileData } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + async function transformDownloadResponse(response: Response, params?: DropboxDownloadParams) { if (!response.ok) { const errorText = await response.text() @@ -28,14 +33,17 @@ async function transformDownloadResponse(response: Response, params?: DropboxDow let temporaryLink: string | undefined if (params?.accessToken) { try { - const linkResponse = await fetch('https://api.dropboxapi.com/2/files/get_temporary_link', { - method: 'POST', - headers: { - Authorization: `Bearer ${params.accessToken}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ path: params.path.trim() }), - }) + const linkResponse = await providerFetch( + 'https://api.dropboxapi.com/2/files/get_temporary_link', + { + method: 'POST', + headers: { + Authorization: `Bearer ${params.accessToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ path: params.path.trim() }), + } + ) if (linkResponse.ok) { const linkData = await linkResponse.json() temporaryLink = linkData.link diff --git a/apps/sim/tools/dropcontact-hosting.test.ts b/apps/sim/tools/dropcontact-hosting.test.ts index 02d92b02c39..d038eae8a66 100644 --- a/apps/sim/tools/dropcontact-hosting.test.ts +++ b/apps/sim/tools/dropcontact-hosting.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, describe, expect, it, vi } from 'vitest' import { dropcontactEnrichContactTool } from '@/tools/dropcontact/enrich_contact' import { DROPCONTACT_CREDIT_USD } from '@/tools/dropcontact/hosting' diff --git a/apps/sim/tools/dropcontact/enrich_contact.ts b/apps/sim/tools/dropcontact/enrich_contact.ts index a82322a0ad4..397e34d8f43 100644 --- a/apps/sim/tools/dropcontact/enrich_contact.ts +++ b/apps/sim/tools/dropcontact/enrich_contact.ts @@ -1,4 +1,5 @@ import { sleep } from '@sim/utils/helpers' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { dropcontactHosting } from '@/tools/dropcontact/hosting' import type { DropcontactEmailEntry, @@ -8,6 +9,10 @@ import type { } from '@/tools/dropcontact/types' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const POLL_INTERVAL_MS = 5000 const MAX_POLL_TIME_MS = 120000 @@ -254,7 +259,7 @@ export const dropcontactEnrichContactTool: ToolConfig< // Poll endpoint: GET https://api.dropcontact.com/v1/enrich/all/{request_id} // Source: https://developer.dropcontact.com (retrieved 2026-05) - const pollResponse = await fetch( + const pollResponse = await providerFetch( `https://api.dropcontact.com/v1/enrich/all/${encodeURIComponent(requestId)}`, { headers: { diff --git a/apps/sim/tools/enrichment-hosting.test.ts b/apps/sim/tools/enrichment-hosting.test.ts index c4d298acd99..83484e73f36 100644 --- a/apps/sim/tools/enrichment-hosting.test.ts +++ b/apps/sim/tools/enrichment-hosting.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, describe, expect, it, vi } from 'vitest' import { ErrorExtractorId } from '@/tools/error-extractors' import { findEmailFromNameTool } from '@/tools/findymail/find_email_from_name' diff --git a/apps/sim/tools/enrow-hosting.test.ts b/apps/sim/tools/enrow-hosting.test.ts index 46e9d7a453a..11cf70861b3 100644 --- a/apps/sim/tools/enrow-hosting.test.ts +++ b/apps/sim/tools/enrow-hosting.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, describe, expect, it, vi } from 'vitest' import { enrowFindEmailTool } from '@/tools/enrow/find_email' import { ENROW_CREDIT_USD } from '@/tools/enrow/hosting' diff --git a/apps/sim/tools/enrow/find_email.test.ts b/apps/sim/tools/enrow/find_email.test.ts index 4077119dada..4a5ade6bc81 100644 --- a/apps/sim/tools/enrow/find_email.test.ts +++ b/apps/sim/tools/enrow/find_email.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' /** @@ -21,6 +22,8 @@ import type { import { enrowVerifyEmailTool } from '@/tools/enrow/verify_email' import type { ToolResponse } from '@/tools/types' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const JOB_ID = 'job-123' /** `postProcess`'s third argument — never invoked by these polling tools. */ diff --git a/apps/sim/tools/enrow/find_email.ts b/apps/sim/tools/enrow/find_email.ts index bcead35e5fa..17bee6c14ba 100644 --- a/apps/sim/tools/enrow/find_email.ts +++ b/apps/sim/tools/enrow/find_email.ts @@ -1,4 +1,5 @@ import { sleep } from '@sim/utils/helpers' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { enrowHosting } from '@/tools/enrow/hosting' import type { EnrowFindEmailParams, @@ -12,6 +13,10 @@ import { } from '@/tools/enrow/types' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const POLL_INTERVAL_MS = 3000 const MAX_POLL_TIME_MS = 120_000 @@ -142,7 +147,7 @@ export const enrowFindEmailTool: ToolConfig = { elapsedTime += POLL_INTERVAL_MS try { - const statusResponse = await fetch(`https://api.exa.ai/agent/runs/${runId}`, { + const statusResponse = await providerFetch(`https://api.exa.ai/agent/runs/${runId}`, { method: 'GET', headers: { 'x-api-key': params.apiKey, diff --git a/apps/sim/tools/file-message-provider-clients.test.ts b/apps/sim/tools/file-message-provider-clients.test.ts index 5f65fef9493..b41c0f58922 100644 --- a/apps/sim/tools/file-message-provider-clients.test.ts +++ b/apps/sim/tools/file-message-provider-clients.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -11,6 +12,7 @@ const mocks = vi.hoisted(() => ({ })) vi.mock('@/lib/core/security/input-validation.server', () => ({ + ...inputValidationMock, MAX_JSON_API_RESPONSE_BYTES: 10 * 1024 * 1024, secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP, secureFetchWithValidation: mocks.secureFetchWithValidation, @@ -29,6 +31,9 @@ describe('file and message provider clients', () => { beforeEach(() => { vi.clearAllMocks() vi.stubGlobal('fetch', mocks.fetch) + mocks.secureFetchWithValidation.mockImplementation((...args: Parameters) => + fetch(...args) + ) mocks.validateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '203.0.113.10' }) }) diff --git a/apps/sim/tools/firecrawl/agent.ts b/apps/sim/tools/firecrawl/agent.ts index 13c77ffda28..bf2e934714b 100644 --- a/apps/sim/tools/firecrawl/agent.ts +++ b/apps/sim/tools/firecrawl/agent.ts @@ -1,9 +1,14 @@ import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { AgentParams, AgentResponse } from '@/tools/firecrawl/types' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('FirecrawlAgentTool') const POLL_INTERVAL_MS = 5000 @@ -119,7 +124,7 @@ export const agentTool: ToolConfig = { while (elapsedTime < MAX_POLL_TIME_MS) { try { - const statusResponse = await fetch(`https://api.firecrawl.dev/v2/agent/${jobId}`, { + const statusResponse = await providerFetch(`https://api.firecrawl.dev/v2/agent/${jobId}`, { method: 'GET', headers: { Authorization: `Bearer ${params.apiKey}`, diff --git a/apps/sim/tools/firecrawl/batch-scrape.ts b/apps/sim/tools/firecrawl/batch-scrape.ts index f2d23d82044..b97a53676c7 100644 --- a/apps/sim/tools/firecrawl/batch-scrape.ts +++ b/apps/sim/tools/firecrawl/batch-scrape.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { firecrawlHosting } from '@/tools/firecrawl/hosting' import { applyFirecrawlFormatModelInput, @@ -15,6 +16,10 @@ import type { import { CRAWLED_PAGE_OUTPUT_PROPERTIES } from '@/tools/firecrawl/types' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('FirecrawlBatchScrapeTool') const POLL_INTERVAL_MS = 5000 @@ -196,13 +201,16 @@ export const batchScrapeTool: ToolConfig while (elapsedTime < MAX_POLL_TIME_MS) { try { - const statusResponse = await fetch(`https://api.firecrawl.dev/v2/crawl/${jobId}`, { + const statusResponse = await providerFetch(`https://api.firecrawl.dev/v2/crawl/${jobId}`, { method: 'GET', headers: { Authorization: `Bearer ${params.apiKey}`, diff --git a/apps/sim/tools/firecrawl/extract.ts b/apps/sim/tools/firecrawl/extract.ts index 1ef2b13a15f..3321d3438eb 100644 --- a/apps/sim/tools/firecrawl/extract.ts +++ b/apps/sim/tools/firecrawl/extract.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { firecrawlHosting } from '@/tools/firecrawl/hosting' import { applyFirecrawlScrapeOptionsModelInput, @@ -9,6 +10,10 @@ import { import type { ExtractParams, ExtractResponse } from '@/tools/firecrawl/types' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('FirecrawlExtractTool') const POLL_INTERVAL_MS = 5000 @@ -168,13 +173,16 @@ export const extractTool: ToolConfig = { while (elapsedTime < MAX_POLL_TIME_MS) { try { - const statusResponse = await fetch(`https://api.firecrawl.dev/v2/extract/${jobId}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${params.apiKey}`, - 'Content-Type': 'application/json', - }, - }) + const statusResponse = await providerFetch( + `https://api.firecrawl.dev/v2/extract/${jobId}`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${params.apiKey}`, + 'Content-Type': 'application/json', + }, + } + ) if (!statusResponse.ok) { throw new Error(`Failed to get extract status: ${statusResponse.statusText}`) diff --git a/apps/sim/tools/github/pr.test.ts b/apps/sim/tools/github/pr.test.ts index 915250edbec..e9071abe39d 100644 --- a/apps/sim/tools/github/pr.test.ts +++ b/apps/sim/tools/github/pr.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterEach, describe, expect, expectTypeOf, it, vi } from 'vitest' import { listPRsV2Tool } from '@/tools/github/list_prs' import { prTool, prV2Tool } from '@/tools/github/pr' @@ -10,6 +11,8 @@ import type { PRV2OperationParams, } from '@/tools/github/types' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + type HasIncludeFiles = 'includeFiles' extends keyof T ? true : false const BASE_PARAMS = { diff --git a/apps/sim/tools/github/pr.ts b/apps/sim/tools/github/pr.ts index 6460911ea2a..870306ae083 100644 --- a/apps/sim/tools/github/pr.ts +++ b/apps/sim/tools/github/pr.ts @@ -1,4 +1,5 @@ import { isRecordLike } from '@sim/utils/object' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { nullableBoolean, nullableString, @@ -21,6 +22,10 @@ import type { import { PR_BRANCH_REF_OUTPUT, PR_FILE_OUTPUT_PROPERTIES, USER_OUTPUT } from '@/tools/github/types' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + type GitHubPullRequest = Omit type PullRequestFilesResult = @@ -149,7 +154,7 @@ async function fetchPullRequestFiles( const maxPages = MAX_PULL_REQUEST_FILES / PULL_REQUEST_FILES_PER_PAGE for (let page = 1; page <= maxPages; page += 1) { - const response = await fetch( + const response = await providerFetch( `https://api.github.com/repos/${params.owner}/${params.repo}/pulls/${pullNumber}/files?per_page=${PULL_REQUEST_FILES_PER_PAGE}&page=${page}`, { headers: { diff --git a/apps/sim/tools/gmail/read.test.ts b/apps/sim/tools/gmail/read.test.ts index e7b1e8a8384..757a9c62467 100644 --- a/apps/sim/tools/gmail/read.test.ts +++ b/apps/sim/tools/gmail/read.test.ts @@ -1,12 +1,15 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { AttachmentDownloadBudget } from '@/lib/uploads/utils/attachment-download-budget' import { gmailReadTool, gmailReadV2Tool } from '@/tools/gmail/read' import { downloadAttachments } from '@/tools/gmail/utils' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const fetchMock = vi.fn() const attachment = { attachmentId: 'file/1', diff --git a/apps/sim/tools/gmail/read.ts b/apps/sim/tools/gmail/read.ts index 93350c6f84c..8b3bd7d2f7b 100644 --- a/apps/sim/tools/gmail/read.ts +++ b/apps/sim/tools/gmail/read.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { AttachmentDownloadBudget, readAttachmentJson, @@ -18,6 +19,10 @@ import { } from '@/tools/gmail/utils' import type { ToolConfig, ToolResponseContext } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('GmailReadTool') export const gmailReadTool: ToolConfig = { @@ -157,7 +162,7 @@ export const gmailReadTool: ToolConfig = { try { // Get the first message details const messageId = data.messages[0].id - const messageResponse = await fetch( + const messageResponse = await providerFetch( `${GMAIL_API_BASE}/messages/${encodeURIComponent(messageId)}?format=full`, { headers: { @@ -204,7 +209,7 @@ export const gmailReadTool: ToolConfig = { const messages: GmailMessage[] = [] for (const msg of data.messages.slice(0, maxResults)) { context?.signal?.throwIfAborted() - const messageResponse = await fetch( + const messageResponse = await providerFetch( `${GMAIL_API_BASE}/messages/${encodeURIComponent(msg.id)}?format=full`, { headers: { Authorization: `Bearer ${params?.accessToken || ''}` }, diff --git a/apps/sim/tools/gmail/search.ts b/apps/sim/tools/gmail/search.ts index b8b83b2e74e..32a6c9cfea7 100644 --- a/apps/sim/tools/gmail/search.ts +++ b/apps/sim/tools/gmail/search.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { GmailSearchParams, GmailToolResponse } from '@/tools/gmail/types' import { createMessagesSummary, @@ -7,6 +8,10 @@ import { } from '@/tools/gmail/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('GmailSearchTool') export const gmailSearchTool: ToolConfig = { @@ -75,12 +80,15 @@ export const gmailSearchTool: ToolConfig = try { // Fetch full message details for each result const messagePromises = data.messages.map(async (msg: any) => { - const messageResponse = await fetch(`${GMAIL_API_BASE}/messages/${msg.id}?format=full`, { - headers: { - Authorization: `Bearer ${params?.accessToken || ''}`, - 'Content-Type': 'application/json', - }, - }) + const messageResponse = await providerFetch( + `${GMAIL_API_BASE}/messages/${msg.id}?format=full`, + { + headers: { + Authorization: `Bearer ${params?.accessToken || ''}`, + 'Content-Type': 'application/json', + }, + } + ) if (!messageResponse.ok) { throw new Error(`Failed to fetch details for message ${msg.id}`) diff --git a/apps/sim/tools/gmail/utils.test.ts b/apps/sim/tools/gmail/utils.test.ts index e21eaddff30..c18629b7752 100644 --- a/apps/sim/tools/gmail/utils.test.ts +++ b/apps/sim/tools/gmail/utils.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { describe, expect, it } from 'vitest' import { buildMimeMessage, @@ -12,6 +13,8 @@ import { sanitizeHeaderValue, } from './utils' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + function decodeSimpleMessage(encoded: string): string { return Buffer.from(encoded, 'base64url').toString('utf-8') } diff --git a/apps/sim/tools/gmail/utils.ts b/apps/sim/tools/gmail/utils.ts index 7e19495e5d6..08c01afa190 100644 --- a/apps/sim/tools/gmail/utils.ts +++ b/apps/sim/tools/gmail/utils.ts @@ -1,5 +1,6 @@ import { generateRandomString } from '@sim/utils/random' import { convert } from 'html-to-text' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { AttachmentDownloadBudget, readAttachmentJson, @@ -12,6 +13,10 @@ import type { GmailToolResponse, } from '@/tools/gmail/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + export const GMAIL_API_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me' /** @@ -29,7 +34,7 @@ export async function fetchThreadingHeaders( subject?: string }> { try { - const messageResponse = await fetch( + const messageResponse = await providerFetch( `${GMAIL_API_BASE}/messages/${messageId}?format=metadata&metadataHeaders=Message-ID&metadataHeaders=References&metadataHeaders=Subject`, { headers: { @@ -242,7 +247,7 @@ export async function downloadAttachments( for (const attachment of attachmentInfo) { try { budget.assertSize(attachment.size, 'Gmail attachments') - const response = await fetch( + const response = await providerFetch( `${GMAIL_API_BASE}/messages/${encodeURIComponent(messageId)}/attachments/${encodeURIComponent(attachment.attachmentId)}`, { headers: { Authorization: `Bearer ${accessToken}` }, diff --git a/apps/sim/tools/google_calendar/invite.ts b/apps/sim/tools/google_calendar/invite.ts index f785870fbf2..9d395f8341d 100644 --- a/apps/sim/tools/google_calendar/invite.ts +++ b/apps/sim/tools/google_calendar/invite.ts @@ -1,3 +1,4 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { CALENDAR_API_BASE, type CalendarAttendee, @@ -8,6 +9,10 @@ import { import { normalizeAttendees } from '@/tools/google_calendar/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + interface InviteResult { data: GoogleCalendarApiEventResponse totalAttendees: number @@ -74,7 +79,7 @@ async function inviteAttendees( queryParams.append('sendUpdates', params?.sendUpdates ?? 'all') const putUrl = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events/${encodeURIComponent(params?.eventId?.trim() ?? '')}?${queryParams.toString()}` - const putResponse = await fetch(putUrl, { + const putResponse = await providerFetch(putUrl, { method: 'PUT', headers: { Authorization: `Bearer ${params?.accessToken}`, diff --git a/apps/sim/tools/google_calendar/quick_add.ts b/apps/sim/tools/google_calendar/quick_add.ts index 32252e986ad..d2274346d48 100644 --- a/apps/sim/tools/google_calendar/quick_add.ts +++ b/apps/sim/tools/google_calendar/quick_add.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { CALENDAR_API_BASE, type GoogleCalendarApiEventResponse, @@ -7,6 +8,10 @@ import { } from '@/tools/google_calendar/types' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('GoogleCalendarQuickAddTool') export const quickAddTool: ToolConfig< @@ -110,7 +115,7 @@ export const quickAddTool: ToolConfig< const updateUrl = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events/${eventId}${updateQueryParams.toString() ? `?${updateQueryParams.toString()}` : ''}` - const updateResponse = await fetch(updateUrl, { + const updateResponse = await providerFetch(updateUrl, { method: 'PATCH', headers: { Authorization: `Bearer ${params.accessToken}`, @@ -239,7 +244,7 @@ export const quickAddV2Tool: ToolConfig< const updateUrl = `${CALENDAR_API_BASE}/calendars/${encodeURIComponent(calendarId)}/events/${eventId}${updateQueryParams.toString() ? `?${updateQueryParams.toString()}` : ''}` - const updateResponse = await fetch(updateUrl, { + const updateResponse = await providerFetch(updateUrl, { method: 'PATCH', headers: { Authorization: `Bearer ${params.accessToken}`, diff --git a/apps/sim/tools/google_drive/create_folder.ts b/apps/sim/tools/google_drive/create_folder.ts index 77ce955c322..ea27a296d4d 100644 --- a/apps/sim/tools/google_drive/create_folder.ts +++ b/apps/sim/tools/google_drive/create_folder.ts @@ -1,8 +1,13 @@ import { createLogger } from '@sim/logger' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { GoogleDriveToolParams, GoogleDriveUploadResponse } from '@/tools/google_drive/types' import { ALL_FILE_FIELDS } from '@/tools/google_drive/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('GoogleDriveCreateFolderTool') export const createFolderTool: ToolConfig = { @@ -86,7 +91,7 @@ export const createFolderTool: ToolConfig = { @@ -90,7 +95,7 @@ export const getContentTool: ToolConfig = { @@ -82,7 +87,7 @@ export const writeTool: ToolConfig inputValidationMock) + import { afterEach, describe, expect, it, vi } from 'vitest' import { icypeasFindEmailTool } from '@/tools/icypeas/find_email' import { ICYPEAS_CREDIT_USD } from '@/tools/icypeas/hosting' diff --git a/apps/sim/tools/icypeas/find_email.ts b/apps/sim/tools/icypeas/find_email.ts index 89fe64ab83e..abb8f48d36b 100644 --- a/apps/sim/tools/icypeas/find_email.ts +++ b/apps/sim/tools/icypeas/find_email.ts @@ -1,4 +1,5 @@ import { sleep } from '@sim/utils/helpers' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { icypeasHosting } from '@/tools/icypeas/hosting' import type { IcypeasFindEmailOutput, @@ -13,6 +14,10 @@ import { } from '@/tools/icypeas/types' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + /** Icypeas statuses that indicate the search has finished (success or failure). */ const TERMINAL_STATUSES = new Set([ 'FOUND', @@ -142,14 +147,17 @@ export const icypeasFindEmailTool: ToolConfig { const commentUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${params!.issueKey?.trim() ?? ''}/comment` - const commentResponse = await fetch(commentUrl, { + const commentResponse = await providerFetch(commentUrl, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/add_watcher.ts b/apps/sim/tools/jira/add_watcher.ts index 1cce8427c08..b4986e26c23 100644 --- a/apps/sim/tools/jira/add_watcher.ts +++ b/apps/sim/tools/jira/add_watcher.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraAddWatcherParams, JiraAddWatcherResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + export const jiraAddWatcherTool: ToolConfig = { id: 'jira_add_watcher', name: 'Jira Add Watcher', @@ -79,7 +84,7 @@ export const jiraAddWatcherTool: ToolConfig { const worklogUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${params!.issueKey?.trim() ?? ''}/worklog` - const worklogResponse = await fetch(worklogUrl, { + const worklogResponse = await providerFetch(worklogUrl, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/assign_issue.ts b/apps/sim/tools/jira/assign_issue.ts index 4daa6d00ae9..2d3964842ed 100644 --- a/apps/sim/tools/jira/assign_issue.ts +++ b/apps/sim/tools/jira/assign_issue.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraAssignIssueParams, JiraAssignIssueResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + /** * Maps user-provided accountId to the Jira API value. * Empty string, "null", "none", or "unassigned" → null (unassign). @@ -88,7 +93,7 @@ export const jiraAssignIssueTool: ToolConfig inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { AttachmentDownloadBudget } from '@/lib/uploads/utils/attachment-download-budget' diff --git a/apps/sim/tools/jira/bulk_read.ts b/apps/sim/tools/jira/bulk_read.ts index 116fbd6fc88..27766698976 100644 --- a/apps/sim/tools/jira/bulk_read.ts +++ b/apps/sim/tools/jira/bulk_read.ts @@ -1,9 +1,14 @@ import { selectAtlassianCloudId } from '@/lib/atlassian/discovery' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraRetrieveBulkParams, JiraRetrieveResponseBulk } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { extractAdfText } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + export const jiraBulkRetrieveTool: ToolConfig = { id: 'jira_bulk_read', name: 'Jira Bulk Read', @@ -60,7 +65,7 @@ export const jiraBulkRetrieveTool: ToolConfig { const cloudId = params?.cloudId || (await getJiraCloudId(params!.domain, params!.accessToken)) - const typesResp = await fetch( + const typesResp = await providerFetch( `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issueLinkType`, { method: 'GET', @@ -115,7 +120,7 @@ export const jiraCreateIssueLinkTool: ToolConfig< } const linkUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issueLink` - const linkResponse = await fetch(linkUrl, { + const linkResponse = await providerFetch(linkUrl, { method: 'POST', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/delete_attachment.ts b/apps/sim/tools/jira/delete_attachment.ts index befb1ecfb02..f54c6e99d9f 100644 --- a/apps/sim/tools/jira/delete_attachment.ts +++ b/apps/sim/tools/jira/delete_attachment.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraDeleteAttachmentParams, JiraDeleteAttachmentResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + export const jiraDeleteAttachmentTool: ToolConfig< JiraDeleteAttachmentParams, JiraDeleteAttachmentResponse @@ -66,7 +71,7 @@ export const jiraDeleteAttachmentTool: ToolConfig< const cloudId = await getJiraCloudId(params!.domain, params!.accessToken) // Make the actual request with the resolved cloudId const attachmentUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/attachment/${params?.attachmentId?.trim() ?? ''}` - const attachmentResponse = await fetch(attachmentUrl, { + const attachmentResponse = await providerFetch(attachmentUrl, { method: 'DELETE', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/delete_comment.ts b/apps/sim/tools/jira/delete_comment.ts index fa516a5be2c..55a30197546 100644 --- a/apps/sim/tools/jira/delete_comment.ts +++ b/apps/sim/tools/jira/delete_comment.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraDeleteCommentParams, JiraDeleteCommentResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + export const jiraDeleteCommentTool: ToolConfig = { id: 'jira_delete_comment', @@ -70,7 +75,7 @@ export const jiraDeleteCommentTool: ToolConfig = { id: 'jira_delete_issue', name: 'Jira Delete Issue', @@ -71,7 +76,7 @@ export const jiraDeleteIssueTool: ToolConfig = { id: 'jira_delete_worklog', @@ -69,7 +74,7 @@ export const jiraDeleteWorklogTool: ToolConfig { const attachmentsUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${params!.issueKey?.trim() ?? ''}?fields=attachment` - const attachmentsResponse = await fetch(attachmentsUrl, { + const attachmentsResponse = await providerFetch(attachmentsUrl, { method: 'GET', signal: context?.signal, headers: { diff --git a/apps/sim/tools/jira/get_comments.ts b/apps/sim/tools/jira/get_comments.ts index 0276b99e158..646a15c5a30 100644 --- a/apps/sim/tools/jira/get_comments.ts +++ b/apps/sim/tools/jira/get_comments.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraGetCommentsParams, JiraGetCommentsResponse } from '@/tools/jira/types' import { COMMENT_ITEM_PROPERTIES, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { extractAdfText, getJiraCloudId, transformUser } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + /** * Transforms a raw Jira comment object into typed output. */ @@ -104,7 +109,7 @@ export const jiraGetCommentsTool: ToolConfig { const fetchFields = async (cloudId: string) => { - const fieldsResponse = await fetch(buildFieldsUrl(cloudId), { + const fieldsResponse = await providerFetch(buildFieldsUrl(cloudId), { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/get_project.ts b/apps/sim/tools/jira/get_project.ts index 7f59c02dc7b..b49017cee28 100644 --- a/apps/sim/tools/jira/get_project.ts +++ b/apps/sim/tools/jira/get_project.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraGetProjectParams, JiraGetProjectResponse } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + function buildProjectUrl(cloudId: string, projectIdOrKey: string): string { return `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/project/${encodeURIComponent(projectIdOrKey)}` } @@ -63,7 +68,7 @@ export const jiraGetProjectTool: ToolConfig { const fetchProject = async (cloudId: string) => { - const projectResponse = await fetch(buildProjectUrl(cloudId, params!.projectId), { + const projectResponse = await providerFetch(buildProjectUrl(cloudId, params!.projectId), { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/get_transitions.ts b/apps/sim/tools/jira/get_transitions.ts index b6847347ca3..8edaf68ca62 100644 --- a/apps/sim/tools/jira/get_transitions.ts +++ b/apps/sim/tools/jira/get_transitions.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraGetTransitionsParams, JiraGetTransitionsResponse } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + function buildTransitionsUrl(cloudId: string, issueKey: string): string { return `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${encodeURIComponent(issueKey)}/transitions` } @@ -66,13 +71,16 @@ export const jiraGetTransitionsTool: ToolConfig< transformResponse: async (response: Response, params?: JiraGetTransitionsParams) => { const fetchTransitions = async (cloudId: string) => { - const transitionsResponse = await fetch(buildTransitionsUrl(cloudId, params!.issueKey), { - method: 'GET', - headers: { - Accept: 'application/json', - Authorization: `Bearer ${params!.accessToken}`, - }, - }) + const transitionsResponse = await providerFetch( + buildTransitionsUrl(cloudId, params!.issueKey), + { + method: 'GET', + headers: { + Accept: 'application/json', + Authorization: `Bearer ${params!.accessToken}`, + }, + } + ) if (!transitionsResponse.ok) { const errorText = await transitionsResponse.text() diff --git a/apps/sim/tools/jira/get_users.ts b/apps/sim/tools/jira/get_users.ts index 4fa1219c0d7..987ebe8001a 100644 --- a/apps/sim/tools/jira/get_users.ts +++ b/apps/sim/tools/jira/get_users.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraGetUsersParams, JiraGetUsersResponse } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT, USER_OUTPUT_PROPERTIES } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + /** * Transforms a raw Jira user API object into typed output. */ @@ -109,7 +114,7 @@ export const jiraGetUsersTool: ToolConfig { const fetchIssueTypes = async (cloudId: string) => { - const issueTypesResponse = await fetch(buildIssueTypesUrl(cloudId), { + const issueTypesResponse = await providerFetch(buildIssueTypesUrl(cloudId), { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/list_projects.ts b/apps/sim/tools/jira/list_projects.ts index b5ddc5befd5..d4ec19242af 100644 --- a/apps/sim/tools/jira/list_projects.ts +++ b/apps/sim/tools/jira/list_projects.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraListProjectsParams, JiraListProjectsResponse } from '@/tools/jira/types' import { TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId, parseAtlassianErrorMessage } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + /** * Transforms a raw Jira project object into typed output. */ @@ -100,7 +105,7 @@ export const jiraListProjectsTool: ToolConfig { const fetchProjects = async (cloudId: string) => { - const projectsResponse = await fetch(buildSearchUrl(cloudId, params!), { + const projectsResponse = await providerFetch(buildSearchUrl(cloudId, params!), { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/remove_watcher.ts b/apps/sim/tools/jira/remove_watcher.ts index ba353fe82df..f2609180e5f 100644 --- a/apps/sim/tools/jira/remove_watcher.ts +++ b/apps/sim/tools/jira/remove_watcher.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraRemoveWatcherParams, JiraRemoveWatcherResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { getJiraCloudId } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + export const jiraRemoveWatcherTool: ToolConfig = { id: 'jira_remove_watcher', @@ -69,7 +74,7 @@ export const jiraRemoveWatcherTool: ToolConfig { const issueUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${params.issueKey?.trim() ?? ''}?expand=renderedFields,names,schema,transitions,operations,editmeta,changelog,versionedRepresentations` - const issueResponse = await fetch(issueUrl, { + const issueResponse = await providerFetch(issueUrl, { method: 'GET', signal: context?.signal, headers: { @@ -297,15 +302,15 @@ export const jiraRetrieveTool: ToolConfig { const base = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${params.issueKey?.trim() ?? ''}` const [commentsResp, worklogResp, watchersResp] = await Promise.all([ - fetch(`${base}/comment?maxResults=100&orderBy=-created`, { + providerFetch(`${base}/comment?maxResults=100&orderBy=-created`, { headers: { Accept: 'application/json', Authorization: `Bearer ${params.accessToken}` }, signal: context?.signal, }), - fetch(`${base}/worklog?maxResults=100`, { + providerFetch(`${base}/worklog?maxResults=100`, { headers: { Accept: 'application/json', Authorization: `Bearer ${params.accessToken}` }, signal: context?.signal, }), - fetch(`${base}/watchers`, { + providerFetch(`${base}/watchers`, { headers: { Accept: 'application/json', Authorization: `Bearer ${params.accessToken}` }, signal: context?.signal, }), diff --git a/apps/sim/tools/jira/search_issues.ts b/apps/sim/tools/jira/search_issues.ts index f3f19c97fd0..756e3cf4dd3 100644 --- a/apps/sim/tools/jira/search_issues.ts +++ b/apps/sim/tools/jira/search_issues.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraSearchIssuesParams, JiraSearchIssuesResponse } from '@/tools/jira/types' import { SEARCH_ISSUE_ITEM_PROPERTIES, TIMESTAMP_OUTPUT } from '@/tools/jira/types' import { extractAdfText, getJiraCloudId, transformUser } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + /** * Transforms a raw Jira search result issue into typed output. */ @@ -167,7 +172,7 @@ export const jiraSearchIssuesTool: ToolConfig = { id: 'jira_search_users', name: 'Jira Search Users', @@ -85,7 +90,7 @@ export const jiraSearchUsersTool: ToolConfig { // First, fetch available transitions to get the name and target status const transitionsUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${params!.issueKey?.trim() ?? ''}/transitions` - const transitionsResp = await fetch(transitionsUrl, { + const transitionsResp = await providerFetch(transitionsUrl, { method: 'GET', headers: { Accept: 'application/json', @@ -114,7 +119,7 @@ export const jiraTransitionIssueTool: ToolConfig< } // Perform the transition - const transitionResponse = await fetch(transitionsUrl, { + const transitionResponse = await providerFetch(transitionsUrl, { method: 'POST', headers: { Accept: 'application/json', @@ -159,7 +164,7 @@ export const jiraTransitionIssueTool: ToolConfig< // Fetch transition metadata for the response try { const transitionsUrl = `https://api.atlassian.com/ex/jira/${params.cloudId}/rest/api/3/issue/${params.issueKey?.trim() ?? ''}/transitions` - const transitionsResp = await fetch(transitionsUrl, { + const transitionsResp = await providerFetch(transitionsUrl, { method: 'GET', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/update_comment.ts b/apps/sim/tools/jira/update_comment.ts index ffe0b0d950e..e8143e3c162 100644 --- a/apps/sim/tools/jira/update_comment.ts +++ b/apps/sim/tools/jira/update_comment.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraUpdateCommentParams, JiraUpdateCommentResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT, USER_OUTPUT_PROPERTIES } from '@/tools/jira/types' import { extractAdfText, getJiraCloudId, toAdf, transformUser } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + /** * Transforms an update comment API response into typed output. */ @@ -107,7 +112,7 @@ export const jiraUpdateCommentTool: ToolConfig { const commentUrl = `https://api.atlassian.com/ex/jira/${cloudId}/rest/api/3/issue/${params!.issueKey?.trim() ?? ''}/comment/${params!.commentId?.trim() ?? ''}` - const commentResponse = await fetch(commentUrl, { + const commentResponse = await providerFetch(commentUrl, { method: 'PUT', headers: { Accept: 'application/json', diff --git a/apps/sim/tools/jira/update_worklog.ts b/apps/sim/tools/jira/update_worklog.ts index 60fb3ddc1b5..7ff499e3766 100644 --- a/apps/sim/tools/jira/update_worklog.ts +++ b/apps/sim/tools/jira/update_worklog.ts @@ -1,3 +1,4 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { JiraUpdateWorklogParams, JiraUpdateWorklogResponse } from '@/tools/jira/types' import { SUCCESS_OUTPUT, TIMESTAMP_OUTPUT, USER_OUTPUT_PROPERTIES } from '@/tools/jira/types' import { @@ -9,6 +10,10 @@ import { } from '@/tools/jira/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + function buildWorklogBody(params: JiraUpdateWorklogParams) { let timeSpentSeconds: number | undefined if ( @@ -144,7 +149,7 @@ export const jiraUpdateWorklogTool: ToolConfig inputValidationMock) + const toolLogger = loggerMock.createLogger.mock.results.at(-1)?.value as ReturnType< typeof createMockLogger > diff --git a/apps/sim/tools/linkedin/share_post.ts b/apps/sim/tools/linkedin/share_post.ts index 002b576e89e..a3cacbd96db 100644 --- a/apps/sim/tools/linkedin/share_post.ts +++ b/apps/sim/tools/linkedin/share_post.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { LinkedInProfileOutput, ProfileIdExtractor, @@ -8,6 +9,10 @@ import type { } from '@/tools/linkedin/types' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('LinkedInSharePost') // Helper function to extract profile ID from various response formats @@ -104,7 +109,7 @@ export const linkedInSharePostTool: ToolConfig ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { executeAddUserAppRoleAssignmentOperation } from '@/lib/internal/microsoft-ad/operations/add-user-app-role-assignment' diff --git a/apps/sim/tools/microsoft_ad/utils.ts b/apps/sim/tools/microsoft_ad/utils.ts index 3a388906870..a60ecb4961d 100644 --- a/apps/sim/tools/microsoft_ad/utils.ts +++ b/apps/sim/tools/microsoft_ad/utils.ts @@ -1,5 +1,10 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { assertGraphNextPageUrl } from '@/tools/sharepoint/utils' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + /** * Splits a comma or newline separated list of identifiers into a trimmed, de-duplicated array. * Used for Microsoft Graph parameters that take GUID collections (SKU IDs, service plan IDs). @@ -189,7 +194,7 @@ export async function resolveGraphUserObjectId( const identifier = userId.trim() if (isGraphObjectId(identifier)) return identifier - const response = await fetch( + const response = await providerFetch( `https://graph.microsoft.com/v1.0/users/${encodeURIComponent(identifier)}?$select=id`, { headers: { Authorization: `Bearer ${accessToken}` }, signal } ) diff --git a/apps/sim/tools/microsoft_excel/format_range.ts b/apps/sim/tools/microsoft_excel/format_range.ts index 9932a4cdec7..6bfe20ae6c7 100644 --- a/apps/sim/tools/microsoft_excel/format_range.ts +++ b/apps/sim/tools/microsoft_excel/format_range.ts @@ -1,3 +1,4 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { ErrorExtractorId } from '@/tools/error-extractors' import type { MicrosoftExcelFormatRangeParams, @@ -11,6 +12,10 @@ import { } from '@/tools/microsoft_excel/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + /** * Builds the font PATCH body from the provided font params, omitting any unset fields. * Returns null when no font property was supplied. @@ -175,7 +180,7 @@ export const formatRangeTool: ToolConfig< if (hasFill) { const basePath = getItemBasePath(spreadsheetId, driveId) const fillUrl = `${buildWorksheetRangeUrl(basePath, params.range, params.sheetName)}/format/fill` - const fillResp = await fetch(fillUrl, { + const fillResp = await providerFetch(fillUrl, { method: 'PATCH', headers: { Authorization: `Bearer ${accessToken}`, diff --git a/apps/sim/tools/microsoft_excel/read.ts b/apps/sim/tools/microsoft_excel/read.ts index fe18103b563..471a16ea203 100644 --- a/apps/sim/tools/microsoft_excel/read.ts +++ b/apps/sim/tools/microsoft_excel/read.ts @@ -1,3 +1,4 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { ErrorExtractorId } from '@/tools/error-extractors' import type { ExcelCellValue, @@ -15,6 +16,10 @@ import { } from '@/tools/microsoft_excel/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const EXCEL_RETRY_CONFIG = { enabled: true, maxRetries: 3, @@ -131,7 +136,7 @@ export const readTool: ToolConfig inputValidationMock) + describe('parseGraphErrorMessage', () => { it('extracts top-level error.message', () => { const body = JSON.stringify({ diff --git a/apps/sim/tools/microsoft_excel/utils.ts b/apps/sim/tools/microsoft_excel/utils.ts index 27650907cf9..da19ea34d5b 100644 --- a/apps/sim/tools/microsoft_excel/utils.ts +++ b/apps/sim/tools/microsoft_excel/utils.ts @@ -1,7 +1,12 @@ import { createLogger } from '@sim/logger' import { validatePathSegment } from '@/lib/core/security/input-validation' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { ExcelCellValue } from '@/tools/microsoft_excel/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('MicrosoftExcelUtils') /** @@ -246,7 +251,7 @@ export async function getSpreadsheetWebUrl( ): Promise { const basePath = getItemBasePath(spreadsheetId, driveId) try { - const response = await fetch(`${basePath}?$select=id,webUrl`, { + const response = await providerFetch(`${basePath}?$select=id,webUrl`, { headers: { Authorization: `Bearer ${accessToken}`, }, diff --git a/apps/sim/tools/microsoft_teams/attachment-downloads.test.ts b/apps/sim/tools/microsoft_teams/attachment-downloads.test.ts index 6f3218f6f0f..28397cd0b14 100644 --- a/apps/sim/tools/microsoft_teams/attachment-downloads.test.ts +++ b/apps/sim/tools/microsoft_teams/attachment-downloads.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { AttachmentDownloadBudget } from '@/lib/uploads/utils/attachment-download-budget' diff --git a/apps/sim/tools/microsoft_teams/utils.ts b/apps/sim/tools/microsoft_teams/utils.ts index e1262bf2c9c..3111d197776 100644 --- a/apps/sim/tools/microsoft_teams/utils.ts +++ b/apps/sim/tools/microsoft_teams/utils.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { AttachmentDownloadBudget, readAttachmentJson, @@ -7,6 +8,10 @@ import { import type { MicrosoftTeamsAttachment } from '@/tools/microsoft_teams/types' import type { ToolFileData } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('MicrosoftTeamsUtils') interface ParsedMention { @@ -78,7 +83,7 @@ async function fetchHostedContents( const results: ToolFileData[] = [] try { budget.signal?.throwIfAborted() - const response = await fetch(path, { + const response = await providerFetch(path, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -95,7 +100,7 @@ async function fetchHostedContents( for (const item of data.value ?? []) { if (!item.id) continue budget.signal?.throwIfAborted() - const content = await fetch(`${path}/${encodeURIComponent(item.id)}/$value`, { + const content = await providerFetch(`${path}/${encodeURIComponent(item.id)}/$value`, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -156,7 +161,7 @@ async function downloadReferenceAttachment( budget.signal?.throwIfAborted() const shareId = `u!${Buffer.from(attachment.contentUrl).toString('base64url')}` const path = `https://graph.microsoft.com/v1.0/shares/${shareId}/driveItem` - const metadataResponse = await fetch(`${path}?$select=name,size,file`, { + const metadataResponse = await providerFetch(`${path}?$select=name,size,file`, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -171,7 +176,7 @@ async function downloadReferenceAttachment( file?: { mimeType?: string } }>(metadataResponse, 'Teams attachment metadata', budget.signal) if (item.size !== undefined) budget.assertSize(item.size, 'Teams attachments') - const content = await fetch(`${path}/content`, { + const content = await providerFetch(`${path}/content`, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -229,7 +234,7 @@ function parseMentions(content: string): ParsedMention[] { } async function fetchChatMembers(chatId: string, accessToken: string): Promise { - const response = await fetch( + const response = await providerFetch( `https://graph.microsoft.com/v1.0/chats/${encodeURIComponent(chatId)}/members`, { headers: { @@ -256,7 +261,7 @@ async function fetchChannelMembers( channelId: string, accessToken: string ): Promise { - const response = await fetch( + const response = await providerFetch( `https://graph.microsoft.com/v1.0/teams/${encodeURIComponent(teamId)}/channels/${encodeURIComponent(channelId)}/members`, { headers: { diff --git a/apps/sim/tools/netsuite/netsuite.test.ts b/apps/sim/tools/netsuite/netsuite.test.ts index 7092e862858..60e236791e0 100644 --- a/apps/sim/tools/netsuite/netsuite.test.ts +++ b/apps/sim/tools/netsuite/netsuite.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { readFileSync } from 'node:fs' import { resolve } from 'node:path' import { afterEach, describe, expect, expectTypeOf, it, vi } from 'vitest' diff --git a/apps/sim/tools/netsuite/utils.test.ts b/apps/sim/tools/netsuite/utils.test.ts index dbfe47ae257..1501f1261f5 100644 --- a/apps/sim/tools/netsuite/utils.test.ts +++ b/apps/sim/tools/netsuite/utils.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterEach, describe, expect, it, vi } from 'vitest' import { executeNetsuiteBatchCreateRecordsOperation } from '@/lib/internal/netsuite/operations/batch-create-records' import { executeNetsuiteCreateRecordOperation } from '@/lib/internal/netsuite/operations/create-record' @@ -15,6 +16,8 @@ import { normalizeSuiteTalkUrl, } from '@/tools/netsuite/utils' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const AUTH: NetSuiteAuthParams = { oauthCredential: 'credential-id', accessToken: 'access-token', diff --git a/apps/sim/tools/netsuite/utils.ts b/apps/sim/tools/netsuite/utils.ts index b215f1b38a6..f6307c4ad45 100644 --- a/apps/sim/tools/netsuite/utils.ts +++ b/apps/sim/tools/netsuite/utils.ts @@ -1,6 +1,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' import { truncate } from '@sim/utils/string' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { DEFAULT_MAX_ERROR_BODY_BYTES, readResponseTextWithLimit, @@ -15,6 +16,10 @@ import type { } from '@/tools/netsuite/types' import type { HttpMethod, ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const SUITETALK_REQUEST_TIMEOUT_MS = 30_000 const DEFAULT_PAGE_LIMIT = 100 const MAX_PAGE_LIMIT = 1_000 @@ -439,7 +444,7 @@ async function sendSuiteTalkRequest( headers['X-NetSuite-PropertyNameValidation'] = 'error' } } - return fetch(url, { + return providerFetch(url, { method: request.method, headers, ...(serializedBody !== undefined ? { body: serializedBody } : {}), diff --git a/apps/sim/tools/notion/read.ts b/apps/sim/tools/notion/read.ts index 00d91321c5e..bfc0a874c2f 100644 --- a/apps/sim/tools/notion/read.ts +++ b/apps/sim/tools/notion/read.ts @@ -1,8 +1,13 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { NotionReadParams, NotionResponse } from '@/tools/notion/types' import { PAGE_OUTPUT_PROPERTIES } from '@/tools/notion/types' import { extractTitle } from '@/tools/notion/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + export const notionReadTool: ToolConfig = { id: 'notion_read', name: 'Notion Reader', @@ -72,7 +77,7 @@ export const notionReadTool: ToolConfig = { } // Fetch page content using blocks endpoint - const blocksResponse = await fetch( + const blocksResponse = await providerFetch( `https://api.notion.com/v1/blocks/${pageId}/children?page_size=100`, { method: 'GET', @@ -206,7 +211,7 @@ export const notionReadV2Tool: ToolConfig inputValidationMock) + import { afterEach, describe, expect, it, vi } from 'vitest' import { executeOktaUpdateGroupOperation } from '@/lib/internal/okta/operations/update-group' import { OktaBlock } from '@/blocks/blocks/okta' diff --git a/apps/sim/tools/outlook/read.test.ts b/apps/sim/tools/outlook/read.test.ts index 435b01f0b37..48dd23c9bba 100644 --- a/apps/sim/tools/outlook/read.test.ts +++ b/apps/sim/tools/outlook/read.test.ts @@ -1,11 +1,14 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { PayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { AttachmentDownloadBudget } from '@/lib/uploads/utils/attachment-download-budget' import { downloadAttachments, outlookReadTool } from '@/tools/outlook/read' +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + const fetchMock = vi.fn() const attachment = { '@odata.type': '#microsoft.graph.fileAttachment', diff --git a/apps/sim/tools/outlook/read.ts b/apps/sim/tools/outlook/read.ts index 988aeb058bc..37be5997e86 100644 --- a/apps/sim/tools/outlook/read.ts +++ b/apps/sim/tools/outlook/read.ts @@ -1,3 +1,4 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { AttachmentDownloadBudget, readAttachmentJson, @@ -13,6 +14,10 @@ import type { import { OUTLOOK_MESSAGE_OUTPUT_PROPERTIES } from '@/tools/outlook/types' import type { ToolConfig, ToolResponseContext } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + interface OutlookDownloadAttachmentMetadata { '@odata.type'?: string id: string @@ -31,7 +36,7 @@ export async function downloadAttachments( const path = `https://graph.microsoft.com/v1.0/me/messages/${encodeURIComponent(messageId)}/attachments` try { budget.signal?.throwIfAborted() - const response = await fetch(`${path}?$select=id,name,contentType,size`, { + const response = await providerFetch(`${path}?$select=id,name,contentType,size`, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) @@ -50,7 +55,7 @@ export async function downloadAttachments( try { if (attachment.size !== undefined) budget.assertSize(attachment.size, 'Outlook attachments') budget.signal?.throwIfAborted() - const content = await fetch(`${path}/${encodeURIComponent(attachment.id)}/$value`, { + const content = await providerFetch(`${path}/${encodeURIComponent(attachment.id)}/$value`, { headers: { Authorization: `Bearer ${accessToken}` }, signal: budget.signal, }) diff --git a/apps/sim/tools/parallel/deep_research.ts b/apps/sim/tools/parallel/deep_research.ts index 5d8193d3cd2..e9f3c3b77bf 100644 --- a/apps/sim/tools/parallel/deep_research.ts +++ b/apps/sim/tools/parallel/deep_research.ts @@ -1,11 +1,16 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/core/execution-limits' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { PlatformEvents } from '@/lib/core/telemetry' import { toList } from '@/tools/parallel/search' import type { ParallelDeepResearchParams } from '@/tools/parallel/types' import type { ToolConfig, ToolResponse } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('ParallelDeepResearchTool') /** @@ -199,7 +204,7 @@ export const deepResearchTool: ToolConfig inputValidationMock) + import { afterEach, describe, expect, it, vi } from 'vitest' import { ParallelBlock } from '@/blocks/blocks/parallel' import { deepResearchTool } from '@/tools/parallel/deep_research' diff --git a/apps/sim/tools/quickbooks/client.ts b/apps/sim/tools/quickbooks/client.ts index 9888ec5392d..47786676d77 100644 --- a/apps/sim/tools/quickbooks/client.ts +++ b/apps/sim/tools/quickbooks/client.ts @@ -1,9 +1,14 @@ +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit, readResponseTextWithLimit, } from '@/lib/core/utils/stream-limits' import { formatQuickBooksFaultDetail, sanitizeQuickBooksFaultData } from '@/tools/quickbooks/fault' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + export const QUICKBOOKS_MINOR_VERSION = '75' export const QUICKBOOKS_MAX_RESPONSE_BYTES = 8 * 1024 * 1024 export const QUICKBOOKS_MAX_USER_INFO_BYTES = 1024 * 1024 @@ -125,7 +130,7 @@ export async function fetchValidatedQuickBooksCompanyInfo( environment: QuickBooksEnvironment ): Promise { const normalizedRealmId = normalizeQuickBooksRealmId(realmId) - const response = await fetch( + const response = await providerFetch( buildQuickBooksCompanyUrl( normalizedRealmId, `companyinfo/${encodeURIComponent(normalizedRealmId)}`, diff --git a/apps/sim/tools/quickbooks/full_update.test.ts b/apps/sim/tools/quickbooks/full_update.test.ts index ad7d308999e..39f339f55c5 100644 --- a/apps/sim/tools/quickbooks/full_update.test.ts +++ b/apps/sim/tools/quickbooks/full_update.test.ts @@ -1,6 +1,10 @@ /** * @vitest-environment node */ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { executeQuickBooksUpdateItemOperation, diff --git a/apps/sim/tools/quickbooks/sales_utils.test.ts b/apps/sim/tools/quickbooks/sales_utils.test.ts index a7f89638606..47d9116017f 100644 --- a/apps/sim/tools/quickbooks/sales_utils.test.ts +++ b/apps/sim/tools/quickbooks/sales_utils.test.ts @@ -1,3 +1,7 @@ +import { inputValidationMock } from '@sim/testing' + +vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) + import { afterEach, describe, expect, it, vi } from 'vitest' import { executeQuickBooksUpdateCustomerPaymentOperation } from '@/lib/internal/quickbooks/provider-operations' diff --git a/apps/sim/tools/quickbooks/utils.ts b/apps/sim/tools/quickbooks/utils.ts index 647c05c40cb..4cd5e245a34 100644 --- a/apps/sim/tools/quickbooks/utils.ts +++ b/apps/sim/tools/quickbooks/utils.ts @@ -1,4 +1,5 @@ import { omit } from '@sim/utils/object' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { ErrorExtractorId, extractErrorMessage } from '@/tools/error-extractors' import { @@ -44,6 +45,10 @@ import { validateQuickBooksPagination, } from '@/tools/quickbooks/values' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + export type QuickBooksQueryEntity = | 'Account' | 'Bill' @@ -484,7 +489,7 @@ export async function executeQuickBooksFullUpdate< const recordId = requiredQuickBooksString(options.recordId, 'recordId') const syncToken = requiredQuickBooksString(options.syncToken, 'syncToken') const patch = options.buildPatch(options.params) - const readResponse = await fetch( + const readResponse = await providerFetch( buildQuickBooksEntityUrl(options.params, options.resource, recordId), { method: 'GET', @@ -525,7 +530,7 @@ export async function executeQuickBooksFullUpdate< * references the item. */ if (options.entity === 'Item') updateUrl.searchParams.set('include', 'donotupdateaccountontxns') - const updateResponse = await fetch(updateUrl, { + const updateResponse = await providerFetch(updateUrl, { method: 'POST', headers: getQuickBooksToolHeaders(options.params.accessToken, 'application/json'), body: JSON.stringify(fullBody), diff --git a/apps/sim/tools/sharepoint/read_page.ts b/apps/sim/tools/sharepoint/read_page.ts index 8785b21a5da..ad37ad9b181 100644 --- a/apps/sim/tools/sharepoint/read_page.ts +++ b/apps/sim/tools/sharepoint/read_page.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { GraphApiResponse, SharepointPageContent, @@ -16,6 +17,10 @@ import { } from '@/tools/sharepoint/utils' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('SharePointReadPage') export const readPageTool: ToolConfig = { @@ -215,7 +220,7 @@ export const readPageTool: ToolConfig ({ + ...inputValidationMock, + secureFetchWithValidation: (...args: Parameters) => fetch(...args), +})) + import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { assertAssistantIntegrationCall, diff --git a/apps/sim/tools/slack/utils.ts b/apps/sim/tools/slack/utils.ts index 3f5aaef314d..8a984dba7fa 100644 --- a/apps/sim/tools/slack/utils.ts +++ b/apps/sim/tools/slack/utils.ts @@ -1,12 +1,17 @@ import { interruptibleSleep } from '@sim/utils/helpers' import { isRecordLike } from '@sim/utils/object' import { parseRetryAfter } from '@sim/utils/retry' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { SlackAgentSessionStatus, SlackCanvasFile, SlackSuggestedPrompt, } from '@/tools/slack/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const SLACK_AGENT_SESSION_STATUSES = new Set([ 'active', 'processing', @@ -258,7 +263,7 @@ export async function fetchSlackMessagesPaginated( let response: Response let attempt = 0 while (true) { - response = await fetch(url.toString(), { + response = await providerFetch(url.toString(), { method: 'GET', headers: { Authorization: `Bearer ${token}` }, signal: opts.signal, diff --git a/apps/sim/tools/twilio_voice/list_calls.ts b/apps/sim/tools/twilio_voice/list_calls.ts index fb2512537fb..275d63ea143 100644 --- a/apps/sim/tools/twilio_voice/list_calls.ts +++ b/apps/sim/tools/twilio_voice/list_calls.ts @@ -1,7 +1,12 @@ import { createLogger } from '@sim/logger' +import { createSsrfGuardedFetchWithDispatcher } from '@/lib/core/security/input-validation.server' import type { TwilioListCallsOutput, TwilioListCallsParams } from '@/tools/twilio_voice/types' import type { ToolConfig } from '@/tools/types' +const { fetch: providerFetch } = createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', +}) + const logger = createLogger('TwilioVoiceListCallsTool') export const listCallsTool: ToolConfig = { @@ -123,7 +128,7 @@ export const listCallsTool: ToolConfig = { @@ -134,7 +139,7 @@ export const xReadTool: ToolConfig = { max_results: '100', // Max allowed }) - const repliesResponse = await fetch( + const repliesResponse = await providerFetch( `https://api.twitter.com/2/tweets/search/recent?${searchParams.toString()}`, { method: 'GET', diff --git a/bun.lock b/bun.lock index 848b03f4155..e8c22546a52 100644 --- a/bun.lock +++ b/bun.lock @@ -247,6 +247,8 @@ "@sim/workflow-persistence": "workspace:*", "@sim/workflow-renderer": "workspace:*", "@sim/workflow-types": "workspace:*", + "@smithy/core": "3.33.3", + "@smithy/node-http-handler": "4.11.3", "@t3-oss/env-nextjs": "0.13.4", "@tanstack/react-query": "5.90.8", "@tanstack/react-virtual": "3.13.24", diff --git a/scripts/check-egress-boundary.test.ts b/scripts/check-egress-boundary.test.ts index bb3109357b3..329b71397eb 100644 --- a/scripts/check-egress-boundary.test.ts +++ b/scripts/check-egress-boundary.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { mayLoadTransport } from './check-egress-boundary' +import { findNativeProviderFetches, mayLoadTransport } from './check-egress-boundary' describe('egress transport candidate scan', () => { it('finds literal transport modules', () => { @@ -14,3 +14,69 @@ describe('egress transport candidate scan', () => { expect(mayLoadTransport('const https = createClient()')).toBe(false) }) }) + +describe('organization-aware provider fetch boundary', () => { + it('detects native calls and explicit global bypasses', () => { + expect( + findNativeProviderFetches('fetch(url)\nglobalThis.fetch(url)\nglobal.fetch(url)') + ).toEqual([1, 2, 3]) + }) + it('leaves routed calls, client methods, strings and comments alone', () => { + expect( + findNativeProviderFetches( + "secureFetchWithValidation(url, options); client.fetch(url); const text = 'fetch(url)'; /* fetch(url) */" + ) + ).toEqual([]) + }) + + it('detects native fetch passed to SDKs or saved under another name', () => { + expect( + findNativeProviderFetches( + [ + 'new Client({ fetch })', + 'new Client({ fetch: globalThis.fetch })', + 'const send = fetch', + "const sendGlobal = globalThis['fetch']", + 'const { fetch: unguarded } = globalThis', + ].join('\n') + ) + ).toEqual([1, 2, 3, 4, 5]) + }) + + it('allows fetch types, guarded SDK options and object method declarations', () => { + expect( + findNativeProviderFetches( + [ + 'type Fetcher = typeof fetch', + 'const client = new Client({ fetch: transport.fetch })', + 'const { fetch: guarded } = transport', + 'const object = { fetch(url: string) { return transport.fetch(url) } }', + ].join('\n') + ) + ).toEqual([]) + }) + + it('exempts only named account lifecycle entry points in their exact modules', () => { + const source = [ + 'export function revokeQuickBooksToken() { fetch(url) }', + 'export function refreshToken() { fetch(url) }', + 'fetch(url)', + ].join('\n') + expect(findNativeProviderFetches(source, 'apps/sim/lib/oauth/quickbooks.ts')).toEqual([2, 3]) + expect(findNativeProviderFetches(source, 'apps/sim/lib/oauth/other.ts')).toEqual([1, 2, 3]) + }) + + it('uses the top-level entry point for nested helpers and callbacks', () => { + const source = [ + 'export function createGitHubRepositoriesProvider() {', + ' return { getToken: () => fetch(url) }', + '}', + 'export function runtimeProvider() {', + ' function createGitHubRepositoriesProvider() { globalThis.fetch(url) }', + '}', + ].join('\n') + expect(findNativeProviderFetches(source, 'apps/sim/lib/oauth/github-repositories.ts')).toEqual([ + 5, + ]) + }) +}) diff --git a/scripts/check-egress-boundary.ts b/scripts/check-egress-boundary.ts index 537fac33da9..adf77192731 100644 --- a/scripts/check-egress-boundary.ts +++ b/scripts/check-egress-boundary.ts @@ -18,7 +18,8 @@ * containing a quote hid a real one — which is what a scanner that does not * understand the grammar will keep doing. * - * Not checked: bare `fetch()`. It is used constantly for same-origin and + * Bare `fetch()` is additionally forbidden in the managed provider modules. + * Elsewhere bare `fetch()` is not checked. It is used constantly for same-origin and * server-action calls where the guard does not apply, so flagging it would be * noise. The transports it can reach are covered by the rules below. * @@ -71,11 +72,13 @@ const TRANSPORTS = new Set([ * guard or predates it for a documented reason. */ const ALLOWED = new Set([ + /** Subprocess-only runtime fixture; never imported by production modules. */ + 'apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts', // The guard itself: resolves, classifies, pins, and follows redirects. 'apps/sim/lib/core/security/input-validation.server.ts', - /** TLS wrapping preserves the validated destination and upstream certificate identity. */ + /** TLS CONNECT implementation; accepts only classified IPs from the shared guard. */ 'apps/sim/lib/core/network/gateway.server.ts', - /** Owns validated direct and environment-proxy pools for the shared HTTP adapters. */ + /** Owns route-specific pools and emits requests for the HTTP and SDK adapters. */ 'apps/sim/lib/core/network/transport.server.ts', // Streaming MCP transport, built on the guard's pinned dispatcher. 'apps/sim/lib/mcp/pinned-fetch.ts', @@ -203,6 +206,98 @@ export function mayLoadTransport(source: string): boolean { return false } +/** Provider modules must keep native fetch behind the organization-aware transport. */ +const MANAGED_FETCH_ROOTS = [ + 'apps/sim/connectors/', + 'apps/sim/providers/', + 'apps/sim/tools/', + 'apps/sim/lib/internal/', + 'apps/sim/lib/atlassian/', + 'apps/sim/lib/oauth/', + 'apps/sim/lib/embeddings/', + 'apps/sim/lib/knowledge/', + 'apps/sim/lib/credentials/', + 'apps/sim/lib/credential-groups/', + 'apps/sim/lib/selectors/server/providers/', + 'apps/sim/lib/webhooks/providers/', + 'apps/sim/lib/webhooks/polling/', + 'apps/sim/lib/media/', + 'apps/sim/lib/messaging/email/providers/', +] + +/** + * Account connection and revocation retain deployment networking: an account can + * span organizations, and initial callbacks have no authorized resource scope. + * Exempt only these entry points; runtime provider calls and refresh remain checked. + */ +const ACCOUNT_LIFECYCLE_FETCHES = new Map>([ + [ + 'apps/sim/lib/oauth/github-repositories.ts', + new Set(['verifyGitHubRepositoriesIdentity', 'createGitHubRepositoriesProvider']), + ], + ['apps/sim/lib/oauth/monday.ts', new Set(['exchangeMondayAuthorizationCode'])], + ['apps/sim/lib/oauth/shopify.ts', new Set(['completeShopifyOAuthConnection'])], + ['apps/sim/lib/oauth/quickbooks.ts', new Set(['revokeQuickBooksToken'])], +]) + +/** Includes native fetch handed to SDKs or aliases; object methods and types are not transports. */ +export function findNativeProviderFetches(source: string, file?: string): number[] { + const ast = ts.createSourceFile('provider.ts', source, ts.ScriptTarget.Latest, true) + const accountLifecycleFunctions = file ? ACCOUNT_LIFECYCLE_FETCHES.get(file) : undefined + const lines: number[] = [] + const visit = (node: ts.Node): void => { + if (ts.isTypeNode(node)) return + const parent = node.parent + const globalFetch = + (ts.isPropertyAccessExpression(node) || ts.isElementAccessExpression(node)) && + ts.isIdentifier(node.expression) && + ['globalThis', 'global'].includes(node.expression.text) && + (ts.isPropertyAccessExpression(node) + ? node.name.text === 'fetch' + : ts.isStringLiteralLike(node.argumentExpression) && + node.argumentExpression.text === 'fetch') + const bareFetch = + ts.isIdentifier(node) && + node.text === 'fetch' && + !ts.isImportSpecifier(parent) && + !ts.isExportSpecifier(parent) && + !ts.isBindingElement(parent) && + !(ts.isPropertyAccessExpression(parent) && parent.name === node) && + !( + (ts.isPropertyAssignment(parent) || + ts.isMethodDeclaration(parent) || + ts.isVariableDeclaration(parent) || + ts.isParameter(parent) || + ts.isFunctionDeclaration(parent) || + ts.isPropertySignature(parent) || + ts.isMethodSignature(parent)) && + parent.name === node + ) + const globalBinding = + ts.isVariableDeclaration(node) && + ts.isObjectBindingPattern(node.name) && + node.initializer && + ts.isIdentifier(node.initializer) && + ['globalThis', 'global'].includes(node.initializer.text) && + node.name.elements.some((element) => { + const key = element.propertyName ?? element.name + return (ts.isIdentifier(key) || ts.isStringLiteralLike(key)) && key.text === 'fetch' + }) + if (globalFetch || bareFetch || globalBinding) { + let boundary: ts.Node = node + while (boundary.parent && !ts.isSourceFile(boundary.parent)) boundary = boundary.parent + const functionName = ts.isFunctionDeclaration(boundary) ? boundary.name?.text : undefined + if (!functionName || !accountLifecycleFunctions?.has(functionName)) { + lines.push(ast.getLineAndCharacterOfPosition(node.getStart(ast)).line + 1) + } + return + } + ts.forEachChild(node, visit) + } + visit(ast) + return lines +} + function main() { const violations: Violation[] = [] let scanned = 0 @@ -213,6 +308,15 @@ function main() { if (ALLOWED.has(rel)) continue scanned++ const source = readFileSync(file, 'utf8') + if (MANAGED_FETCH_ROOTS.some((root) => rel.startsWith(root))) { + for (const line of findNativeProviderFetches(source, rel)) + violations.push({ + file: rel, + line, + kind: 'native fetch', + specifier: 'Use secureFetchWithValidation or a profile-aware guarded fetch factory', + }) + } if (!mayLoadTransport(source)) continue for (const load of findTransportLoads(rel, source)) { violations.push({ file: rel, ...load }) diff --git a/scripts/check-tool-registry-boundary.baseline.json b/scripts/check-tool-registry-boundary.baseline.json index e4de733eac0..1dcf50dad0a 100644 --- a/scripts/check-tool-registry-boundary.baseline.json +++ b/scripts/check-tool-registry-boundary.baseline.json @@ -402,11 +402,11 @@ "gateways": {} }, "app/workspace/[workspaceId]/settings/[section]/page.tsx": { - "modules": 2341, + "modules": 2348, "gateways": { - "apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx": 736, + "apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx": 738, "apps/sim/triggers/registry.ts": 485, - "apps/sim/lib/auth/index.ts": 378, + "apps/sim/lib/auth/index.ts": 379, "apps/sim/blocks/registry.ts": 354, "apps/sim/lib/webhooks/providers/index.ts": 117, "apps/sim/lib/webhooks/providers/registry.ts": 115, From 0fb8b92f362d0c505d8e1f519a217569ceee31ce Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 12 Sep 2026 09:49:58 -0700 Subject: [PATCH 2/2] chore(network): clarify Request body coverage --- apps/sim/lib/core/network/gateway.server.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sim/lib/core/network/gateway.server.test.ts b/apps/sim/lib/core/network/gateway.server.test.ts index 0c6c5a4f8e1..fdbaa455b3d 100644 --- a/apps/sim/lib/core/network/gateway.server.test.ts +++ b/apps/sim/lib/core/network/gateway.server.test.ts @@ -237,7 +237,7 @@ describe('organization gateways over real TLS CONNECT sockets', () => { await transport.dispatcher.destroy() } }) - it('streams Request bodies, encodes multipart data, and honors manual redirects', async () => { + it('preserves Request bodies, encodes multipart data, and honors manual redirects', async () => { const transport = createPinnedFetchWithDispatcher('127.0.0.1', options) try { await runWithOutboundOrganization('org_a', async () => {