diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index f50b20410aa..6c93954f998 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -16,6 +16,8 @@ paths-ignore: - '**/__tests__/**' - '**/__mocks__/**' - '**/__fixtures__/**' + - '**/__integration__/**' + - '**/*.integration.ts' - '**/e2e/**' # Deliberately no '**/test/**' or '**/tests/**'. A directory named `test` is a # routable Next.js path segment, not necessarily test code: those globs diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index eb467bfea33..dacc08540b9 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -187,6 +187,7 @@ jobs: bunx vitest run --mode integration lib/knowledge/__integration__/search-source-progress.integration.ts lib/knowledge/__integration__/search-source-pagination.integration.ts + lib/knowledge/__integration__/search-reference-batching.integration.ts lib/core/outbox/service.integration.ts lib/knowledge/__integration__/connector-upload.integration.ts diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 22c00ff4e27..054a8290273 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -3008,7 +3008,7 @@ sim selectors get [options] | Option | Required | Description | | --- | --- | --- | -| `--selector-key ` | Yes | Registered selector key for discovering this field’s destination options. Accepted values: `airtable.bases`, `airtable.tables`, `asana.workspaces`, `attio.lists`, `attio.objects`, `bigquery.datasets`, `bigquery.tables`, `bitbucket.workspaces`, `bitbucket.repositories`, `calcom.eventTypes`, `calcom.schedules`, `clickup.workspaces`, `clickup.spaces`, `clickup.folders`, `clickup.lists`, `confluence.spaces`, `confluence.spacesById`, `confluence.pages`, `google.tasks.lists`, `gmail.labels`, `google.calendar`, `google.drive`, `google.sheets`, `harmonic.savedSearches`, `hubspot.lists`, `hubspot.owners`, `hubspot.pipelines`, `hubspot.pipelineStages`, `hubspot.properties`, `jsm.requestTypes`, `jsm.serviceDesks`, `microsoft.planner.plans`, `notion.databases`, `notion.pages`, `netsuite.recordTypes`, `netsuite.asyncTasks`, `pipedrive.pipelines`, `sharepoint.lists`, `trello.boards`, `zoho_desk.organizations`, `zoho_desk.departments`, `zoho_desk.agents`, `zoom.meetings`, `slack.channels`, `snowflake.databases`, `snowflake.schemas`, `snowflake.tables`, `snowflake.warehouses`, `snowflake.roles`, `snowflake.fileFormats`, `snowflake.procedures`, `slack.users`, `outlook.folders`, `outlook.calendars`, `microsoft.teams`, `microsoft.chats`, `microsoft.channels`, `microsoft.planner`, `onedrive.files`, `onedrive.folders`, `sharepoint.sites`, `microsoft.excel`, `microsoft.excel.drives`, `microsoft.excel.sheets`, `microsoft.word`, `wealthbox.contacts`, `jira.issues`, `jira.projects`, `linear.projects`, `linear.teams`, `monday.boards`, `monday.groups`, `webflow.sites`, `webflow.collections`, `webflow.items`, `cloudwatch.logGroups`, `cloudwatch.logStreams`, `imap.mailboxes`, `mcp.tools`, `managedAgent.agents`, `managedAgent.environments`, `managedAgent.vaults`, `managedAgent.memoryStores`, `knowledge.documents`, `sim.workflows`, `table.columns`, `table.outputColumns`, `workspace.secretNames`, `workspace.sandboxes`, `providers.ollamaEmbeddingModels`, `providers.openrouterEmbeddingModels`. | +| `--selector-key ` | Yes | Registered selector key for discovering this field’s destination options. Accepted values: `airtable.bases`, `airtable.tables`, `asana.workspaces`, `attio.lists`, `attio.objects`, `bigquery.datasets`, `bigquery.tables`, `bitbucket.workspaces`, `bitbucket.repositories`, `calcom.eventTypes`, `calcom.schedules`, `clickup.workspaces`, `clickup.spaces`, `clickup.folders`, `clickup.lists`, `confluence.spaces`, `confluence.spacesById`, `confluence.pages`, `google.tasks.lists`, `gmail.labels`, `google.calendar`, `google.drive`, `google.sheets`, `harmonic.savedSearches`, `hubspot.lists`, `hubspot.owners`, `hubspot.pipelines`, `hubspot.pipelineStages`, `hubspot.properties`, `jsm.requestTypes`, `jsm.serviceDesks`, `microsoft.planner.plans`, `notion.databases`, `notion.pages`, `netsuite.recordTypes`, `netsuite.asyncTasks`, `pipedrive.pipelines`, `sharepoint.lists`, `trello.boards`, `zoho_desk.organizations`, `zoho_desk.departments`, `zoho_desk.agents`, `zoom.meetings`, `slack.channels`, `snowflake.databases`, `snowflake.schemas`, `snowflake.tables`, `snowflake.warehouses`, `snowflake.roles`, `snowflake.fileFormats`, `snowflake.procedures`, `slack.users`, `outlook.folders`, `outlook.calendars`, `microsoft.teams`, `microsoft.chats`, `microsoft.channels`, `microsoft.planner`, `onedrive.files`, `onedrive.folders`, `sharepoint.sites`, `microsoft.excel`, `microsoft.excel.drives`, `microsoft.excel.sheets`, `microsoft.word`, `wealthbox.contacts`, `jira.issues`, `jira.projects`, `jira.projectKeys`, `linear.projects`, `linear.teams`, `monday.boards`, `monday.groups`, `webflow.sites`, `webflow.collections`, `webflow.items`, `cloudwatch.logGroups`, `cloudwatch.logStreams`, `imap.mailboxes`, `mcp.tools`, `managedAgent.agents`, `managedAgent.environments`, `managedAgent.vaults`, `managedAgent.memoryStores`, `knowledge.documents`, `sim.workflows`, `table.columns`, `table.outputColumns`, `workspace.secretNames`, `workspace.sandboxes`, `providers.ollamaEmbeddingModels`, `providers.openrouterEmbeddingModels`. | | `--context ` | No | Only the dependencies declared by the selector, such as oauthCredential and channelId. Missing OAuth connections require human authorization. (JSON, or @path / @- to read a file or stdin). | | `--id ` | Yes | Resource identifier. | @@ -3028,7 +3028,7 @@ sim selectors list [options] | Option | Required | Description | | --- | --- | --- | -| `--selector-key ` | Yes | Registered selector key for discovering this field’s destination options. Accepted values: `airtable.bases`, `airtable.tables`, `asana.workspaces`, `attio.lists`, `attio.objects`, `bigquery.datasets`, `bigquery.tables`, `bitbucket.workspaces`, `bitbucket.repositories`, `calcom.eventTypes`, `calcom.schedules`, `clickup.workspaces`, `clickup.spaces`, `clickup.folders`, `clickup.lists`, `confluence.spaces`, `confluence.spacesById`, `confluence.pages`, `google.tasks.lists`, `gmail.labels`, `google.calendar`, `google.drive`, `google.sheets`, `harmonic.savedSearches`, `hubspot.lists`, `hubspot.owners`, `hubspot.pipelines`, `hubspot.pipelineStages`, `hubspot.properties`, `jsm.requestTypes`, `jsm.serviceDesks`, `microsoft.planner.plans`, `notion.databases`, `notion.pages`, `netsuite.recordTypes`, `netsuite.asyncTasks`, `pipedrive.pipelines`, `sharepoint.lists`, `trello.boards`, `zoho_desk.organizations`, `zoho_desk.departments`, `zoho_desk.agents`, `zoom.meetings`, `slack.channels`, `snowflake.databases`, `snowflake.schemas`, `snowflake.tables`, `snowflake.warehouses`, `snowflake.roles`, `snowflake.fileFormats`, `snowflake.procedures`, `slack.users`, `outlook.folders`, `outlook.calendars`, `microsoft.teams`, `microsoft.chats`, `microsoft.channels`, `microsoft.planner`, `onedrive.files`, `onedrive.folders`, `sharepoint.sites`, `microsoft.excel`, `microsoft.excel.drives`, `microsoft.excel.sheets`, `microsoft.word`, `wealthbox.contacts`, `jira.issues`, `jira.projects`, `linear.projects`, `linear.teams`, `monday.boards`, `monday.groups`, `webflow.sites`, `webflow.collections`, `webflow.items`, `cloudwatch.logGroups`, `cloudwatch.logStreams`, `imap.mailboxes`, `mcp.tools`, `managedAgent.agents`, `managedAgent.environments`, `managedAgent.vaults`, `managedAgent.memoryStores`, `knowledge.documents`, `sim.workflows`, `table.columns`, `table.outputColumns`, `workspace.secretNames`, `workspace.sandboxes`, `providers.ollamaEmbeddingModels`, `providers.openrouterEmbeddingModels`. | +| `--selector-key ` | Yes | Registered selector key for discovering this field’s destination options. Accepted values: `airtable.bases`, `airtable.tables`, `asana.workspaces`, `attio.lists`, `attio.objects`, `bigquery.datasets`, `bigquery.tables`, `bitbucket.workspaces`, `bitbucket.repositories`, `calcom.eventTypes`, `calcom.schedules`, `clickup.workspaces`, `clickup.spaces`, `clickup.folders`, `clickup.lists`, `confluence.spaces`, `confluence.spacesById`, `confluence.pages`, `google.tasks.lists`, `gmail.labels`, `google.calendar`, `google.drive`, `google.sheets`, `harmonic.savedSearches`, `hubspot.lists`, `hubspot.owners`, `hubspot.pipelines`, `hubspot.pipelineStages`, `hubspot.properties`, `jsm.requestTypes`, `jsm.serviceDesks`, `microsoft.planner.plans`, `notion.databases`, `notion.pages`, `netsuite.recordTypes`, `netsuite.asyncTasks`, `pipedrive.pipelines`, `sharepoint.lists`, `trello.boards`, `zoho_desk.organizations`, `zoho_desk.departments`, `zoho_desk.agents`, `zoom.meetings`, `slack.channels`, `snowflake.databases`, `snowflake.schemas`, `snowflake.tables`, `snowflake.warehouses`, `snowflake.roles`, `snowflake.fileFormats`, `snowflake.procedures`, `slack.users`, `outlook.folders`, `outlook.calendars`, `microsoft.teams`, `microsoft.chats`, `microsoft.channels`, `microsoft.planner`, `onedrive.files`, `onedrive.folders`, `sharepoint.sites`, `microsoft.excel`, `microsoft.excel.drives`, `microsoft.excel.sheets`, `microsoft.word`, `wealthbox.contacts`, `jira.issues`, `jira.projects`, `jira.projectKeys`, `linear.projects`, `linear.teams`, `monday.boards`, `monday.groups`, `webflow.sites`, `webflow.collections`, `webflow.items`, `cloudwatch.logGroups`, `cloudwatch.logStreams`, `imap.mailboxes`, `mcp.tools`, `managedAgent.agents`, `managedAgent.environments`, `managedAgent.vaults`, `managedAgent.memoryStores`, `knowledge.documents`, `sim.workflows`, `table.columns`, `table.outputColumns`, `workspace.secretNames`, `workspace.sandboxes`, `providers.ollamaEmbeddingModels`, `providers.openrouterEmbeddingModels`. | | `--context ` | No | Only the dependencies declared by the selector, such as oauthCredential and channelId. Missing OAuth connections require human authorization. (JSON, or @path / @- to read a file or stdin). | | `--search ` | No | Provider option search text. | | `--cursor ` | No | Continue from nextCursor returned by a previous result. | diff --git a/apps/docs/content/docs/cli/selectors.mdx b/apps/docs/content/docs/cli/selectors.mdx index 4a9dda8556d..e728cb93a0e 100644 --- a/apps/docs/content/docs/cli/selectors.mdx +++ b/apps/docs/content/docs/cli/selectors.mdx @@ -21,7 +21,7 @@ Get Selector Option (OAuth login or personal API key required) | Option | Required | Description | | --- | --- | --- | -| `--selector-key ` | Yes | Registered selector key for discovering this field’s destination options. Accepted values: `airtable.bases`, `airtable.tables`, `asana.workspaces`, `attio.lists`, `attio.objects`, `bigquery.datasets`, `bigquery.tables`, `bitbucket.workspaces`, `bitbucket.repositories`, `calcom.eventTypes`, `calcom.schedules`, `clickup.workspaces`, `clickup.spaces`, `clickup.folders`, `clickup.lists`, `confluence.spaces`, `confluence.spacesById`, `confluence.pages`, `google.tasks.lists`, `gmail.labels`, `google.calendar`, `google.drive`, `google.sheets`, `harmonic.savedSearches`, `hubspot.lists`, `hubspot.owners`, `hubspot.pipelines`, `hubspot.pipelineStages`, `hubspot.properties`, `jsm.requestTypes`, `jsm.serviceDesks`, `microsoft.planner.plans`, `notion.databases`, `notion.pages`, `netsuite.recordTypes`, `netsuite.asyncTasks`, `pipedrive.pipelines`, `sharepoint.lists`, `trello.boards`, `zoho_desk.organizations`, `zoho_desk.departments`, `zoho_desk.agents`, `zoom.meetings`, `slack.channels`, `snowflake.databases`, `snowflake.schemas`, `snowflake.tables`, `snowflake.warehouses`, `snowflake.roles`, `snowflake.fileFormats`, `snowflake.procedures`, `slack.users`, `outlook.folders`, `outlook.calendars`, `microsoft.teams`, `microsoft.chats`, `microsoft.channels`, `microsoft.planner`, `onedrive.files`, `onedrive.folders`, `sharepoint.sites`, `microsoft.excel`, `microsoft.excel.drives`, `microsoft.excel.sheets`, `microsoft.word`, `wealthbox.contacts`, `jira.issues`, `jira.projects`, `linear.projects`, `linear.teams`, `monday.boards`, `monday.groups`, `webflow.sites`, `webflow.collections`, `webflow.items`, `cloudwatch.logGroups`, `cloudwatch.logStreams`, `imap.mailboxes`, `mcp.tools`, `managedAgent.agents`, `managedAgent.environments`, `managedAgent.vaults`, `managedAgent.memoryStores`, `knowledge.documents`, `sim.workflows`, `table.columns`, `table.outputColumns`, `workspace.secretNames`, `workspace.sandboxes`, `providers.ollamaEmbeddingModels`, `providers.openrouterEmbeddingModels`. | +| `--selector-key ` | Yes | Registered selector key for discovering this field’s destination options. Accepted values: `airtable.bases`, `airtable.tables`, `asana.workspaces`, `attio.lists`, `attio.objects`, `bigquery.datasets`, `bigquery.tables`, `bitbucket.workspaces`, `bitbucket.repositories`, `calcom.eventTypes`, `calcom.schedules`, `clickup.workspaces`, `clickup.spaces`, `clickup.folders`, `clickup.lists`, `confluence.spaces`, `confluence.spacesById`, `confluence.pages`, `google.tasks.lists`, `gmail.labels`, `google.calendar`, `google.drive`, `google.sheets`, `harmonic.savedSearches`, `hubspot.lists`, `hubspot.owners`, `hubspot.pipelines`, `hubspot.pipelineStages`, `hubspot.properties`, `jsm.requestTypes`, `jsm.serviceDesks`, `microsoft.planner.plans`, `notion.databases`, `notion.pages`, `netsuite.recordTypes`, `netsuite.asyncTasks`, `pipedrive.pipelines`, `sharepoint.lists`, `trello.boards`, `zoho_desk.organizations`, `zoho_desk.departments`, `zoho_desk.agents`, `zoom.meetings`, `slack.channels`, `snowflake.databases`, `snowflake.schemas`, `snowflake.tables`, `snowflake.warehouses`, `snowflake.roles`, `snowflake.fileFormats`, `snowflake.procedures`, `slack.users`, `outlook.folders`, `outlook.calendars`, `microsoft.teams`, `microsoft.chats`, `microsoft.channels`, `microsoft.planner`, `onedrive.files`, `onedrive.folders`, `sharepoint.sites`, `microsoft.excel`, `microsoft.excel.drives`, `microsoft.excel.sheets`, `microsoft.word`, `wealthbox.contacts`, `jira.issues`, `jira.projects`, `jira.projectKeys`, `linear.projects`, `linear.teams`, `monday.boards`, `monday.groups`, `webflow.sites`, `webflow.collections`, `webflow.items`, `cloudwatch.logGroups`, `cloudwatch.logStreams`, `imap.mailboxes`, `mcp.tools`, `managedAgent.agents`, `managedAgent.environments`, `managedAgent.vaults`, `managedAgent.memoryStores`, `knowledge.documents`, `sim.workflows`, `table.columns`, `table.outputColumns`, `workspace.secretNames`, `workspace.sandboxes`, `providers.ollamaEmbeddingModels`, `providers.openrouterEmbeddingModels`. | | `--context ` | No | Only the dependencies declared by the selector, such as oauthCredential and channelId. Missing OAuth connections require human authorization. (JSON, or @path / @- to read a file or stdin). | | `--id ` | Yes | Resource identifier. | @@ -41,7 +41,7 @@ List Selector Options (OAuth login or personal API key required) | Option | Required | Description | | --- | --- | --- | -| `--selector-key ` | Yes | Registered selector key for discovering this field’s destination options. Accepted values: `airtable.bases`, `airtable.tables`, `asana.workspaces`, `attio.lists`, `attio.objects`, `bigquery.datasets`, `bigquery.tables`, `bitbucket.workspaces`, `bitbucket.repositories`, `calcom.eventTypes`, `calcom.schedules`, `clickup.workspaces`, `clickup.spaces`, `clickup.folders`, `clickup.lists`, `confluence.spaces`, `confluence.spacesById`, `confluence.pages`, `google.tasks.lists`, `gmail.labels`, `google.calendar`, `google.drive`, `google.sheets`, `harmonic.savedSearches`, `hubspot.lists`, `hubspot.owners`, `hubspot.pipelines`, `hubspot.pipelineStages`, `hubspot.properties`, `jsm.requestTypes`, `jsm.serviceDesks`, `microsoft.planner.plans`, `notion.databases`, `notion.pages`, `netsuite.recordTypes`, `netsuite.asyncTasks`, `pipedrive.pipelines`, `sharepoint.lists`, `trello.boards`, `zoho_desk.organizations`, `zoho_desk.departments`, `zoho_desk.agents`, `zoom.meetings`, `slack.channels`, `snowflake.databases`, `snowflake.schemas`, `snowflake.tables`, `snowflake.warehouses`, `snowflake.roles`, `snowflake.fileFormats`, `snowflake.procedures`, `slack.users`, `outlook.folders`, `outlook.calendars`, `microsoft.teams`, `microsoft.chats`, `microsoft.channels`, `microsoft.planner`, `onedrive.files`, `onedrive.folders`, `sharepoint.sites`, `microsoft.excel`, `microsoft.excel.drives`, `microsoft.excel.sheets`, `microsoft.word`, `wealthbox.contacts`, `jira.issues`, `jira.projects`, `linear.projects`, `linear.teams`, `monday.boards`, `monday.groups`, `webflow.sites`, `webflow.collections`, `webflow.items`, `cloudwatch.logGroups`, `cloudwatch.logStreams`, `imap.mailboxes`, `mcp.tools`, `managedAgent.agents`, `managedAgent.environments`, `managedAgent.vaults`, `managedAgent.memoryStores`, `knowledge.documents`, `sim.workflows`, `table.columns`, `table.outputColumns`, `workspace.secretNames`, `workspace.sandboxes`, `providers.ollamaEmbeddingModels`, `providers.openrouterEmbeddingModels`. | +| `--selector-key ` | Yes | Registered selector key for discovering this field’s destination options. Accepted values: `airtable.bases`, `airtable.tables`, `asana.workspaces`, `attio.lists`, `attio.objects`, `bigquery.datasets`, `bigquery.tables`, `bitbucket.workspaces`, `bitbucket.repositories`, `calcom.eventTypes`, `calcom.schedules`, `clickup.workspaces`, `clickup.spaces`, `clickup.folders`, `clickup.lists`, `confluence.spaces`, `confluence.spacesById`, `confluence.pages`, `google.tasks.lists`, `gmail.labels`, `google.calendar`, `google.drive`, `google.sheets`, `harmonic.savedSearches`, `hubspot.lists`, `hubspot.owners`, `hubspot.pipelines`, `hubspot.pipelineStages`, `hubspot.properties`, `jsm.requestTypes`, `jsm.serviceDesks`, `microsoft.planner.plans`, `notion.databases`, `notion.pages`, `netsuite.recordTypes`, `netsuite.asyncTasks`, `pipedrive.pipelines`, `sharepoint.lists`, `trello.boards`, `zoho_desk.organizations`, `zoho_desk.departments`, `zoho_desk.agents`, `zoom.meetings`, `slack.channels`, `snowflake.databases`, `snowflake.schemas`, `snowflake.tables`, `snowflake.warehouses`, `snowflake.roles`, `snowflake.fileFormats`, `snowflake.procedures`, `slack.users`, `outlook.folders`, `outlook.calendars`, `microsoft.teams`, `microsoft.chats`, `microsoft.channels`, `microsoft.planner`, `onedrive.files`, `onedrive.folders`, `sharepoint.sites`, `microsoft.excel`, `microsoft.excel.drives`, `microsoft.excel.sheets`, `microsoft.word`, `wealthbox.contacts`, `jira.issues`, `jira.projects`, `jira.projectKeys`, `linear.projects`, `linear.teams`, `monday.boards`, `monday.groups`, `webflow.sites`, `webflow.collections`, `webflow.items`, `cloudwatch.logGroups`, `cloudwatch.logStreams`, `imap.mailboxes`, `mcp.tools`, `managedAgent.agents`, `managedAgent.environments`, `managedAgent.vaults`, `managedAgent.memoryStores`, `knowledge.documents`, `sim.workflows`, `table.columns`, `table.outputColumns`, `workspace.secretNames`, `workspace.sandboxes`, `providers.ollamaEmbeddingModels`, `providers.openrouterEmbeddingModels`. | | `--context ` | No | Only the dependencies declared by the selector, such as oauthCredential and channelId. Missing OAuth connections require human authorization. (JSON, or @path / @- to read a file or stdin). | | `--search ` | No | Provider option search text. | | `--cursor ` | No | Continue from nextCursor returned by a previous result. | diff --git a/apps/docs/content/docs/integrations/atlassian-service-account.mdx b/apps/docs/content/docs/integrations/atlassian-service-account.mdx index b0dbd6d5b3c..cf1c15f2711 100644 --- a/apps/docs/content/docs/integrations/atlassian-service-account.mdx +++ b/apps/docs/content/docs/integrations/atlassian-service-account.mdx @@ -33,7 +33,7 @@ See Atlassian's [service-account setup](https://support.atlassian.com/user-manag ## Choose scopes -Start with the connection and read scopes for each product you will use. These cover Sim's account validation, pickers, and common read operations; individual operations may need additional scopes. +Start with the connection and read scopes for each product, then check the scope requirements for your workflow operations. ### Jira and Jira Service Management @@ -65,17 +65,17 @@ Add scopes for the actions your workflow performs: | Create or update Jira issues | `write:jira-work` | | Create or update Service Management requests | `write:servicedesk-request` | | Manage Service Management customers | `manage:servicedesk-customer` | -| Create or update Confluence content | `write:confluence-content`, `write:page:confluence` | +| Create or update Confluence pages | `write:confluence-content`, `write:page:confluence` | -Delete, webhook, Assets, and other operations can require additional scopes. Check the specific endpoint in the [Jira](https://developer.atlassian.com/cloud/jira/platform/rest/v3/intro/), [Jira Service Management](https://developer.atlassian.com/cloud/jira/service-desk/rest/intro/), or [Confluence](https://developer.atlassian.com/cloud/confluence/rest/v2/intro/) API reference. Include the complete scope set for that endpoint; a classic scope does not cover every API in its product. +Deletes, webhooks, Assets, and other operations can require additional scopes. Check the endpoint's complete scope set in the [Jira](https://developer.atlassian.com/cloud/jira/platform/rest/v3/intro/), [Jira Service Management](https://developer.atlassian.com/cloud/jira/service-desk/rest/intro/), or [Confluence](https://developer.atlassian.com/cloud/confluence/rest/v2/intro/) API reference. Use the **App** and **Scope type** filters to find both classic and granular scopes. Scopes and account permissions are separate: the account must also have access to the project, space, or content. Atlassian scope picker filtered to Jira classic scopes ## Add the credential to Sim @@ -86,7 +86,7 @@ Use the **App** and **Scope type** filters to find both classic and granular sco 4. Select **Add service account**. Sim checks the token against the selected product's current-user endpoint; resolve any error before continuing. 5. In your workflow's Jira, Jira Service Management, or Confluence block, select the credential and configure the operation. -The credential appears on all three integration pages, but adding it only validates the selected product. Check account access and scopes before using another product. Sim encrypts the token at rest and calls Atlassian as the service account. +The credential appears on all three integration pages, but adding it only validates the selected product. Check account access and scopes before using another product. ## Troubleshooting and rotation diff --git a/apps/docs/content/docs/search/confluence.mdx b/apps/docs/content/docs/search/confluence.mdx index 308ca638a00..f818dcba9de 100644 --- a/apps/docs/content/docs/search/confluence.mdx +++ b/apps/docs/content/docs/search/confluence.mdx @@ -7,7 +7,7 @@ import { Callout } from 'fumadocs-ui/components/callout' import { Step, Steps } from 'fumadocs-ui/components/steps' import { Image } from '@/components/ui/image' -Search pages and blog posts from selected Confluence Cloud spaces. A Sim organization admin approves the provider; **each teammate connects their own Confluence account**, including when a service account supplies the content. +Search pages and blog posts from selected Confluence Cloud spaces. A Sim organization admin enables Confluence; **each teammate connects their own account**. | Method | How it works | | --- | --- | @@ -16,11 +16,13 @@ Search pages and blog posts from selected Confluence Cloud spaces. A Sim organiz ## Before you start -- Use **Confluence Cloud** and a site hostname such as `your-team.atlassian.net`. Server and Data Center are not supported. +- Use **Confluence Cloud**. Server and Data Center are not supported. - Each teammate needs a verified Sim email matching their active Atlassian account's email. - For a central source, an Atlassian organization admin creates a service account with Confluence access. Grant it access to the chosen spaces and restricted pages, plus permission to read space permissions and the user/group directory. Admin status alone does not bypass page restrictions. -On hosted Sim, teammates authorize Sim's existing OAuth app. Self-hosted deployments must configure the [shared OAuth app](#self-hosted-operator-setup), including when a service account supplies content. + +Teammates authorize Sim's shared Confluence app, which also requests workflow permissions, including writes. Search does not edit pages. Hosted Sim provides this app; self-hosted deployments must [configure it](#self-hosted-operator-setup), even when a service account supplies content. + ## Set up a central source @@ -39,10 +41,10 @@ Open your organization's **Settings → Sources**, turn on **Confluence**, then Under **Indexing account**, select a service account or [add one](#using-a-service-account). Enter the same **Confluence Domain** as the credential, then choose **Spaces**. To enter comma-separated keys such as `ENG, PRODUCT`, use the switch beside the Spaces field. Confluence source setup with a service account, site domain, and spaces Open **More options** for content type, labels, and metadata tags. The default is **Pages only**; choose **All content** to include blog posts. @@ -52,9 +54,9 @@ Open **More options** for content type, labels, and metadata tags. The default i ### Sync and connect your identity -Select **Connect & Sync**. Setup samples permission access in one selected space. Each document's access is verified during sync. Then open **Integrations** in the main sidebar and select **Connect** on the Confluence source. In the new tab, authorize the configured site using the Atlassian email matching your verified Sim email. +Select **Connect & Sync** to start indexing. Then open **Integrations** in the main sidebar and select **Connect** on the Confluence source. Authorize the configured site using the Atlassian email matching your verified Sim email. -Each teammate completes this identity connection. An existing authorized account may already be connected. Return to Integrations to check indexing status and your searchable document count. +Each teammate completes this identity connection. Existing authorized accounts may already be connected. Integrations shows indexing status and your searchable document count. @@ -90,7 +92,7 @@ read:user:confluence read:group:confluence ``` -These scopes cover Search's account check, pickers, content, permissions, and directory reads. `read:confluence-user` is needed for the [current-user check](https://developer.atlassian.com/cloud/confluence/rest/v1/api-group-users/#api-wiki-rest-api-user-current-get); `read:space:confluence` is needed for the [space picker](https://developer.atlassian.com/cloud/confluence/rest/v2/api-group-space/#api-spaces-get). Workflow write scopes are not needed for central Search. +Use all 12 scopes for account validation, pickers, content, permissions, and directory reads. Central indexing does not need write scopes. 4. Review and create the token, then copy it. Atlassian shows it only once. 5. In Sim's source form, use **Indexing account** to add a service account. Paste the **API token**, enter **Site domain** (hostname only), and select **Add service account**. Continue in the source form with the same domain. @@ -102,11 +104,7 @@ These scopes cover Search's account check, pickers, content, permissions, and di height={551} /> -See Atlassian's [account setup](https://support.atlassian.com/user-management/docs/manage-your-service-accounts/) and [token instructions](https://support.atlassian.com/user-management/docs/manage-api-tokens-for-service-accounts/). Scopes do not grant space or page access themselves. Before expiry, or when scopes must change, create a replacement token and add it as a new credential in the source's **Settings**. Select **Change indexing account**, verify a sync, then revoke the old token. - - -Personal OAuth connections use Sim's shared Confluence app, which also requests permissions for workflow actions, including writes. Search reads content and permissions; it does not edit pages. - +See Atlassian's [account setup](https://support.atlassian.com/user-management/docs/manage-your-service-accounts/) and [token instructions](https://support.atlassian.com/user-management/docs/manage-api-tokens-for-service-accounts/). Scopes do not grant space or page access. To replace an expiring token or change scopes, add a new credential in the source's **Settings**, select **Change indexing account**, and verify a sync before revoking the old token. ## Configuration and indexed content @@ -114,7 +112,7 @@ Personal OAuth connections use Sim's shared Confluence app, which also requests | --- | --- | | **Confluence Domain** | Cloud hostname only, such as `your-team.atlassian.net`; omit page URLs and `/wiki`. | | **Spaces / Space Keys** | Required spaces. The picker and manual input set the same scope. | -| **Content Type** | **Pages only** by default; **All content** includes pages and blog posts. | +| **Content Type** | **Pages only** (default), **Blog posts only**, or **All content** for both. | | **Filter by Label** | Optional comma-separated labels; content can match any listed label. | | **Metadata tags** | Labels, version, and last-modified tags. | @@ -122,9 +120,9 @@ Search manages the schedule and hides item limits. It indexes published/current ## Manage access and sync -Central sources combine space permissions, page and ancestor restrictions, and group membership. Member sources use each person's provider listing. Sim admin status does not grant access to all pages, and permission changes take effect after syncing and processing. +Central sources combine space permissions, page and ancestor restrictions, and group membership. Before returning central content, Sim uses your personal connection to check that you still have access to the configured Confluence site. If Atlassian cannot confirm that access, the content is hidden. Member sources use each person's provider listing. Sim admin status does not grant access to all pages, and permission changes take effect after syncing and processing. -Open **Settings → Sources → Confluence → Manage**, then a source's **Documents**, **Settings**, or **Sync history**. Teammates use **Integrations** to connect or reconnect. Invite new people to the Sim organization through Members settings or SSO first; **Accounts → Request connections** requests a provider connection, not organization membership. +Open **Settings → Sources → Confluence → Manage**, then a source's **Documents**, **Settings**, or **Sync history**. Invite teammates through **Settings → Members → Invite** or SSO, then have them connect through **Integrations**. **Accounts → Request connections** only requests a provider connection; it does not invite people to the organization. ## Troubleshooting @@ -134,6 +132,7 @@ Open **Settings → Sources → Confluence → Manage**, then a source's **Docum | Space picker is empty or fails | Check the domain, account's space access, and `read:space:confluence` scope. Manual space keys are also supported. | | Service-account validation fails | Check token expiry, site, Confluence app access, and the full scope list above, including `read:confluence-user`. | | Content syncs but Search is empty | Connect your personal Confluence identity. Check permission/directory sync errors and group-read scopes. | +| A new page, blog post, or label is missing | Confluence search can take time to update. Once the content appears in Confluence search with the selected label, sync again. | | A restricted page is missing | Both your account and the crawling account need access to the page and its ancestors. | | Embedded content is missing | Index the referenced page separately; remote macro output is excluded. | | **Reconnect** or email mismatch | Authorize with the Atlassian account matching your verified Sim email and grant all requested permissions. | @@ -145,9 +144,9 @@ Open a missing page as the affected teammate. **Share → General access** shows Configure one shared Confluence OAuth app for teammates' connections: 1. In the [Atlassian developer console](https://developer.atlassian.com/console/myapps/), select or create the deployment's **OAuth 2.0 integration**. -2. Under **Authorization → OAuth 2.0 (3LO)**, save `https:///api/auth/oauth2/callback/confluence` as a callback, preserving callbacks used by other deployments. +2. Under **Authorization → OAuth 2.0 (3LO)**, add `https:///api/auth/oauth2/callback/confluence` as a callback. 3. Under **Permissions**, add the Confluence API and its full `confluence` scope list from [Sim's OAuth configuration](https://github.com/simstudioai/sim/blob/staging/apps/sim/lib/oauth/oauth.ts), including `read:group:confluence`. Add **User Identity API → read:me**. Sim requests `offline_access` for refresh tokens; the service-account list above does not replace this shared OAuth scope set. 4. Enable sharing under **Distribution**. Set `CONFLUENCE_CLIENT_ID` and `CONFLUENCE_CLIENT_SECRET` from the app's **Settings**, verify `NEXT_PUBLIC_APP_URL`, and restart Sim. 5. Connect from **Integrations** and select the configured site. After changing the OAuth client or requested scopes, use **Settings → Sources → More → Refresh connection settings**, then have affected teammates reconnect. -The callback must exactly match Sim's URL, including scheme, hostname, port, and path. For `http://localhost:3000`, register `http://localhost:3000/api/auth/oauth2/callback/confluence`. If only the app owner can connect, check **Distribution**. See the [Atlassian OAuth guide](https://developer.atlassian.com/cloud/confluence/oauth-2-3lo-apps/) and [Sim deployment reference](/platform/self-hosting/integrations-oauth). +The callback must match Sim's scheme, hostname, port, and path exactly. If only the app owner can connect, check **Distribution**. See the [Atlassian OAuth guide](https://developer.atlassian.com/cloud/confluence/oauth-2-3lo-apps/) and [Sim deployment reference](/platform/self-hosting/integrations-oauth). diff --git a/apps/docs/content/docs/search/gmail.mdx b/apps/docs/content/docs/search/gmail.mdx index 8945c941261..b4ce9e2e53f 100644 --- a/apps/docs/content/docs/search/gmail.mdx +++ b/apps/docs/content/docs/search/gmail.mdx @@ -36,7 +36,7 @@ Open **Settings → Sources** and turn on **Gmail**. This allows personal connec ### Connect your account -Open **Integrations** and select **Connect** beside Gmail. Authorize the Google account matching your verified Sim email. The first connection creates the default sync configuration: the last 6 months across all labels, excluding Promotions, Social, Spam, and Trash. +Join the Sim organization, then open **Integrations** and select **Connect** beside Gmail. Authorize the Google account matching your verified Sim email. The first connection creates the default sync configuration: the last 6 months across all labels, excluding Promotions, Social, Spam, and Trash. Return to Integrations to see indexing status and your searchable document count. @@ -52,6 +52,8 @@ Configurations are additive: a narrower one does not restrict an existing broade +Admins can request member connections from **Manage → Accounts → Request connections**. These requests do not invite recipients to the Sim organization. See [Connect your account](/search/connect-your-account) for the shared connection and recovery steps. + ## Set up a central service account Open **Settings → Sources**, enable **Gmail**, and select **Manage → Advanced → Add sync configuration**. If personal connections are disabled for your organization, select **Add source** from the provider page instead. @@ -100,16 +102,6 @@ Teammates join the Sim organization with their matching verified primary email. -## Connect your account - -These steps apply to **Member accounts**. A central service-account source does not require a personal Gmail connection. - -1. Join the Sim organization and verify your Sim email address. Open **Integrations** and click **Connect** beside Gmail. -2. Complete the connection in the tab that opens. Choose the Google account whose verified email matches your Sim email, and grant the requested permissions. -3. Return to Integrations. The source shows its indexing status and the number of documents you can search. - -Teammates follow these same steps after joining the organization. Once an admin allows Gmail, the first connection can create its source with default filters. Admins can edit those filters afterward or request connections from **Manage → Accounts → Request connections**. A connection request does not invite the recipient to the Sim organization. - ## Source options An admin opens **Settings → Sources** and selects **Manage** beside **Gmail** to open its configuration list. Each row shows **Member accounts** or **Service account** beside its sync status. Open a configuration's **Settings** tab to edit its filters, then select **Save**. Filters apply separately to each mailbox in the source. **Documents** shows indexed threads and **Sync history** shows recent runs. @@ -121,11 +113,11 @@ An admin opens **Settings → Sources** and selects **Manage** beside **Gmail** | Labels | Optional comma-separated names or system IDs, such as `Engineering, INBOX`. A thread matching any listed label is included. Leave empty for all labels. Custom IDs such as `Label_7` belong to one mailbox and cannot be used for member or central setup. | | Directory administrator email | Required for central indexing. An active Workspace administrator who can read Directory users; this does not limit the crawl to the administrator's mailbox. | | Users | Central indexing only. Optional primary Workspace email addresses (up to 100); blank includes all active users in the customer. This selects which mailboxes to crawl. Each mailbox remains searchable only by its owner. | -| Date Range | Last 6 months by default for Search sources. Choose the last 7, 30, or 90 days, a year, or all time. A knowledge-base connector outside Search defaults to all time. | +| Date Range | Last 6 months (180 days) by default. Other options are rolling windows of 7, 30, or 90 days, 1 year (365 days), or all time. | | Exclude Promotions / Exclude Social | Both enabled by default. Choose **No** to include either category. | | Search Filter | Optional [Gmail query](https://developers.google.com/workspace/gmail/api/guides/filtering), such as `from:team@example.com subject:release`. This filters what is indexed; it is not a Sim Search query. Member-account sources with a search filter relist the mailbox on every sync instead of using Gmail's change history. | -In the add-source form, **More options** contains optional **Metadata tags**. Sync frequency and the general knowledge-base **Max Threads** setting are hidden in Search. +In the add-source form, **More options** also contains optional **Metadata tags**. ## What gets indexed @@ -137,13 +129,9 @@ Search schedules syncs hourly. The first sync lists every thread in scope and ca **Member accounts:** later syncs use each mailbox's Gmail change history, unless the configuration has a search filter. A full relisting runs about weekly, or sooner if Gmail no longer retains the saved history. -**Service account:** each sync revisits the selected active mailboxes and resumes unfinished listings. It does not reuse one mailbox's history cursor across the company. Only new or changed threads need their bodies fetched. Failed mailbox reads leave the crawl incomplete; they are not treated as an empty mailbox for deletion reconciliation. - -Updates, removals, and access refresh in the background, rather than being checked live for each search. - -An empty mailbox or filters with no matching threads complete normally with zero documents. +**Service account:** each sync revisits the selected active mailboxes and resumes unfinished listings. A failed mailbox read leaves the crawl incomplete; it does not cause existing indexed mail to be deleted from Search. -Threads that exceed indexing size limits are skipped and reconsidered when the thread changes. +Updates, removals, and access refresh in the background. Empty mailboxes and filters with no matches complete normally with zero documents. Threads exceeding indexing size limits are skipped and reconsidered when they change. ## Troubleshooting @@ -155,7 +143,7 @@ Threads that exceed indexing size limits are skipped and reconsidered when the t | Reconnect | Click **Reconnect** and authorize the same account again. | | Unavailable or needs admin attention | Ask your Sim admin to check source status and the deployment's Google OAuth configuration. | | Directory or delegation error | Check both central crawl scopes, the service-account key, and the Directory administrator's user-read privileges. A normal OAuth account cannot replace the central service account. | -| Gmail access fails for a selected user | Verify Gmail is enabled for that primary Workspace account and delegation is authorized. Narrow **Users** to accounts with Gmail enabled. Aliases and external accounts cannot be selected. | +| Gmail access fails for a selected user | Verify delegation is authorized and [Gmail is enabled](https://knowledge.workspace.google.com/admin/gmail/control-gmail-access-for-your-organizations-users) for that primary Workspace account. Set **Users** to accounts with Gmail enabled; leaving it blank includes all active users and can stop sync on a service-access error. Aliases and external accounts cannot be selected. | | A central source indexes mail but a teammate sees no results | Confirm their verified Sim email is the mailbox's primary email and they belong to the Sim organization. Administrators do not receive other people's mailbox access. | ## Self-hosted operator setup diff --git a/apps/docs/content/docs/search/google-calendar.mdx b/apps/docs/content/docs/search/google-calendar.mdx index 5435e41ebd8..37f427a369e 100644 --- a/apps/docs/content/docs/search/google-calendar.mdx +++ b/apps/docs/content/docs/search/google-calendar.mdx @@ -29,7 +29,7 @@ These are alternative setup paths. When only a central Calendar source is config ### Allow and connect Google Calendar -An admin opens **Settings → Sources** and turns on **Google Calendar**. Then each person opens **Integrations**, selects **Connect** beside Google Calendar, and authorizes their matching Google account. The first connection creates the default member-account sync configuration. +An admin opens **Settings → Sources** and turns on **Google Calendar**. Each person joins the Sim organization, opens **Integrations**, selects **Connect** beside Google Calendar, and authorizes the Google account matching their verified Sim email. The first connection creates the default member-account sync configuration. Return to Integrations to see indexing status and your searchable document count. @@ -50,6 +50,8 @@ The default date range covers the previous and next 30 days. Save any changes to +Admins can request member connections from **Manage → Accounts → Request connections**. These requests do not grant organization membership. Connecting Gmail or Drive does not replace the Calendar connection. See [Connect your account](/search/connect-your-account) for the shared connection and recovery steps. + `primary` means the connected or impersonated person's main calendar. A calendar selected from the list is a specific calendar ID, even when it is your main calendar. That same ID applies to every selected user, and only users with access to it can search its events. @@ -102,16 +104,6 @@ Sim verifies Directory access and selected users, then probes one selected user' -## Connect your account - -These steps apply to **Member accounts**. When only a central Calendar source is configured, teammates use Search or Home directly and are not offered a personal Calendar connection for that source. - -1. Join the Sim organization and verify your Sim email. Open **Integrations** and click **Connect** beside Google Calendar. -2. In the connection tab, choose the Google account whose verified email matches your Sim email. Grant the requested permissions. -3. Return to Integrations to see indexing status and your searchable document count. - -Teammates repeat only these connection steps after joining the organization. They do not need to configure the source. An admin can send connection requests from **Settings → Sources**: select **Manage** beside **Google Calendar**, then **Accounts → Request connections**. These requests do not grant organization membership. Connecting Gmail or Google Drive does not replace the Calendar connection. - ## Source options An admin opens **Settings → Sources** and selects **Manage** beside **Google Calendar** to open its configuration list. Each row shows **Member accounts** or **Service account** beside its sync status. Open a configuration's **Settings** tab to edit its filters, then select **Save**. **Documents** shows indexed events and **Sync history** shows recent runs. @@ -127,13 +119,13 @@ An admin opens **Settings → Sources** and selects **Manage** beside **Google C | Search Query | Optional text filter applied by Google to event titles, descriptions, locations, and organizer or attendee names and emails. Leave empty to include all matching events in the date range. | | Include Attendees | **Yes** by default. **No** omits organizer and attendee identity fields and keeps the attendee count. It does not redact names written into titles or descriptions. | -In the add-source form, **More options** contains optional **Metadata tags**. Search hides sync frequency and the general knowledge-base **Max Events** setting. +In the add-source form, **More options** also contains optional **Metadata tags**. ## What gets indexed -Sim indexes event titles, descriptions, times, locations, and the selected attendee information. All-day events and individual occurrences of recurring meetings are supported. An invitation you declined stays searchable and is marked `Response: declined`. Results link back to Google Calendar. +Sim indexes event titles, descriptions, times, locations, and the selected attendee information. All-day events and individual occurrences of recurring meetings are supported. Declined invitations returned by Google stay searchable and are marked `Response: declined`; hidden invitations are not requested. Results link back to Google Calendar. -Cancelled events, attachment contents, meeting recordings, and transcripts are not indexed. Status entries such as working location, out of office, focus time, and birthdays, and automatically generated reservation events from Gmail are not indexed. A shared calendar where you can see only free or busy times contributes nothing, since those blocks have no title or description. Events outside the selected date window are excluded. Private event details that Google withholds are not available in Search; see [Google's calendar sharing rules](https://developers.google.com/workspace/calendar/api/concepts/sharing). +Cancelled events, attachment contents, meeting recordings, and transcripts are not indexed. Status entries such as working location, out of office, focus time, and birthdays, and automatically generated reservation events from Gmail are not indexed. Events Google returns only as free/busy blocks, without searchable details, are not indexed. Events outside the selected date window are excluded. Private event details that Google withholds are not available in Search; see [Google's calendar sharing rules](https://developers.google.com/workspace/calendar/api/concepts/sharing). Search schedules syncs hourly. Event edits, cancellations, access changes, inactive or removed users, and events moving outside the date window are reconciled during completed background syncs. Central crawls page through each selected user and resume unfinished work before removing documents no longer listed. Authorization, quota, and provider failures stop the sync rather than treating unread calendars as empty. The first sync may take longer, and results appear as indexing progresses; Search is not a live Calendar read. @@ -148,6 +140,7 @@ Search schedules syncs hourly. Event edits, cancellations, access changes, inact | Reconnect | Click **Reconnect** and complete Google authorization again. Allow pop-ups if the connection tab does not open. | | Unavailable or needs admin attention | Ask your Sim admin to check source status and the deployment's Google OAuth configuration. | | Service-account authorization or Directory error | Confirm both delegated scopes, enabled APIs, and the Directory administrator's user-read privilege. Check whether delegation still awaits approval or propagation. | +| Calendar is disabled for a selected user | An active Workspace user may have Calendar turned off. [Enable Calendar](https://knowledge.workspace.google.com/admin/users/access/turn-calendar-on-or-off-for-users) for them, or set **Users** to accounts with Calendar enabled. Leaving **Users** blank includes all active users and can stop sync on a service-access error. | | User not found or inactive | Use an active primary email in the same Workspace customer. Aliases, external or guest accounts, suspended users, and archived users cannot be selected. | | A central source has no results for a teammate | Confirm their primary Workspace email matches their verified Sim email, they belong to the Sim organization, and they are included in **Users**. Check calendar IDs and **Sync history**. | diff --git a/apps/docs/content/docs/search/google-drive.mdx b/apps/docs/content/docs/search/google-drive.mdx index 8cde905b3e9..52ecfc4864b 100644 --- a/apps/docs/content/docs/search/google-drive.mdx +++ b/apps/docs/content/docs/search/google-drive.mdx @@ -21,7 +21,7 @@ Admin setup uses your organization's **Settings → Sources** page. Teammates co These are alternative setup paths. When only a central Drive source is configured, Integrations does not offer a personal Drive **Connect** action. Teammates use Search or Home directly. Existing member-account sources keep their connection actions. - A central crawl reads each selected employee's Drive through domain-wide delegation, including private My Drive files and shared-drive files they can access. Leave **Users** blank to include all active users in your Google Workspace customer, including secondary domains. Files keep their original user and group permissions; indexing a private file does not make it visible to other employees. + A central crawl includes each selected employee's private My Drive files and shared-drive files they can access. Files keep their original user and group permissions; indexing a private file does not make it visible to other employees. ## Connect member accounts @@ -38,7 +38,7 @@ An organization admin opens **Settings → Sources** and turns on **Google Drive ### Connect your account -Open **Integrations** in the main sidebar and select **Connect** beside Google Drive. Use the Google account matching your verified Sim email. The first personal connection can create a source with default filters. Teammates follow the same [connection steps](/search/connect-your-account). +Join the Sim organization, then open **Integrations** and select **Connect** beside Google Drive. Use the Google account matching your verified Sim email. The first personal connection can create a source with default filters. Return to Integrations to see indexing status and your searchable document count. @@ -52,6 +52,8 @@ Keep **Sync documents with → Connected members** unless a dedicated account sh +Admins can request member connections from **Manage → Accounts → Request connections**. These requests do not grant organization membership. See [Connect your account](/search/connect-your-account) for the shared connection and recovery steps. + ## Set up a central service account Open **Settings → Sources** and turn on **Google Drive**. Select **Manage → Advanced → Add sync configuration** to open the central service-account form directly. If personal connections are disabled for your organization, select **Add source** from the provider page instead. Teammates do not need a personal Drive connection for this source. @@ -80,8 +82,6 @@ Open the service account's **Keys** tab and choose **Add key → Create new key In the service account's **Details**, expand **Advanced settings** and copy its numeric **Client ID**. Sign in to the [Workspace Admin Console](https://admin.google.com/ac/owl/domainwidedelegation) as a super administrator. Open **Security → Access and data control → API controls → Manage Domain Wide Delegation → Add new**. -Google Workspace Admin Console Add a new client ID dialog with Client ID and OAuth scopes fields - Paste that Client ID into **Client ID**, then enter these exact scopes as a comma-separated list under **OAuth scopes**: ```text diff --git a/apps/docs/content/docs/search/jira.mdx b/apps/docs/content/docs/search/jira.mdx index e47e217091f..b793135c408 100644 --- a/apps/docs/content/docs/search/jira.mdx +++ b/apps/docs/content/docs/search/jira.mdx @@ -7,9 +7,9 @@ import { Callout } from 'fumadocs-ui/components/callout' import { Step, Steps } from 'fumadocs-ui/components/steps' import { Image } from '@/components/ui/image' -Search issue titles, descriptions, and metadata from selected **Jira Cloud** projects. An organization admin approves Jira and defines the source; **each teammate connects their own Jira account** to search issues they can access. +Search issue titles, descriptions, and metadata from selected **Jira Cloud** projects. **Each teammate connects their own Jira account** to search issues they can access. -Jira Search uses **Member accounts**. Service accounts are supported for Jira workflows, but do not provide a central Jira Search crawl. Comments, attachment contents, dashboards, and saved filters are not indexed. +Jira Search uses **Member accounts**. Service accounts can run workflows and populate the project picker, but cannot centrally index Jira for Search. Comments, attachment contents, dashboards, and saved filters are not indexed. ## Before you start @@ -39,13 +39,13 @@ Open your organization's **Settings → Sources**, turn on **Jira**, then select Enter **Jira Domain**. Under **Account for browsing**, select an account or choose **Connect Jira account**, then select **Projects**. To enter keys such as `ENG, SUPPORT` manually, use the switch beside Projects; this works without a browsing account. -**Account for browsing** only populates the project picker. You can use OAuth or **Add service account** for this step; neither enrolls that account for Search or enables central indexing. Each teammate still connects their own Jira account. +**Account for browsing** only loads the project picker, using OAuth or **Add service account**. It does not enroll anyone for Search. A service account needs Jira access and the [Jira read scopes](/integrations/atlassian-service-account#jira-and-jira-service-management); Confluence-only tokens do not work here. Jira source setup with an account for browsing, site domain, and projects @@ -64,7 +64,7 @@ Select **Add source**. This saves the shared scope; it does not authorize accoun Open **Integrations** in the main sidebar and select **Connect** on the Jira source. In the new tab, authorize the configured site using the Atlassian email matching your verified Sim email. -Each teammate follows this step. An existing authorized account may already be connected. Return to Integrations to check indexing status and your searchable document count. +Each teammate follows this step. Existing authorized accounts may already be connected. Integrations shows indexing status and your searchable document count. @@ -73,11 +73,11 @@ For workspace Search, start from **Search → Add source**; its source-creation ## Manage the source -Admins open **Settings → Sources → Jira → Manage**, then a source's **Documents**, **Settings**, or **Sync history**. Settings include the site, projects, JQL filter, and optional issue type, status, priority, labels, assignee, and last-updated tags. +Admins open **Settings → Sources → Jira → Manage**, then a source's **Documents**, **Settings**, or **Sync history**. Metadata tags include issue type, status, priority, labels, assignee, and last updated. Teammates share the configured scope and do not choose projects again. For another site or scope, add another source, or use **Connect** beside **Connect a different site or content scope** in Integrations. -Invite new teammates to the Sim organization through Members settings or SSO, then have them connect Jira. **Jira → Accounts → Request connections** requests a provider connection; it does not add people to the organization. +Invite teammates through **Settings → Members → Invite** or SSO, then have them connect Jira through **Integrations**. **Jira → Accounts → Request connections** only requests a provider connection; it does not invite people to the organization. Sim checks Jira separately for each connected person. Content becomes searchable as indexing finishes; issue changes and lost access are reflected after later syncs. @@ -88,6 +88,7 @@ Sim checks Jira separately for each connected person. Content becomes searchable | No source setup controls | Ask a Sim organization admin to approve Jira. | | Projects are empty or disabled | Enter the correct domain, connect a browsing account with project access, or switch to manual keys. | | Connected, but no issues | Check the authorized site, project access, issue security, and JQL. An admin's Jira access does not grant access to teammates. | +| Projects appear but issues do not sync | Ask the Atlassian admin whether a [data security policy](https://support.atlassian.com/security-and-access-policies/docs/block-app-access/) blocks Sim from the selected projects. Project visibility does not prove that an app may read its issues. | | Email mismatch | Use the Atlassian email matching your verified Sim email. If switching accounts fails, sign out of Atlassian and retry **Connect**. | | **Reconnect** | Reauthorize and grant all requested permissions. A revoked grant or changed scope list can require a new connection. | | Connection tab does not open | Allow pop-ups for Sim and retry. | @@ -100,9 +101,9 @@ Open a missing issue in Jira using the connected account. For company-managed pr Configure one shared Jira OAuth app for the deployment: 1. In the [Atlassian developer console](https://developer.atlassian.com/console/myapps/), select or create an **OAuth 2.0 integration**. -2. Under **Authorization → OAuth 2.0 (3LO)**, save `https:///api/auth/oauth2/callback/jira` as a callback, preserving callbacks used by other deployments. +2. Under **Authorization → OAuth 2.0 (3LO)**, add `https:///api/auth/oauth2/callback/jira` as a callback. 3. Under **Permissions**, add **Jira API** and configure the full `jira` scope list from [Sim's OAuth configuration](https://github.com/simstudioai/sim/blob/staging/apps/sim/lib/oauth/oauth.ts), including its Jira Service Management and Assets scopes. Add **User Identity API → read:me**. Sim requests `offline_access` for refresh tokens; Search's three scopes above are only a subset of this shared app's permissions. 4. Enable sharing under **Distribution**. Set `JIRA_CLIENT_ID` and `JIRA_CLIENT_SECRET` from the app's **Settings**, verify `NEXT_PUBLIC_APP_URL`, and restart Sim. 5. Connect from **Integrations** and select the configured site. After changing the OAuth client or requested scopes, use **Settings → Sources → More → Refresh connection settings**, then have affected teammates reconnect. -The callback must match exactly, including scheme, hostname, port, and path. For `http://localhost:3000`, register `http://localhost:3000/api/auth/oauth2/callback/jira`. Use a separate development app when production callbacks must stay unchanged. If only the app owner can connect, check **Distribution**. See the [Atlassian OAuth guide](https://developer.atlassian.com/cloud/jira/platform/oauth-2-3lo-apps/) and [Sim deployment reference](/platform/self-hosting/integrations-oauth). +The callback must match Sim's scheme, hostname, port, and path exactly. If only the app owner can connect, check **Distribution**. See the [Atlassian OAuth guide](https://developer.atlassian.com/cloud/jira/platform/oauth-2-3lo-apps/) and [Sim deployment reference](/platform/self-hosting/integrations-oauth). diff --git a/apps/docs/content/docs/search/slack.mdx b/apps/docs/content/docs/search/slack.mdx index 42917bc7438..2b63bb88e82 100644 --- a/apps/docs/content/docs/search/slack.mdx +++ b/apps/docs/content/docs/search/slack.mdx @@ -40,7 +40,7 @@ You can also open this wizard from **Settings → Sim Search in Slack → Set up Return to Sim and select **Continue**. In step 2, copy **Client ID**, **Client Secret**, and **Signing Secret** from the new app’s **Basic Information → App Credentials**: -Sim Search in Slack setup asking for Client ID, Client Secret, and Signing Secret +Sim Search in Slack setup with placeholders for Client ID, Client Secret, and Signing Secret Select **Continue**, then **Install in Slack** in step 3. Approve the installation in Slack. Sim saves the bot connection and opens **Settings → Sim Search in Slack**. Complete any required Slack administrator approval before continuing. diff --git a/apps/docs/openapi-v2-workflows.json b/apps/docs/openapi-v2-workflows.json index d25aaf9460e..b557a9b373e 100644 --- a/apps/docs/openapi-v2-workflows.json +++ b/apps/docs/openapi-v2-workflows.json @@ -15882,6 +15882,7 @@ "wealthbox.contacts", "jira.issues", "jira.projects", + "jira.projectKeys", "linear.projects", "linear.teams", "monday.boards", @@ -16047,6 +16048,7 @@ "wealthbox.contacts", "jira.issues", "jira.projects", + "jira.projectKeys", "linear.projects", "linear.teams", "monday.boards", diff --git a/apps/docs/public/static/credentials/atlassian/admin-scope-picker.png b/apps/docs/public/static/credentials/atlassian/admin-scope-picker.png index 27b160adb16..120db7118c5 100644 Binary files a/apps/docs/public/static/credentials/atlassian/admin-scope-picker.png and b/apps/docs/public/static/credentials/atlassian/admin-scope-picker.png differ diff --git a/apps/docs/public/static/search/confluence-setup.jpg b/apps/docs/public/static/search/confluence-setup.jpg deleted file mode 100644 index 8efc63e456d..00000000000 Binary files a/apps/docs/public/static/search/confluence-setup.jpg and /dev/null differ diff --git a/apps/docs/public/static/search/confluence-setup.png b/apps/docs/public/static/search/confluence-setup.png new file mode 100644 index 00000000000..8ce6c182f59 Binary files /dev/null and b/apps/docs/public/static/search/confluence-setup.png differ diff --git a/apps/docs/public/static/search/google-domain-delegation.png b/apps/docs/public/static/search/google-domain-delegation.png deleted file mode 100644 index b68680d0b82..00000000000 Binary files a/apps/docs/public/static/search/google-domain-delegation.png and /dev/null differ diff --git a/apps/docs/public/static/search/jira-setup.jpg b/apps/docs/public/static/search/jira-setup.jpg deleted file mode 100644 index bd6afc94d83..00000000000 Binary files a/apps/docs/public/static/search/jira-setup.jpg and /dev/null differ diff --git a/apps/docs/public/static/search/jira-setup.png b/apps/docs/public/static/search/jira-setup.png new file mode 100644 index 00000000000..393b062c641 Binary files /dev/null and b/apps/docs/public/static/search/jira-setup.png differ diff --git a/apps/docs/public/static/search/slack-setup.jpg b/apps/docs/public/static/search/slack-setup.jpg deleted file mode 100644 index c9ee55a27df..00000000000 Binary files a/apps/docs/public/static/search/slack-setup.jpg and /dev/null differ diff --git a/apps/docs/public/static/search/slack-setup.png b/apps/docs/public/static/search/slack-setup.png new file mode 100644 index 00000000000..15b28f24555 Binary files /dev/null and b/apps/docs/public/static/search/slack-setup.png differ diff --git a/apps/sim/app/api/files/authorization.test.ts b/apps/sim/app/api/files/authorization.test.ts index 13aaf73db66..22525582e75 100644 --- a/apps/sim/app/api/files/authorization.test.ts +++ b/apps/sim/app/api/files/authorization.test.ts @@ -9,7 +9,13 @@ * * @vitest-environment node */ -import { dbChainMockFns } from '@sim/testing' +import { + dbChainMockFns, + hasMockCondition, + queueTableRows, + resetDbChainMock, + schemaMock, +} from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockGetFileMetadataByKey, mockGetUserEntityPermissions, mockGetFileMetadata } = vi.hoisted( @@ -46,7 +52,7 @@ vi.mock('@/executor/constants', () => ({ isUuid: vi.fn(() => false), })) -import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' +import { type KnowledgeAccessProvider, SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' import { verifyFileAccess, verifyKBFileWriteAccess } from '@/app/api/files/authorization' const CLOUD_KEY = 'kb/1780162789495-secret.txt' @@ -392,3 +398,65 @@ describe('organization connector cache access', () => { ).resolves.toBe(false) }) }) + +describe('KB file live source authorization', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockGetFileMetadataByKey.mockResolvedValue({ workspaceId: 'ws-1', deletedAt: null }) + mockGetUserEntityPermissions.mockResolvedValue('read') + }) + + it.each([true, false])( + 'returns live permission %s after an ordinary file lookup misses', + async (allowed) => { + const scope = { kind: 'user' as const, userId: USER_ID, tokens: ['reader-token'] } + const getForConnectors = vi.fn().mockResolvedValue(scope) + const access: KnowledgeAccessProvider = { + get: async () => scope, + getForConnectors, + getForDocuments: async () => scope, + } + queueTableRows(schemaMock.document, []) + queueTableRows(schemaMock.document, [{ connectorId: 'confluence-source' }]) + queueTableRows(schemaMock.document, allowed ? [{ id: 'doc-1' }] : []) + await expect( + verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'knowledge-base', false, { + knowledgeAccess: access, + }) + ).resolves.toBe(allowed) + expect(getForConnectors).toHaveBeenCalledExactlyOnceWith(['confluence-source'], undefined) + for (const [condition] of dbChainMockFns.where.mock.calls) { + expect( + hasMockCondition( + condition, + (node) => + node.type === 'eq' && + node.left === schemaMock.document.storageKey && + node.right === CLOUD_KEY + ) + ).toBe(true) + expect( + hasMockCondition( + condition, + (node) => + node.type === 'eq' && + node.left === schemaMock.knowledgeBase.workspaceId && + node.right === 'ws-1' + ) + ).toBe(true) + } + } + ) + + it('rejects a missing ownership permission before resolving the reader', async () => { + mockGetUserEntityPermissions.mockResolvedValue(null) + const get = vi.fn() + await expect( + verifyFileAccess(CLOUD_KEY, USER_ID, undefined, 'knowledge-base', false, { + knowledgeAccess: { get, getForConnectors: vi.fn(), getForDocuments: vi.fn() }, + }) + ).resolves.toBe(false) + expect(get).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/files/authorization.ts b/apps/sim/app/api/files/authorization.ts index a847a5748ac..618b4e9711e 100644 --- a/apps/sim/app/api/files/authorization.ts +++ b/apps/sim/app/api/files/authorization.ts @@ -6,12 +6,11 @@ import { and, eq, isNotNull, isNull, or } from 'drizzle-orm' import { NextResponse } from 'next/server' import type { ResourceScope } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' -import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import { - resolveUserKnowledgeAccessScope, + createUserKnowledgeAccessProvider, WORKSPACE_ACCESS_SCOPE, } from '@/lib/knowledge/access/scope' -import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/access/types' +import { type KnowledgeReadAccess, knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' import { getFileMetadata } from '@/lib/uploads' import type { StorageContext } from '@/lib/uploads/config' import type { StorageConfig } from '@/lib/uploads/core/storage-client' @@ -494,61 +493,55 @@ async function verifyCopilotFileAccess( } /** - * Whether an active KB document (non-archived/excluded/deleted, in a - * non-deleted KB) in the owning workspace references exactly `cloudKey`, matched - * on the document's persisted canonical `storageKey`. This is an exact, indexed - * lookup — no URL parsing or wildcard matching at read time. It is a lifecycle - * signal only: it reflects whether the file is still part of a live KB, not who - * owns it (ownership comes from the binding). + * Checks whether a readable, active document references the exact storage key + * within the binding's canonical scope. Live source proof uses only candidate + * IDs; document existence is checked with the complete access predicate. */ -/** A reader once resolved: a person's or the workspace's tokens, or the system reading its own rows. */ -type ResolvedKnowledgeFileAccess = KnowledgeAccessScope | SystemAccessScope - async function hasActiveKbDocumentForKey( cloudKey: string, scope: ResourceScope, - access: ResolvedKnowledgeFileAccess + access: KnowledgeReadAccess ): Promise { - const rows = await db - .select({ id: document.id }) - .from(document) - .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id)) - .where( - and( - resourceScopeCondition(knowledgeBase, scope), - eq(document.storageKey, cloudKey), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - isNull(knowledgeBase.deletedAt), - access.kind === 'system' - ? undefined - : or(isNull(document.connectorId), isNotNull(document.contentHash)), - knowledgeAccessCondition(access) - ) - ) - .limit(1) - - return rows.length > 0 + const conditions = [ + resourceScopeCondition(knowledgeBase, scope), + eq(document.storageKey, cloudKey), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + isNull(knowledgeBase.deletedAt), + !('get' in access) && access.kind === 'system' + ? undefined + : or(isNull(document.connectorId), isNotNull(document.contentHash)), + ] + for await (const accessCondition of knowledgeReadAccessBatches(access, conditions)) { + const rows = await db + .select({ id: document.id }) + .from(document) + .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id)) + .where(and(...conditions, accessCondition)) + .limit(1) + if (rows.length > 0) return true + } + return false } /** * How a KB file read identifies the reader for document access. `'user'` is - * for a session-authenticated person; a resolved scope is for a caller that - * already holds one (an execution with a principal). The system scope is for + * for a session-authenticated person; an access provider carries the principal + * behind an execution and resolves current source permissions. The system scope is for * a background job reading a connector-owned row it is processing, which in * members mode is hidden until the sync materializes its readers. Anything * else — an internal token, a tool running with the workflow owner's id — * reads as the workspace, never as the person whose id it happens to carry. */ -export type KnowledgeFileAccess = 'user' | ResolvedKnowledgeFileAccess +export type KnowledgeFileAccess = 'user' | KnowledgeReadAccess async function resolveKnowledgeFileAccess( knowledgeAccess: KnowledgeFileAccess | undefined, userId: string, workspaceId: string -): Promise { - if (knowledgeAccess === 'user') return resolveUserKnowledgeAccessScope(userId, workspaceId) +): Promise { + if (knowledgeAccess === 'user') return createUserKnowledgeAccessProvider(userId, { workspaceId }) return knowledgeAccess ?? WORKSPACE_ACCESS_SCOPE } @@ -559,9 +552,8 @@ async function resolveKnowledgeFileAccess( * 1. Ownership — the trusted `workspace_files` binding (exact key) names the * owning workspace; the caller must have permission on it. Ownership is * never inferred from an attacker-authorable `document.fileUrl`. - * 2. Liveness — an active document must still reference the exact key, so the - * retained bytes of an archived document or soft-deleted KB are not - * downloadable (the liveness document is not an authorization signal). + * 2. Readability — an active document must reference the exact key and satisfy + * the caller's complete document and live source access predicates. * * A missing binding denies (the ownership backfill populates bindings for * pre-existing objects before this path is deployed). @@ -589,6 +581,7 @@ async function verifyKBFileAccess( if ( binding.workspaceId || typeof knowledgeAccess !== 'object' || + 'get' in knowledgeAccess || knowledgeAccess.kind !== 'system' ) return false @@ -624,7 +617,7 @@ async function verifyKBFileAccess( logger.warn('KB file access denied: no readable document references the file', { userId, cloudKey, - accessScopeKind: access.kind, + accessScopeKind: 'get' in access ? 'reader' : access.kind, }) return false } diff --git a/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts b/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts index 7ad5558439e..35a7a1d1a9b 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts @@ -14,7 +14,7 @@ import { performDeleteKnowledgeDocument } from '@/lib/knowledge/orchestration' import { handleError, resolveKnowledgeBase, - resolveV1KnowledgeAccessScope, + resolveV1KnowledgeReadAccess, serializeDate, } from '@/app/api/v1/knowledge/utils' import { authenticateRequest, v1ValidationErrorResponse } from '@/app/api/v1/middleware' @@ -52,7 +52,7 @@ export const GET = withRouteHandler( const doc = await getKnowledgeDocument( knowledgeBaseId, documentId, - await resolveV1KnowledgeAccessScope(userId, rateLimit, parsed.data.query.workspaceId) + await resolveV1KnowledgeReadAccess(userId, rateLimit, parsed.data.query.workspaceId) ) if (!doc) { @@ -116,7 +116,7 @@ export const DELETE = withRouteHandler( const doc = await getKnowledgeDocument( knowledgeBaseId, documentId, - await resolveV1KnowledgeAccessScope(userId, rateLimit, parsed.data.query.workspaceId) + await resolveV1KnowledgeReadAccess(userId, rateLimit, parsed.data.query.workspaceId) ) if (!doc) { diff --git a/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts b/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts index ba403a072e8..8ee583d3640 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts @@ -29,7 +29,7 @@ import { validateFileType } from '@/lib/uploads/utils/validation' import { handleError, resolveKnowledgeBase, - resolveV1KnowledgeAccessScope, + resolveV1KnowledgeReadAccess, serializeDate, } from '@/app/api/v1/knowledge/utils' import { authenticateRequest, v1ValidationErrorResponse } from '@/app/api/v1/middleware' @@ -79,7 +79,7 @@ export const GET = withRouteHandler(async (request: NextRequest, context: Docume sortOrder: sortOrder as SortOrder, }, requestId, - await resolveV1KnowledgeAccessScope(userId, rateLimit, workspaceId) + await resolveV1KnowledgeReadAccess(userId, rateLimit, workspaceId) ) return NextResponse.json({ diff --git a/apps/sim/app/api/v1/knowledge/search/route.test.ts b/apps/sim/app/api/v1/knowledge/search/route.test.ts index 54c23374a28..7be0518423e 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.test.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.test.ts @@ -12,18 +12,22 @@ import { getErrorMessage } from '@sim/utils/errors' import { beforeEach, describe, expect, it, vi } from 'vitest' const { + mockResolveV1KnowledgeReadAccess, mockExecuteKnowledgeSearch, mockGenerateSearchEmbedding, mockGetDocumentMetadataByIds, + mockGetDocumentTagDefinitions, mockAuthenticateRequest, mockValidateWorkspaceAccess, mockResolveBillingAttribution, mockResolveSystemBillingAttribution, mockRecordSearchEmbeddingUsage, } = vi.hoisted(() => ({ + mockResolveV1KnowledgeReadAccess: vi.fn(), mockExecuteKnowledgeSearch: vi.fn(), mockGenerateSearchEmbedding: vi.fn(), mockGetDocumentMetadataByIds: vi.fn(), + mockGetDocumentTagDefinitions: vi.fn(), mockAuthenticateRequest: vi.fn(), mockValidateWorkspaceAccess: vi.fn(), mockResolveBillingAttribution: vi.fn(), @@ -81,9 +85,7 @@ vi.mock('@/app/api/v1/middleware', () => ({ })) vi.mock('@/app/api/v1/knowledge/utils', () => ({ - resolveV1KnowledgeAccessScope: vi - .fn() - .mockResolvedValue({ kind: 'workspace', tokens: ['pub', 'ws'] }), + resolveV1KnowledgeReadAccess: mockResolveV1KnowledgeReadAccess, handleError: (e: unknown) => new Response(JSON.stringify({ error: getErrorMessage(e, 'error') }), { status: 500, @@ -91,7 +93,7 @@ vi.mock('@/app/api/v1/knowledge/utils', () => ({ })) vi.mock('@/lib/knowledge/tags/service', () => ({ - getDocumentTagDefinitions: vi.fn().mockResolvedValue([]), + getDocumentTagDefinitions: mockGetDocumentTagDefinitions, })) import { POST } from '@/app/api/v1/knowledge/search/route' @@ -111,6 +113,7 @@ const baseKb = (id: string, embeddingModel: string, embeddingDimension = 1536) = describe('v1 knowledge search route — per-KB embedding model', () => { beforeEach(() => { vi.clearAllMocks() + mockResolveV1KnowledgeReadAccess.mockResolvedValue({ kind: 'workspace', tokens: ['pub', 'ws'] }) mockAuthenticateRequest.mockResolvedValue({ requestId: 'req-1', userId: 'user-1', @@ -123,6 +126,7 @@ describe('v1 knowledge search route — per-KB embedding model', () => { }) mockExecuteKnowledgeSearch.mockResolvedValue([]) mockGetDocumentMetadataByIds.mockResolvedValue({}) + mockGetDocumentTagDefinitions.mockResolvedValue([]) mockResolveBillingAttribution.mockImplementation( ({ actorUserId, workspaceId }: { actorUserId: string; workspaceId: string }) => Promise.resolve({ @@ -135,6 +139,111 @@ describe('v1 knowledge search route — per-KB embedding model', () => { mockRecordSearchEmbeddingUsage.mockResolvedValue(undefined) }) + it('retains the reader provider for ranked results and returned document metadata', async () => { + const access = { kind: 'user' as const, userId: 'user-1', tokens: ['reader-token'] } + const provider = { + get: vi.fn().mockResolvedValue(access), + getForConnectors: vi.fn(), + getForDocuments: vi.fn(), + } + mockResolveV1KnowledgeReadAccess.mockResolvedValue(provider) + mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({ + hasAccess: true, + knowledgeBase: baseKb('kb-1', 'text-embedding-3-small'), + }) + const response = await POST( + createMockRequest('POST', { + workspaceId: 'ws-1', + knowledgeBaseIds: 'kb-1', + query: 'hello', + }) + ) + expect(response.status).toBe(200) + expect(mockExecuteKnowledgeSearch).toHaveBeenCalledWith( + expect.objectContaining({ + access, + accessProvider: provider, + }) + ) + expect(mockGetDocumentMetadataByIds).toHaveBeenCalledWith([], access, provider) + }) + + it.each([ + ['query', false], + ['query', true], + ['filters', false], + ['filters', true], + ] as const)( + 'omits newly denied content from %s results and counts when all denied is %s', + async (mode, allDenied) => { + const access = { kind: 'user' as const, userId: 'user-1', tokens: ['reader-token'] } + const provider = { + get: vi.fn().mockResolvedValue(access), + getForConnectors: vi.fn(), + getForDocuments: vi.fn(), + } + mockResolveV1KnowledgeReadAccess.mockResolvedValue(provider) + mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({ + hasAccess: true, + knowledgeBase: baseKb('kb-1', 'text-embedding-3-small'), + }) + mockGetDocumentTagDefinitions.mockResolvedValue([ + { tagSlot: 'tag1', displayName: 'category', fieldType: 'text' }, + ]) + mockExecuteKnowledgeSearch.mockResolvedValue([ + { + documentId: 'revoked-document', + knowledgeBaseId: 'kb-1', + content: 'revoked page content', + tag1: 'revoked tag', + chunkIndex: 0, + distance: 0.1, + }, + { + documentId: 'allowed-document', + knowledgeBaseId: 'kb-1', + content: 'allowed page content', + tag1: 'docs', + chunkIndex: 0, + distance: 0.2, + }, + ]) + mockGetDocumentMetadataByIds.mockResolvedValue( + allDenied ? {} : { 'allowed-document': { filename: 'Allowed page', sourceUrl: null } } + ) + const response = await POST( + createMockRequest('POST', { + workspaceId: 'ws-1', + knowledgeBaseIds: 'kb-1', + ...(mode === 'query' + ? { query: 'docs' } + : { tagFilters: [{ tagName: 'category', operator: 'eq', value: 'docs' }] }), + }) + ) + const body = await response.json() + expect(response.status).toBe(200) + expect(mockGetDocumentMetadataByIds).toHaveBeenCalledWith( + ['revoked-document', 'allowed-document'], + access, + provider + ) + expect(body.data.results).toEqual( + allDenied + ? [] + : [ + expect.objectContaining({ + documentId: 'allowed-document', + documentName: 'Allowed page', + content: 'allowed page content', + metadata: { category: 'docs' }, + }), + ] + ) + expect(body.data.totalResults).toBe(allDenied ? 0 : 1) + expect(JSON.stringify(body)).not.toContain('revoked') + } + ) + it('passes the KB embedding model into generateSearchEmbedding', async () => { mockCheckKnowledgeBaseAccess.mockResolvedValueOnce({ hasAccess: true, diff --git a/apps/sim/app/api/v1/knowledge/search/route.ts b/apps/sim/app/api/v1/knowledge/search/route.ts index 44b347c4160..b8209d400cf 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.ts @@ -24,7 +24,7 @@ import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service' import { buildUndefinedTagsError, validateTagValue } from '@/lib/knowledge/tags/utils' import type { StructuredFilter } from '@/lib/knowledge/types' import { checkKnowledgeBaseAccess, type KnowledgeBaseAccessResult } from '@/app/api/knowledge/utils' -import { handleError, resolveV1KnowledgeAccessScope } from '@/app/api/v1/knowledge/utils' +import { handleError, resolveV1KnowledgeReadAccess } from '@/app/api/v1/knowledge/utils' import { authenticateRequest, capabilityGovernedUserId, @@ -228,8 +228,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let results: SearchResult[] let queryEmbeddingIsBYOK: boolean | null = null - const [access, { searchMode, boostRecency }] = await Promise.all([ - resolveV1KnowledgeAccessScope(userId, rateLimit, workspaceId), + const [readAccess, { searchMode, boostRecency }] = await Promise.all([ + resolveV1KnowledgeReadAccess(userId, rateLimit, workspaceId), resolveKnowledgeSearchDefaults({ workspaceId, /** A personal key acts as its user; a workspace key has no person behind it. */ @@ -238,11 +238,15 @@ export const POST = withRouteHandler(async (request: NextRequest) => { }), ]) + const accessProvider = 'get' in readAccess ? readAccess : undefined + const access = 'get' in readAccess ? await readAccess.get() : readAccess + if (!hasQuery && hasFilters) { results = await executeKnowledgeSearch({ knowledgeBaseIds: accessibleKbIds, topK, access, + accessProvider, searchMode, boostRecency, structuredFilters, @@ -258,6 +262,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { knowledgeBaseIds: accessibleKbIds, topK, access, + accessProvider, searchMode, boostRecency, query, @@ -306,12 +311,13 @@ export const POST = withRouteHandler(async (request: NextRequest) => { }) const documentIds = results.map((r) => r.documentId) - const documentMetadataMap = await getDocumentMetadataByIds(documentIds, access) + const documentMetadataMap = await getDocumentMetadataByIds(documentIds, access, accessProvider) + const readableResults = results.filter((result) => documentMetadataMap[result.documentId]) return NextResponse.json({ success: true, data: { - results: results.map((result) => { + results: readableResults.map((result) => { const kbTagMap = tagDefinitionsMap[result.knowledgeBaseId] || {} const tags: Record = {} @@ -337,7 +343,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { query: query || '', knowledgeBaseIds: accessibleKbIds, topK, - totalResults: results.length, + totalResults: readableResults.length, }, }) } catch (error) { diff --git a/apps/sim/app/api/v1/knowledge/utils.test.ts b/apps/sim/app/api/v1/knowledge/utils.test.ts new file mode 100644 index 00000000000..877c4d1e091 --- /dev/null +++ b/apps/sim/app/api/v1/knowledge/utils.test.ts @@ -0,0 +1,35 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { createUserProvider } = vi.hoisted(() => ({ createUserProvider: vi.fn() })) +vi.mock('@/lib/knowledge/access/scope', () => ({ + createUserKnowledgeAccessProvider: createUserProvider, + WORKSPACE_ACCESS_SCOPE: { kind: 'workspace', tokens: ['pub', 'ws'] }, +})) +vi.mock('@/lib/knowledge/service', () => ({ getKnowledgeBaseById: vi.fn() })) +vi.mock('@/app/api/v1/middleware', () => ({ validateWorkspaceAccess: vi.fn() })) + +import { resolveV1KnowledgeReadAccess } from '@/app/api/v1/knowledge/utils' + +beforeEach(() => vi.clearAllMocks()) + +describe('v1 knowledge reader identity', () => { + it.each(['personal', 'oauth_access_token'] as const)( + 'retains live source proof for a %s caller', + async (keyType) => { + const provider = { get: vi.fn(), getForConnectors: vi.fn(), getForDocuments: vi.fn() } + createUserProvider.mockReturnValue(provider) + await expect( + resolveV1KnowledgeReadAccess('reader', { keyType }, 'workspace-1') + ).resolves.toBe(provider) + expect(createUserProvider).toHaveBeenCalledWith('reader', { workspaceId: 'workspace-1' }) + } + ) + + it('keeps workspace keys actorless without borrowing their creator identity', async () => { + await expect( + resolveV1KnowledgeReadAccess('key-creator', { keyType: 'workspace' }, 'workspace-1') + ).resolves.toMatchObject({ kind: 'workspace' }) + expect(createUserProvider).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/v1/knowledge/utils.ts b/apps/sim/app/api/v1/knowledge/utils.ts index 8d721d0ec4a..34fcc72784c 100644 --- a/apps/sim/app/api/v1/knowledge/utils.ts +++ b/apps/sim/app/api/v1/knowledge/utils.ts @@ -2,10 +2,10 @@ import { createLogger } from '@sim/logger' import { NextResponse } from 'next/server' import { validationErrorResponseFromError } from '@/lib/api/server' import { - resolveUserKnowledgeAccessScope, + createUserKnowledgeAccessProvider, WORKSPACE_ACCESS_SCOPE, } from '@/lib/knowledge/access/scope' -import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' +import type { KnowledgeAccessProvider, KnowledgeAccessScope } from '@/lib/knowledge/access/types' import { getKnowledgeBaseById } from '@/lib/knowledge/service' import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' import { @@ -53,16 +53,16 @@ export async function resolveKnowledgeBase( } /** - * The document-access scope of a v1 API caller. A personal key acts as its + * The document reader of a v1 API caller. A personal key acts as its * user; a workspace key has no person behind it and reads as the workspace. */ -export async function resolveV1KnowledgeAccessScope( +export async function resolveV1KnowledgeReadAccess( userId: string, rateLimit: { keyType?: 'personal' | 'workspace' | 'oauth_access_token' }, workspaceId: string | undefined -): Promise { +): Promise { if (rateLimit.keyType === 'workspace') return WORKSPACE_ACCESS_SCOPE - return resolveUserKnowledgeAccessScope(userId, workspaceId) + return createUserKnowledgeAccessProvider(userId, { workspaceId }) } /** diff --git a/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts b/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts index 6289e012046..3971a4c4ca5 100644 --- a/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts +++ b/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts @@ -48,7 +48,8 @@ vi.mock('@/lib/knowledge/application/contexts', () => ({ })) vi.mock('@/lib/knowledge/service', () => ({ - getActiveKnowledgeBaseReference: mocks.getKnowledgeBase, + getActiveKnowledgeBaseReferences: (ids: string[]) => + Promise.all(ids.map((id) => mocks.getKnowledgeBase(id))), })) vi.mock('@/lib/knowledge/embeddings', () => ({ @@ -63,7 +64,8 @@ vi.mock('@/lib/knowledge/search/queries', () => ({ })) vi.mock('@/lib/knowledge/tags/service', () => ({ - getDocumentTagDefinitions: mocks.getTagDefinitions, + getDocumentTagDefinitionsByKnowledgeBaseIds: async (ids: string[]) => + new Map(await Promise.all(ids.map(async (id) => [id, await mocks.getTagDefinitions(id)]))), })) vi.mock('@/lib/knowledge/tags/utils', () => ({ diff --git a/apps/sim/app/invite/[id]/invite.test.tsx b/apps/sim/app/invite/[id]/invite.test.tsx index 1a966c3f386..b04b3123473 100644 --- a/apps/sim/app/invite/[id]/invite.test.tsx +++ b/apps/sim/app/invite/[id]/invite.test.tsx @@ -276,6 +276,104 @@ afterEach(() => { }) describe('Invite', () => { + it.each([ + { status: 401, body: { error: 'Unauthorized' }, authRequired: true }, + { status: 403, body: { error: 'Forbidden' }, authRequired: true }, + { + status: 401, + code: 'UNRECOGNIZED_AUTH_CODE', + body: { error: 'Please authenticate' }, + authRequired: true, + }, + { status: 404, body: { error: 'Invitation not found' }, authRequired: false }, + ])('uses HTTP $status when the response has no known invitation code', async (response) => { + mockRequestJson.mockRejectedValue( + new ApiClientError({ ...response, message: 'Request failed' }) + ) + await renderInvite() + + if (response.authRequired) { + expect(container.textContent).toContain('Authentication Required') + expect(container.textContent).not.toContain('Invitation Error') + await clickAction('Sign in to continue') + expect(mockPush).toHaveBeenCalledWith( + `/login?invite_flow=true&callbackUrl=${encodeURIComponent('/invite/invitation-1?token=token-1')}` + ) + } else { + expect(container.textContent).toContain('This invitation is invalid or no longer exists.') + expect(actionLabels()).not.toContain('Try Again') + } + }) + + it.each([ + { code: 'disclosure-outdated', body: { error: 'Your workspaces changed' } }, + { code: undefined, body: { error: 'disclosure-outdated' } }, + ])('preserves a known invitation error before the HTTP fallback: %j', async (response) => { + mockRequestJson.mockRejectedValue( + new ApiClientError({ ...response, status: 409, message: 'Review the updated disclosure' }) + ) + await renderInvite() + + expect(container.textContent).toContain('Review the updated notice and accept again.') + expect(container.textContent).not.toContain('Already Part of a Team') + expect(actionLabels()).toContain('Try Again') + }) + + it('offers sign-in if the session expires while accepting an invitation', async () => { + await renderInvite() + mockRequestJson.mockRejectedValueOnce( + new ApiClientError({ status: 401, body: { error: 'Unauthorized' }, message: 'Unauthorized' }) + ) + await clickAction('Accept Invitation') + + expect(container.textContent).toContain('Authentication Required') + expect(container.textContent).not.toContain('Welcome!') + expect(actionLabels()).not.toContain('Accept Invitation') + await clickAction('Sign in to continue') + expect(mockPush).toHaveBeenCalledWith( + `/login?invite_flow=true&callbackUrl=${encodeURIComponent('/invite/invitation-1?token=token-1')}` + ) + }) + + it.each(['internal', 'external'] as const)( + 'offers an account switch to a token holder who is not the %s invitee', + async (intent) => { + membershipIntent = intent + joinPreview = null + mockUseSession.mockReturnValue({ + data: { user: { id: 'other-user', email: 'other@example.com' } }, + isPending: false, + }) + await renderInvite() + + expect(container.textContent).toContain('Wrong Account') + expect(container.textContent).not.toContain('We could not load how this invitation affects') + expect(actionLabels()).not.toContain('Accept Invitation') + expect(actionLabels()).not.toContain('Refresh invitation') + expect(mockRequestJson).not.toHaveBeenCalledWith( + expect.objectContaining({ method: 'POST' }), + expect.anything() + ) + await clickAction('Sign in with a different account') + expect(mockSignOut).toHaveBeenCalledOnce() + expect(mockClearUserData).toHaveBeenCalledOnce() + expect(mockPush).toHaveBeenCalledWith( + `/login?invite_flow=true&callbackUrl=${encodeURIComponent('/invite/invitation-1?token=token-1')}` + ) + } + ) + + it('matches the invitation email with the same normalization as the server', async () => { + mockUseSession.mockReturnValue({ + data: { user: { id: 'user-1', email: ' INVITEE@EXAMPLE.COM ' } }, + isPending: false, + }) + await renderInvite() + + expect(container.textContent).not.toContain('Wrong Account') + expect(actionLabels()).toContain('Accept Invitation') + }) + it('clears the previous account cache before navigating to the invitation sign-in', async () => { mockRequestJson.mockRejectedValue( new ApiClientError({ diff --git a/apps/sim/app/invite/[id]/invite.tsx b/apps/sim/app/invite/[id]/invite.tsx index 09fe4ed825d..288c3f19eec 100644 --- a/apps/sim/app/invite/[id]/invite.tsx +++ b/apps/sim/app/invite/[id]/invite.tsx @@ -3,7 +3,7 @@ import { useEffect, useState } from 'react' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { formatQuotedNameList } from '@sim/utils/string' +import { formatQuotedNameList, normalizeEmail } from '@sim/utils/string' import { useQueryClient } from '@tanstack/react-query' import { useParams, useRouter, useSearchParams } from 'next/navigation' import { ApiClientError } from '@/lib/api/client/errors' @@ -263,9 +263,11 @@ function codeFromStatus(status: number): InviteErrorCode { } function codeFromApiClientError(error: ApiClientError): string { + if (error.code && getInviteError(error.code).code !== 'unknown') return error.code + if (error.body && typeof error.body === 'object') { const code = (error.body as { error?: unknown }).error - if (typeof code === 'string' && code.length > 0) return code + if (typeof code === 'string' && getInviteError(code).code !== 'unknown') return code } return codeFromStatus(error.status) @@ -316,6 +318,11 @@ export default function Invite({ registrationDisabled }: InviteProps) { }) const invitation = invitationQuery.data?.invitation ?? null const joinPreview = invitationQuery.data?.joinPreview ?? null + const isWrongAccount = Boolean( + invitation && + session?.user && + normalizeEmail(session.user.email || '') !== normalizeEmail(invitation.email) + ) const isDisclosureMissing = invitation?.membershipIntent === 'internal' && !joinPreview const isLoading = Boolean(session?.user) && (!isTokenResolved || invitationQuery.isPending) @@ -330,10 +337,15 @@ export default function Invite({ registrationDisabled }: InviteProps) { * Action errors (accept failures) outrank fetch errors; the URL error param * only shows until the invitation loads successfully. */ - const error = actionError ?? fetchError ?? (invitationQuery.data ? null : urlError) + const error = + actionError ?? + fetchError ?? + (isWrongAccount ? getInviteError('email-mismatch') : null) ?? + (invitationQuery.data ? null : urlError) const handleAcceptInvitation = async () => { - if (!session?.user || !invitation || isDisclosureMissing || isAccepting) return + if (!session?.user || !invitation || isWrongAccount || isDisclosureMissing || isAccepting) + return setIsAccepting(true) try { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.test.tsx index 1fca7ec50c9..3c23dda2671 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.test.tsx @@ -89,6 +89,31 @@ afterEach(async () => { }) describe('connection method selection', () => { + it('does not label a saved method unavailable while availability is loading', async () => { + await render({ + scope: { kind: 'organization', organizationId: 'org-1' }, + value: { accessMode: 'admin' }, + lockAccessMode: true, + allowAdmin: false, + isAvailabilityReady: false, + }) + expect(container.textContent).not.toContain('This connection method is not available') + expect(container.querySelector('[aria-label="Sync using: Service account"]')).toBeDisabled() + }) + + it('shows a real unavailable method after availability finishes loading', async () => { + await render({ + scope: { kind: 'organization', organizationId: 'org-1' }, + value: { accessMode: 'admin' }, + lockAccessMode: true, + allowAdmin: false, + isAvailabilityReady: true, + }) + expect(container.textContent).toContain( + 'This connection method is not available in this organization.' + ) + }) + it.each([ { mode: 'members', label: 'Member accounts' }, { mode: 'admin', label: 'Service account' }, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx index 9137754ea32..1f355ac085e 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx @@ -67,6 +67,7 @@ interface ConnectorAccessFieldProps { disabled?: boolean /** Existing Search sources retain the sync method chosen during setup. */ lockAccessMode?: boolean + isAvailabilityReady?: boolean /** Whether member accounts may be chosen; an existing selection remains visible for recovery. */ allowMembers?: boolean /** Whether administrator access may be chosen; it needs a connector that mirrors source permissions. */ @@ -89,6 +90,7 @@ export function ConnectorAccessField({ canAdmin, disabled = false, lockAccessMode = false, + isAvailabilityReady = true, allowMembers = true, allowAdmin = false, allowWorkspace = true, @@ -147,7 +149,7 @@ export function ConnectorAccessField({ title={slackSetupOnly ? 'Slack app' : allowWorkspace ? 'Connection method' : 'Sync using'} error={canAdmin && !showSlackSetup ? accountsQuery.error?.message : undefined} hint={ - canAdmin && !currentMode?.allowed + canAdmin && isAvailabilityReady && !currentMode?.allowed ? `This connection method is not available in this ${scope.kind}.` : lockAccessMode ? 'Create a new source to change the sync method.' diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.test.tsx index 6006d640dc5..bb51c991365 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.test.tsx @@ -102,7 +102,7 @@ import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta' function fieldProps(connectorConfig: ConnectorMeta): ConnectorSettingsFieldsProps { return { - availability: { error: null, isFetching: false, refetch: vi.fn() }, + availability: { error: null, isFetching: false, isReady: true, refetch: vi.fn() }, isSearchIndex: true, connectorConfig, selectionLabels: {}, @@ -195,6 +195,29 @@ describe('connector settings service-account choices', () => { } ) + it('keeps a failed availability check actionable before methods are known', async () => { + const refetch = vi.fn() + await render(confluenceConnectorMeta, { + availability: { + error: new Error('Could not load connection availability'), + isFetching: false, + isReady: false, + refetch, + }, + allowAdmin: false, + }) + expect(container.textContent).toContain('Could not load connection availability') + expect(mocks.accessField).toHaveBeenLastCalledWith( + expect.objectContaining({ isAvailabilityReady: false, allowAdmin: false }) + ) + const retry = Array.from(container.querySelectorAll('button')).find( + (button) => button.textContent === 'Try again' + ) + expect(retry).toBeEnabled() + await act(async () => retry!.click()) + expect(refetch).toHaveBeenCalledOnce() + }) + it('shows the acting user’s managed connection for browsing member sources', async () => { mocks.credentials = [ { id: 'managed-1', name: 'My Confluence', provider: 'confluence', type: 'managed_oauth' }, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx index 5d1719e982f..6e2f1c9612a 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx @@ -56,6 +56,7 @@ export interface ConnectorSettingsFieldsProps { availability: { error: Error | null isFetching: boolean + isReady: boolean refetch: () => unknown } isSearchIndex: boolean @@ -225,6 +226,7 @@ export function ConnectorSettingsFields({ onChange={onAccessChange} canAdmin={canAdmin} lockAccessMode={isSearchIndex} + isAvailabilityReady={availability.isReady} allowMembers={allowMembers} allowAdmin={allowAdmin} allowWorkspace={allowWorkspace} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts index 495ede16381..2a172992b1e 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts @@ -414,6 +414,7 @@ export function useConnectorSettingsForm({ availability: { error: integrationAvailabilityError, isFetching: isIntegrationAvailabilityFetching, + isReady: isIntegrationAvailabilityReady, refetch: refetchIntegrationAvailability, }, isSearchIndex, diff --git a/apps/sim/components/integrations/slack-search-setup-wizard.tsx b/apps/sim/components/integrations/slack-search-setup-wizard.tsx index c584cb7d30d..9473b345fab 100644 --- a/apps/sim/components/integrations/slack-search-setup-wizard.tsx +++ b/apps/sim/components/integrations/slack-search-setup-wizard.tsx @@ -156,6 +156,11 @@ export function SlackSearchSetupWizard({ title='Client ID' value={clientId} onChange={setClientId} + placeholder={ + installationId + ? 'Leave blank to keep the saved value' + : 'Paste your Slack app’s client ID' + } required={!installationId} /> diff --git a/apps/sim/connectors/gmail/gmail.test.ts b/apps/sim/connectors/gmail/gmail.test.ts index 0623b164f61..346021bb07b 100644 --- a/apps/sim/connectors/gmail/gmail.test.ts +++ b/apps/sim/connectors/gmail/gmail.test.ts @@ -110,7 +110,7 @@ describe('gmail listDocuments with maxThreads 0 (unlimited, a per-member sync)', describe('Gmail listing checkpoints', () => { it('keeps a resumed query fixed when a relative date range crosses midnight', async () => { vi.useFakeTimers() - vi.setSystemTime(new Date(2026, 8, 1, 23, 59, 59)) + vi.setSystemTime(new Date('2026-09-01T23:59:59Z')) const urls = mockPages([ { threads: [{ id: 'thread-1', historyId: '10' }], nextPageToken: 'page-2' }, { threads: [{ id: 'thread-2', historyId: '20' }], nextPageToken: 'page-3' }, @@ -124,9 +124,9 @@ describe('Gmail listing checkpoints', () => { memberContext('alice') ) const initialQuery = new URL(urls[0]).searchParams.get('q') - expect(initialQuery).toContain('after:2026/08/25') + expect(initialQuery).toContain(`after:${Date.parse('2026-08-25T23:59:59Z') / 1000}`) - vi.setSystemTime(new Date(2026, 8, 2, 0, 0, 1)) + vi.setSystemTime(new Date('2026-09-02T00:00:01Z')) const resumed = await gmailConnector.listDocuments( 'token', sourceConfig, @@ -141,7 +141,9 @@ describe('Gmail listing checkpoints', () => { }) await gmailConnector.listDocuments('token', sourceConfig, undefined, memberContext('alice')) - expect(new URL(urls[2]).searchParams.get('q')).toContain('after:2026/08/26') + expect(new URL(urls[2]).searchParams.get('q')).toContain( + `after:${Date.parse('2026-08-26T00:00:01Z') / 1000}` + ) }) it('resumes with the saved label query rather than resolving a renamed label again', async () => { @@ -181,6 +183,19 @@ describe('Gmail listing checkpoints', () => { expect(new URL(urls[1]).searchParams.get('pageToken')).toBe('page-2') }) + it('replays an existing date-based checkpoint without changing its provider query', async () => { + const urls = mockPages([{ threads: [] }]) + const searchQuery = 'after:2026/08/25 -category:promotions -category:social' + await gmailConnector.listDocuments( + 'token', + { dateRange: '7d' }, + JSON.stringify({ pageToken: 'saved-page', searchQuery }), + memberContext('alice') + ) + expect(new URL(urls[0]).searchParams.get('q')).toBe(searchQuery) + expect(new URL(urls[0]).searchParams.get('pageToken')).toBe('saved-page') + }) + it('still accepts a legacy raw page token and upgrades the next checkpoint', async () => { const urls = mockPages([{ threads: [], nextPageToken: 'page-3' }]) const result = await gmailConnector.listDocuments('token', {}, 'page-2') @@ -1307,6 +1322,107 @@ describe('Gmail change feed', () => { }) }) + it('uses the same timezone-independent cutoff for full listings and history', async () => { + vi.setSystemTime(new Date('2026-09-10T12:00:00.987Z')) + const cutoff = new Date('2026-09-03T12:00:00Z').getTime() + const config = { dateRange: '7d', maxThreads: 0 } + const urls = mockPages([{ threads: [{ id: 'recent', historyId: '77' }] }]) + const listing = await gmailConnector.listDocuments( + 'token', + config, + undefined, + memberContext('member-a') + ) + expect(new URL(urls[0]).searchParams.get('q')).toContain(`after:${cutoff / 1000}`) + + mockFeed([historyPage(['earlier-that-day', 'recent'])], { + 'earlier-that-day': metadataThread('earlier-that-day', [ + { labelIds: ['INBOX'], internalDate: String(cutoff - 2 * 60 * 60 * 1000) }, + ]), + recent: metadataThread('recent', [ + { labelIds: ['INBOX'], internalDate: String(cutoff + 1000) }, + ]), + }) + const history = await gmailConnector.listChanges!( + 'token', + config, + '500', + memberContext('member-a') + ) + expect( + history.changes.filter(({ kind }) => kind === 'upsert').map(({ externalId }) => externalId) + ).toEqual(listing.documents.map(({ externalId }) => externalId)) + expect(history.changes).toContainEqual({ + kind: 'removed', + externalId: 'member:member-a:earlier-that-day', + }) + }) + + it('keeps a missing member label empty when moving from a full listing to history', async () => { + const config = { label: 'Engineering', maxThreads: 0 } + mockFetchWithRetry + .mockResolvedValueOnce(Response.json({ labels: [{ id: 'INBOX', name: 'INBOX' }] })) + .mockResolvedValueOnce(Response.json({ threads: [] })) + const listing = await gmailConnector.listDocuments( + 'token', + config, + undefined, + memberContext('member-a') + ) + expect(listing.documents).toEqual([]) + expect(listing.hasMore).toBe(false) + + mockFeed([historyPage(['unlabelled'])], { + unlabelled: metadataThread('unlabelled', [{ labelIds: ['INBOX'] }]), + }) + const page = await gmailConnector.listChanges!( + 'token', + config, + '500', + memberContext('member-a') + ) + expect(page.changes).toEqual([{ kind: 'removed', externalId: 'member:member-a:unlabelled' }]) + expect(JSON.parse(page.nextCursor)).toEqual({ historyId: '900' }) + expect(page.hasMore).toBe(false) + }) + + it('matches existing labels in an OR filter even when another label is absent', async () => { + mockFeed( + [historyPage(['match', 'unlabelled'])], + { + match: metadataThread('match', [{ labelIds: ['Label_7'] }]), + unlabelled: metadataThread('unlabelled', [{ labelIds: ['INBOX'] }]), + }, + [{ id: 'Label_7', name: 'Engineering' }] + ) + const page = await gmailConnector.listChanges!( + 'token', + { label: ['Engineering', 'Missing label'] }, + '500', + memberContext('member-a') + ) + expect(page.changes.map(({ kind, externalId }) => ({ kind, externalId }))).toEqual([ + { kind: 'upsert', externalId: 'member:member-a:match' }, + { kind: 'removed', externalId: 'member:member-a:unlabelled' }, + ]) + }) + + it.each([403, 503])( + 'does not treat a failed label lookup (%i) as an empty history scope', + async (status) => { + mockFetchWithRetry.mockResolvedValueOnce(new Response(null, { status })) + await expect( + gmailConnector.listChanges!( + 'token', + { label: 'Engineering' }, + '500', + memberContext('member-a') + ) + ).rejects.toThrow('cannot resolve the configured label filter') + expect(mockFetchWithRetry).toHaveBeenCalledOnce() + } + ) + it('keeps the start history id while paging and advances it once the feed drains', async () => { const requests = mockFeed( [ diff --git a/apps/sim/connectors/gmail/gmail.ts b/apps/sim/connectors/gmail/gmail.ts index 1f65f4354fe..c85f7df81c0 100644 --- a/apps/sim/connectors/gmail/gmail.ts +++ b/apps/sim/connectors/gmail/gmail.ts @@ -308,7 +308,7 @@ function buildSearchQuery( } const after = dateRangeStart(sourceConfig, new Date()) - if (after) parts.push(`after:${formatGmailDate(after)}`) + if (after) parts.push(`after:${after.getTime() / 1000}`) const excludePromotions = sourceConfig.excludePromotions !== 'false' if (excludePromotions) { @@ -350,21 +350,12 @@ function isBoundedDateRange(value: unknown): value is keyof typeof DATE_RANGE_DA return typeof value === 'string' && Object.hasOwn(DATE_RANGE_DAYS, value) } -/** The earliest message date the configured range admits, or undefined for all time. */ +/** Uses second precision so Gmail's epoch query and local history filtering share one cutoff. */ function dateRangeStart(sourceConfig: Record, now: Date): Date | undefined { const range = sourceConfig.dateRange - return isBoundedDateRange(range) ? daysAgo(now, DATE_RANGE_DAYS[range]) : undefined -} - -function daysAgo(now: Date, days: number): Date { - return new Date(now.getTime() - days * 24 * 60 * 60 * 1000) -} - -function formatGmailDate(date: Date): string { - const y = date.getFullYear() - const m = String(date.getMonth() + 1).padStart(2, '0') - const d = String(date.getDate()).padStart(2, '0') - return `${y}/${m}/${d}` + if (!isBoundedDateRange(range)) return undefined + const cutoffSeconds = Math.floor(now.getTime() / 1000) - DATE_RANGE_DAYS[range] * 24 * 60 * 60 + return new Date(cutoffSeconds * 1000) } /** @@ -766,8 +757,7 @@ function buildChangeScope( const labelIds = new Set() for (const value of configuredLabels) { const id = labelIndex.byId[value] ? value : labelIndex.idByLowerName[value.toLowerCase()] - if (!id) throw new Error(`Gmail label "${value}" does not exist in this mailbox`) - labelIds.add(id) + if (id) labelIds.add(id) } scope.labelIds = labelIds } diff --git a/apps/sim/connectors/jira/jira.test.ts b/apps/sim/connectors/jira/jira.test.ts index 695239763c3..a2c08e7a0ff 100644 --- a/apps/sim/connectors/jira/jira.test.ts +++ b/apps/sim/connectors/jira/jira.test.ts @@ -74,7 +74,7 @@ describe('Jira Search member documents', () => { expect( jiraConnectorMeta.configFields.filter((field) => field.canonicalParamId === 'projectKey') ).toEqual([ - expect.objectContaining({ selectorKey: 'jira.projects', mode: 'basic', required: true }), + expect.objectContaining({ selectorKey: 'jira.projectKeys', mode: 'basic', required: true }), expect.objectContaining({ id: 'projectKey', mode: 'advanced', required: true }), ]) }) diff --git a/apps/sim/connectors/jira/meta.ts b/apps/sim/connectors/jira/meta.ts index 6cf85dc1cb8..43f05f39fa3 100644 --- a/apps/sim/connectors/jira/meta.ts +++ b/apps/sim/connectors/jira/meta.ts @@ -25,7 +25,7 @@ export const jiraConnectorMeta: ConnectorMeta = { id: 'projectSelector', title: 'Projects', type: 'selector', - selectorKey: 'jira.projects', + selectorKey: 'jira.projectKeys', canonicalParamId: 'projectKey', mode: 'basic', multi: true, diff --git a/apps/sim/hooks/queries/kb/connectors-cache.test.tsx b/apps/sim/hooks/queries/kb/connectors-cache.test.tsx index 7bce795ad62..89ee868601c 100644 --- a/apps/sim/hooks/queries/kb/connectors-cache.test.tsx +++ b/apps/sim/hooks/queries/kb/connectors-cache.test.tsx @@ -13,6 +13,16 @@ const mocks = vi.hoisted(() => ({ requestJson: vi.fn() })) vi.mock('@/lib/api/client/request', () => ({ requestJson: mocks.requestJson })) import { + type ConnectorDetailData, + getKnowledgeConnectorContract, + listKnowledgeConnectorsContract, + triggerKnowledgeConnectorSyncContract, +} from '@/lib/api/contracts/knowledge/connectors' +import { + CONNECTOR_SYNC_POLL_INTERVAL_MS, + connectorKeys, + useConnectorDetail, + useConnectorList, useConnectSimSearchConnector, useCreateConnector, useDeleteConnector, @@ -20,6 +30,7 @@ import { usePrepareSearchSource, useRestoreConnectorDocument, useStartConnectorMemberEnrollment, + useTriggerSync, useUpdateConnector, useUpdateConnectorAccess, } from '@/hooks/queries/kb/connectors' @@ -123,6 +134,117 @@ afterEach(() => { for (const root of mountedRoots.splice(0)) root.unmount() }) for (const queryClient of queryClients.splice(0)) queryClient.clear() + vi.useRealTimers() +}) + +describe('manual sync history reconciliation', () => { + it.each([false, true])( + 'resumes polling after an early idle response with connector list mounted=%s', + async (showList) => { + vi.useFakeTimers() + const client = createQueryClient() + const detailKey = connectorKeys.detail(KNOWLEDGE_BASE_ID, CONNECTOR_ID) + let serverDetail: ConnectorDetailData = { + id: CONNECTOR_ID, + knowledgeBaseId: KNOWLEDGE_BASE_ID, + connectorType: 'google_drive', + credentialId: 'credential-1', + sourceConfig: {}, + syncMode: 'full', + syncIntervalMinutes: 60, + status: 'active', + lastSyncAt: null, + lastSyncError: null, + lastSyncDocCount: 0, + nextSyncAt: null, + consecutiveFailures: 0, + accessMode: 'admin', + viewerMembership: null, + credentialGroupId: null, + credentialGroupOptionId: null, + memberSyncStatus: 'idle', + lastMemberSyncAt: null, + nextMemberSyncAt: null, + lastMemberSyncError: null, + memberSyncConsecutiveFailures: 0, + accessRewritePending: false, + createdAt: '2026-09-09T00:00:00Z', + updatedAt: '2026-09-09T00:00:00Z', + syncLogs: [], + memberSyncLogs: [], + members: { active: 0, suspended: 0, stale: 0 }, + } + client.setQueryData(detailKey, serverDetail) + client.setQueryData(connectorKeys.lists(KNOWLEDGE_BASE_ID), [serverDetail]) + const trigger = Promise.withResolvers() + mocks.requestJson.mockImplementation(async (contract) => { + if (contract === triggerKnowledgeConnectorSyncContract) return trigger.promise + if (contract === getKnowledgeConnectorContract) return { data: serverDetail } + if (contract === listKnowledgeConnectorsContract) return { data: [serverDetail] } + throw new Error('Unexpected request') + }) + const current = renderMutation(client, () => ({ + detail: useConnectorDetail(KNOWLEDGE_BASE_ID, CONNECTOR_ID), + list: useConnectorList(showList ? KNOWLEDGE_BASE_ID : undefined), + sync: useTriggerSync(), + })) + let mutation: Promise | undefined + await act(async () => { + mutation = current().sync.mutateAsync({ + knowledgeBaseId: KNOWLEDGE_BASE_ID, + connectorId: CONNECTOR_ID, + }) + await vi.advanceTimersByTimeAsync(0) + }) + expect(client.getQueryData(detailKey)?.status).toBe('pending') + + await act(async () => { + await vi.advanceTimersByTimeAsync(CONNECTOR_SYNC_POLL_INTERVAL_MS + 1) + }) + expect(current().detail.data?.status).toBe('active') + expect(current().sync.isPending).toBe(true) + serverDetail = { ...serverDetail, status: 'pending' } + await act(async () => { + trigger.resolve({ success: true }) + await mutation + await vi.advanceTimersByTimeAsync(1) + }) + expect(current().detail.data?.status).toBe('pending') + if (showList) expect(current().list.data?.[0].status).toBe('pending') + + serverDetail = { + ...serverDetail, + status: 'active', + syncLogs: [ + { + id: 'new-run', + connectorId: CONNECTOR_ID, + status: 'completed', + startedAt: '2026-09-10T00:00:00Z', + completedAt: '2026-09-10T00:01:00Z', + docsAdded: 0, + docsUpdated: 0, + docsDeleted: 0, + docsUnchanged: 3, + docsSkipped: 0, + docsFailed: 0, + errorMessage: null, + }, + ], + } + await act(async () => { + await vi.advanceTimersByTimeAsync(CONNECTOR_SYNC_POLL_INTERVAL_MS + 1) + }) + expect(current().detail.data?.syncLogs.map((log) => log.id)).toEqual(['new-run']) + expect(current().detail.data?.status).toBe('active') + const requestsAtCompletion = mocks.requestJson.mock.calls.length + await act(async () => { + await vi.advanceTimersByTimeAsync(CONNECTOR_SYNC_POLL_INTERVAL_MS * 2) + }) + expect(mocks.requestJson).toHaveBeenCalledTimes(requestsAtCompletion) + expectInvalidated(client, UNRELATED_KEY, false) + } + ) }) describe('connector account cache reconciliation', () => { diff --git a/apps/sim/hooks/queries/kb/connectors.test.ts b/apps/sim/hooks/queries/kb/connectors.test.ts index dac82a534f6..457a1c648fe 100644 --- a/apps/sim/hooks/queries/kb/connectors.test.ts +++ b/apps/sim/hooks/queries/kb/connectors.test.ts @@ -211,7 +211,11 @@ describe('useTriggerSync optimistic state', () => { function capturedMutationOptions() { return mocks.useMutation.mock.calls.at(-1)?.[0] as { onMutate: (vars: { knowledgeBaseId: string; connectorId: string }) => Promise - onSettled: () => Promise + onSettled: ( + data: undefined, + error: Error | null, + vars: { knowledgeBaseId: string; connectorId: string } + ) => Promise onSuccess: (data: undefined, vars: { knowledgeBaseId: string; connectorId: string }) => void onError: ( error: unknown, @@ -297,7 +301,10 @@ describe('useTriggerSync optimistic state', () => { it('reconciles server source summaries after either sync outcome', async () => { useTriggerSync() - await capturedMutationOptions().onSettled() + await capturedMutationOptions().onSettled(undefined, null, { + knowledgeBaseId: KB_ID, + connectorId: 'connector-1', + }) expect(mocks.invalidateQueries).toHaveBeenCalledWith({ queryKey: searchSourceKeys.lists() }) }) diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index 675d6e3e4f1..8c524b253fb 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -807,7 +807,18 @@ export function useTriggerSync() { queryKey: connectorKeys.progresses(knowledgeBaseId, connectorId), }) }, - onSettled: () => queryClient.invalidateQueries({ queryKey: searchSourceKeys.lists() }), + /** An early poll can read idle before dispatch marks pending; reconcile after the request settles. */ + onSettled: (_data, error, { knowledgeBaseId, connectorId }) => + Promise.all([ + queryClient.invalidateQueries({ queryKey: searchSourceKeys.lists() }), + queryClient.invalidateQueries({ + queryKey: connectorKeys.detail(knowledgeBaseId, connectorId), + exact: true, + }), + ...(!error + ? [queryClient.invalidateQueries({ queryKey: connectorKeys.lists(knowledgeBaseId) })] + : []), + ]), }) } diff --git a/apps/sim/hooks/queries/utils/reset-organization-search-access.test.ts b/apps/sim/hooks/queries/utils/reset-organization-search-access.test.ts index 8e6a96e4a35..7040c040f72 100644 --- a/apps/sim/hooks/queries/utils/reset-organization-search-access.test.ts +++ b/apps/sim/hooks/queries/utils/reset-organization-search-access.test.ts @@ -1,10 +1,63 @@ /** @vitest-environment node */ -import { QueryClient } from '@tanstack/react-query' +import { QueryClient, QueryObserver } from '@tanstack/react-query' import { expect, it, vi } from 'vitest' import type { WorkspaceKnowledgeSearchResult } from '@/lib/api/contracts/knowledge/search' import { resourceScopeKey } from '@/lib/core/resource-scope' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' import { resetOrganizationSearchAccess } from '@/hooks/queries/utils/reset-organization-search-access' +import { searchSourceKeys } from '@/hooks/queries/utils/search-source-keys' + +it.each([true, false])( + 'keeps administrative rows visible while revalidating access, refresh success=%s', + async (success) => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const scope = { kind: 'organization', organizationId: 'org-1' } as const + const adminKey = searchSourceKeys.organizationOverview(scope.organizationId) + const otherKey = searchSourceKeys.organizationOverview('org-2') + const viewerKeys = [ + searchSourceKeys.list(scope), + searchSourceKeys.overview(scope), + searchSourceKeys.pages(scope, { search: '', mine: false }), + ] + const before = { providers: [{ connectorType: 'gmail', approved: true }] } + const after = { providers: [{ connectorType: 'gmail', approved: false }] } + const response = Promise.withResolvers() + const fetchOverview = vi.fn(() => response.promise) + client.setQueryData(adminKey, before) + client.setQueryData(otherKey, before) + for (const key of viewerKeys) client.setQueryData(key, { privateContent: 'previous access' }) + const observer = new QueryObserver(client, { + queryKey: adminKey, + queryFn: fetchOverview, + staleTime: Number.POSITIVE_INFINITY, + }) + const observed = vi.fn() + const unsubscribe = observer.subscribe(observed) + try { + const refreshing = resetOrganizationSearchAccess(client, scope.organizationId) + expect(fetchOverview).toHaveBeenCalledOnce() + expect(observer.getCurrentResult()).toMatchObject({ data: before, isPending: false }) + for (const key of viewerKeys) expect(client.getQueryData(key)).toBeUndefined() + expect(client.getQueryState(otherKey)?.isInvalidated).toBe(false) + + if (success) response.resolve(after) + else response.reject(new Error('Could not refresh sources')) + await refreshing + + expect(observer.getCurrentResult()).toMatchObject({ + data: success ? after : before, + isError: !success, + isFetching: false, + }) + expect(observed.mock.calls.every(([result]) => result.data && !result.isPending)).toBe(true) + expect(client.getQueryData(otherKey)).toEqual(before) + } finally { + response.resolve(after) + unsubscribe() + client.clear() + } + } +) it.each([ { name: 'document', key: knowledgeKeys.document('kb-direct', 'document-direct') }, diff --git a/apps/sim/hooks/queries/utils/reset-organization-search-access.ts b/apps/sim/hooks/queries/utils/reset-organization-search-access.ts index 00a2c91cb97..53f7b3a5cfb 100644 --- a/apps/sim/hooks/queries/utils/reset-organization-search-access.ts +++ b/apps/sim/hooks/queries/utils/reset-organization-search-access.ts @@ -1,4 +1,4 @@ -import type { QueryClient } from '@tanstack/react-query' +import { matchQuery, type QueryClient } from '@tanstack/react-query' import { resourceScopeKey } from '@/lib/core/resource-scope' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' import { searchSourceKeys } from '@/hooks/queries/utils/search-source-keys' @@ -9,12 +9,20 @@ export async function resetOrganizationSearchAccess( organizationId: string ) { const scope = { kind: 'organization', organizationId } as const + const adminOverview = { + queryKey: searchSourceKeys.organizationOverview(organizationId), + exact: true, + } await Promise.all([ queryClient.resetQueries({ queryKey: [...knowledgeKeys.searches(), resourceScopeKey(scope)], }), /** Document keys carry no resource scope and may exist without source or result caches. */ queryClient.resetQueries({ queryKey: knowledgeKeys.details() }), - queryClient.resetQueries({ queryKey: searchSourceKeys.list(scope) }), + queryClient.resetQueries({ + queryKey: searchSourceKeys.list(scope), + predicate: (query) => !matchQuery(adminOverview, query), + }), + queryClient.invalidateQueries(adminOverview), ]) } diff --git a/apps/sim/hooks/use-member-enrollment.test.tsx b/apps/sim/hooks/use-member-enrollment.test.tsx index 6efb3ce8138..d279c8a818c 100644 --- a/apps/sim/hooks/use-member-enrollment.test.tsx +++ b/apps/sim/hooks/use-member-enrollment.test.tsx @@ -160,6 +160,252 @@ describe('useMemberEnrollment', () => { expect(mocks.connectionError).not.toHaveBeenCalled() }) + it.each(['existing', 'new'] as const)( + 'does not expire a superseded %s source authorization after its retry connects', + (source) => { + mount(new Set(), true, mocks.connectionError) + const mutation = source === 'existing' ? mocks.enrollmentMutate : mocks.sourceConnectionMutate + for (let index = 0; index < 2; index += 1) { + act(() => { + if (source === 'existing') enrollment().connect('kb-1', 'connector-1') + else enrollment().connectSource('workspace-1', 'jira') + }) + act(() => + mutation.mock.calls[index][1].onSuccess({ + url: `https://provider.test/attempt-${index}`, + connectorId: 'connector-1', + }) + ) + } + act(() => mocks.channels[1].onmessage?.(new MessageEvent('message', { data: 'connected' }))) + act(() => vi.advanceTimersByTime(10 * 60_000)) + act(() => + mocks.channels[0].onmessage?.(new MessageEvent('message', { data: 'account_mismatch' })) + ) + expect(mocks.connectionError).not.toHaveBeenCalled() + expect(enrollment().error).toBeNull() + expect(enrollment().isAwaiting('connector-1')).toBe(false) + expect(mocks.channels[0].close).toHaveBeenCalledOnce() + expect(mocks.channels[1].close).toHaveBeenCalledOnce() + } + ) + + it('reopening one connector leaves a different connector authorization active', () => { + mount(new Set(), true, mocks.connectionError) + for (const connectorId of ['connector-1', 'connector-2', 'connector-1']) { + act(() => enrollment().connect('kb-1', connectorId)) + const index = mocks.enrollmentMutate.mock.calls.length - 1 + act(() => + mocks.enrollmentMutate.mock.calls[index][1].onSuccess({ + url: `https://provider.test/attempt-${index}`, + }) + ) + } + expect(mocks.channels[0].close).toHaveBeenCalledOnce() + expect(mocks.channels[1].close).not.toHaveBeenCalled() + act(() => mocks.channels[2].onmessage?.(new MessageEvent('message', { data: 'connected' }))) + expect(enrollment().isAwaiting('connector-2')).toBe(true) + act(() => mocks.channels[1].onmessage?.(new MessageEvent('message', { data: 'denied' }))) + expect(mocks.connectionError).toHaveBeenCalledExactlyOnceWith( + 'Authorization was canceled. Try connecting your account again.' + ) + }) + + it.each([ + ['existing', 'account_mismatch'], + ['existing', 'denied'], + ['existing', 'expired'], + ['new', 'account_mismatch'], + ['new', 'denied'], + ['new', 'expired'], + ] as const)( + 'ignores the previous %s source’s %s while its retry request is pending', + (source, failure) => { + mount(new Set(), true, mocks.connectionError) + const mutation = source === 'existing' ? mocks.enrollmentMutate : mocks.sourceConnectionMutate + const connect = () => { + if (source === 'existing') enrollment().connect('kb-1', 'connector-1') + else enrollment().connectSource('workspace-1', 'jira', { projectKey: 'ENG' }) + } + act(connect) + act(() => + mutation.mock.calls[0][1].onSuccess({ + url: 'https://provider.test/previous', + connectorId: 'connector-1', + }) + ) + act(() => vi.advanceTimersByTime(9 * 60_000)) + act(connect) + if (failure !== 'expired') { + act(() => mocks.channels[0].onmessage?.(new MessageEvent('message', { data: failure }))) + } + act(() => vi.advanceTimersByTime(60_000)) + expect(mocks.connectionError).not.toHaveBeenCalled() + expect(enrollment().error).toBeNull() + act(() => + mutation.mock.calls[1][1].onSuccess({ + url: 'https://provider.test/retry', + connectorId: 'connector-1', + }) + ) + expect(enrollment().isAwaiting('connector-1')).toBe(true) + expect(mocks.channels[1].close).not.toHaveBeenCalled() + } + ) + + it.each([ + ['existing', 'success'], + ['existing', 'failure'], + ['new', 'success'], + ['new', 'failure'], + ] as const)('ignores a superseded %s source request’s late %s', (source, outcome) => { + mount(new Set(), true, mocks.connectionError) + const mutation = source === 'existing' ? mocks.enrollmentMutate : mocks.sourceConnectionMutate + const retryTab = { location: { href: '' }, closed: false, close: vi.fn() } + vi.mocked(window.open) + .mockReturnValueOnce(enrollmentTab as unknown as Window) + .mockReturnValueOnce(retryTab as unknown as Window) + for (let index = 0; index < 2; index += 1) { + act(() => { + if (source === 'existing') enrollment().connect('kb-1', 'connector-1') + else enrollment().connectSource('workspace-1', 'jira', { projectKey: 'ENG' }) + }) + } + act(() => + mutation.mock.calls[1][1].onSuccess({ + url: 'https://provider.test/retry', + connectorId: 'connector-1', + }) + ) + act(() => { + if (outcome === 'failure') { + mutation.mock.calls[0][1].onError(new Error('Previous request failed')) + } else { + mutation.mock.calls[0][1].onSuccess({ + url: 'https://provider.test/previous', + connectorId: 'connector-1', + }) + } + }) + expect(enrollmentTab.location.href).toBe('') + expect(retryTab.location.href).toBe('https://provider.test/retry') + expect(retryTab.close).not.toHaveBeenCalled() + expect(enrollment().isAwaiting('connector-1')).toBe(true) + expect(mocks.channels[1].close).not.toHaveBeenCalled() + expect(mocks.connectionError).not.toHaveBeenCalled() + expect(enrollment().error).toBeNull() + }) + + it('retires a first-source authorization when retrying its resolved connector', () => { + mount(new Set(), true, mocks.connectionError) + act(() => enrollment().connectSource('workspace-1', 'jira', { projectKey: 'ENG' })) + act(() => + mocks.sourceConnectionMutate.mock.calls[0][1].onSuccess({ + url: 'https://provider.test/previous', + connectorId: 'connector-1', + }) + ) + act(() => enrollment().connect('kb-1', 'connector-1')) + act(() => mocks.channels[0].onmessage?.(new MessageEvent('message', { data: 'denied' }))) + expect(mocks.channels[0].close).toHaveBeenCalledOnce() + expect(mocks.connectionError).not.toHaveBeenCalled() + }) + + it('does not let a delayed first-source response replace its newer connector authorization', () => { + mount(new Set(), true, mocks.connectionError) + act(() => enrollment().connectSource('workspace-1', 'jira', { projectKey: 'ENG' })) + act(() => enrollment().connect('kb-1', 'connector-1')) + act(() => + mocks.enrollmentMutate.mock.calls[0][1].onSuccess({ url: 'https://provider.test/retry' }) + ) + act(() => + mocks.sourceConnectionMutate.mock.calls[0][1].onSuccess({ + url: 'https://provider.test/previous', + connectorId: 'connector-1', + }) + ) + expect(enrollmentTab.location.href).toBe('https://provider.test/retry') + expect(mocks.channels[1].close).not.toHaveBeenCalled() + expect(enrollment().isAwaiting('connector-1')).toBe(true) + }) + + it('ignores first-source success while a newer request for its connector is still pending', () => { + mount(new Set(), true, mocks.connectionError) + const retryTab = { location: { href: '' }, closed: false, close: vi.fn() } + vi.mocked(window.open) + .mockReturnValueOnce(enrollmentTab as unknown as Window) + .mockReturnValueOnce(retryTab as unknown as Window) + act(() => enrollment().connectSource('workspace-1', 'jira', { projectKey: 'ENG' })) + act(() => enrollment().connect('kb-1', 'connector-1')) + act(() => + mocks.sourceConnectionMutate.mock.calls[0][1].onSuccess({ + url: 'https://provider.test/previous', + connectorId: 'connector-1', + }) + ) + expect(enrollmentTab.location.href).toBe('') + expect(enrollment().isAwaiting('connector-1')).toBe(false) + act(() => mocks.channels[0].onmessage?.(new MessageEvent('message', { data: 'denied' }))) + expect(mocks.connectionError).not.toHaveBeenCalled() + act(() => + mocks.enrollmentMutate.mock.calls[0][1].onSuccess({ url: 'https://provider.test/retry' }) + ) + expect(retryTab.location.href).toBe('https://provider.test/retry') + expect(enrollment().isAwaiting('connector-1')).toBe(true) + expect(mocks.channels[1].close).not.toHaveBeenCalled() + }) + + it('retires a pending connector request when a newer first-source request resolves to it', () => { + mount(new Set(), true, mocks.connectionError) + act(() => enrollment().connect('kb-1', 'connector-1')) + act(() => enrollment().connectSource('workspace-1', 'jira', { projectKey: 'ENG' })) + act(() => + mocks.sourceConnectionMutate.mock.calls[0][1].onSuccess({ + url: 'https://provider.test/retry', + connectorId: 'connector-1', + }) + ) + act(() => mocks.enrollmentMutate.mock.calls[0][1].onError(new Error('Previous request failed'))) + expect(mocks.connectionError).not.toHaveBeenCalled() + expect(enrollment().isAwaiting('connector-1')).toBe(true) + expect(mocks.channels[0].close).toHaveBeenCalledOnce() + expect(mocks.channels[1].close).not.toHaveBeenCalled() + }) + + it('keeps the previous authorization active when the retry popup is blocked', () => { + mount(new Set(), true, mocks.connectionError) + act(() => enrollment().connect('kb-1', 'connector-1')) + act(() => + mocks.enrollmentMutate.mock.calls[0][1].onSuccess({ url: 'https://provider.test/previous' }) + ) + vi.mocked(window.open).mockReturnValueOnce(null) + act(() => enrollment().connect('kb-1', 'connector-1')) + expect(mocks.channels[0].close).not.toHaveBeenCalled() + expect(enrollment().isAwaiting('connector-1')).toBe(true) + expect(mocks.enrollmentMutate).toHaveBeenCalledOnce() + }) + + it('keeps pending source requests with different scopes or configurations independent', () => { + mount(new Set(), true, mocks.connectionError) + for (const [owner, projectKey] of [ + ['workspace-1', 'ENG'], + ['workspace-1', 'SUPPORT'], + ['workspace-2', 'ENG'], + ]) { + act(() => enrollment().connectSource(owner, 'jira', { projectKey })) + } + for (let index = 0; index < 3; index += 1) { + act(() => + mocks.sourceConnectionMutate.mock.calls[index][1].onSuccess({ + url: `https://provider.test/attempt-${index}`, + connectorId: `connector-${index}`, + }) + ) + expect(enrollment().isAwaiting(`connector-${index}`)).toBe(true) + expect(mocks.channels[index].close).not.toHaveBeenCalled() + } + }) + it('reports a blocked popup once without starting a connection', () => { mount(new Set(), true, mocks.connectionError) vi.mocked(window.open).mockReturnValueOnce(null) diff --git a/apps/sim/hooks/use-member-enrollment.ts b/apps/sim/hooks/use-member-enrollment.ts index 75826480eb1..07eb32ba0dd 100644 --- a/apps/sim/hooks/use-member-enrollment.ts +++ b/apps/sim/hooks/use-member-enrollment.ts @@ -4,7 +4,11 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import { type QueryKey, useQueryClient } from '@tanstack/react-query' -import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' +import { + type ResourceScope, + resourceScopeFields, + resourceScopeKey, +} from '@/lib/core/resource-scope' import { CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES, credentialGroupOAuthCompletionChannel, @@ -131,7 +135,15 @@ export function useMemberEnrollment({ }: UseMemberEnrollmentProps) { const connectedRef = useRef(connectedConnectorIds) const oauthPopups = useRef( - new Map }>() + new Map< + string, + { + channel: BroadcastChannel + timer: ReturnType + attemptKey: string + connectorId?: string + } + >() ) const queryClient = useQueryClient() const enrollment = useStartConnectorMemberEnrollment() @@ -149,9 +161,9 @@ export function useMemberEnrollment({ void queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) }, [membershipQueryKeys, queryClient]) - const finishOAuth = (completionId: string, error: string | null) => { + const clearOAuth = (completionId: string) => { const popup = oauthPopups.current.get(completionId) - if (!popup) return + if (!popup) return false clearTimeout(popup.timer) popup.channel.close() oauthPopups.current.delete(completionId) @@ -159,6 +171,11 @@ export function useMemberEnrollment({ (current) => new Map([...current].filter(([, entry]) => entry.oauthCompletionId !== completionId)) ) + return true + } + + const finishOAuth = (completionId: string, error: string | null) => { + if (!clearOAuth(completionId)) return setOAuthError(error) if (error) onConnectionError?.(error) refreshMemberships() @@ -209,10 +226,11 @@ export function useMemberEnrollment({ /** Opens the tab inside the click, then sends it wherever `start` mints. */ const openEnrollment = ( + attemptKey: string, start: (handlers: { oauthCompletionId?: string onSuccess: (url: string, connectorId: string, connectorType?: string) => boolean - onError: () => void + onError: () => boolean }) => void ) => { const tab = window.open('about:blank', '_blank') @@ -226,6 +244,14 @@ export function useMemberEnrollment({ setOAuthError(null) const oauthCompletionId = directOAuth ? generateId() : undefined if (oauthCompletionId) { + for (const [previousId, previous] of oauthPopups.current) { + if ( + previous.attemptKey === attemptKey || + (previous.connectorId && `connector:${previous.connectorId}` === attemptKey) + ) { + clearOAuth(previousId) + } + } const channel = new BroadcastChannel(credentialGroupOAuthCompletionChannel(oauthCompletionId)) channel.onmessage = ({ data }: MessageEvent) => { if (data === 'connected') finishOAuth(oauthCompletionId, null) @@ -235,7 +261,7 @@ export function useMemberEnrollment({ const timer = setTimeout(() => { finishOAuth(oauthCompletionId, CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES.expired) }, AWAITING_CONNECTION_TIMEOUT_MS) - oauthPopups.current.set(oauthCompletionId, { channel, timer }) + oauthPopups.current.set(oauthCompletionId, { channel, timer, attemptKey }) } start({ ...(oauthCompletionId ? { oauthCompletionId } : {}), @@ -245,6 +271,31 @@ export function useMemberEnrollment({ finishOAuth(oauthCompletionId, CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES.denied) return false } + if (oauthCompletionId) { + const popup = oauthPopups.current.get(oauthCompletionId)! + const connectorAttemptKey = `connector:${connectorId}` + const latestAttempt = [...oauthPopups.current] + .reverse() + .find( + ([id, entry]) => + id === oauthCompletionId || + entry.connectorId === connectorId || + entry.attemptKey === connectorAttemptKey + ) + if (latestAttempt?.[0] !== oauthCompletionId) { + clearOAuth(oauthCompletionId) + return false + } + for (const [previousId, previous] of oauthPopups.current) { + if ( + previousId === oauthCompletionId || + (previous.connectorId !== connectorId && previous.attemptKey !== connectorAttemptKey) + ) + continue + clearOAuth(previousId) + } + popup.connectorId = connectorId + } tab.location.href = url setAwaitingSince((current) => new Map(current).set(connectorId, { @@ -257,20 +308,22 @@ export function useMemberEnrollment({ return true }, onError: () => { + const active = !oauthCompletionId || oauthPopups.current.has(oauthCompletionId) if (oauthCompletionId) finishOAuth(oauthCompletionId, null) tab.close() + return active }, }) } const connect = (knowledgeBaseId: string, connectorId: string) => - openEnrollment(({ onSuccess, onError, oauthCompletionId }) => { + openEnrollment(`connector:${connectorId}`, ({ onSuccess, onError, oauthCompletionId }) => { enrollment.mutate( { knowledgeBaseId, connectorId, ...(oauthCompletionId ? { oauthCompletionId } : {}) }, { onSuccess: ({ url }) => onSuccess(url, connectorId), onError: (err) => { - onError() + if (!onError()) return onConnectionError?.(err.message) logger.error('Failed to start member enrollment', { error: err.message }) }, @@ -287,27 +340,36 @@ export function useMemberEnrollment({ owner: string | ResourceScope, connectorType: string, sourceConfig?: Record - ) => - openEnrollment(({ onSuccess, onError, oauthCompletionId }) => { - sourceConnection.mutate( - { - ...(typeof owner === 'string' ? { workspaceId: owner } : resourceScopeFields(owner)), - connectorType, - sourceConfig, - ...(oauthCompletionId ? { oauthCompletionId } : {}), - }, - { - onSuccess: ({ url, connectorId }) => { - if (onSuccess(url, connectorId, connectorType)) setSetupConnector(null) - }, - onError: (err) => { - onError() - onConnectionError?.(err.message) - logger.error('Failed to connect a Sim Search source', { error: err.message }) + ) => { + const scope = + typeof owner === 'string' ? { kind: 'workspace' as const, workspaceId: owner } : owner + const configKey = JSON.stringify( + Object.entries(sourceConfig ?? {}).sort(([left], [right]) => left.localeCompare(right)) + ) + openEnrollment( + `source:${resourceScopeKey(scope)}:${connectorType}:${configKey}`, + ({ onSuccess, onError, oauthCompletionId }) => { + sourceConnection.mutate( + { + ...resourceScopeFields(scope), + connectorType, + sourceConfig, + ...(oauthCompletionId ? { oauthCompletionId } : {}), }, - } - ) - }) + { + onSuccess: ({ url, connectorId }) => { + if (onSuccess(url, connectorId, connectorType)) setSetupConnector(null) + }, + onError: (err) => { + if (!onError()) return + onConnectionError?.(err.message) + logger.error('Failed to connect a Sim Search source', { error: err.message }) + }, + } + ) + } + ) + } const [setupConnector, setSetupConnector] = useState(null) diff --git a/apps/sim/lib/billing/core/ingestion-usage-gate.test.ts b/apps/sim/lib/billing/core/ingestion-usage-gate.test.ts new file mode 100644 index 00000000000..dbecf11abf0 --- /dev/null +++ b/apps/sim/lib/billing/core/ingestion-usage-gate.test.ts @@ -0,0 +1,98 @@ +/** + * @vitest-environment node + */ +import { sleep } from '@sim/utils/helpers' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCheck } = vi.hoisted(() => ({ mockCheck: vi.fn() })) + +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + checkAttributedUsageLimits: mockCheck, +})) + +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { + checkIngestionUsageLimits, + INGESTION_USAGE_GATE_TTL_MS, + resetIngestionUsageGateCache, +} from '@/lib/billing/core/ingestion-usage-gate' + +const ATTRIBUTION: BillingAttributionSnapshot = { + actorUserId: 'member-1', + workspaceId: null, + organizationId: 'org-1', + billedAccountUserId: 'owner-1', + billingEntity: { type: 'organization', id: 'org-1' }, + billingPeriod: { + start: '2026-09-01T00:00:00.000Z', + end: '2026-10-01T00:00:00.000Z', + source: 'stripe', + }, + payerSubscription: null, +} + +describe('checkIngestionUsageLimits', () => { + beforeEach(() => { + resetIngestionUsageGateCache() + mockCheck.mockReset().mockResolvedValue({ isExceeded: false }) + }) + afterEach(() => vi.restoreAllMocks()) + + it('reads the ledger once per payer, period and actor within the TTL', async () => { + await checkIngestionUsageLimits(ATTRIBUTION) + await checkIngestionUsageLimits(ATTRIBUTION) + await checkIngestionUsageLimits({ ...ATTRIBUTION, workspaceId: 'workspace-9' }) + expect(mockCheck).toHaveBeenCalledTimes(1) + }) + + it('collapses concurrent misses onto one ledger read', async () => { + let resolve!: (value: { isExceeded: boolean }) => void + mockCheck.mockReturnValueOnce(new Promise((r) => (resolve = r))) + const pending = Promise.all([ + checkIngestionUsageLimits(ATTRIBUTION), + checkIngestionUsageLimits(ATTRIBUTION), + checkIngestionUsageLimits(ATTRIBUTION), + ]) + await sleep(0) + resolve({ isExceeded: false }) + const results = await pending + expect(results.every((result) => result.isExceeded === false)).toBe(true) + expect(mockCheck).toHaveBeenCalledTimes(1) + }) + + it('keeps a refusal for the same bounded window, then reads fresh', async () => { + mockCheck.mockResolvedValueOnce({ isExceeded: true, scope: 'payer', message: 'over' }) + expect((await checkIngestionUsageLimits(ATTRIBUTION)).isExceeded).toBe(true) + expect((await checkIngestionUsageLimits(ATTRIBUTION)).isExceeded).toBe(true) + expect(mockCheck).toHaveBeenCalledTimes(1) + + /** `lru-cache` reads `performance.now()` and debounces it behind a real 1 ms timer. */ + const start = performance.now() + vi.spyOn(performance, 'now').mockReturnValue(start + INGESTION_USAGE_GATE_TTL_MS + 1) + await sleep(5) + expect((await checkIngestionUsageLimits(ATTRIBUTION)).isExceeded).toBe(false) + expect(mockCheck).toHaveBeenCalledTimes(2) + }) + + it('separates answers by actor, period and payer', async () => { + await checkIngestionUsageLimits(ATTRIBUTION) + await checkIngestionUsageLimits({ ...ATTRIBUTION, actorUserId: 'member-2' }) + await checkIngestionUsageLimits({ + ...ATTRIBUTION, + billingPeriod: { ...ATTRIBUTION.billingPeriod, start: '2026-10-01T00:00:00.000Z' }, + }) + await checkIngestionUsageLimits({ + ...ATTRIBUTION, + billedAccountUserId: 'owner-2', + billingEntity: { type: 'user', id: 'owner-2' }, + }) + expect(mockCheck).toHaveBeenCalledTimes(4) + }) + + it('does not cache a failed read', async () => { + mockCheck.mockRejectedValueOnce(new Error('ledger unavailable')) + await expect(checkIngestionUsageLimits(ATTRIBUTION)).rejects.toThrow('ledger unavailable') + await checkIngestionUsageLimits(ATTRIBUTION) + expect(mockCheck).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/sim/lib/billing/core/ingestion-usage-gate.ts b/apps/sim/lib/billing/core/ingestion-usage-gate.ts new file mode 100644 index 00000000000..2f6f1c5f5f8 --- /dev/null +++ b/apps/sim/lib/billing/core/ingestion-usage-gate.ts @@ -0,0 +1,69 @@ +import { LRUCache } from 'lru-cache' +import { + type AttributedUsageLimitsResult, + type BillingAttributionSnapshot, + checkAttributedUsageLimits, +} from '@/lib/billing/core/billing-attribution' +import { coalesceLocally } from '@/lib/concurrency/singleflight' + +/** + * How long a usage-gate answer stays usable on the ingestion path. + * + * The gate sums the payer's usage ledger for the billing period, which grows + * with every indexed document, so a bulk sync that re-checks per document + * reads the whole period's ledger tens of thousands of times. Staleness fails + * in the harmless direction: a payer at their limit keeps indexing for at most + * this long, and a payer whose limit was just raised waits at most this long. + * Nothing on this path has a person waiting for the answer. + */ +export const INGESTION_USAGE_GATE_TTL_MS = 60 * 1000 + +/** Recent gate answers, with `LRUCache` supplying the TTL and the size bound. */ +const gateCache = new LRUCache({ + max: 10_000, + ttl: INGESTION_USAGE_GATE_TTL_MS, +}) + +/** + * The gate depends on who pays, for which period, and which member acts: the + * payer pool and the per-member cap are both part of the answer. + */ +function gateKey(attribution: BillingAttributionSnapshot): string { + return [ + attribution.billingEntity.type, + attribution.billingEntity.id, + attribution.billingPeriod.start, + attribution.billingPeriod.end, + attribution.billedAccountUserId, + attribution.actorUserId, + ].join(':') +} + +/** + * {@link checkAttributedUsageLimits} for background ingestion, with bounded + * staleness. Interactive callers (uploads, search, the settings surfaces) keep + * reading the gate fresh so a limit change is visible at once. + * + * `coalesceLocally` collapses the concurrent misses of a batch onto one ledger + * read and bounds a hung read at its settle deadline. The cache write stays on + * the value this caller received, so a producer that timed out and later + * resolved cannot overwrite a fresher answer. + */ +export async function checkIngestionUsageLimits( + attribution: BillingAttributionSnapshot +): Promise { + const key = gateKey(attribution) + const cached = gateCache.get(key) + if (cached !== undefined) return cached + + const result = await coalesceLocally(`ingestion-usage-gate:${key}`, () => + checkAttributedUsageLimits(attribution) + ) + gateCache.set(key, result) + return result +} + +/** Drops every cached gate answer. Test seam; never called in production code. */ +export function resetIngestionUsageGateCache(): void { + gateCache.clear() +} diff --git a/apps/sim/lib/core/rate-limiter/provider-admission.test.ts b/apps/sim/lib/core/rate-limiter/provider-admission.test.ts index 0307ed6f683..76d71ce4a81 100644 --- a/apps/sim/lib/core/rate-limiter/provider-admission.test.ts +++ b/apps/sim/lib/core/rate-limiter/provider-admission.test.ts @@ -1,6 +1,7 @@ /** * @vitest-environment node */ +import { resetEnvMock, setEnv } from '@sim/testing/mocks/env.mock' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { consumeTokens, getCooldownUntil, setCooldownUntil } = vi.hoisted(() => ({ @@ -35,7 +36,10 @@ describe('provider admission', () => { consumeTokens.mockResolvedValue({ allowed: true, tokensRemaining: 1, resetAt: new Date() }) }) - afterEach(() => vi.useRealTimers()) + afterEach(() => { + vi.useRealTimers() + resetEnvMock() + }) it('shares both credential dimensions in one reservation across concurrent callers', async () => { await Promise.all([waitForProviderAdmission(INPUT), waitForProviderAdmission(INPUT)]) @@ -86,6 +90,47 @@ describe('provider admission', () => { ) }) + it('caps bulk work below the aggregate budget so interactive callers keep headroom', async () => { + await waitForProviderAdmission({ ...INPUT, bulk: true }) + const [reservations, options] = consumeTokens.mock.calls[0] + expect(reservations).toMatchObject([ + { key: 'provider:embedding:openai:hashed-credential:tokens', config: { maxTokens: 600_000 } }, + { key: 'provider:embedding:openai:hashed-credential:requests', config: { maxTokens: 64 } }, + { + key: 'provider:embedding:openai:hashed-credential:bulk:tokens', + cost: 50, + config: { maxTokens: 540_000, refillRate: 9_000 }, + }, + { + key: 'provider:embedding:openai:hashed-credential:bulk:requests', + config: { maxTokens: 57, refillRate: 9 }, + }, + ]) + expect(options.cooldownKeys).toEqual([ + 'provider:embedding:openai:hashed-credential:cooldown', + 'provider:embedding:openai:hashed-credential:quota', + ]) + }) + + it('rejects a bulk batch the lane can never hold and keeps one request slot at a minimal burst', async () => { + setEnv({ + KB_CONFIG_EMBEDDING_REQUESTS_PER_MINUTE: '1', + KB_CONFIG_EMBEDDING_TOKENS_PER_MINUTE: '100', + }) + await expect( + waitForProviderAdmission({ ...INPUT, inputTokens: 95, bulk: true }) + ).rejects.toThrow('exceeds the configured per-credential token budget') + await waitForProviderAdmission({ ...INPUT, inputTokens: 95 }) + await waitForProviderAdmission({ ...INPUT, inputTokens: 90, bulk: true }) + expect(consumeTokens.mock.calls[1][0].slice(2)).toMatchObject([ + { key: 'provider:embedding:openai:hashed-credential:bulk:tokens', config: { maxTokens: 90 } }, + { + key: 'provider:embedding:openai:hashed-credential:bulk:requests', + config: { maxTokens: 1 }, + }, + ]) + }) + it('isolates another credential and does not impose token costs on OCR', async () => { await waitForProviderAdmission({ ...INPUT, diff --git a/apps/sim/lib/core/rate-limiter/provider-admission.ts b/apps/sim/lib/core/rate-limiter/provider-admission.ts index 895ebf33289..b9b191c2f81 100644 --- a/apps/sim/lib/core/rate-limiter/provider-admission.ts +++ b/apps/sim/lib/core/rate-limiter/provider-admission.ts @@ -12,10 +12,20 @@ export interface ProviderIdentity { operation: 'embedding' | 'ocr' | 'rerank' } +/** + * Share of a credential's budget the bulk lane may use. Every caller reserves + * from the aggregate buckets, so the budget is never exceeded; bulk callers + * also reserve from buckets capped at this share, which leaves an interactive + * caller headroom instead of a queue behind a crawl's batches. + */ +const BULK_LANE_SHARE = 0.9 + interface ProviderAdmissionInput extends ProviderIdentity { inputTokens?: number signal?: AbortSignal maxWaitMs: number + /** Bulk work is capped at {@link BULK_LANE_SHARE}; cooldown and quota gates still stop every caller. */ + bulk?: boolean } /** @@ -55,36 +65,47 @@ export async function waitForProviderAdmission(input: ProviderAdmissionInput): P : input.operation === 'ocr' ? envNumber(env.KB_CONFIG_OCR_REQUESTS_PER_MINUTE, 60, { min: 1 }) : envNumber(env.KB_CONFIG_RERANK_REQUESTS_PER_MINUTE, 60, { min: 1 }) + const tokenBudget = + input.operation === 'embedding' && input.inputTokens + ? { + cost: input.inputTokens, + perMinute: envNumber(env.KB_CONFIG_EMBEDDING_TOKENS_PER_MINUTE, 600_000, { min: 1 }), + } + : undefined + const laneShare = input.bulk ? BULK_LANE_SHARE : 1 + if (tokenBudget && tokenBudget.cost > Math.floor(tokenBudget.perMinute * laneShare)) { + throw new Error('Embedding request exceeds the configured per-credential token budget') + } + const requestBurst = Math.min( + input.operation === 'embedding' ? EMBEDDING_REQUEST_BURST : DEFAULT_REQUEST_BURST, + requestsPerMinute + ) const reservations: TokenBucketReservation[] = [] - if (input.operation === 'embedding' && input.inputTokens) { - const tokensPerMinute = envNumber(env.KB_CONFIG_EMBEDDING_TOKENS_PER_MINUTE, 600_000, { - min: 1, - }) - if (input.inputTokens > tokensPerMinute) { - throw new Error('Embedding request exceeds the configured per-credential token budget') + const reserveBuckets = (bucketKey: string, share: number) => { + if (tokenBudget) { + reservations.push({ + key: `${bucketKey}:tokens`, + cost: tokenBudget.cost, + config: { + maxTokens: Math.floor(tokenBudget.perMinute * share), + refillRate: (tokenBudget.perMinute * share) / 60, + refillIntervalMs: 1000, + }, + }) } reservations.push({ - key: `${key}:tokens`, - cost: input.inputTokens, + key: `${bucketKey}:requests`, + cost: 1, config: { - maxTokens: tokensPerMinute, - refillRate: tokensPerMinute / 60, + /** A burst of one leaves no share to carve out, so the lane then matches the aggregate. */ + maxTokens: Math.max(1, Math.floor(requestBurst * share)), + refillRate: (requestsPerMinute * share) / 60, refillIntervalMs: 1000, }, }) } - reservations.push({ - key: `${key}:requests`, - cost: 1, - config: { - maxTokens: Math.min( - input.operation === 'embedding' ? EMBEDDING_REQUEST_BURST : DEFAULT_REQUEST_BURST, - requestsPerMinute - ), - refillRate: requestsPerMinute / 60, - refillIntervalMs: 1000, - }, - }) + reserveBuckets(key, 1) + if (input.bulk) reserveBuckets(`${key}:bulk`, BULK_LANE_SHARE) /** When the bucket last said capacity returns, so a deadline hit after a sleep reports the wait still left. */ let capacityAvailableAt: number | undefined diff --git a/apps/sim/lib/embeddings/client.test.ts b/apps/sim/lib/embeddings/client.test.ts index 70ff9e97646..5efc3104276 100644 --- a/apps/sim/lib/embeddings/client.test.ts +++ b/apps/sim/lib/embeddings/client.test.ts @@ -1807,13 +1807,14 @@ describe('durable embedding batches', () => { expect(KNOWLEDGE_EMBEDDING_ADMISSION_WAIT_MS).toBeLessThan(EMBEDDING_RETRY_BUDGET_MS) }) - it('limits checkpointed admission waits while retaining the interactive request budget', async () => { + it('limits checkpointed admission waits and keeps interactive callers off the bulk lane', async () => { fetchMock.mockImplementation(() => Promise.resolve(jsonResponse(openAIBody([[1]], 7)))) await embed(['text'], { apiKey: 'fixture-key', checkpoints: memoryCheckpoints() }) expect(mockAdmit).toHaveBeenLastCalledWith( - expect.objectContaining({ maxWaitMs: KNOWLEDGE_EMBEDDING_ADMISSION_WAIT_MS }) + expect.objectContaining({ maxWaitMs: KNOWLEDGE_EMBEDDING_ADMISSION_WAIT_MS, bulk: true }) ) await embed(['text'], { apiKey: 'fixture-key' }) + expect(mockAdmit).toHaveBeenLastCalledWith(expect.objectContaining({ bulk: false })) expect(mockAdmit.mock.lastCall?.[0].maxWaitMs).toBeGreaterThan( KNOWLEDGE_EMBEDDING_ADMISSION_WAIT_MS ) diff --git a/apps/sim/lib/embeddings/client.ts b/apps/sim/lib/embeddings/client.ts index 0c6728d35e7..846ba759f0d 100644 --- a/apps/sim/lib/embeddings/client.ts +++ b/apps/sim/lib/embeddings/client.ts @@ -544,8 +544,10 @@ async function callEmbeddingAPI( expectedDimensions: number | undefined, isBYOK: boolean, signal?: AbortSignal, - admissionWaitMs = EMBEDDING_RETRY_BUDGET_MS + /** Bulk indexing waits briefly and is capped below the credential budget; everything else has a person waiting on it. */ + bulk = false ): Promise<{ embeddings: number[][]; totalTokens: number; dimensions: number }> { + const admissionWaitMs = bulk ? KNOWLEDGE_EMBEDDING_ADMISSION_WAIT_MS : EMBEDDING_RETRY_BUDGET_MS const admissionIdentity = embeddingAdmissionIdentity({ providerId, quotaCircuitIdentity, isBYOK }) return retryWithExponentialBackoff( async (operationSignal, deadlineAt) => { @@ -563,6 +565,7 @@ async function callEmbeddingAPI( ), signal: operationSignal, maxWaitMs: Math.min(admissionWaitMs, Math.max(0, deadlineAt - Date.now())), + bulk, }) } catch (error) { if (error instanceof ProviderQuotaExhaustedError) @@ -795,6 +798,7 @@ async function mapEmbeddingBatches( return results.map((result) => result!.value) } +/** Checkpoints mark the bulk indexing path; every other caller is interactive. */ async function callCheckpointedEmbeddingBatch( batch: string[], batchIndex: number, @@ -848,7 +852,7 @@ async function callCheckpointedEmbeddingBatch( provider.dimensions, provider.isBYOK, signal, - checkpoints ? KNOWLEDGE_EMBEDDING_ADMISSION_WAIT_MS : undefined + checkpoints !== undefined ) if (identity) await checkpoints!.save(identity, result, signal) return result diff --git a/apps/sim/lib/execution/payloads/materialization.server.test.ts b/apps/sim/lib/execution/payloads/materialization.server.test.ts index 7b55b117813..cdcdcd225d3 100644 --- a/apps/sim/lib/execution/payloads/materialization.server.test.ts +++ b/apps/sim/lib/execution/payloads/materialization.server.test.ts @@ -61,6 +61,39 @@ describe('readUserFileContent', () => { expect(generatedPdf.size).toBe(PDF_BYTES.length) }) + it('carries the actual execution principal through live knowledge-file authorization', async () => { + const principal = { kind: 'session' as const, userId: 'reader', sessionId: 'session-1' } + const file: UserFile = { + id: 'kb-file', + name: 'page.txt', + url: '', + size: 4, + type: 'text/plain', + key: 'kb/page.txt', + context: 'knowledge-base', + } + await readUserFileContent(file, { + userId: 'reader', + workspaceId: 'workspace-1', + principal, + encoding: 'text', + }) + expect(mockVerifyFileAccess).toHaveBeenCalledWith( + 'kb/page.txt', + 'reader', + undefined, + 'knowledge-base', + false, + { + knowledgeAccess: expect.objectContaining({ + get: expect.any(Function), + getForConnectors: expect.any(Function), + getForDocuments: expect.any(Function), + }), + } + ) + }) + it('authorizes execution-scoped files without inventing a human subject', async () => { const executionFile: UserFile = { id: 'file-2', diff --git a/apps/sim/lib/execution/payloads/materialization.server.ts b/apps/sim/lib/execution/payloads/materialization.server.ts index 64ade013d81..c5eafdedc6b 100644 --- a/apps/sim/lib/execution/payloads/materialization.server.ts +++ b/apps/sim/lib/execution/payloads/materialization.server.ts @@ -17,7 +17,7 @@ import { MAX_INLINE_MATERIALIZATION_BYTES, } from '@/lib/execution/payloads/limits' import { ExecutionResourceLimitError } from '@/lib/execution/resource-errors' -import { resolveKnowledgeAccessScope } from '@/lib/knowledge/access/scope' +import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' import type { StorageContext } from '@/lib/uploads' import type { WorkspaceFileSecretProvenanceIdentity } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' import { @@ -321,7 +321,7 @@ export async function assertUserFileContentAccess( */ const knowledgeAccess = context === 'knowledge-base' && options.principal - ? await resolveKnowledgeAccessScope(options.principal, { workspaceId: options.workspaceId }) + ? createKnowledgeAccessProvider(options.principal, { workspaceId: options.workspaceId }) : undefined const hasAccess = await verifyFileAccess(file.key, options.userId, undefined, context, false, { knowledgeAccess, diff --git a/apps/sim/lib/integrations/credential-display.test.ts b/apps/sim/lib/integrations/credential-display.test.ts index 456332bb029..cc138c9d705 100644 --- a/apps/sim/lib/integrations/credential-display.test.ts +++ b/apps/sim/lib/integrations/credential-display.test.ts @@ -279,9 +279,9 @@ describe('resolveCredentialDisplay', () => { }) describe('getServiceAccountCoverageSentence', () => { - it('tells the user up front that one Atlassian token spans all three products', () => { + it('explains that reusing an Atlassian token requires scopes and app access for each product', () => { expect(getServiceAccountCoverageSentence('atlassian-service-account')).toBe( - 'One token works across Confluence, Jira, and Jira Service Management.' + 'Reuse one token across Confluence, Jira, and Jira Service Management. Each product requires its own scopes and app access.' ) }) diff --git a/apps/sim/lib/integrations/credential-display.ts b/apps/sim/lib/integrations/credential-display.ts index 0ecadc9e980..2ad42a6b617 100644 --- a/apps/sim/lib/integrations/credential-display.ts +++ b/apps/sim/lib/integrations/credential-display.ts @@ -154,6 +154,9 @@ export function getServiceAccountCoverageSentence(providerId: string): string | const familyName = getServiceAccountFamilyName(providerId) if (!familyName) return null const covered = getIntegrationsForCredentialProvider(providerId) + if (providerId === 'atlassian-service-account') { + return `Reuse one token across ${formatList(covered.map((i) => i.name))}. Each product requires its own scopes and app access.` + } if (covered.length > MAX_ENUMERATED_INTEGRATIONS) { return `One token works across all ${covered.length} ${familyName} integrations.` } diff --git a/apps/sim/lib/knowledge/__integration__/application-acl.integration.ts b/apps/sim/lib/knowledge/__integration__/application-acl.integration.ts index 142b82b0ae2..48d30b587e3 100644 --- a/apps/sim/lib/knowledge/__integration__/application-acl.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/application-acl.integration.ts @@ -3,6 +3,7 @@ * vectors and the storage root is temporary; no database, principal, scope, * authorization, parser, chunking, ACL persistence, or search code is mocked. */ +import { createHash } from 'node:crypto' import { mkdtempSync } from 'node:fs' import { rm } from 'node:fs/promises' import { tmpdir } from 'node:os' @@ -11,6 +12,7 @@ import type { Principal } from '@sim/auth/principal' import { db } from '@sim/db' import { credential, + credentialGroup, credentialGroupEnrollment, document, embedding, @@ -18,6 +20,7 @@ import { knowledgeConnectorMember, knowledgeDocumentObservation, knowledgeExternalGroup, + knowledgeExternalGroupMember, permissions, user, workspace, @@ -49,6 +52,10 @@ vi.mock('@/lib/embeddings', async () => ({ })) import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' +import { env } from '@/lib/core/config/env' +import { encryptSecret } from '@/lib/core/security/encryption' +import { getCredentialGroupProviderAdapterByProviderId } from '@/lib/credential-groups/provider-registry' +import { encryptManagedOAuthTokenSet } from '@/lib/credentials/managed-oauth' import { createKnowledgeAclFixtureIds, seedKnowledgeAclFixture, @@ -56,6 +63,7 @@ import { } from '@/lib/knowledge/__integration__/seed-source-access-fixture' import { confluencePageAcl } from '@/lib/knowledge/access/confluence-permissions' import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' +import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' import { searchKnowledge } from '@/lib/knowledge/application/search' @@ -71,6 +79,11 @@ import { processDocumentAsync } from '@/lib/knowledge/documents/service' import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' describe('indexed source content through real application access', () => { + const previousConfluenceClient = { + id: env.CONFLUENCE_CLIENT_ID, + secret: env.CONFLUENCE_CLIENT_SECRET, + } + const revokedSiteReaders = new Set() const ids = createKnowledgeAclFixtureIds() const { aliceId, bobId, workspaceId, knowledgeBaseId, connectorId, lockId, groups, groupIds } = ids @@ -94,11 +107,121 @@ describe('indexed source content through real application access', () => { }) beforeAll(async () => { - vi.stubGlobal('fetch', async () => { + Object.assign(env, { + CONFLUENCE_CLIENT_ID: 'isolated-confluence-fixture-client', + CONFLUENCE_CLIENT_SECRET: 'isolated-confluence-fixture-secret', + }) + vi.stubGlobal('fetch', async (url: string, options?: RequestInit) => { + const authorization = new Headers(options?.headers).get('Authorization') + const reader = [aliceId, bobId].find( + (id) => authorization === `Bearer fixture-confluence-${id}` + ) + if ( + url === + 'https://api.atlassian.com/ex/confluence/fixture-tenant/wiki/rest/api/user/current' && + reader + ) + return revokedSiteReaders.has(reader) + ? new Response(null, { status: 403 }) + : Response.json({ type: 'known', accountId: reader }) throw new Error('Unexpected outbound request in isolated application integration test') }) fixtures.storageRoot = mkdtempSync(path.join(tmpdir(), 'sim-acl-integration-')) await seedKnowledgeAclFixture(ids) + const policy = await getCredentialGroupProviderAdapterByProviderId('confluence').getPolicy( + undefined, + { workspaceId } + ) + const groupId = generateId() + const optionId = generateId() + const crawlerId = generateId() + await db.insert(credentialGroup).values({ + id: groupId, + workspaceId, + publicId: generateId(), + name: 'Connected accounts', + options: [ + { + id: optionId, + provider: 'confluence', + label: 'Confluence', + required: false, + status: 'active', + authorizationAppId: policy.authorizationAppId, + requiredScopes: policy.requiredScopes, + scopeVersion: policy.scopeVersion, + }, + { + id: generateId(), + provider: 'google-drive', + label: 'Drive fixture', + authorizationAppId: 'fixture-app', + requiredScopes: ['drive.readonly'], + scopeVersion: 1, + required: false, + status: 'active', + }, + ], + }) + await db.insert(credential).values({ + id: crawlerId, + workspaceId, + type: 'service_account', + providerId: 'atlassian-service-account', + displayName: 'Confluence crawler', + createdBy: aliceId, + encryptedServiceAccountKey: ( + await encryptSecret( + JSON.stringify({ + type: 'atlassian_service_account', + cloudId: 'fixture-tenant', + domain: 'fixture.atlassian.net', + apiToken: 'fixture-crawler-never-used-for-reading', + }) + ) + ).encrypted, + }) + await db + .update(knowledgeConnector) + .set({ credentialId: crawlerId, sourceConfig: { domain: 'fixture.atlassian.net' } }) + .where(eq(knowledgeConnector.id, connectorId)) + for (const userId of [aliceId, bobId]) { + const enrollmentId = generateId() + await db.insert(credentialGroupEnrollment).values({ + id: enrollmentId, + credentialGroupId: groupId, + userId, + email: `${userId}@fixture.test`, + status: 'completed', + invitationTokenHash: createHash('sha256').update(generateId()).digest('hex'), + invitationExpiresAt: new Date(Date.now() + 3600000), + invitedAt: new Date(), + }) + await db.insert(credential).values({ + id: generateId(), + workspaceId, + type: 'managed_oauth', + displayName: 'Personal Confluence', + createdBy: userId, + providerId: 'confluence', + providerSubjectId: userId, + authorizationAppId: policy.authorizationAppId, + credentialGroupEnrollmentId: enrollmentId, + credentialGroupOptionId: optionId, + managedOauthScopeVersion: policy.scopeVersion, + managedOauthStatus: 'active', + grantedScopes: policy.requiredScopes, + grantedAt: new Date(), + encryptedOauthTokenSet: await encryptManagedOAuthTokenSet({ + accessToken: `fixture-confluence-${userId}`, + }), + accessTokenExpiresAt: new Date(Date.now() + 3600000), + }) + await db + .update(knowledgeExternalGroupMember) + .set({ subjectToken: `s:confluence:-:${userId}` }) + .where(eq(knowledgeExternalGroupMember.subjectToken, `u:${userId}@fixture.test`)) + } const doc = await addDocument( knowledgeBaseId, connectorId, @@ -133,9 +256,26 @@ describe('indexed source content through real application access', () => { expect(persisted).toEqual({ status: 'completed', error: null }) expect(await search(alice)).toEqual([]) await persistDocumentAcls(connectorId, new Map([['page-1', sourceAcl]])) + const readerAccess = createKnowledgeAccessProvider(alice, { + workspaceId, + knowledgeBaseIds: [knowledgeBaseId], + }) + expect((await readerAccess.get()).tokens).toEqual( + expect.arrayContaining([`s:confluence:-:${aliceId}`, 'g:confluence:fixture-tenant:page']) + ) + const proof = await readerAccess.getForConnectors([connectorId]) + expect(proof).toMatchObject({ + confluenceSiteGrants: [ + expect.objectContaining({ connectorId, readerSubjectToken: `s:confluence:-:${aliceId}` }), + ], + }) }) afterAll(async () => { + Object.assign(env, { + CONFLUENCE_CLIENT_ID: previousConfluenceClient.id, + CONFLUENCE_CLIENT_SECRET: previousConfluenceClient.secret, + }) await db.delete(workspace).where(eq(workspace.id, workspaceId)) await db.delete(user).where(eq(user.id, aliceId)) await db.delete(user).where(eq(user.id, bobId)) @@ -242,6 +382,25 @@ describe('indexed source content through real application access', () => { expect(fixtures.calls).toBeGreaterThan(0) }) + it('denies a revoked Confluence site across search, chunks, document reads, and files without rewriting ACLs', async () => { + revokedSiteReaders.add(aliceId) + try { + expect(await search(alice)).toEqual([]) + await expect( + readKnowledgeDocument.execute({ principal: alice, input: { knowledgeBaseId, documentId } }) + ).rejects.toThrow('Document not found') + await expect( + listKnowledgeChunks.execute({ principal: alice, input: { knowledgeBaseId, documentId } }) + ).rejects.toThrow('Document not found') + await expect( + downloadFileFromUrl(fileUrl, { userId: aliceId, knowledgeAccess: 'user' }) + ).rejects.toThrow('Access denied') + } finally { + revokedSiteReaders.delete(aliceId) + } + expect(await search(alice)).toEqual([documentId]) + }) + async function refreshDirectory(bobCanReadPage: boolean, complete = true) { await db .update(knowledgeExternalGroup) @@ -258,8 +417,8 @@ describe('indexed source content through real application access', () => { group, complete, memberTokens: [ - `u:${aliceId}@fixture.test`, - ...(group.id !== 'page' || bobCanReadPage ? [`u:${bobId}@fixture.test`] : []), + `s:confluence:-:${aliceId}`, + ...(group.id !== 'page' || bobCanReadPage ? [`s:confluence:-:${bobId}`] : []), ], }), }, @@ -325,25 +484,36 @@ describe('indexed source content through real application access', () => { }) it('permits fresh public grants for workspace keys but expires public evidence too', async () => { - await persistDocumentAcls(connectorId, new Map([['page-1', ['pub']]])) - expect(await search(workspaceKey)).toEqual([documentId]) - expect( - ( - await readKnowledgeDocument.execute({ + await db + .update(knowledgeConnector) + .set({ connectorType: 'google_drive' }) + .where(eq(knowledgeConnector.id, connectorId)) + try { + await persistDocumentAcls(connectorId, new Map([['page-1', ['pub']]])) + expect(await search(workspaceKey)).toEqual([documentId]) + expect( + ( + await readKnowledgeDocument.execute({ + principal: workspaceKey, + input: { knowledgeBaseId, documentId }, + }) + ).document.id + ).toBe(documentId) + await db.update(document).set({ aclVerifiedAt: null }).where(eq(document.id, documentId)) + expect(await search(workspaceKey)).toEqual([]) + await expect( + readKnowledgeDocument.execute({ principal: workspaceKey, input: { knowledgeBaseId, documentId }, }) - ).document.id - ).toBe(documentId) - await db.update(document).set({ aclVerifiedAt: null }).where(eq(document.id, documentId)) - expect(await search(workspaceKey)).toEqual([]) - await expect( - readKnowledgeDocument.execute({ - principal: workspaceKey, - input: { knowledgeBaseId, documentId }, - }) - ).rejects.toThrow('Document not found') - await persistDocumentAcls(connectorId, new Map([['page-1', sourceAcl]])) + ).rejects.toThrow('Document not found') + await persistDocumentAcls(connectorId, new Map([['page-1', sourceAcl]])) + } finally { + await db + .update(knowledgeConnector) + .set({ connectorType: 'confluence' }) + .where(eq(knowledgeConnector.id, connectorId)) + } }) it('requires current verified identity and actual workspace membership even when source groups grant access', async () => { diff --git a/apps/sim/lib/knowledge/__integration__/confluence-identity.integration.ts b/apps/sim/lib/knowledge/__integration__/confluence-identity.integration.ts index b5714c3ff5d..ea7b12dc25f 100644 --- a/apps/sim/lib/knowledge/__integration__/confluence-identity.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/confluence-identity.integration.ts @@ -1,10 +1,21 @@ -/** Real source identity, directory persistence and protected document reads; no provider calls. */ +/** Real source identity, directory persistence and protected reads; only the provider response is fixed. */ import type { Principal } from '@sim/auth/principal' import { db } from '@sim/db' -import { credential, credentialGroup, document, user, workspace } from '@sim/db/schema' +import { + credential, + credentialGroup, + document, + knowledgeConnector, + user, + workspace, +} from '@sim/db/schema' import { generateId } from '@sim/utils/id' import { eq } from 'drizzle-orm' -import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import { env } from '@/lib/core/config/env' +import { encryptSecret } from '@/lib/core/security/encryption' +import { getCredentialGroupProviderAdapterByProviderId } from '@/lib/credential-groups/provider-registry' +import { encryptManagedOAuthTokenSet } from '@/lib/credentials/managed-oauth' import { createKnowledgeAclFixtureIds, seedKnowledgeAclFixture, @@ -17,6 +28,10 @@ import { syncExternalDirectoryGroups } from '@/lib/knowledge/connectors/external describe('Confluence identities with hidden directory email', () => { const ids = createKnowledgeAclFixtureIds() const accountId = 'Opaque:CaseSensitiveAccount' + const previousConfluenceClient = { + CONFLUENCE_CLIENT_ID: env.CONFLUENCE_CLIENT_ID, + CONFLUENCE_CLIENT_SECRET: env.CONFLUENCE_CLIENT_SECRET, + } const sourceSubject = subjectToken({ providerId: 'confluence', providerTenantId: null, @@ -39,21 +54,80 @@ describe('Confluence identities with hidden directory email', () => { let member: Awaited>['members'][number] beforeAll(async () => { + Object.assign(env, { + CONFLUENCE_CLIENT_ID: 'isolated-confluence-fixture-client', + CONFLUENCE_CLIENT_SECRET: 'isolated-confluence-fixture-secret', + }) + vi.stubGlobal('fetch', async (url: string, options?: RequestInit) => { + if ( + url === + 'https://api.atlassian.com/ex/confluence/fixture-cloud/wiki/rest/api/user/current' && + new Headers(options?.headers).get('Authorization') === 'Bearer fixture-confluence-reader' + ) + return Response.json({ type: 'known', accountId }) + throw new Error('Unexpected outbound Confluence identity fixture request') + }) await seedKnowledgeAclFixture(ids) const managed = await seedKnowledgeMemberFixture(ids) member = managed.members[0] + const policy = await getCredentialGroupProviderAdapterByProviderId('confluence').getPolicy( + undefined, + { workspaceId: ids.workspaceId } + ) const [accounts] = await db .select() .from(credentialGroup) .where(eq(credentialGroup.id, managed.groupId)) await db .update(credentialGroup) - .set({ options: accounts.options.map((option) => ({ ...option, provider: 'confluence' })) }) + .set({ + options: accounts.options.map((option) => ({ + ...option, + provider: 'confluence', + authorizationAppId: policy.authorizationAppId, + requiredScopes: policy.requiredScopes, + scopeVersion: policy.scopeVersion, + })), + }) .where(eq(credentialGroup.id, managed.groupId)) await db .update(credential) - .set({ providerId: 'confluence', providerTenantId: null, providerSubjectId: accountId }) + .set({ + providerId: 'confluence', + providerTenantId: null, + providerSubjectId: accountId, + authorizationAppId: policy.authorizationAppId, + managedOauthScopeVersion: policy.scopeVersion, + grantedScopes: policy.requiredScopes, + encryptedOauthTokenSet: await encryptManagedOAuthTokenSet({ + accessToken: 'fixture-confluence-reader', + }), + accessTokenExpiresAt: new Date(Date.now() + 3600000), + }) .where(eq(credential.id, member.credentialId)) + const crawlerId = generateId() + await db.insert(credential).values({ + id: crawlerId, + workspaceId: ids.workspaceId, + type: 'service_account', + providerId: 'atlassian-service-account', + displayName: 'Confluence crawler fixture', + createdBy: ids.aliceId, + encryptedServiceAccountKey: ( + await encryptSecret( + JSON.stringify({ + type: 'atlassian_service_account', + cloudId: 'fixture-cloud', + domain: 'fixture.atlassian.net', + apiToken: 'fixture-crawler-never-used-for-reading', + }) + ) + ).encrypted, + }) + await db + .update(knowledgeConnector) + .set({ credentialId: crawlerId, sourceConfig: { domain: 'fixture.atlassian.net' } }) + .where(eq(knowledgeConnector.id, ids.connectorId)) await syncExternalDirectoryGroups({ workspaceId: ids.workspaceId, force: true, @@ -85,6 +159,8 @@ describe('Confluence identities with hidden directory email', () => { }) afterAll(async () => { + vi.unstubAllGlobals() + Object.assign(env, previousConfluenceClient) await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) await db.delete(user).where(eq(user.id, ids.aliceId)) await db.delete(user).where(eq(user.id, ids.bobId)) diff --git a/apps/sim/lib/knowledge/__integration__/embedding-processing-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/embedding-processing-recovery.integration.ts index ab5d891779a..141d16eeaa6 100644 --- a/apps/sim/lib/knowledge/__integration__/embedding-processing-recovery.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/embedding-processing-recovery.integration.ts @@ -63,7 +63,7 @@ describe('embedding progress survives a processing slice', () => { .where(eq(outboxEvent.eventType, EMBEDDING_CHECKPOINT_CLEANUP_EVENT)) .limit(5000)) priorCheckpointIds.add(row.id) - await seedKnowledgeAclFixture(ids) + await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) }) afterAll(async () => { vi.restoreAllMocks() @@ -91,7 +91,7 @@ describe('embedding progress survives a processing slice', () => { const file = await addDocument( ids.knowledgeBaseId, ids.connectorId, - 'confluence', + 'google_drive', { externalId: 'large-text', title: 'Synthetic operations.txt', diff --git a/apps/sim/lib/knowledge/__integration__/filtered-search.integration.ts b/apps/sim/lib/knowledge/__integration__/filtered-search.integration.ts index 95c0409fd8a..6a0b41448de 100644 --- a/apps/sim/lib/knowledge/__integration__/filtered-search.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/filtered-search.integration.ts @@ -39,7 +39,7 @@ describe.each([384, 768, 1024, 1536, 3072] as const)( keyId: 'fixture-key', } const restrictedAcl = confluencePageAcl({ - providerId: 'confluence', + providerId: 'google-drive', tenantId: 'fixture-tenant', spacePrincipals: [{ kind: 'group', id: 'space' }], restrictionChain: [[{ kind: 'group', id: 'parent' }], [{ kind: 'group', id: 'page' }]], @@ -62,7 +62,7 @@ describe.each([384, 768, 1024, 1536, 3072] as const)( ].map((fixture) => ({ ...fixture, documentId: generateId(), embeddingId: generateId() })) beforeAll(async () => { - await seedKnowledgeAclFixture(ids) + await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) await db .update(knowledgeBase) .set({ embeddingModel, embeddingDimension: dimensions }) @@ -91,7 +91,7 @@ describe.each([384, 768, 1024, 1536, 3072] as const)( 'workspace' in fixture ? ['ws'] : 'denied' in fixture - ? ['g:confluence:fixture-tenant:missing'] + ? ['g:google-drive:fixture-tenant:missing'] : [...restrictedAcl.acl], aclRequirements: 'workspace' in fixture ? [] : restrictedAcl.requirements.map((clause) => [...clause]), diff --git a/apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts b/apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts index 05ae14ad403..a3abd7da45f 100644 --- a/apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/gmail-member.integration.ts @@ -121,6 +121,7 @@ describe('Gmail member ingestion and ACLs in PostgreSQL (provider fixtures)', () } if (url.pathname === '/gmail/v1/users/me/threads') { expect(url.searchParams.get('maxResults')).toBe('100') + expect(url.searchParams.get('q')).toContain('subject:Orion') if (cursor === 'empty' && mailbox.failSecondPage) { return Promise.resolve( Response.json( @@ -237,7 +238,8 @@ describe('Gmail member ingestion and ACLs in PostgreSQL (provider fixtures)', () .update(knowledgeConnector) .set({ connectorType: 'gmail', - sourceConfig: { maxThreads: 0 }, + /** A fixed query keeps this full-listing suite separate from history-feed ingestion. */ + sourceConfig: { maxThreads: 0, query: 'subject:Orion' }, status: 'active', memberSyncStatus: 'idle', memberSyncLockToken: null, diff --git a/apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts b/apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts index e81d086e876..f00ea34e458 100644 --- a/apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/google-calendar-member.integration.ts @@ -451,14 +451,14 @@ describe('Google Calendar member indexing and authorization in PostgreSQL', () = it('isolates owner and reader projections across two calendars and follows empty pages', async () => { const rows = await stored() - expect(rows).toHaveLength(6) + expect(rows).toHaveLength(5) const expected = [ 'primary:collision', `${SHARED_CALENDAR}:collision`, `${SHARED_CALENDAR}:planning`, ] await assertAccess(alice, expected) - await assertAccess(bob, expected) + await assertAccess(bob, ['primary:collision', `${SHARED_CALENDAR}:planning`]) for (const person of people) { const pages = requests.filter((request) => request.token === person.accessToken) expect(pages.map(({ calendarId, page }) => [calendarId, page])).toEqual([ @@ -479,7 +479,7 @@ describe('Google Calendar member indexing and authorization in PostgreSQL', () = rows.map((row) => row.id) ) ) - expect(observations).toHaveLength(6) + expect(observations).toHaveLength(5) for (const row of rows) { const evidence = observations.filter((observation) => observation.documentId === row.id) expect(evidence).toHaveLength(1) @@ -489,16 +489,10 @@ describe('Google Calendar member indexing and authorization in PostgreSQL', () = const own = await content(alice, `${SHARED_CALENDAR}:collision`) expect(own).toContain('Confidential renewal terms') expect(own).not.toContain('

') - const reader = await content(bob, `${SHARED_CALENDAR}:collision`) - for (const hidden of [ - 'Confidential renewal', - 'confidential renewal', - 'organizer@fixture.test', - 'Provider Attendee', - 'Private meeting room', - ]) - expect(reader).not.toContain(hidden) - expect(reader).toContain('Date:') + const reader = await member(bob) + expect( + rows.some((row) => row.externalId === `member:${reader.id}:${SHARED_CALENDAR}:collision`) + ).toBe(false) expect(await content(alice, 'primary:collision')).not.toContain('Bob personal notes') expect(await content(bob, 'primary:collision')).not.toContain('Alice personal notes') const search = await searchKnowledge.execute({ @@ -541,7 +535,7 @@ describe('Google Calendar member indexing and authorization in PostgreSQL', () = return chunks.chunks.map((chunk) => chunk.content).join('\n') } - it('replaces a reader projection with provider-redacted free/busy data without an event update', async () => { + it('withdraws a reader projection reduced to free/busy data without an event update', async () => { const previous = await rowFor(bob, `${SHARED_CALENDAR}:planning`) readerRole = 'freeBusyReader' listing @@ -551,14 +545,17 @@ describe('Google Calendar member indexing and authorization in PostgreSQL', () = redactedEvent(planningEvent), ]) await sync() - const current = await rowFor(bob, `${SHARED_CALENDAR}:planning`) - expect(current.id).toBe(previous.id) - expect(current.contentHash).not.toBe(previous.contentHash) - const redacted = await content(bob, `${SHARED_CALENDAR}:planning`) - expect(redacted).not.toContain('Planning roadmap') - expect(redacted).not.toContain('Provider Organizer') - expect(redacted).not.toContain('Private meeting room') - expect(redacted).toContain('Date:') + await assertAccess(bob, ['primary:collision']) + const withdrawn = await rowFor(bob, `${SHARED_CALENDAR}:planning`) + expect(withdrawn.acl).toEqual([]) + expect(withdrawn.contentHash).toBe(previous.contentHash) + expect(withdrawn.deletedAt).not.toBeNull() + expect( + await db + .select() + .from(knowledgeDocumentObservation) + .where(eq(knowledgeDocumentObservation.documentId, previous.id)) + ).toEqual([]) expect(await content(alice, `${SHARED_CALENDAR}:planning`)).toContain('Planning roadmap') }) @@ -582,7 +579,7 @@ describe('Google Calendar member indexing and authorization in PostgreSQL', () = .set(SHARED_CALENDAR, [listing.get(bob.accessToken)!.get(SHARED_CALENDAR)![0], cancelled]) await sync() await assertAccess(alice, [`${SHARED_CALENDAR}:collision`]) - await assertAccess(bob, ['primary:collision', `${SHARED_CALENDAR}:collision`]) + await assertAccess(bob, ['primary:collision']) expect(await content(alice, `${SHARED_CALENDAR}:collision`)).toContain( 'Updated confidential renewal' ) @@ -596,7 +593,7 @@ describe('Google Calendar member indexing and authorization in PostgreSQL', () = rows.map((row) => row.id) ) ) - expect(observations).toHaveLength(3) + expect(observations).toHaveLength(2) }) it('withdraws a lost shared calendar while preserving the member primary calendar', async () => { @@ -652,6 +649,6 @@ describe('Google Calendar member indexing and authorization in PostgreSQL', () = .select() .from(document) .where(and(eq(document.connectorId, connectorId), isNull(document.deletedAt))) - ).toHaveLength(6) + ).toHaveLength(5) }) }) diff --git a/apps/sim/lib/knowledge/__integration__/google-drive-shortcuts.integration.ts b/apps/sim/lib/knowledge/__integration__/google-drive-shortcuts.integration.ts index 11a8836ba91..80b569631bf 100644 --- a/apps/sim/lib/knowledge/__integration__/google-drive-shortcuts.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/google-drive-shortcuts.integration.ts @@ -70,6 +70,16 @@ describe('Drive shortcuts through indexing and search', () => { let targetDownloads = 0 let aliasDownloads = 0 let enrolled: Awaited> + const directoryUser = { + id: 'fixture-admin', + primaryEmail: 'admin@fixture.test', + customerId: 'fixture-customer', + suspended: false, + } + const delegatedToken = vi.fn(async (email: string) => { + expect(email).toBe(directoryUser.primaryEmail) + return 'fixture-delegated-admin' + }) const principal = (userId: string) => ({ kind: 'session' as const, userId, @@ -114,10 +124,13 @@ describe('Drive shortcuts through indexing and search', () => { ) const headers = new Headers(init?.headers) if (url.hostname === 'admin.googleapis.com') { + if (url.pathname.endsWith('/users')) return json({ users: [directoryUser] }) + if (url.pathname.endsWith(`/users/${directoryUser.id}`)) return json(directoryUser) if (url.pathname.endsWith('/groups')) return json({ groups: [] }) if (url.pathname.endsWith('/domains')) return json({ domains: [] }) } if (url.hostname !== 'www.googleapis.com') throw new Error('Unexpected fixture provider') + if (url.pathname.endsWith('/drives')) return json({ drives: [] }) if (url.pathname.endsWith('/changes/startPageToken')) return json({ startPageToken: 'start' }) if (url.pathname.endsWith('/changes')) return json({ changes: [], newStartPageToken: 'resume' }) if (url.pathname.endsWith('/files')) return json({ files: aliasPresent ? [alias()] : [] }) @@ -198,6 +211,7 @@ describe('Drive shortcuts through indexing and search', () => { .where(eq(knowledgeConnector.id, ids.connectorId)) vi.spyOn(connectorTokens, 'resolveConnectorAccessToken').mockResolvedValue({ accessToken: 'fixture-admin', + getDelegatedAccessToken: delegatedToken, }) vi.stubGlobal('fetch', providerFetch) }) @@ -275,6 +289,7 @@ describe('Drive shortcuts through indexing and search', () => { .orderBy(embedding.id) const downloaded = targetDownloads await sync() + expect(delegatedToken).toHaveBeenCalledWith(directoryUser.primaryEmail) const indexed = await row() expect(indexed.aclRequirements).toHaveLength(2) expect(indexed.aclRequirements).toEqual( diff --git a/apps/sim/lib/knowledge/__integration__/organization-mcp-search.integration.ts b/apps/sim/lib/knowledge/__integration__/organization-mcp-search.integration.ts index 51dec08bd0e..8f8374ebb9f 100644 --- a/apps/sim/lib/knowledge/__integration__/organization-mcp-search.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/organization-mcp-search.integration.ts @@ -241,7 +241,7 @@ describe('organization Search MCP with real ingestion and current access', () => throw new Error('Unexpected outbound organization MCP fixture request') }) fixtures.storageRoot = mkdtempSync(path.join(tmpdir(), 'sim-organization-mcp-integration-')) - await seedKnowledgeAclFixture(ids) + await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) await db.insert(user).values( [outsiderId, otherAdminId].map((id) => ({ id, @@ -297,7 +297,7 @@ describe('organization Search MCP with real ingestion and current access', () => }) await db.insert(organizationSearchIntegration).values({ organizationId, - connectorType: 'confluence', + connectorType: 'google_drive', approved: true, }) await db.insert(apiKey).values( @@ -350,7 +350,7 @@ describe('organization Search MCP with real ingestion and current access', () => const doc = await addDocument( knowledgeBaseId, connectorId, - 'confluence', + 'google_drive', { externalId: 'organization-mcp-page', mimeType: 'text/plain', @@ -387,7 +387,7 @@ describe('organization Search MCP with real ingestion and current access', () => [ 'organization-mcp-page', confluencePageAcl({ - providerId: 'confluence', + providerId: 'google-drive', tenantId: 'fixture-tenant', spacePrincipals: [{ kind: 'group', id: 'space' }], restrictionChain: [[{ kind: 'group', id: 'page' }], [{ kind: 'group', id: 'parent' }]], @@ -582,7 +582,7 @@ describe('organization Search MCP with real ingestion and current access', () => try { const included = await value(alice, 'search', { query: 'Orion', - source: 'confluence', + source: 'google_drive', modifiedAfter: '2026-01-01T00:00:00Z', documentIds: [documentId], }) @@ -639,7 +639,7 @@ describe('organization Search MCP with real ingestion and current access', () => it('hides source content immediately when organization approval is disabled', async () => { const approval = and( eq(organizationSearchIntegration.organizationId, organizationId), - eq(organizationSearchIntegration.connectorType, 'confluence') + eq(organizationSearchIntegration.connectorType, 'google_drive') ) await db.update(organizationSearchIntegration).set({ approved: false }).where(approval) try { diff --git a/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts index 0e9bd56a690..98831b8765b 100644 --- a/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/provider-processing-recovery.integration.ts @@ -114,15 +114,13 @@ describe('provider throttling resumes the shared indexing pipeline', () => { vi.useRealTimers() const ids = createKnowledgeAclFixtureIds() seeded.push(ids) - await seedKnowledgeAclFixture(ids) + await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) let connectorId = ids.connectorId - let connectorType = 'confluence' let lease = createContentSyncLease(connectorId, ids.lockId) let memberFixture: Awaited> | undefined if (scope === 'member source') { memberFixture = await seedKnowledgeMemberFixture(ids) connectorId = memberFixture.connectorId - connectorType = 'google_drive' lease = createMemberSyncLease(connectorId, memberFixture.runId) } const orgOwned = scope === 'organization Search' @@ -150,7 +148,7 @@ describe('provider throttling resumes the shared indexing pipeline', () => { const file = await addDocument( ids.knowledgeBaseId, connectorId, - connectorType, + 'google_drive', { externalId: 'orion-scan', title: 'Orion scan.png', diff --git a/apps/sim/lib/knowledge/__integration__/read-indexed-document.integration.ts b/apps/sim/lib/knowledge/__integration__/read-indexed-document.integration.ts index ac5b5ba01b6..a4cb6ff0afa 100644 --- a/apps/sim/lib/knowledge/__integration__/read-indexed-document.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/read-indexed-document.integration.ts @@ -72,7 +72,7 @@ describe('indexed document references', () => { const doc = await addDocument( ids.knowledgeBaseId, ids.connectorId, - 'confluence', + 'google_drive', { externalId, mimeType: 'text/plain', @@ -117,8 +117,8 @@ describe('indexed document references', () => { vi.stubGlobal('fetch', async () => { throw new Error('Indexed document reads must never fetch provider URLs') }) - await seedKnowledgeAclFixture(ids) - await seedKnowledgeAclFixture(other) + await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) + await seedKnowledgeAclFixture(other, { connectorType: 'google_drive' }) await db .update(knowledgeBase) .set({ workspaceId: null, organizationId: ids.organizationId, isSearchIndex: true }) @@ -133,11 +133,11 @@ describe('indexed document references', () => { .where(inArray(knowledgeExternalGroup.id, ids.groupIds)) await db .insert(organizationSearchIntegration) - .values({ organizationId: ids.organizationId, connectorType: 'confluence', approved: true }) + .values({ organizationId: ids.organizationId, connectorType: 'google_drive', approved: true }) documentId = await ingest('visible-target') hiddenDocumentId = await ingest('hidden-target') const sourceAcl = confluencePageAcl({ - providerId: 'confluence', + providerId: 'google-drive', tenantId: 'fixture-tenant', spacePrincipals: [{ kind: 'group', id: 'space' }], restrictionChain: [[{ kind: 'group', id: 'page' }]], diff --git a/apps/sim/lib/knowledge/__integration__/search-index-policy.integration.ts b/apps/sim/lib/knowledge/__integration__/search-index-policy.integration.ts index de11b572eb9..988ef9fe054 100644 --- a/apps/sim/lib/knowledge/__integration__/search-index-policy.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-index-policy.integration.ts @@ -69,7 +69,7 @@ describe('canonical search knowledge-base policy', () => { vi.stubGlobal('fetch', async () => { throw new Error('Unexpected provider request in index policy tests') }) - await seedKnowledgeAclFixture(ids) + await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) await db .update(knowledgeBase) .set({ isSearchIndex: true, name: indexName, userId: ids.bobId }) diff --git a/apps/sim/lib/knowledge/__integration__/search-reference-batching.integration.ts b/apps/sim/lib/knowledge/__integration__/search-reference-batching.integration.ts new file mode 100644 index 00000000000..a08d83b195a --- /dev/null +++ b/apps/sim/lib/knowledge/__integration__/search-reference-batching.integration.ts @@ -0,0 +1,123 @@ +import { db } from '@sim/db' +import { + knowledgeBase, + knowledgeBaseTagDefinitions, + organization, + user, + workspace, +} from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray } from 'drizzle-orm' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import { + createKnowledgeAclFixtureIds, + seedKnowledgeAclFixture, +} from '@/lib/knowledge/__integration__/seed-source-access-fixture' +import { + getActiveKnowledgeBaseReference, + getActiveKnowledgeBaseReferences, +} from '@/lib/knowledge/service' +import { + getDocumentTagDefinitions, + getDocumentTagDefinitionsByKnowledgeBaseIds, +} from '@/lib/knowledge/tags/service' + +describe('batched search reference reads', () => { + const fixture = createKnowledgeAclFixtureIds() + const baseIds = [fixture.knowledgeBaseId, ...Array.from({ length: 19 }, () => generateId())] + const deletedBaseId = generateId() + + beforeAll(async () => { + await seedKnowledgeAclFixture(fixture) + await db.insert(knowledgeBase).values( + [...baseIds.slice(1), deletedBaseId].map((id, index) => ({ + id, + userId: fixture.aliceId, + workspaceId: fixture.workspaceId, + name: `Batched search fixture ${index}`, + deletedAt: id === deletedBaseId ? new Date() : null, + })) + ) + await db.insert(knowledgeBaseTagDefinitions).values( + baseIds.slice(0, -1).flatMap((knowledgeBaseId) => + (['tag3', 'tag2'] as const).map((tagSlot) => ({ + id: generateId(), + knowledgeBaseId, + tagSlot, + displayName: tagSlot, + fieldType: 'text' as const, + })) + ) + ) + }) + + afterAll(async () => { + vi.restoreAllMocks() + await db.delete(workspace).where(eq(workspace.id, fixture.workspaceId)) + await db.delete(organization).where(eq(organization.id, fixture.organizationId)) + await db.delete(user).where(inArray(user.id, [fixture.aliceId, fixture.bobId])) + await db.$client.end() + }) + + it('returns identical active references with one query instead of twenty', async () => { + const select = vi.spyOn(db, 'select') + try { + const expected = await Promise.all(baseIds.map(getActiveKnowledgeBaseReference)) + expect(select).toHaveBeenCalledTimes(20) + select.mockClear() + expect(await getActiveKnowledgeBaseReferences(baseIds)).toEqual(expected) + expect(select).toHaveBeenCalledOnce() + } finally { + select.mockRestore() + } + }) + + it('preserves input order, duplicates, missing identities, and soft deletion', async () => { + const ids = [baseIds[8], generateId(), baseIds[0], deletedBaseId, baseIds[8]] + expect(await getActiveKnowledgeBaseReferences(ids)).toEqual( + await Promise.all(ids.map(getActiveKnowledgeBaseReference)) + ) + }) + + it('returns identical ordered tag definitions with one query instead of twenty', async () => { + const select = vi.spyOn(db, 'select') + try { + const expected = new Map( + await Promise.all( + baseIds.map(async (id) => [id, await getDocumentTagDefinitions(id)] as const) + ) + ) + expect(select).toHaveBeenCalledTimes(20) + select.mockClear() + expect(await getDocumentTagDefinitionsByKnowledgeBaseIds(baseIds)).toEqual(expected) + expect(select).toHaveBeenCalledOnce() + expect(expected.get(baseIds.at(-1)!)).toEqual([]) + } finally { + select.mockRestore() + } + }) + + it('does not cache updated references or tag definitions across reads', async () => { + await getActiveKnowledgeBaseReferences(baseIds) + await getDocumentTagDefinitionsByKnowledgeBaseIds(baseIds) + await db + .update(knowledgeBase) + .set({ name: 'Updated reference' }) + .where(eq(knowledgeBase.id, baseIds[0])) + await db + .update(knowledgeBaseTagDefinitions) + .set({ displayName: 'Updated definition' }) + .where( + and( + eq(knowledgeBaseTagDefinitions.knowledgeBaseId, baseIds[0]), + eq(knowledgeBaseTagDefinitions.tagSlot, 'tag1') + ) + ) + const references = await getActiveKnowledgeBaseReferences(baseIds) + const tags = await getDocumentTagDefinitionsByKnowledgeBaseIds(baseIds) + expect(references[0]?.name).toBe('Updated reference') + expect( + tags.get(baseIds[0])?.find((definition) => definition.tagSlot === 'tag1')?.displayName + ).toBe('Updated definition') + }) +}) diff --git a/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts b/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts index c14ee7c7a17..5a82344815a 100644 --- a/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts @@ -27,8 +27,9 @@ const failedId = generateId() const pendingId = generateId() const input = { workspaceId: ids.workspaceId, connectorIds: [ids.connectorId] } +/** Drive models the mirrored email grants exercised by these provider-independent progress tests. */ beforeAll(async () => { - await seedKnowledgeAclFixture(ids) + await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) await db .update(knowledgeBase) .set({ isSearchIndex: true }) @@ -144,7 +145,7 @@ describe('connector document filename search and document sets', () => { const scope = { knowledgeBaseId: fixture.knowledgeBaseId, connectorId: fixture.connectorId } beforeAll(async () => { - await seedKnowledgeAclFixture(fixture) + await seedKnowledgeAclFixture(fixture, { connectorType: 'google_drive' }) const rows: Array & { id: string; filename: string }> = [ ...Array.from({ length: 221 }, (_, index) => ({ id: generateId(), diff --git a/apps/sim/lib/knowledge/__integration__/seed-source-access-fixture.ts b/apps/sim/lib/knowledge/__integration__/seed-source-access-fixture.ts index 7970a713756..3c0749defc0 100644 --- a/apps/sim/lib/knowledge/__integration__/seed-source-access-fixture.ts +++ b/apps/sim/lib/knowledge/__integration__/seed-source-access-fixture.ts @@ -35,7 +35,10 @@ export function createKnowledgeAclFixtureIds() { } /** Inserts only unique fixture rows, and refuses the developer's ordinary database. */ -export async function seedKnowledgeAclFixture(ids = createKnowledgeAclFixtureIds()) { +export async function seedKnowledgeAclFixture( + ids = createKnowledgeAclFixtureIds(), + options: { connectorType?: 'confluence' | 'google_drive' } = {} +) { const target = new URL(process.env.DATABASE_URL ?? '') if ( !['localhost', '127.0.0.1'].includes(target.hostname) || @@ -46,6 +49,8 @@ export async function seedKnowledgeAclFixture(ids = createKnowledgeAclFixtureIds const { aliceId, bobId, workspaceId, knowledgeBaseId, connectorId, lockId, groups, groupIds } = ids const now = new Date() + const connectorType = options.connectorType ?? 'confluence' + const providerId = connectorType === 'google_drive' ? 'google-drive' : 'confluence' await db.insert(user).values([ { id: aliceId, @@ -109,7 +114,7 @@ export async function seedKnowledgeAclFixture(ids = createKnowledgeAclFixtureIds await db.insert(knowledgeConnector).values({ id: connectorId, knowledgeBaseId, - connectorType: 'confluence', + connectorType, sourceConfig: {}, accessMode: 'admin', status: 'syncing', @@ -119,7 +124,7 @@ export async function seedKnowledgeAclFixture(ids = createKnowledgeAclFixtureIds groups.map((name, index) => ({ id: groupIds[index], workspaceId, - providerId: 'confluence', + providerId, tenantId: 'fixture-tenant', externalGroupId: name, lastSyncedAt: now, diff --git a/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts b/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts index 7bef350ec38..80c4fcb7172 100644 --- a/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/stored-document-recovery.integration.ts @@ -65,7 +65,7 @@ const old = () => new Date(Date.now() - QUEUED_DISPATCH_GRACE_MS - 60_000) async function seed() { const ids = createKnowledgeAclFixtureIds() fixtures.push(ids) - await seedKnowledgeAclFixture(ids) + await seedKnowledgeAclFixture(ids, { connectorType: 'google_drive' }) return ids } async function eventsFor(ids: ReturnType) { @@ -86,7 +86,7 @@ async function failedFile( const file = await addDocument( ids.knowledgeBaseId, ids.connectorId, - 'confluence', + 'google_drive', { externalId: generateId(), title: 'Retained fixture.txt', diff --git a/apps/sim/lib/knowledge/access/confluence-site.test.ts b/apps/sim/lib/knowledge/access/confluence-site.test.ts new file mode 100644 index 00000000000..543c63a168f --- /dev/null +++ b/apps/sim/lib/knowledge/access/confluence-site.test.ts @@ -0,0 +1,459 @@ +/** @vitest-environment node */ +import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + CONFLUENCE_READ_ATTEMPT_TIMEOUT_MS, + CONFLUENCE_READ_CONCURRENCY, + CONFLUENCE_READ_CREDENTIAL_ALTERNATIVES, + CONFLUENCE_READ_RESPONSE_MAX_BYTES, + CONFLUENCE_READ_TIMEOUT_MS, + resolveConfluenceSiteReadGrants, +} from '@/lib/knowledge/access/confluence-site' +import { MAX_KNOWLEDGE_ACCESS_CANDIDATES } from '@/lib/knowledge/access/types' + +const mocks = vi.hoisted(() => ({ token: vi.fn(), decrypt: vi.fn(), fetch: vi.fn() })) +vi.mock('@/lib/credentials/managed-oauth', () => ({ resolveManagedOAuthToken: mocks.token })) +vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: mocks.decrypt })) + +const input = { + scope: { kind: 'organization' as const, organizationId: 'org-1' }, + readers: [{ credentialId: 'alice-credential', subjectToken: 's:confluence:-:alice' }], + knowledgeBaseIds: ['index-1'], + connectorIds: ['source-1'], +} +const source = { + connectorId: 'source-1', + contentCredentialId: 'crawler-credential', + readerCredentialId: 'alice-credential', + providerSubjectId: 'alice', + domain: 'company.atlassian.net', +} +const binding = { + type: 'atlassian_service_account', + cloudId: 'cloud-1', + domain: source.domain, + apiToken: 'crawler-token-must-never-authenticate-reader', +} +const contentCredential = { id: 'crawler-credential', key: 'encrypted-crawler' } +const grant = { + connectorId: source.connectorId, + contentCredentialId: source.contentCredentialId, + readerCredentialId: source.readerCredentialId, + readerSubjectToken: input.readers[0].subjectToken, + domain: source.domain, + cloudId: binding.cloudId, +} + +function queueSources(rows = [source], credentials = [contentCredential]) { + input.connectorIds = rows.map((row) => row.connectorId) + queueTableRows(schemaMock.knowledgeConnector, rows) + queueTableRows(schemaMock.credential, [ + ...new Map( + rows.map((row) => [ + row.readerCredentialId, + { id: row.readerCredentialId, providerSubjectId: row.providerSubjectId }, + ]) + ).values(), + ]) + queueTableRows(schemaMock.credential, credentials) +} + +beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + vi.stubGlobal('fetch', mocks.fetch) + mocks.token.mockResolvedValue({ accessToken: 'alice-oauth-token' }) + mocks.decrypt.mockResolvedValue({ decrypted: JSON.stringify(binding) }) + mocks.fetch.mockImplementation(async () => Response.json({ accountId: 'alice', type: 'known' })) +}) +afterEach(() => vi.restoreAllMocks()) + +describe('current Confluence site access', () => { + it('requires the actual reader’s Can use permission at the crawler’s immutable site', async () => { + queueSources() + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([grant]) + expect(mocks.token).toHaveBeenCalledWith({ + credentialId: 'alice-credential', + organizationId: 'org-1', + expectedProviderId: 'confluence', + requiredScopes: ['read:confluence-user'], + }) + expect(mocks.fetch).toHaveBeenCalledWith( + 'https://api.atlassian.com/ex/confluence/cloud-1/wiki/rest/api/user/current', + expect.objectContaining({ + headers: { Authorization: 'Bearer alice-oauth-token', Accept: 'application/json' }, + cache: 'no-store', + redirect: 'error', + signal: expect.any(AbortSignal), + }) + ) + expect(JSON.stringify(mocks.fetch.mock.calls)).not.toContain(binding.apiToken) + }) + + it('denies newly revoked site access even if a direct space grant remains indexed', async () => { + queueSources() + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([grant]) + queueSources() + mocks.fetch.mockResolvedValueOnce(new Response(null, { status: 403 })) + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([]) + expect(mocks.token).toHaveBeenCalledTimes(2) + expect(mocks.fetch).toHaveBeenCalledTimes(2) + }) + + it.each([401, 403, 404, 429, 500, 503])( + 'denies unconfirmed access on HTTP %i', + async (status) => { + queueSources() + mocks.fetch.mockResolvedValueOnce(new Response(null, { status })) + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([]) + } + ) + + it.each([ + { accountId: 'bob', type: 'known' }, + { accountId: 'alice', type: 'anonymous' }, + { accountId: 'alice' }, + null, + ])('denies another identity and incomplete provider data', async (profile) => { + queueSources() + mocks.fetch.mockResolvedValueOnce(Response.json(profile)) + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([]) + }) + + it('never substitutes another reader or a crawler when the personal connection is missing', async () => { + queueSources([{ ...source, providerSubjectId: 'bob' }]) + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([]) + expect(mocks.token).not.toHaveBeenCalled() + expect(mocks.fetch).not.toHaveBeenCalled() + await expect(resolveConfluenceSiteReadGrants({ ...input, readers: [] })).resolves.toEqual([]) + }) + + it('denies an unavailable scoped crawler credential before checking any reader token', async () => { + queueSources([source], []) + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([]) + expect(mocks.token).not.toHaveBeenCalled() + expect(mocks.fetch).not.toHaveBeenCalled() + }) + + it.each([ + { ...binding, domain: 'another.atlassian.net' }, + { ...binding, cloudId: '../other-cloud' }, + { ...binding, type: 'google_service_account' }, + ])('rejects mismatched or malformed site bindings', async (value) => { + queueSources() + mocks.decrypt.mockResolvedValueOnce({ decrypted: JSON.stringify(value) }) + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([]) + expect(mocks.token).not.toHaveBeenCalled() + expect(mocks.fetch).not.toHaveBeenCalled() + }) + + it('deduplicates one reader and site only within the current admission', async () => { + queueSources([source, { ...source, connectorId: 'source-2' }]) + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toHaveLength(2) + expect(mocks.token).toHaveBeenCalledTimes(1) + expect(mocks.fetch).toHaveBeenCalledTimes(1) + }) + + it('never carries an allowed site proof to a denied site', async () => { + queueSources( + [source, { ...source, connectorId: 'source-2', contentCredentialId: 'other-crawler' }], + [contentCredential, { id: 'other-crawler', key: 'encrypted-other-crawler' }] + ) + mocks.decrypt.mockImplementation(async (key: string) => ({ + decrypted: JSON.stringify({ + ...binding, + cloudId: key === 'encrypted-crawler' ? 'cloud-1' : 'cloud-2', + }), + })) + mocks.fetch.mockImplementation(async (url: string) => + url.includes('/cloud-2/') + ? new Response(null, { status: 403 }) + : Response.json({ accountId: 'alice', type: 'known' }) + ) + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([grant]) + expect(mocks.fetch).toHaveBeenCalledTimes(2) + }) + + it('keeps a full bounded batch without enumerating the organization', async () => { + queueSources( + Array.from({ length: MAX_KNOWLEDGE_ACCESS_CANDIDATES }, (_, index) => ({ + ...source, + connectorId: `source-${index}`, + })) + ) + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toHaveLength( + MAX_KNOWLEDGE_ACCESS_CANDIDATES + ) + expect(dbChainMockFns.limit).toHaveBeenCalledWith(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + expect(mocks.fetch).toHaveBeenCalledTimes(1) + await expect( + resolveConfluenceSiteReadGrants({ ...input, connectorIds: [...input.connectorIds, 'extra'] }) + ).rejects.toThrow('bounded pages') + }) + + it('does not let multiple reader rows truncate later source candidates', async () => { + const rows = Array.from({ length: 301 }, (_, index) => ({ + ...source, + connectorId: `source-${index}`, + })) + queueTableRows(schemaMock.knowledgeConnector, rows) + queueTableRows(schemaMock.credential, [ + { id: 'alice-credential', providerSubjectId: 'alice' }, + { id: 'bob-credential', providerSubjectId: 'bob' }, + ]) + queueTableRows(schemaMock.credential, [contentCredential]) + mocks.token.mockImplementation(async ({ credentialId }: { credentialId: string }) => ({ + accessToken: credentialId, + })) + mocks.fetch.mockImplementation(async (_url: string, options: RequestInit) => + options.headers && + new Headers(options.headers).get('Authorization') === 'Bearer bob-credential' + ? Response.json({ accountId: 'bob', type: 'known' }) + : new Response(null, { status: 403 }) + ) + const grants = await resolveConfluenceSiteReadGrants({ + ...input, + connectorIds: rows.map((row) => row.connectorId), + readers: [ + ...input.readers, + { credentialId: 'bob-credential', subjectToken: 's:confluence:-:bob' }, + ], + }) + expect(grants).toHaveLength(301) + expect(grants.every((entry) => entry.readerSubjectToken === 's:confluence:-:bob')).toBe(true) + expect(grants.map((entry) => entry.connectorId)).toContain('source-300') + expect(mocks.fetch).toHaveBeenCalledTimes(2) + }) + + it('bounds failed site proofs across a source and reader cross product', async () => { + const rows = Array.from({ length: MAX_KNOWLEDGE_ACCESS_CANDIDATES }, (_, index) => ({ + ...source, + connectorId: `source-${index}`, + contentCredentialId: `crawler-${index}`, + })) + queueTableRows(schemaMock.knowledgeConnector, rows) + queueTableRows(schemaMock.credential, [ + { id: 'alice-credential', providerSubjectId: 'alice' }, + { id: 'bob-credential', providerSubjectId: 'bob' }, + ]) + queueTableRows( + schemaMock.credential, + rows.map((row, index) => ({ id: row.contentCredentialId, key: `cloud-${index}` })) + ) + mocks.decrypt.mockImplementation(async (key: string) => ({ + decrypted: JSON.stringify({ ...binding, cloudId: key }), + })) + mocks.fetch.mockImplementation(async () => new Response(null, { status: 403 })) + await expect( + resolveConfluenceSiteReadGrants({ + ...input, + connectorIds: rows.map((row) => row.connectorId), + readers: [ + ...input.readers, + { credentialId: 'bob-credential', subjectToken: 's:confluence:-:bob' }, + ], + }) + ).resolves.toEqual([]) + expect(mocks.fetch).toHaveBeenCalledTimes(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + }) + + it.each(['refresh', 'site request'])( + 'uses a working same-subject credential while an older %s stalls for all 400 same-site sources', + async (stall) => { + const rows = Array.from({ length: MAX_KNOWLEDGE_ACCESS_CANDIDATES }, (_, index) => ({ + ...source, + connectorId: `source-${index}`, + })) + queueTableRows(schemaMock.knowledgeConnector, rows) + queueTableRows(schemaMock.credential, [ + { id: 'old-credential', providerSubjectId: 'alice' }, + { id: 'working-credential', providerSubjectId: 'alice' }, + ]) + queueTableRows(schemaMock.credential, [contentCredential]) + mocks.token.mockImplementation(async ({ credentialId }: { credentialId: string }) => { + if (stall === 'refresh' && credentialId === 'old-credential') return new Promise(() => {}) + return { accessToken: credentialId } + }) + let stalledSignal: AbortSignal | undefined + mocks.fetch.mockImplementation(async (_url: string, options: RequestInit) => { + if (new Headers(options.headers).get('Authorization') === 'Bearer old-credential') { + stalledSignal = options.signal ?? undefined + return new Promise(() => {}) + } + return Response.json({ accountId: 'alice', type: 'known' }) + }) + const grants = await resolveConfluenceSiteReadGrants({ + ...input, + connectorIds: rows.map((row) => row.connectorId), + readers: ['old-credential', 'working-credential'].map((credentialId) => ({ + credentialId, + subjectToken: 's:confluence:-:alice', + })), + }) + expect(grants).toHaveLength(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + expect( + grants.every( + (entry) => + entry.readerCredentialId === 'working-credential' && entry.cloudId === 'cloud-1' + ) + ).toBe(true) + expect(grants.map((entry) => entry.connectorId)).toContain('source-399') + expect(mocks.fetch).toHaveBeenCalledTimes(stall === 'refresh' ? 1 : 2) + expect(mocks.token).toHaveBeenCalledTimes(2) + if (stall === 'site request') expect(stalledSignal?.aborted).toBe(true) + } + ) + + it('bounds same-subject alternatives without trying unrelated or unlimited credentials', async () => { + const credentials = Array.from( + { length: CONFLUENCE_READ_CREDENTIAL_ALTERNATIVES + 2 }, + (_, index) => ({ id: `reader-${index}`, providerSubjectId: 'alice' }) + ) + queueTableRows(schemaMock.knowledgeConnector, [source]) + queueTableRows(schemaMock.credential, credentials) + queueTableRows(schemaMock.credential, [contentCredential]) + mocks.fetch.mockImplementation(async () => new Response(null, { status: 403 })) + await expect( + resolveConfluenceSiteReadGrants({ + ...input, + connectorIds: [source.connectorId], + readers: credentials.map((reader) => ({ + credentialId: reader.id, + subjectToken: 's:confluence:-:alice', + })), + }) + ).resolves.toEqual([]) + expect(mocks.fetch).toHaveBeenCalledTimes(CONFLUENCE_READ_CREDENTIAL_ALTERNATIVES) + expect(mocks.token).toHaveBeenCalledTimes(CONFLUENCE_READ_CREDENTIAL_ALTERNATIVES) + }) + + it('keeps concurrent credential alternatives bounded while later sites still make progress', async () => { + const rows = Array.from({ length: 6 }, (_, index) => ({ + ...source, + connectorId: `source-${index}`, + contentCredentialId: `crawler-${index}`, + })) + queueTableRows(schemaMock.knowledgeConnector, rows) + queueTableRows(schemaMock.credential, [ + { id: 'old-credential', providerSubjectId: 'alice' }, + { id: 'working-credential', providerSubjectId: 'alice' }, + ]) + queueTableRows( + schemaMock.credential, + rows.map((row, index) => ({ id: row.contentCredentialId, key: `cloud-${index}` })) + ) + mocks.decrypt.mockImplementation(async (key: string) => ({ + decrypted: JSON.stringify({ ...binding, cloudId: key }), + })) + mocks.token.mockImplementation(async ({ credentialId }: { credentialId: string }) => ({ + accessToken: credentialId, + })) + let active = 0 + let peak = 0 + mocks.fetch.mockImplementation(async (_url: string, options: RequestInit) => { + active += 1 + peak = Math.max(peak, active) + if (new Headers(options.headers).get('Authorization') === 'Bearer old-credential') + return new Promise((_resolve, reject) => + options.signal?.addEventListener( + 'abort', + () => { + active -= 1 + reject(options.signal?.reason) + }, + { once: true } + ) + ) + await Promise.resolve() + active -= 1 + return Response.json({ accountId: 'alice', type: 'known' }) + }) + const grants = await resolveConfluenceSiteReadGrants({ + ...input, + connectorIds: rows.map((row) => row.connectorId), + readers: ['old-credential', 'working-credential'].map((credentialId) => ({ + credentialId, + subjectToken: 's:confluence:-:alice', + })), + }) + expect(grants).toHaveLength(6) + expect(grants.map((entry) => entry.connectorId)).toContain('source-5') + expect(grants.every((entry) => entry.readerCredentialId === 'working-credential')).toBe(true) + expect(peak).toBeLessThanOrEqual(CONFLUENCE_READ_CONCURRENCY) + expect(active).toBe(0) + }) + + it('rejects oversized provider responses', async () => { + queueSources() + mocks.fetch.mockResolvedValueOnce( + new Response('x'.repeat(CONFLUENCE_READ_RESPONSE_MAX_BYTES + 1)) + ) + await expect(resolveConfluenceSiteReadGrants(input)).resolves.toEqual([]) + }) + + it.each(['source', 'admission'])( + 'preserves completed sites when a stalled %s deadline expires', + async (deadline) => { + const overall = new AbortController() + const timers: AbortController[] = [] + vi.spyOn(AbortSignal, 'timeout').mockImplementation((duration) => { + if (duration === CONFLUENCE_READ_TIMEOUT_MS) return overall.signal + expect(duration).toBe(CONFLUENCE_READ_ATTEMPT_TIMEOUT_MS) + const timer = new AbortController() + timers.push(timer) + return timer.signal + }) + const count = CONFLUENCE_READ_CONCURRENCY + 2 + queueSources( + Array.from({ length: count }, (_, index) => ({ + ...source, + connectorId: `source-${index}`, + contentCredentialId: `crawler-${index}`, + })), + Array.from({ length: count }, (_, index) => ({ + id: `crawler-${index}`, + key: `cloud-${index}`, + })) + ) + mocks.decrypt.mockImplementation(async (key: string) => ({ + decrypted: JSON.stringify({ ...binding, cloudId: key }), + })) + let lastFastCheck: (() => void) | undefined + const allFastChecks = new Promise((resolve) => { + lastFastCheck = resolve + }) + let active = 0 + let peak = 0 + mocks.fetch.mockImplementation(async (url: string) => { + active += 1 + peak = Math.max(peak, active) + if (url.includes('/cloud-0/')) return new Promise(() => {}) + await Promise.resolve() + active -= 1 + if (url.includes(`/cloud-${count - 1}/`)) lastFastCheck?.() + return Response.json({ accountId: 'alice', type: 'known' }) + }) + const pending = resolveConfluenceSiteReadGrants(input) + await allFastChecks + for (let turn = 0; turn < 20; turn++) await Promise.resolve() + ;(deadline === 'source' ? timers[0] : overall).abort(new Error('deadline')) + const grants = await pending + expect(grants).toHaveLength(count - 1) + expect(grants.map((entry) => entry.connectorId)).not.toContain('source-0') + expect(peak).toBeLessThanOrEqual(CONFLUENCE_READ_CONCURRENCY) + } + ) + + it('stops on caller cancellation while a reader token refresh remains pending', async () => { + queueSources() + const controller = new AbortController() + mocks.token.mockImplementation(() => { + controller.abort(new Error('cancelled')) + return new Promise(() => {}) + }) + await expect( + resolveConfluenceSiteReadGrants({ ...input, signal: controller.signal }) + ).rejects.toThrow('cancelled') + expect(mocks.fetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/access/confluence-site.ts b/apps/sim/lib/knowledge/access/confluence-site.ts new file mode 100644 index 00000000000..afd80d523d5 --- /dev/null +++ b/apps/sim/lib/knowledge/access/confluence-site.ts @@ -0,0 +1,366 @@ +import { db } from '@sim/db' +import { credential, knowledgeBase, knowledgeConnector } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { isPlainRecord } from '@sim/utils/object' +import { and, asc, eq, inArray, isNull, sql } from 'drizzle-orm' +import { normalizeAtlassianSiteUrl } from '@/lib/atlassian/discovery' +import { type ResourceScope, resourceScopeFields } from '@/lib/core/resource-scope' +import { resourceScopeCondition } from '@/lib/core/resource-scope.server' +import { decryptSecret } from '@/lib/core/security/encryption' +import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { resolveManagedOAuthToken } from '@/lib/credentials/managed-oauth' +import { confluenceSubjectToken } from '@/lib/knowledge/access/confluence-permissions' +import { + type ConfluenceSiteReadGrant, + MAX_KNOWLEDGE_ACCESS_CANDIDATES, +} from '@/lib/knowledge/access/types' +import { + ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID, + ATLASSIAN_SERVICE_ACCOUNT_SECRET_TYPE, +} from '@/lib/oauth/types' + +const logger = createLogger('ConfluenceSiteReadAccess') +export const CONFLUENCE_READ_CONCURRENCY = 4 +export const CONFLUENCE_READ_TIMEOUT_MS = 8000 +export const CONFLUENCE_READ_ATTEMPT_TIMEOUT_MS = 4000 +export const CONFLUENCE_READ_CREDENTIAL_ALTERNATIVES = 4 +const CONFLUENCE_READ_ALTERNATIVE_CONCURRENCY = 2 +export const CONFLUENCE_READ_RESPONSE_MAX_BYTES = 64 * 1024 +const SITE_BINDING_MAX_BYTES = 32 * 1024 + +export interface ConfluenceReaderCredential { + credentialId: string + subjectToken: string +} + +interface ConfluenceReadSource { + connectorId: string + contentCredentialId: string | null + domain: string | null +} + +interface SiteBinding { + cloudId: string + domain: string +} + +interface BoundConfluenceSource { + connectorId: string + contentCredentialId: string + domain: string + cloudId: string +} + +interface ConfluenceReaderIdentity { + accountId: string + subjectToken: string + credentialIds: string[] +} + +interface ConfirmedConfluenceReader { + credentialId: string + subjectToken: string +} + +/** A pending token refresh must not hold the caller after its authorization deadline. */ +function withinAdmission(pending: Promise, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const abort = () => reject(signal.reason) + signal.addEventListener('abort', abort, { once: true }) + pending.then(resolve, reject).finally(() => signal.removeEventListener('abort', abort)) + if (signal.aborted) abort() + }) +} + +async function readSiteBinding(key: string): Promise { + const { decrypted } = await decryptSecret(key) + const value: unknown = JSON.parse(decrypted) + if ( + !isPlainRecord(value) || + value.type !== ATLASSIAN_SERVICE_ACCOUNT_SECRET_TYPE || + typeof value.cloudId !== 'string' || + !/^[A-Za-z0-9_-]{1,100}$/.test(value.cloudId) || + typeof value.domain !== 'string' || + value.domain.length > 255 + ) + throw new Error('Confluence site binding is invalid') + return { cloudId: value.cloudId, domain: value.domain } +} + +/** Space grants do not prove that the reader still has Confluence's site-level Can use permission. */ +async function verifySite( + binding: SiteBinding, + accountId: string, + accessToken: string, + signal: AbortSignal +): Promise { + signal.throwIfAborted() + const response = await fetch( + `https://api.atlassian.com/ex/confluence/${binding.cloudId}/wiki/rest/api/user/current`, + { + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + redirect: 'error', + cache: 'no-store', + signal, + } + ) + if (!response.ok) { + await response.body?.cancel() + return false + } + const profile = await readResponseJsonWithLimit(response, { + maxBytes: CONFLUENCE_READ_RESPONSE_MAX_BYTES, + label: 'Confluence site authorization response', + signal, + }) + return isPlainRecord(profile) && profile.accountId === accountId && profile.type === 'known' +} + +/** + * Checks only candidate central sources with credentials already bound to the current reader. + * Site identity comes from the scoped indexing credential, but its token never authenticates + * a reader. Sources, readers, provider proofs, and returned grants each have a 400-row cap. + * Sources share proofs by site and Atlassian subject, with bounded concurrent credential + * alternatives. A subject tries at most four credentials; an admission issues at most 400 + * site requests. Additional site/subject combinations fail closed when those budgets or the + * deadline are exhausted. Proofs and token reuse last only for this admission. + */ +export async function resolveConfluenceSiteReadGrants(input: { + scope: ResourceScope + readers: readonly ConfluenceReaderCredential[] + connectorIds: readonly string[] + knowledgeBaseIds?: readonly string[] + signal?: AbortSignal +}): Promise { + input.signal?.throwIfAborted() + if (input.connectorIds.length > MAX_KNOWLEDGE_ACCESS_CANDIDATES) + throw new Error('Knowledge access candidates must be authorized in bounded pages') + if (!input.readers.length || !input.connectorIds.length || input.knowledgeBaseIds?.length === 0) + return [] + const readers = new Map( + input.readers + .slice(0, MAX_KNOWLEDGE_ACCESS_CANDIDATES) + .map((reader) => [reader.credentialId, reader.subjectToken]) + ) + const sources: ConfluenceReadSource[] = await db + .select({ + connectorId: knowledgeConnector.id, + contentCredentialId: knowledgeConnector.credentialId, + domain: sql`left(${knowledgeConnector.sourceConfig}->>'domain', 256)`, + }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + resourceScopeCondition(knowledgeBase, input.scope), + inArray(knowledgeConnector.id, [...new Set(input.connectorIds)]), + input.knowledgeBaseIds ? inArray(knowledgeBase.id, [...input.knowledgeBaseIds]) : undefined, + isNull(knowledgeBase.deletedAt), + eq(knowledgeConnector.connectorType, 'confluence'), + eq(knowledgeConnector.accessMode, 'admin'), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .orderBy(asc(knowledgeConnector.id)) + .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + if (!sources.length) return [] + const readerCredentials = await db + .select({ id: credential.id, providerSubjectId: credential.providerSubjectId }) + .from(credential) + .where( + and( + resourceScopeCondition(credential, input.scope), + inArray(credential.id, [...readers.keys()]), + eq(credential.type, 'managed_oauth'), + eq(credential.providerId, 'confluence'), + eq(credential.managedOauthStatus, 'active'), + isNull(credential.revokedAt) + ) + ) + .orderBy(asc(credential.id)) + .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + if (!readerCredentials.length) return [] + const contentCredentialIds = [ + ...new Set( + sources.flatMap((source) => (source.contentCredentialId ? [source.contentCredentialId] : [])) + ), + ] + if (!contentCredentialIds.length) return [] + const credentials = await db + .select({ id: credential.id, key: credential.encryptedServiceAccountKey }) + .from(credential) + .where( + and( + inArray(credential.id, contentCredentialIds), + resourceScopeCondition(credential, input.scope), + eq(credential.type, 'service_account'), + eq(credential.providerId, ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID), + isNull(credential.revokedAt), + sql`octet_length(${credential.encryptedServiceAccountKey}) <= ${SITE_BINDING_MAX_BYTES}` + ) + ) + .limit(MAX_KNOWLEDGE_ACCESS_CANDIDATES) + const timeout = AbortSignal.timeout(CONFLUENCE_READ_TIMEOUT_MS) + const admissionSignal = input.signal ? AbortSignal.any([input.signal, timeout]) : timeout + const bindings = new Map() + for (const content of credentials) { + if (!content.key || admissionSignal.aborted) continue + try { + bindings.set(content.id, await withinAdmission(readSiteBinding(content.key), admissionSignal)) + } catch { + logger.warn('Confluence Search site binding is unavailable', { credentialId: content.id }) + } + } + const boundSources: BoundConfluenceSource[] = [] + const sites = new Map() + for (const source of sources) { + if (!source.contentCredentialId || !source.domain || source.domain.length > 255) continue + const binding = bindings.get(source.contentCredentialId) + if ( + !binding || + normalizeAtlassianSiteUrl(binding.domain) !== normalizeAtlassianSiteUrl(source.domain) + ) + continue + sites.set(binding.cloudId, binding) + boundSources.push({ + connectorId: source.connectorId, + contentCredentialId: source.contentCredentialId, + domain: source.domain, + cloudId: binding.cloudId, + }) + } + const identities = new Map() + for (const reader of readerCredentials) { + if (!reader.providerSubjectId) continue + try { + const subjectToken = confluenceSubjectToken(reader.providerSubjectId) + if (readers.get(reader.id) !== subjectToken) continue + let identity = identities.get(subjectToken) + if (!identity) { + identity = { accountId: reader.providerSubjectId, subjectToken, credentialIds: [] } + identities.set(subjectToken, identity) + } + if (identity.credentialIds.length < CONFLUENCE_READ_CREDENTIAL_ALTERNATIVES) + identity.credentialIds.push(reader.id) + } catch { + logger.warn('Confluence Search reader identity is unavailable', { credentialId: reader.id }) + } + } + const siteList = [...sites.values()] + const identityList = [...identities.values()] + const proofCount = Math.min( + MAX_KNOWLEDGE_ACCESS_CANDIDATES, + siteList.length * identityList.length + ) + const tokens = new Map>() + const confirmed = new Map() + let requests = 0 + let nextProof = 0 + const attempt = async ( + binding: SiteBinding, + identity: ConfluenceReaderIdentity, + credentialId: string, + cancellation: AbortSignal + ): Promise => { + const signal = AbortSignal.any([ + admissionSignal, + cancellation, + AbortSignal.timeout(CONFLUENCE_READ_ATTEMPT_TIMEOUT_MS), + ]) + signal.throwIfAborted() + let token = tokens.get(credentialId) + if (!token) { + const tokenSignal = AbortSignal.any([ + admissionSignal, + AbortSignal.timeout(CONFLUENCE_READ_ATTEMPT_TIMEOUT_MS), + ]) + token = withinAdmission( + resolveManagedOAuthToken({ + credentialId, + ...resourceScopeFields(input.scope), + expectedProviderId: 'confluence', + requiredScopes: ['read:confluence-user'], + }).then(({ accessToken }) => accessToken), + tokenSignal + ) + tokens.set(credentialId, token) + } + const accessToken = await withinAdmission(token, signal) + signal.throwIfAborted() + if (requests >= MAX_KNOWLEDGE_ACCESS_CANDIDATES) + throw new Error('Confluence read verification budget exhausted') + requests += 1 + if ( + !(await withinAdmission(verifySite(binding, identity.accountId, accessToken, signal), signal)) + ) + throw new Error('Confluence did not confirm current site access') + return { credentialId, subjectToken: identity.subjectToken } + } + const verifyIdentity = async (binding: SiteBinding, identity: ConfluenceReaderIdentity) => { + for ( + let offset = 0; + offset < identity.credentialIds.length && !admissionSignal.aborted; + offset += CONFLUENCE_READ_ALTERNATIVE_CONCURRENCY + ) { + const cancellation = new AbortController() + try { + return await Promise.any( + identity.credentialIds + .slice(offset, offset + CONFLUENCE_READ_ALTERNATIVE_CONCURRENCY) + .map((credentialId) => attempt(binding, identity, credentialId, cancellation.signal)) + ) + } catch { + if (requests >= MAX_KNOWLEDGE_ACCESS_CANDIDATES) return undefined + } finally { + cancellation.abort() + } + } + return undefined + } + const worker = async () => { + while (nextProof < proofCount && !admissionSignal.aborted) { + const index = nextProof++ + const siteIndex = index % siteList.length + /** Rotate subjects across sites before revisiting either dimension under the proof budget. */ + const identityIndex = (Math.floor(index / siteList.length) + siteIndex) % identityList.length + const binding = siteList[siteIndex] + const reader = await verifyIdentity(binding, identityList[identityIndex]) + if (reader) { + const siteReaders = confirmed.get(binding.cloudId) ?? [] + siteReaders.push(reader) + confirmed.set(binding.cloudId, siteReaders) + } + } + } + await Promise.all( + Array.from( + { + length: Math.min( + proofCount, + CONFLUENCE_READ_CONCURRENCY / CONFLUENCE_READ_ALTERNATIVE_CONCURRENCY + ), + }, + worker + ) + ) + input.signal?.throwIfAborted() + const grants: ConfluenceSiteReadGrant[] = [] + for ( + let round = 0; + round < identityList.length && grants.length < MAX_KNOWLEDGE_ACCESS_CANDIDATES; + round++ + ) { + for (const source of boundSources) { + const reader = confirmed.get(source.cloudId)?.[round] + if (!reader) continue + grants.push({ + ...source, + readerCredentialId: reader.credentialId, + readerSubjectToken: reader.subjectToken, + }) + if (grants.length === MAX_KNOWLEDGE_ACCESS_CANDIDATES) break + } + } + return grants +} diff --git a/apps/sim/lib/knowledge/access/predicate.postgres.test.ts b/apps/sim/lib/knowledge/access/predicate.postgres.test.ts index 286d8199904..0cab491d968 100644 --- a/apps/sim/lib/knowledge/access/predicate.postgres.test.ts +++ b/apps/sim/lib/knowledge/access/predicate.postgres.test.ts @@ -5,7 +5,10 @@ import { readFile } from 'node:fs/promises' import type postgres from 'postgres' import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' import { createEnterpriseSearchMigrationFixture } from '@/lib/knowledge/__integration__/migration-fixture' -import type { GitHubInstallationReadGrant } from '@/lib/knowledge/access/types' +import type { + ConfluenceSiteReadGrant, + GitHubInstallationReadGrant, +} from '@/lib/knowledge/access/types' vi.unmock('drizzle-orm') vi.unmock('@sim/db/schema') @@ -82,10 +85,17 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => { documentId: string, join = false, githubInstallationGrants?: GitHubInstallationReadGrant[], - userId = 'reader' + userId = 'reader', + confluenceSiteGrants?: ConfluenceSiteReadGrant[] ): Promise { const query = new PgDialect().sqlToQuery( - knowledgeAccessCondition({ kind: 'user', userId, tokens, githubInstallationGrants }) + knowledgeAccessCondition({ + kind: 'user', + userId, + tokens, + githubInstallationGrants, + confluenceSiteGrants, + }) ) const values = query.params.map((value: unknown) => { if (typeof value === 'string' || typeof value === 'number') return value @@ -206,6 +216,154 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => { expect(await readable([token], 'github-document', true)).toBe(false) }) + it('requires current Confluence site proof as well as space and inherited page grants', async () => { + const token = 's:confluence:-:cf-alice' + await connection.unsafe(` + INSERT INTO organization(id) VALUES ('cf-org'); + INSERT INTO "user"(id,email,email_verified) VALUES ('cf-reader','cf-alice@example.com',true), ('cf-bob','cf-bob@example.com',true); + INSERT INTO member VALUES ('cf-alice-membership','cf-org','cf-reader'), ('cf-bob-membership','cf-org','cf-bob'); + INSERT INTO knowledge_base(id,organization_id,name,is_search_index) VALUES ('cf-index','cf-org','Search',true); + INSERT INTO credential_group(id,organization_id,name,status,options) + VALUES ('cf-group','cf-org','Confluence','active','[{"id":"cf-option","status":"active"}]'); + INSERT INTO credential_group_enrollment(id,credential_group_id,email,status,user_id) + VALUES ('cf-enrollment','cf-group','cf-alice@example.com','completed','cf-reader'); + INSERT INTO credential(id,organization_id,type,provider_id,encrypted_service_account_key) + VALUES ('cf-crawler','cf-org','service_account','atlassian-service-account','encrypted'); + INSERT INTO credential(id,organization_id,type,provider_id,provider_subject_id,authorization_app_id, + managed_oauth_status,granted_scopes,encrypted_oauth_token_set,granted_at,credential_group_enrollment_id,credential_group_option_id) + VALUES ('cf-personal','cf-org','managed_oauth','confluence','cf-alice','cf-app','active', + ARRAY['read:confluence-user'],'encrypted',now(),'cf-enrollment','cf-option'); + INSERT INTO knowledge_connector(id,knowledge_base_id,connector_type,access_mode,credential_id,source_config) + VALUES ('cf-source','cf-index','confluence','admin','cf-crawler','{"domain":"company.atlassian.net"}'); + INSERT INTO document(id,knowledge_base_id,connector_id,acl,acl_requirements,acl_verified_at) + VALUES ('cf-document','cf-index','cf-source',ARRAY['${token}'],'[["${token}"]]',now()); + INSERT INTO embedding(id,document_id,content) VALUES ('cf-chunk','cf-document','restricted content'); + `) + const grant = { + connectorId: 'cf-source', + contentCredentialId: 'cf-crawler', + readerCredentialId: 'cf-personal', + readerSubjectToken: token, + domain: 'company.atlassian.net', + cloudId: 'cloud-1', + } + const check = (grants?: ConfluenceSiteReadGrant[], userId = 'cf-reader', join = false) => + readable([token], 'cf-document', join, undefined, userId, grants) + for (const join of [false, true]) { + expect(await check(undefined, 'cf-reader', join)).toBe(false) + expect(await check([grant], 'cf-reader', join)).toBe(true) + expect(await check([grant], 'cf-bob', join)).toBe(false) + for (const changed of [ + { ...grant, connectorId: 'other-source' }, + { ...grant, contentCredentialId: 'other-crawler' }, + { ...grant, readerCredentialId: 'other-reader' }, + { ...grant, readerSubjectToken: 's:confluence:-:cf-bob' }, + { ...grant, domain: 'another.atlassian.net' }, + ]) + expect(await check([changed], 'cf-reader', join)).toBe(false) + } + await connection.unsafe( + `UPDATE document SET acl_requirements='[["s:confluence:-:cf-bob"]]' WHERE id='cf-document'` + ) + expect(await check([grant])).toBe(false) + await connection.unsafe( + `UPDATE document SET acl_requirements='[["${token}"]]' WHERE id='cf-document'` + ) + const mutations = [ + [ + "UPDATE credential SET revoked_at=now() WHERE id='cf-personal'", + "UPDATE credential SET revoked_at=NULL WHERE id='cf-personal'", + ], + [ + "UPDATE credential SET managed_oauth_status='revoked' WHERE id='cf-personal'", + "UPDATE credential SET managed_oauth_status='active' WHERE id='cf-personal'", + ], + [ + "UPDATE credential SET provider_subject_id='cf-bob' WHERE id='cf-personal'", + "UPDATE credential SET provider_subject_id='cf-alice' WHERE id='cf-personal'", + ], + [ + "UPDATE credential_group_enrollment SET revoked_at=now() WHERE id='cf-enrollment'", + "UPDATE credential_group_enrollment SET revoked_at=NULL WHERE id='cf-enrollment'", + ], + [ + "UPDATE credential_group SET status='archived' WHERE id='cf-group'", + "UPDATE credential_group SET status='active' WHERE id='cf-group'", + ], + [ + "UPDATE credential_group SET options='[]' WHERE id='cf-group'", + `UPDATE credential_group SET options='[{"id":"cf-option","status":"active"}]' WHERE id='cf-group'`, + ], + [ + "UPDATE credential SET revoked_at=now() WHERE id='cf-crawler'", + "UPDATE credential SET revoked_at=NULL WHERE id='cf-crawler'", + ], + [ + "UPDATE credential SET provider_id='other-provider' WHERE id='cf-crawler'", + "UPDATE credential SET provider_id='atlassian-service-account' WHERE id='cf-crawler'", + ], + [ + `UPDATE knowledge_connector SET source_config='{"domain":"another.atlassian.net"}' WHERE id='cf-source'`, + `UPDATE knowledge_connector SET source_config='{"domain":"company.atlassian.net"}' WHERE id='cf-source'`, + ], + [ + "UPDATE knowledge_connector SET credential_id=NULL WHERE id='cf-source'", + "UPDATE knowledge_connector SET credential_id='cf-crawler' WHERE id='cf-source'", + ], + [ + "DELETE FROM member WHERE id='cf-alice-membership'", + "INSERT INTO member VALUES ('cf-alice-membership','cf-org','cf-reader')", + ], + ] + for (const [revoke, restore] of mutations) { + await connection.unsafe(revoke) + expect(await check([grant], 'cf-reader', true)).toBe(false) + await connection.unsafe(restore) + expect(await check([grant], 'cf-reader', true)).toBe(true) + } + const secondToken = 's:confluence:-:cf-bob' + const groupToken = 'g:confluence:cloud-1:restricted-group' + await connection.unsafe(` + INSERT INTO credential(id,organization_id,type,provider_id,provider_subject_id,authorization_app_id, + managed_oauth_status,granted_scopes,encrypted_oauth_token_set,granted_at,credential_group_enrollment_id,credential_group_option_id) + VALUES ('cf-second-personal','cf-org','managed_oauth','confluence','cf-bob','cf-app','active', + ARRAY['read:confluence-user'],'encrypted',now(),'cf-enrollment','cf-option'); + INSERT INTO knowledge_external_group(id,organization_id,provider_id,tenant_id,external_group_id,last_synced_at) + VALUES ('cf-external-group','cf-org','confluence','cloud-1','restricted-group',now()); + INSERT INTO knowledge_external_group_member(group_id,subject_token) VALUES ('cf-external-group','${secondToken}'); + UPDATE document SET acl=ARRAY['${secondToken}'], acl_requirements='[]' WHERE id='cf-document'; + `) + const secondGrant = { + ...grant, + readerCredentialId: 'cf-second-personal', + readerSubjectToken: secondToken, + } + const mixedIdentityRead = (grants: ConfluenceSiteReadGrant[]) => + readable( + [token, secondToken, groupToken], + 'cf-document', + true, + undefined, + 'cf-reader', + grants + ) + expect(await mixedIdentityRead([grant])).toBe(false) + expect(await mixedIdentityRead([secondGrant])).toBe(true) + await connection.unsafe(`UPDATE document SET acl=ARRAY['${groupToken}'] WHERE id='cf-document'`) + expect(await mixedIdentityRead([grant])).toBe(false) + expect(await mixedIdentityRead([secondGrant])).toBe(true) + expect(await mixedIdentityRead([{ ...secondGrant, cloudId: 'another-cloud' }])).toBe(false) + await connection.unsafe( + `UPDATE document SET acl_requirements='[["${token}"]]' WHERE id='cf-document'` + ) + expect(await mixedIdentityRead([grant, secondGrant])).toBe(false) + await connection.unsafe(`UPDATE document SET acl_requirements='[]' WHERE id='cf-document'`) + await connection.unsafe( + "UPDATE knowledge_external_group SET last_synced_at=now()-interval '25 hours' WHERE id='cf-external-group'" + ) + expect(await mixedIdentityRead([secondGrant])).toBe(false) + }) + it('revokes every source of one integration without changing ACLs or another organization', async () => { await connection.unsafe("INSERT INTO organization(id) VALUES ('approval-org'), ('other-org')") await connection.unsafe( diff --git a/apps/sim/lib/knowledge/access/predicate.test.ts b/apps/sim/lib/knowledge/access/predicate.test.ts index 044ccc3e9b8..2ff74b38a51 100644 --- a/apps/sim/lib/knowledge/access/predicate.test.ts +++ b/apps/sim/lib/knowledge/access/predicate.test.ts @@ -38,6 +38,31 @@ describe('knowledgeAccessCondition', () => { for (const param of params) expect(Array.isArray(param)).toBe(false) }) + it('binds central Confluence evidence to scalar source, crawler, reader, subject, and site values', () => { + const { sql, params } = render( + knowledgeAccessCondition({ + kind: 'user', + userId: 'user-1', + tokens: ['s:confluence:-:alice'], + confluenceSiteGrants: [ + { + connectorId: 'source-1', + contentCredentialId: 'crawler-1', + readerCredentialId: 'reader-1', + readerSubjectToken: 's:confluence:-:alice', + domain: 'company.atlassian.net', + cloudId: 'cloud-1', + }, + ], + }) + ) + expect(sql).toContain('confluence_read_grant') + expect(params).toEqual( + expect.arrayContaining(['source-1', 'crawler-1', 'reader-1', 'company.atlassian.net']) + ) + for (const param of params) expect(Array.isArray(param)).toBe(false) + }) + it('renders the workspace pair for actorless callers', () => { const { sql, params } = render( knowledgeAccessCondition({ kind: 'workspace', tokens: ['pub', 'ws'] }) diff --git a/apps/sim/lib/knowledge/access/predicate.ts b/apps/sim/lib/knowledge/access/predicate.ts index fb46dd48f86..8de2cb47f79 100644 --- a/apps/sim/lib/knowledge/access/predicate.ts +++ b/apps/sim/lib/knowledge/access/predicate.ts @@ -7,14 +7,96 @@ import { knowledgeConnector, knowledgeConnectorMember, knowledgeDocumentObservation, + knowledgeExternalGroup, + knowledgeExternalGroupMember, member, user, } from '@sim/db/schema' import { type SQL, sql } from 'drizzle-orm' +import { EXTERNAL_GROUP_STALE_AFTER_MS } from '@/lib/knowledge/access/external-groups' import { SOURCE_ACL_MAX_AGE_MS } from '@/lib/knowledge/access/freshness' import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/access/types' import { searchIntegrationAccessCondition } from '@/lib/knowledge/search/integration-policy' import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' +import { ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID } from '@/lib/oauth/types' + +/** Every Confluence clause must match the same confirmed reader, including that reader's groups. */ +function confluenceReaderClause(hasToken: (token: SQL) => SQL): SQL { + return sql`(${hasToken(sql`confluence_read_grant.reader_subject_token`)} OR EXISTS ( + SELECT 1 FROM ${knowledgeExternalGroup} + JOIN ${knowledgeExternalGroupMember} ON ${knowledgeExternalGroupMember.groupId} = ${knowledgeExternalGroup.id} + WHERE ${knowledgeExternalGroupMember.subjectToken} = confluence_read_grant.reader_subject_token + AND ${knowledgeExternalGroup.providerId} = 'confluence' + AND ${knowledgeExternalGroup.tenantId} = confluence_read_grant.cloud_id + AND ${knowledgeExternalGroup.organizationId} IS NOT DISTINCT FROM ${knowledgeBase.organizationId} + AND ${knowledgeExternalGroup.workspaceId} IS NOT DISTINCT FROM ${knowledgeBase.workspaceId} + AND ${knowledgeExternalGroup.lastSyncedAt} >= statement_timestamp() - (${EXTERNAL_GROUP_STALE_AFTER_MS} * interval '1 millisecond') + AND ${hasToken(sql`('g:confluence:' || confluence_read_grant.cloud_id || ':' || ${knowledgeExternalGroup.externalGroupId})`)} + ))` +} + +/** A cached space grant cannot substitute for the reader's current Confluence site access. */ +function confluenceSiteAccessCondition(scope: KnowledgeAccessScope): SQL { + const grants = scope.kind === 'user' ? (scope.confluenceSiteGrants ?? []) : [] + const allowed = + scope.kind !== 'user' || grants.length === 0 + ? sql`false` + : sql`EXISTS ( + SELECT 1 FROM (VALUES ${sql.join( + grants.map( + (grant) => sql`( + ${grant.connectorId}, ${grant.contentCredentialId}, ${grant.readerCredentialId}, ${grant.readerSubjectToken}, ${grant.domain}, ${grant.cloudId} + )` + ), + sql`, ` + )}) AS confluence_read_grant(connector_id, content_credential_id, reader_credential_id, reader_subject_token, domain, cloud_id) + JOIN ${knowledgeBase} ON ${knowledgeBase.id} = ${knowledgeConnector.knowledgeBaseId} + JOIN ${credential} ON ${credential.id} = confluence_read_grant.content_credential_id + WHERE confluence_read_grant.connector_id = ${knowledgeConnector.id} + AND confluence_read_grant.content_credential_id = ${knowledgeConnector.credentialId} + AND confluence_read_grant.domain = ${knowledgeConnector.sourceConfig}->>'domain' + AND ${confluenceReaderClause((token) => sql`${token} = ANY(${document.acl})`)} + AND NOT EXISTS ( + SELECT 1 FROM jsonb_array_elements(${document.aclRequirements}) AS confluence_required_clause(tokens) + WHERE NOT ${confluenceReaderClause((token) => sql`confluence_required_clause.tokens ? ${token}`)} + ) + AND ${knowledgeConnector.archivedAt} IS NULL AND ${knowledgeConnector.deletedAt} IS NULL + AND ${knowledgeBase.deletedAt} IS NULL + AND ${credential.type} = 'service_account' + AND ${credential.providerId} = ${ATLASSIAN_SERVICE_ACCOUNT_PROVIDER_ID} + AND ${credential.revokedAt} IS NULL + AND ${credential.organizationId} IS NOT DISTINCT FROM ${knowledgeBase.organizationId} + AND ${credential.workspaceId} IS NOT DISTINCT FROM ${knowledgeBase.workspaceId} + AND (${knowledgeBase.organizationId} IS NULL OR EXISTS ( + SELECT 1 FROM ${member} WHERE ${member.organizationId} = ${knowledgeBase.organizationId} + AND ${member.userId} = ${scope.userId} + )) + AND EXISTS ( + SELECT 1 FROM ${credential} + JOIN ${credentialGroupEnrollment} ON ${credentialGroupEnrollment.id} = ${credential.credentialGroupEnrollmentId} + JOIN ${credentialGroup} ON ${credentialGroup.id} = ${credentialGroupEnrollment.credentialGroupId} + JOIN ${user} ON ${user.id} = ${scope.userId} + WHERE ${credential.id} = confluence_read_grant.reader_credential_id + AND ${credential.type} = 'managed_oauth' AND ${credential.providerId} = 'confluence' + AND ${credential.managedOauthStatus} = 'active' AND ${credential.revokedAt} IS NULL + AND ('s:confluence:' || COALESCE(NULLIF(${credential.providerTenantId}, ''), '-') || ':' || ${credential.providerSubjectId}) = confluence_read_grant.reader_subject_token + AND ${credential.organizationId} IS NOT DISTINCT FROM ${knowledgeBase.organizationId} + AND ${credential.workspaceId} IS NOT DISTINCT FROM ${knowledgeBase.workspaceId} + AND ${credentialGroup.organizationId} IS NOT DISTINCT FROM ${knowledgeBase.organizationId} + AND ${credentialGroup.workspaceId} IS NOT DISTINCT FROM ${knowledgeBase.workspaceId} + AND ${credentialGroup.status} = 'active' + AND ${credentialGroupEnrollment.status} IN ('in_progress', 'completed') + AND ${credentialGroupEnrollment.revokedAt} IS NULL + AND ${user.emailVerified} = true + AND ((${knowledgeBase.organizationId} IS NOT NULL AND ${credentialGroupEnrollment.userId} = ${scope.userId}) + OR (${knowledgeBase.workspaceId} IS NOT NULL AND ${credentialGroupEnrollment.email} = lower(btrim(${user.email})))) + AND EXISTS (SELECT 1 FROM jsonb_array_elements(${credentialGroup.options}) AS option + WHERE option->>'id' = ${credential.credentialGroupOptionId} AND option->>'status' = 'active') + ) + )` + return sql`(${knowledgeConnector.connectorType} IS DISTINCT FROM 'confluence' + OR ${knowledgeConnector.accessMode} <> 'admin' OR ${allowed})` +} /** Missing credentials or missing live evidence must never downgrade an installation source. */ function githubInstallationAccessCondition(scope: KnowledgeAccessScope): SQL { @@ -99,7 +181,9 @@ function githubInstallationAccessCondition(scope: KnowledgeAccessScope): SQL { export function knowledgeAccessCondition(scope: KnowledgeAccessScope | SystemAccessScope): SQL { return storedKnowledgeAccessCondition( scope, - scope.kind === 'system' ? sql`true` : githubInstallationAccessCondition(scope) + scope.kind === 'system' + ? sql`true` + : sql`(${githubInstallationAccessCondition(scope)} AND ${confluenceSiteAccessCondition(scope)})` ) } diff --git a/apps/sim/lib/knowledge/access/scope.test.ts b/apps/sim/lib/knowledge/access/scope.test.ts index d1f936e2004..bb46c448f15 100644 --- a/apps/sim/lib/knowledge/access/scope.test.ts +++ b/apps/sim/lib/knowledge/access/scope.test.ts @@ -6,10 +6,16 @@ import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@s import { eq, inArray } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockAvailability, mockCheckWorkspaceAccess, mockGitHubReadGrants } = vi.hoisted(() => ({ +const { + mockAvailability, + mockCheckWorkspaceAccess, + mockGitHubReadGrants, + mockConfluenceReadGrants, +} = vi.hoisted(() => ({ mockAvailability: vi.fn(async () => ({ memberScoped: true, sourceMirrored: true })), mockCheckWorkspaceAccess: vi.fn(async () => ({ hasAccess: true })), mockGitHubReadGrants: vi.fn(async () => []), + mockConfluenceReadGrants: vi.fn(async () => []), })) vi.mock('@/lib/knowledge/access/availability', () => ({ @@ -18,12 +24,16 @@ vi.mock('@/lib/knowledge/access/availability', () => ({ vi.mock('@/lib/workspaces/permissions/utils', () => ({ checkWorkspaceAccess: mockCheckWorkspaceAccess, })) +vi.mock('@/lib/knowledge/access/confluence-site', () => ({ + resolveConfluenceSiteReadGrants: mockConfluenceReadGrants, +})) vi.mock('@/lib/knowledge/access/github-installation', () => ({ resolveGitHubInstallationReadGrants: mockGitHubReadGrants, })) import { createKnowledgeAccessProvider, + createUserKnowledgeAccessProvider, resolveKnowledgeAccessScope, WORKSPACE_ACCESS_SCOPE, } from '@/lib/knowledge/access/scope' @@ -484,6 +494,71 @@ describe('organization document ACL scope', () => { }) expect(scope).toMatchObject({ githubInstallationGrants: [] }) }) + it('checks only the enrolled Confluence reader after ranking canonical document candidates', async () => { + queueTableRows(schemaMock.member, [{ id: 'membership-1' }]) + queueSubjects([ + { + email: 'viewer@example.com', + credentialId: 'personal-confluence', + providerId: 'confluence', + providerSubjectId: 'alice', + providerTenantId: null, + }, + ]) + const provider = createKnowledgeAccessProvider(SESSION, { + ...organization, + knowledgeBaseIds: ['index-1'], + }) + expect(await provider.get()).not.toHaveProperty('confluenceSiteGrants') + expect(mockConfluenceReadGrants).not.toHaveBeenCalled() + queueTableRows(schemaMock.document, [{ connectorId: 'confluence-source' }]) + expect(await provider.getForDocuments(['selected-document'])).toMatchObject({ + confluenceSiteGrants: [], + }) + expect(mockConfluenceReadGrants).toHaveBeenCalledWith({ + scope: { kind: 'organization', organizationId: 'org-1' }, + readers: [{ credentialId: 'personal-confluence', subjectToken: 's:confluence:-:alice' }], + knowledgeBaseIds: ['index-1'], + connectorIds: ['confluence-source'], + signal: undefined, + }) + expect(mockGitHubReadGrants).not.toHaveBeenCalled() + }) + it('preserves candidate admission for an already-authenticated personal-key or session user', async () => { + queueTableRows(schemaMock.member, [{ id: 'membership-1' }]) + queueSubjects([ + { + credentialId: 'personal-confluence', + providerId: 'confluence', + providerSubjectId: 'alice', + providerTenantId: null, + }, + ]) + const provider = createUserKnowledgeAccessProvider('user-1', organization) + expect(await provider.get()).toMatchObject({ kind: 'user', userId: 'user-1' }) + expect(mockConfluenceReadGrants).not.toHaveBeenCalled() + await provider.getForConnectors(['source-1']) + expect(mockConfluenceReadGrants).toHaveBeenCalledWith( + expect.objectContaining({ + connectorIds: ['source-1'], + readers: [{ credentialId: 'personal-confluence', subjectToken: 's:confluence:-:alice' }], + }) + ) + }) + it('cannot check a retained Confluence connection after organization removal', async () => { + queueTableRows(schemaMock.member, []) + queueSubjects([ + { + credentialId: 'personal-confluence', + providerId: 'confluence', + providerSubjectId: 'alice', + providerTenantId: null, + }, + ]) + const provider = createKnowledgeAccessProvider(SESSION, organization) + expect(await provider.getForConnectors(['confluence-source'])).toMatchObject({ tokens: [] }) + expect(mockConfluenceReadGrants).not.toHaveBeenCalled() + }) it('does not live-check retained provider credentials after organization removal', async () => { queueTableRows(schemaMock.member, []) queueSubjects([ diff --git a/apps/sim/lib/knowledge/access/scope.ts b/apps/sim/lib/knowledge/access/scope.ts index 63ad6d25dd3..0fa1a187299 100644 --- a/apps/sim/lib/knowledge/access/scope.ts +++ b/apps/sim/lib/knowledge/access/scope.ts @@ -20,6 +20,10 @@ import { type ResourceScope, resourceScopeFromOwner } from '@/lib/core/resource- import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { LIVE_ENROLLMENT_STATUSES } from '@/lib/credential-groups/credentials' import { resolveKnowledgeAccessAvailability } from '@/lib/knowledge/access/availability' +import { + type ConfluenceReaderCredential, + resolveConfluenceSiteReadGrants, +} from '@/lib/knowledge/access/confluence-site' import { domainMemberWildcard, EXTERNAL_GROUP_STALE_AFTER_MS, @@ -145,7 +149,11 @@ export interface KnowledgeAccessScopeContext { async function loadUserAccess( userId: string, context: KnowledgeAccessScopeContext -): Promise<{ tokens: readonly string[]; githubReaders?: GitHubReaderCredential[] }> { +): Promise<{ + tokens: readonly string[] + githubReaders?: GitHubReaderCredential[] + confluenceReaders?: ConfluenceReaderCredential[] +}> { const { workspaceId, organizationId } = context const scope = resourceScopeFromOwner(context) const baseline = organizationId ? ORGANIZATION_ACCESS_TOKENS : WORKSPACE_ACCESS_TOKENS @@ -235,6 +243,7 @@ async function loadUserAccess( const identityTokens = new Set() const githubReaders: GitHubReaderCredential[] = [] + const confluenceReaders: ConfluenceReaderCredential[] = [] for (const row of rows) { if (!availability.memberScoped || !row.providerSubjectId) continue try { @@ -242,6 +251,8 @@ async function loadUserAccess( identityTokens.add(token) if (row.providerId === 'github-repositories' && row.credentialId) githubReaders.push({ credentialId: row.credentialId, subjectToken: token }) + if (row.providerId === 'confluence' && row.credentialId) + confluenceReaders.push({ credentialId: row.credentialId, subjectToken: token }) } catch (error) { logger.warn('Skipping malformed managed credential subject', { userId, @@ -275,6 +286,7 @@ async function loadUserAccess( return { tokens: sortAccessTokens(new Set([...baseline, ...identityTokens])), githubReaders, + confluenceReaders, } } @@ -295,7 +307,11 @@ export async function resolveKnowledgeAccessScope( async function resolveKnowledgeIdentity( principal: Principal, context: KnowledgeAccessScopeContext -): Promise<{ access: KnowledgeAccessScope; githubReaders: readonly GitHubReaderCredential[] }> { +): Promise<{ + access: KnowledgeAccessScope + githubReaders: readonly GitHubReaderCredential[] + confluenceReaders: readonly ConfluenceReaderCredential[] +}> { if (principal.kind === 'credential_group_enrollment') { throw new OrchestrationError( 'forbidden', @@ -307,12 +323,22 @@ async function resolveKnowledgeIdentity( if (subject?.kind !== 'sim_user') { if (context.organizationId) throw new OrchestrationError('forbidden', 'Organization search requires a user subject') - return { access: WORKSPACE_ACCESS_SCOPE, githubReaders: [] } + return { access: WORKSPACE_ACCESS_SCOPE, githubReaders: [], confluenceReaders: [] } } - const { tokens, githubReaders = [] } = await loadUserAccess(subject.userId, context) + return resolveUserKnowledgeIdentity(subject.userId, context) +} + +async function resolveUserKnowledgeIdentity(userId: string, context: KnowledgeAccessScopeContext) { + resourceScopeFromOwner(context) + const { + tokens, + githubReaders = [], + confluenceReaders = [], + } = await loadUserAccess(userId, context) return { - access: { kind: 'user', userId: subject.userId, tokens }, + access: { kind: 'user' as const, userId, tokens }, githubReaders, + confluenceReaders, } } @@ -333,10 +359,25 @@ export async function resolveUserKnowledgeAccessScope( export function createKnowledgeAccessProvider( principal: Principal, context: KnowledgeAccessScopeContext +): KnowledgeAccessProvider { + return createAccessProvider(() => resolveKnowledgeIdentity(principal, context), context) +} + +/** Candidate access for a user already authenticated by a session or personal-key adapter. */ +export function createUserKnowledgeAccessProvider( + userId: string, + context: KnowledgeAccessScopeContext +): KnowledgeAccessProvider { + return createAccessProvider(() => resolveUserKnowledgeIdentity(userId, context), context) +} + +function createAccessProvider( + resolveIdentity: () => ReturnType, + context: KnowledgeAccessScopeContext ): KnowledgeAccessProvider { let pending: ReturnType | undefined const identity = () => { - pending ??= resolveKnowledgeIdentity(principal, context).catch((error: unknown) => { + pending ??= resolveIdentity().catch((error: unknown) => { pending = undefined throw error }) @@ -358,26 +399,41 @@ export function createKnowledgeAccessProvider( ? AbortSignal.any([context.signal, signal]) : (signal ?? context.signal) cancellation?.throwIfAborted() - const { access, githubReaders } = await identity() + const { access, githubReaders, confluenceReaders } = await identity() cancellation?.throwIfAborted() - if (access.kind !== 'user' || !githubReaders.length || !ids.length) return access - return { - ...access, - githubInstallationGrants: await resolveGitHubInstallationReadGrants({ - scope: resourceScopeFromOwner(context), - readers: githubReaders, - knowledgeBaseIds: context.knowledgeBaseIds, - connectorIds: ids, - signal: cancellation, - }), + if ( + access.kind !== 'user' || + (!githubReaders.length && !confluenceReaders.length) || + !ids.length + ) + return access + const input = { + scope: resourceScopeFromOwner(context), + knowledgeBaseIds: context.knowledgeBaseIds, + connectorIds: ids, + signal: cancellation, } + const [githubInstallationGrants, confluenceSiteGrants] = await Promise.all([ + githubReaders.length + ? resolveGitHubInstallationReadGrants({ ...input, readers: githubReaders }) + : Promise.resolve([]), + confluenceReaders.length + ? resolveConfluenceSiteReadGrants({ ...input, readers: confluenceReaders }) + : Promise.resolve([]), + ]) + return { ...access, githubInstallationGrants, confluenceSiteGrants } }, async getForDocuments(documentIds, signal) { const ids = boundedIds(documentIds) signal?.throwIfAborted() context.signal?.throwIfAborted() - const { access, githubReaders } = await identity() - if (access.kind !== 'user' || !githubReaders.length || !ids.length) return access + const { access, githubReaders, confluenceReaders } = await identity() + if ( + access.kind !== 'user' || + (!githubReaders.length && !confluenceReaders.length) || + !ids.length + ) + return access const candidates = await db .select({ connectorId: document.connectorId }) .from(document) diff --git a/apps/sim/lib/knowledge/access/types.ts b/apps/sim/lib/knowledge/access/types.ts index 400486649a9..88995f2d3eb 100644 --- a/apps/sim/lib/knowledge/access/types.ts +++ b/apps/sim/lib/knowledge/access/types.ts @@ -41,6 +41,17 @@ export interface UserAccessScope { tokens: readonly string[] /** Live user-token evidence, scoped to the installation source's immutable repository. */ githubInstallationGrants?: readonly GitHubInstallationReadGrant[] + /** Current reader access to the immutable Confluence site behind a central crawl. */ + confluenceSiteGrants?: readonly ConfluenceSiteReadGrant[] +} + +export interface ConfluenceSiteReadGrant { + cloudId: string + connectorId: string + contentCredentialId: string + readerCredentialId: string + readerSubjectToken: string + domain: string } export interface GitHubInstallationReadGrant { diff --git a/apps/sim/lib/knowledge/application/connectors.test.ts b/apps/sim/lib/knowledge/application/connectors.test.ts index c2183e73dda..b631ac8fb93 100644 --- a/apps/sim/lib/knowledge/application/connectors.test.ts +++ b/apps/sim/lib/knowledge/application/connectors.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ -import { document, member } from '@sim/db/schema' +import { document, knowledgeConnector, member } from '@sim/db/schema' import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' @@ -27,6 +27,10 @@ const mocks = vi.hoisted(() => ({ provision: vi.fn(), decryptApiKey: vi.fn(), requireApproval: vi.fn(), + resolveWorkspace: vi.fn(), + viewerMemberships: vi.fn(), + getAccess: vi.fn(), + getForConnectors: vi.fn(), })) vi.mock('@sim/audit', () => ({ @@ -57,6 +61,7 @@ vi.mock('@sim/platform-authz/workspace', () => ({ })) vi.mock('@/lib/knowledge/application/contexts', () => ({ + resolveKnowledgeWorkspaceContext: mocks.resolveWorkspace, resolveActiveKnowledgeBaseContext: mocks.resolveKnowledgeBase, resolveActiveKnowledgeResourceContext: mocks.resolveKnowledgeBase, resolveActiveKnowledgeConnectorContext: mocks.resolveConnector, @@ -67,11 +72,18 @@ vi.mock('@/lib/knowledge/orchestration/connector-access', () => ({ })) vi.mock('@/lib/knowledge/connectors/member-provisioning', () => ({ provisionKnowledgeConnectorMembersBinding: mocks.provision, - resolveViewerConnectorMemberships: async () => new Map(), + resolveViewerConnectorMemberships: mocks.viewerMemberships, })) vi.mock('@/lib/knowledge/connectors/mirrored-access', () => ({ assertConnectorMirrorsSourceAcls: async () => undefined, })) +vi.mock('@/lib/knowledge/access/scope', () => ({ + WORKSPACE_ACCESS_SCOPE: { kind: 'workspace', tokens: ['pub', 'ws'] }, + createKnowledgeAccessProvider: () => ({ + get: mocks.getAccess, + getForConnectors: mocks.getForConnectors, + }), +})) vi.mock('@/lib/knowledge/access/availability', () => ({ requireKnowledgeMemberAccessAvailable: async () => undefined, })) @@ -145,6 +157,7 @@ import { createKnowledgeConnector, deleteKnowledgeConnector, listKnowledgeConnectorDocuments, + listWorkspaceMemberConnectors, resolveConnectorCredentialAccessToken, syncKnowledgeConnector, updateKnowledgeConnector, @@ -223,6 +236,8 @@ describe('knowledge connector application use cases', () => { vi.clearAllMocks() resetDbChainMock() mocks.resolvePermission.mockResolvedValue('write') + mocks.resolveWorkspace.mockResolvedValue(crossWorkspaceContext) + mocks.viewerMemberships.mockResolvedValue(new Map()) mocks.resolveKnowledgeBase.mockResolvedValue(crossWorkspaceContext) mocks.resolveConnector.mockResolvedValue(connectorContext) mocks.getCredentialActorContext.mockResolvedValue({ @@ -246,6 +261,55 @@ describe('knowledge connector application use cases', () => { afterAll(resetDbChainMock) + it('counts workspace central Confluence documents only after candidate site admission', async () => { + const identity = { + kind: 'user' as const, + userId: 'reader', + tokens: ['ws', 's:confluence:-:alice'], + } + mocks.getAccess.mockResolvedValue(identity) + mocks.getForConnectors.mockResolvedValue({ + ...identity, + confluenceSiteGrants: [ + { + connectorId: 'cf-source', + contentCredentialId: 'crawler', + readerCredentialId: 'personal', + readerSubjectToken: 's:confluence:-:alice', + domain: 'company.atlassian.net', + cloudId: 'cloud-1', + }, + ], + }) + mocks.viewerMemberships.mockResolvedValue(new Map([['cf-source', 'connected']])) + queueTableRows(knowledgeConnector, [ + { + id: 'cf-source', + knowledgeBaseId: 'knowledge-b', + knowledgeBaseName: 'Search', + knowledgeBaseIsSearchIndex: true, + connectorType: 'confluence', + accessMode: 'admin', + sourceConfig: { domain: 'company.atlassian.net', spaceKey: ['DEMO'] }, + memberSyncStatus: 'idle', + }, + ]) + queueTableRows(document, []) + queueTableRows(document, [{ connectorId: 'cf-source' }]) + queueTableRows(document, [{ connectorId: 'cf-source', count: 2 }]) + const result = await listWorkspaceMemberConnectors.execute({ + principal: { kind: 'session', userId: 'reader', sessionId: 'test' }, + input: { workspaceId: 'workspace-b' }, + }) + expect(mocks.getForConnectors).toHaveBeenCalledWith(['cf-source'], undefined) + expect(result.connectors).toEqual([ + expect.objectContaining({ connectorId: 'cf-source', viewerDocumentCount: 2 }), + ]) + expect(JSON.stringify(dbChainMockFns.where.mock.calls.at(-1)?.[0])).toContain( + 'confluence_read_grant' + ) + }) + it('rejects a forged OAuth credential for central Drive creation before using its token', async () => { mocks.resolvePermission.mockResolvedValue('admin') mocks.resolveKnowledgeBase.mockResolvedValue({ diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index 5eb12fbef2c..ab8f4a2ff5a 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -31,7 +31,6 @@ import { resolveCredentialTokenIdentity } from '@/lib/credentials/access' import { requireKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' -import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveKnowledgeAttributedUserId, @@ -86,6 +85,7 @@ import type { KnowledgeOperationSource, KnowledgeOrchestrationResult, } from '@/lib/knowledge/orchestration/shared' +import { type KnowledgeReadAccess, knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' import { requireOrganizationSearchApproval } from '@/lib/knowledge/search/integration-policy' import { escapeLikePattern } from '@/lib/knowledge/tags/utils' import { isMemberSyncStatus } from '@/lib/knowledge/types' @@ -491,23 +491,28 @@ export interface ListWorkspaceMemberConnectorsInput { /** Live documents per connector that the viewer's tokens match, for the Search tab's counts. */ async function countViewerDocuments( connectorIds: readonly string[], - access: KnowledgeAccessScope + access: KnowledgeReadAccess ): Promise> { - if (connectorIds.length === 0) return new Map() - const rows = await db - .select({ connectorId: document.connectorId, count: sql`count(*)::int` }) - .from(document) - .where( - and( - inArray(document.connectorId, [...connectorIds]), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - knowledgeAccessCondition(access) - ) - ) - .groupBy(document.connectorId) - return new Map(rows.flatMap((row) => (row.connectorId ? [[row.connectorId, row.count]] : []))) + const counts = new Map() + if (connectorIds.length === 0) return counts + const conditions = [ + inArray(document.connectorId, [...connectorIds]), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + ] + for await (const accessCondition of knowledgeReadAccessBatches(access, conditions)) { + const rows = await db + .select({ connectorId: document.connectorId, count: sql`count(*)::int` }) + .from(document) + .where(and(...conditions, accessCondition)) + .groupBy(document.connectorId) + for (const row of rows) { + if (row.connectorId) + counts.set(row.connectorId, (counts.get(row.connectorId) ?? 0) + row.count) + } + } + return counts } /** Live workspace sources that let the viewer connect a crawl account or a mirrored-ACL identity. */ @@ -564,7 +569,7 @@ export const listWorkspaceMemberConnectors = defineAuthorizedKnowledgeUseCase({ }), countViewerDocuments( rows.map((row) => row.id), - await createKnowledgeAccessProvider(principal, { workspaceId: context.workspaceId }).get() + createKnowledgeAccessProvider(principal, { workspaceId: context.workspaceId }) ), ]) return { diff --git a/apps/sim/lib/knowledge/application/documents.test.ts b/apps/sim/lib/knowledge/application/documents.test.ts index 751ea4c7426..dfb43acb1e4 100644 --- a/apps/sim/lib/knowledge/application/documents.test.ts +++ b/apps/sim/lib/knowledge/application/documents.test.ts @@ -14,6 +14,8 @@ const mocks = vi.hoisted(() => ({ resolveSystemBilling: vi.fn(), checkUsage: vi.fn(), getDocuments: vi.fn(), + bulkDocumentOperation: vi.fn(), + bulkDocumentOperationByFilter: vi.fn(), createDocument: vi.fn(), deleteDocument: vi.fn(), updateDocument: vi.fn(), @@ -68,6 +70,8 @@ vi.mock('@/lib/knowledge/application/contexts', () => ({ vi.mock('@/lib/knowledge/documents/service', () => ({ getDocuments: mocks.getDocuments, + bulkDocumentOperation: mocks.bulkDocumentOperation, + bulkDocumentOperationByFilter: mocks.bulkDocumentOperationByFilter, createSingleDocument: mocks.createDocument, createDocumentRecords: mocks.createDocumentRecords, deleteDocument: mocks.deleteDocumentById, @@ -79,6 +83,10 @@ vi.mock('@/lib/knowledge/documents/service', () => ({ vi.mock('@/lib/knowledge/tags/service', () => ({ getDocumentTagDefinitions: mocks.getDocumentTagDefinitions, + getDocumentTagDefinitionsByKnowledgeBaseIds: async (ids: string[]) => + new Map( + await Promise.all(ids.map(async (id) => [id, await mocks.getDocumentTagDefinitions(id)])) + ), })) vi.mock('@/lib/knowledge/orchestration/documents', () => ({ @@ -106,6 +114,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' import { bulkDeleteKnowledgeDocuments, + bulkUpdateKnowledgeDocuments, createKnowledgeDocuments, deleteKnowledgeDocument, listKnowledgeDocuments, @@ -340,6 +349,90 @@ describe('knowledge document application use cases', () => { expect(mocks.resolvePermission.mock.invocationCallOrder[0]).toBeLessThan( mocks.getDocuments.mock.invocationCallOrder[0] ) + expect(mocks.getDocuments).toHaveBeenCalledWith( + 'knowledge-1', + expect.any(Object), + expect.any(String), + knowledgeAccess + ) + }) + + it.each([true, false])( + 'retains live workspace access for bulk document selection (all=%s)', + async (selectAll) => { + const result = { success: true, successCount: 1, updatedDocuments: [{ id: 'document-1' }] } + mocks.bulkDocumentOperation.mockResolvedValue(result) + mocks.bulkDocumentOperationByFilter.mockResolvedValue(result) + await bulkUpdateKnowledgeDocuments.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + operation: 'enable', + selectAll, + documentIds: ['document-1'], + }, + }) + if (selectAll) { + expect(mocks.bulkDocumentOperationByFilter).toHaveBeenCalledWith( + 'knowledge-1', + 'enable', + undefined, + knowledgeAccess, + expect.any(String) + ) + } else { + expect(mocks.bulkDocumentOperation).toHaveBeenCalledWith( + 'knowledge-1', + 'enable', + ['document-1'], + knowledgeAccess, + expect.any(String) + ) + } + } + ) + + it('finds a workspace upsert replacement after live candidate authorization', async () => { + const scope = { kind: 'user' as const, userId: 'reader', tokens: ['reader-token'] } + const getForConnectors = vi.fn().mockResolvedValue(scope) + const getForDocuments = vi.fn().mockResolvedValue(scope) + mocks.resolveKnowledgeBase.mockResolvedValue({ + ...context, + access: { + get: async () => scope, + getForConnectors, + getForDocuments, + }, + }) + queueTableRows(schemaMock.document, []) + queueTableRows(schemaMock.document, [{ connectorId: 'confluence-source' }]) + queueTableRows(schemaMock.document, [{ id: 'existing-1' }]) + const result = await upsertKnowledgeDocument.execute({ + principal: { kind: 'session', userId: 'reader', sessionId: 'session-1' }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + resolveBillingAttribution: async () => ({ + actorUserId: 'reader', + workspaceId: 'workspace-1', + }), + resolveSecretProvenances: () => undefined, + }, + }) + expect(result).toMatchObject({ isUpdate: true, previousDocumentId: 'existing-1' }) + expect(getForConnectors).toHaveBeenCalledExactlyOnceWith(['confluence-source'], undefined) + expect(getForDocuments).toHaveBeenCalledExactlyOnceWith(['existing-1']) + expect(mocks.deleteDocument).toHaveBeenCalledWith( + 'knowledge-1', + 'existing-1', + expect.any(String), + scope + ) }) it('resolves current workspace-key billing while retaining key audit attribution', async () => { diff --git a/apps/sim/lib/knowledge/application/documents.ts b/apps/sim/lib/knowledge/application/documents.ts index 90e9e5f04cd..dc584acb7ac 100644 --- a/apps/sim/lib/knowledge/application/documents.ts +++ b/apps/sim/lib/knowledge/application/documents.ts @@ -329,7 +329,7 @@ export const listKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ tagFilters: tagFilters.length > 0 ? tagFilters : undefined, }, generateRequestId(), - context.organizationId ? context.access : await context.access.get() + context.access ) return { ...result, @@ -682,7 +682,7 @@ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ ] let existingDocumentId: string | null = null for await (const accessCondition of knowledgeReadAccessBatches( - context.organizationId ? context.access : await context.access.get(), + context.access, lookupConditions )) { const [existing] = await db @@ -1027,7 +1027,7 @@ export const bulkUpdateKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ context.knowledgeBaseId, input.operation, input.enabledFilter, - context.organizationId ? context.access : await context.access.get(), + context.access, generateRequestId() ) : input.documentIds?.length @@ -1035,7 +1035,7 @@ export const bulkUpdateKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ context.knowledgeBaseId, input.operation, input.documentIds, - context.organizationId ? context.access : await context.access.get(), + context.access, generateRequestId() ) : null diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts index c4ecf6a6a05..a0167a92c25 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.test.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.test.ts @@ -12,6 +12,7 @@ const mocks = vi.hoisted(() => ({ resolveArchivedKnowledgeBase: vi.fn(), resolvePermission: vi.fn(), resolveAccess: vi.fn(), + createAccessProvider: vi.fn(), attachConnectors: vi.fn(), resolveFolderPath: vi.fn(), createRecord: vi.fn(), @@ -49,7 +50,7 @@ vi.mock('@sim/platform-authz/workspace', () => ({ })) vi.mock('@/lib/knowledge/access/scope', () => ({ - resolveKnowledgeAccessScope: mocks.resolveAccess, + createKnowledgeAccessProvider: mocks.createAccessProvider, })) vi.mock('@/lib/core/telemetry', () => ({ @@ -143,6 +144,7 @@ describe('knowledge base application use cases', () => { beforeEach(() => { vi.clearAllMocks() mocks.resolveAccess.mockResolvedValue({ kind: 'workspace', tokens: ['workspace', 'public'] }) + mocks.createAccessProvider.mockReturnValue({ get: mocks.resolveAccess }) mocks.attachConnectors.mockImplementation(async (kb) => kb) mocks.resolveWorkspace.mockResolvedValue(context) mocks.loadAuthorizationWorkspace.mockResolvedValue(context) @@ -226,7 +228,7 @@ describe('knowledge base application use cases', () => { { forUpdate: undefined } ) expect(mocks.listRecords).toHaveBeenCalledWith('workspace-1', 'archived', { - access: expect.objectContaining({ kind: 'workspace' }), + access: expect.objectContaining({ get: mocks.resolveAccess }), }) }) @@ -247,7 +249,7 @@ describe('knowledge base application use cases', () => { ).resolves.toEqual({ knowledgeBases: [knowledgeBase] }) expect(mocks.listRecords).toHaveBeenCalledWith('workspace-1', 'active', { - access: expect.objectContaining({ kind: 'workspace' }), + access: expect.objectContaining({ get: mocks.resolveAccess }), }) }) @@ -674,7 +676,7 @@ describe('knowledge base application use cases', () => { }) expect(mocks.listRecords).toHaveBeenCalledWith('workspace-1', 'archived', { - access: expect.objectContaining({ kind: 'workspace' }), + access: expect.objectContaining({ get: mocks.resolveAccess }), folderId: undefined, search: 'docs', sortBy: 'updatedAt', diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.ts b/apps/sim/lib/knowledge/application/knowledge-bases.ts index 95e7be71eda..76fea95ae96 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.ts @@ -16,7 +16,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { loadActiveFolderPathIndex, resolveFolderPathFilter } from '@/lib/folders/queries' -import { resolveKnowledgeAccessScope } from '@/lib/knowledge/access/scope' +import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' import { knowledgeDelegationPolicy } from '@/lib/knowledge/application/authorization' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { @@ -277,7 +277,7 @@ async function executeListKnowledgeBases(args: { sortOrder: args.input.sortOrder, limit: args.input.limit, cursorKeys: args.input.cursorKeys, - access: await resolveKnowledgeAccessScope(args.principal, args.context), + access: createKnowledgeAccessProvider(args.principal, args.context), }) return { knowledgeBases: page.data.map((knowledgeBase) => ({ @@ -349,7 +349,7 @@ async function executeReadKnowledgeBase(args: { return { knowledgeBase: await attachKnowledgeBaseConnectors( args.context.knowledgeBase, - await args.context.access.get() + args.context.access ), folderPath: knowledgeFolderPathForId(index, args.context.knowledgeBase.folderId), } @@ -389,10 +389,7 @@ async function executeUpdateKnowledgeBase(args: { knowledgeBaseId: knowledgeBase.id, }) return { - knowledgeBase: await attachKnowledgeBaseConnectors( - knowledgeBase, - await args.context.access.get() - ), + knowledgeBase: await attachKnowledgeBaseConnectors(knowledgeBase, args.context.access), folderPath: knowledgeFolderPathForId(index, knowledgeBase.folderId), } } @@ -505,7 +502,7 @@ export const restoreKnowledgeBase = defineAuthorizedKnowledgeUseCase({ return { knowledgeBase: await attachKnowledgeBaseConnectors( knowledgeBase, - await resolveKnowledgeAccessScope(principal, context) + createKnowledgeAccessProvider(principal, context) ), folderPath: knowledgeFolderPathForId(index, knowledgeBase.folderId), restored, @@ -549,7 +546,7 @@ export const listInternalKnowledgeBases = { const { data: knowledgeBases } = await getWorkspaceKnowledgeBases( context.workspaceId, input.scope, - { access: await resolveKnowledgeAccessScope(principal, context) } + { access: createKnowledgeAccessProvider(principal, context) } ) return { knowledgeBases } }, @@ -735,7 +732,7 @@ export const readInternalKnowledgeBase = { return { knowledgeBase: await attachKnowledgeBaseConnectors( knowledgeBase, - await resolveKnowledgeAccessScope(principal, { + createKnowledgeAccessProvider(principal, { workspaceId: knowledgeBase.workspaceId, }) ), @@ -794,7 +791,7 @@ export const updateInternalKnowledgeBase = { return { knowledgeBase: await attachKnowledgeBaseConnectors( outcome.knowledgeBase, - await resolveKnowledgeAccessScope(principal, { + createKnowledgeAccessProvider(principal, { workspaceId: outcome.knowledgeBase.workspaceId ?? undefined, }) ), diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index 185e4d64760..abbb70f08f9 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -13,6 +13,7 @@ const mocks = vi.hoisted(() => ({ requireOrganizationSearch: vi.fn(), resolvePermission: vi.fn(), getKnowledgeBase: vi.fn(), + getKnowledgeBases: vi.fn(), resolveBilling: vi.fn(), checkUsage: vi.fn(), checkActorUsage: vi.fn(), @@ -20,6 +21,7 @@ const mocks = vi.hoisted(() => ({ executeSearch: vi.fn(), getDocumentMetadata: vi.fn(), getTagDefinitions: vi.fn(), + getTagDefinitionsBatch: vi.fn(), recordEmbeddingUsage: vi.fn(), importProvenance: vi.fn(), rerank: vi.fn(), @@ -72,7 +74,7 @@ vi.mock('@/lib/permission-groups/resolve.server', () => ({ })) vi.mock('@/lib/knowledge/service', () => ({ - getActiveKnowledgeBaseReference: mocks.getKnowledgeBase, + getActiveKnowledgeBaseReferences: mocks.getKnowledgeBases, })) vi.mock('@/lib/knowledge/embeddings', () => ({ @@ -87,7 +89,7 @@ vi.mock('@/lib/knowledge/search/queries', () => ({ })) vi.mock('@/lib/knowledge/tags/service', () => ({ - getDocumentTagDefinitions: mocks.getTagDefinitions, + getDocumentTagDefinitionsByKnowledgeBaseIds: mocks.getTagDefinitionsBatch, })) vi.mock('@/lib/knowledge/tags/utils', () => ({ @@ -130,6 +132,13 @@ describe('knowledge search application use case', () => { mocks.resolveWorkspace.mockResolvedValue(workspace) mocks.resolvePermission.mockResolvedValue('read') mocks.getKnowledgeBase.mockResolvedValue(knowledgeBase) + mocks.getKnowledgeBases.mockImplementation((ids: string[]) => + Promise.all(ids.map((id) => mocks.getKnowledgeBase(id))) + ) + mocks.getTagDefinitionsBatch.mockImplementation( + async (ids: string[]) => + new Map(await Promise.all(ids.map(async (id) => [id, await mocks.getTagDefinitions(id)]))) + ) mocks.resolveBilling.mockResolvedValue({ actorUserId: 'user-1', workspaceId: 'workspace-1', @@ -554,6 +563,118 @@ describe('knowledge search application use case', () => { expect(mocks.resolveWorkspace).not.toHaveBeenCalled() expect(mocks.getKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.getKnowledgeBases).not.toHaveBeenCalled() + expect(mocks.getTagDefinitionsBatch).not.toHaveBeenCalled() + }) + + it('loads references and tags once for twenty bases while preserving requested order', async () => { + const ids = Array.from({ length: 20 }, (_, index) => `knowledge-${20 - index}`) + mocks.getKnowledgeBases.mockResolvedValue(ids.map((id) => ({ ...knowledgeBase, id }))) + mocks.getTagDefinitionsBatch.mockResolvedValue(new Map(ids.map((id) => [id, []]))) + + const result = await searchKnowledge.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { knowledgeBaseIds: ids, query: 'answer', topK: 5 }, + }) + + expect(mocks.getKnowledgeBases).toHaveBeenCalledExactlyOnceWith(ids) + expect(mocks.getTagDefinitionsBatch).toHaveBeenCalledExactlyOnceWith(ids) + expect(result.knowledgeBaseIds).toEqual(ids) + expect(result.knowledgeBaseId).toBe(ids[0]) + expect(result.knowledgeBases.map((base) => base.id)).toEqual(ids) + expect(mocks.executeSearch).toHaveBeenCalledWith( + expect.objectContaining({ knowledgeBaseIds: ids }) + ) + }) + + it('preserves duplicate requested bases in retrieval and the response', async () => { + const ids = ['knowledge-2', 'knowledge-1', 'knowledge-2'] + mocks.getKnowledgeBases.mockResolvedValue(ids.map((id) => ({ ...knowledgeBase, id }))) + + const result = await searchKnowledge.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { knowledgeBaseIds: ids, query: 'answer', topK: 5 }, + }) + + expect(result.knowledgeBaseIds).toEqual(ids) + expect(mocks.executeSearch).toHaveBeenCalledWith( + expect.objectContaining({ knowledgeBaseIds: ids }) + ) + }) + + it('preserves missing-id order and duplicates in the concealed error before authorization or billing', async () => { + mocks.getKnowledgeBases.mockResolvedValue([null, knowledgeBase, null, null]) + + await expect( + searchKnowledge.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseIds: ['missing-2', 'knowledge-1', 'missing-1', 'missing-2'], + query: 'answer', + topK: 5, + }, + }) + ).rejects.toMatchObject({ + code: 'not_found', + message: 'Knowledge bases not found or access denied: missing-2, missing-1, missing-2', + }) + expect(mocks.resolvePermission).not.toHaveBeenCalled() + expect(mocks.resolveBilling).not.toHaveBeenCalled() + expect(mocks.executeSearch).not.toHaveBeenCalled() + }) + + it('rejects a batch spanning different canonical workspaces before billing', async () => { + mocks.getKnowledgeBases.mockResolvedValue([ + knowledgeBase, + { ...knowledgeBase, id: 'knowledge-2', workspaceId: 'workspace-2' }, + ]) + + await expect( + searchKnowledge.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { knowledgeBaseIds: ['knowledge-1', 'knowledge-2'], query: 'answer', topK: 5 }, + }) + ).rejects.toMatchObject({ + code: 'validation', + message: 'Selected knowledge bases must belong to the same workspace', + }) + expect(mocks.resolveBilling).not.toHaveBeenCalled() + expect(mocks.executeSearch).not.toHaveBeenCalled() + }) + + it('reuses the tag filter batch when naming result metadata', async () => { + const ids = ['knowledge-1', 'knowledge-2'] + mocks.getKnowledgeBases.mockResolvedValue(ids.map((id) => ({ ...knowledgeBase, id }))) + mocks.getTagDefinitionsBatch.mockResolvedValue( + new Map( + ids.map((id) => [ + id, + [{ knowledgeBaseId: id, tagSlot: 'tag1', displayName: 'team', fieldType: 'text' }], + ]) + ) + ) + mocks.executeSearch.mockResolvedValue([ + { + id: 'chunk-1', + documentId: 'document-1', + knowledgeBaseId: ids[0], + content: 'answer', + tag1: 'docs', + }, + ]) + + const result = await searchKnowledge.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseIds: ids, + topK: 5, + tagFilters: [{ tagName: 'team', operator: 'eq', value: 'docs' }], + }, + }) + + expect(mocks.getTagDefinitionsBatch).toHaveBeenCalledExactlyOnceWith(ids) + expect(result.results[0].metadata).toEqual({ team: 'docs' }) + expect(mocks.generateEmbedding).not.toHaveBeenCalled() }) it('rejects multi-knowledge-base tag filters without embedding spend', async () => { @@ -629,7 +750,7 @@ describe('knowledge search application use case', () => { expect(mocks.getDocumentMetadata).toHaveBeenCalledWith( ['document-1'], expect.anything(), - undefined, + expect.objectContaining({ getForDocuments: expect.any(Function) }), undefined ) expect(result.results[0]).toMatchObject({ diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index a47c6f7141c..7e1d6d0da95 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -49,13 +49,13 @@ import { import { importKnowledgeSearchResultSecretProvenance } from '@/lib/knowledge/secret-provenance' import { type ActiveKnowledgeBaseReference, - getActiveKnowledgeBaseReference, + getActiveKnowledgeBaseReferences, } from '@/lib/knowledge/service' import { type KnowledgeTagNameFilter, resolveKnowledgeTagFilters, } from '@/lib/knowledge/tags/filter-resolution' -import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service' +import { getDocumentTagDefinitionsByKnowledgeBaseIds } from '@/lib/knowledge/tags/service' import type { DocumentTagDefinition } from '@/lib/knowledge/tags/types' import type { StructuredFilter } from '@/lib/knowledge/types' import { estimateTokenCount } from '@/lib/tokenization/estimators' @@ -189,9 +189,7 @@ async function resolveKnowledgeSearchContext( `topK must be an integer between 1 and ${KNOWLEDGE_SEARCH_COST_POLICY.maxTopK}` ) } - const knowledgeBases = await Promise.all( - input.knowledgeBaseIds.map(getActiveKnowledgeBaseReference) - ) + const knowledgeBases = await getActiveKnowledgeBaseReferences(input.knowledgeBaseIds) const missingIds = input.knowledgeBaseIds.filter((_, index) => { const knowledgeBase = knowledgeBases[index] return !knowledgeBase || (!knowledgeBase.workspaceId && !knowledgeBase.organizationId) @@ -374,7 +372,7 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ topK: candidateTopK, filters: input.filters, access, - accessProvider: context.organizationId ? context.access : undefined, + accessProvider: context.access, signal: input.signal, searchMode: searchDefaults.searchMode, boostRecency: searchDefaults.boostRecency, @@ -558,18 +556,16 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ } } - const tagDefinitionEntries = await Promise.all( - knowledgeBaseIds.map(async (knowledgeBaseId) => { - const definitions = - definitionsByKnowledgeBase.get(knowledgeBaseId) ?? - (await getDocumentTagDefinitions(knowledgeBaseId)) - return [ - knowledgeBaseId, - new Map(definitions.map((definition) => [definition.tagSlot, definition.displayName])), - ] as const - }) + if (filters.length === 0) { + definitionsByKnowledgeBase = + await getDocumentTagDefinitionsByKnowledgeBaseIds(knowledgeBaseIds) + } + const tagMaps = new Map( + [...definitionsByKnowledgeBase].map(([knowledgeBaseId, definitions]) => [ + knowledgeBaseId, + new Map(definitions.map((definition) => [definition.tagSlot, definition.displayName])), + ]) ) - const tagMaps = new Map(tagDefinitionEntries) /** * Always read: the provenance snapshot vouches for the name, URL, and tags * a model may see, but the source card's modified time and connector type @@ -578,7 +574,7 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ const basicDocumentMetadata = await getDocumentMetadataByIds( rows.map((row) => row.documentId), access, - context.organizationId ? context.access : undefined, + context.access, input.signal ) const results = rows diff --git a/apps/sim/lib/knowledge/application/tags.test.ts b/apps/sim/lib/knowledge/application/tags.test.ts index 19101ed687f..5dbd277113d 100644 --- a/apps/sim/lib/knowledge/application/tags.test.ts +++ b/apps/sim/lib/knowledge/application/tags.test.ts @@ -16,6 +16,7 @@ const mocks = vi.hoisted(() => ({ updateTag: vi.fn(), deleteTag: vi.fn(), readUsage: vi.fn(), + readDetailedUsage: vi.fn(), saveTags: vi.fn(), cleanupTags: vi.fn(), deleteAllTags: vi.fn(), @@ -53,6 +54,7 @@ vi.mock('@/lib/knowledge/tags/service', () => ({ updateTagDefinition: mocks.updateTag, deleteTagDefinition: mocks.deleteTag, getTagUsageStats: mocks.readUsage, + getTagUsage: mocks.readDetailedUsage, normalizeDisplayName: (displayName: string) => displayName.trim().toLowerCase(), createOrUpdateTagDefinitionsBulk: mocks.saveTags, cleanupUnusedTagDefinitions: mocks.cleanupTags, @@ -65,6 +67,7 @@ import { deleteKnowledgeDocumentTagDefinitions, deleteKnowledgeTag, listKnowledgeTags, + readDetailedKnowledgeTagUsage, readKnowledgeTagUsage, readNextKnowledgeTagSlot, saveKnowledgeDocumentTagDefinitions, @@ -72,7 +75,11 @@ import { } from '@/lib/knowledge/application/tags' /** Every mocked context carries the workspace read scope the resolvers would attach. */ -const knowledgeAccess = { get: async () => WORKSPACE_ACCESS_SCOPE } +const knowledgeAccess = { + get: async () => WORKSPACE_ACCESS_SCOPE, + getForConnectors: async () => WORKSPACE_ACCESS_SCOPE, + getForDocuments: async () => WORKSPACE_ACCESS_SCOPE, +} const crossWorkspaceContext = { access: knowledgeAccess, @@ -136,6 +143,18 @@ describe('knowledge tag application use cases', () => { mocks.listTags.mockResolvedValue([]) }) + it('retains live workspace access in summary and detailed tag counts', async () => { + const input = { knowledgeBaseId: 'knowledge-b', assertedWorkspaceId: 'workspace-b' } + await readKnowledgeTagUsage.execute({ principal: sessionPrincipal, input }) + await readDetailedKnowledgeTagUsage.execute({ principal: sessionPrincipal, input }) + expect(mocks.readUsage).toHaveBeenCalledWith('knowledge-b', knowledgeAccess, expect.any(String)) + expect(mocks.readDetailedUsage).toHaveBeenCalledWith( + 'knowledge-b', + expect.any(String), + knowledgeAccess + ) + }) + it.each([ [ 'list', diff --git a/apps/sim/lib/knowledge/application/tags.ts b/apps/sim/lib/knowledge/application/tags.ts index dd8b9a56c8f..9bf78e6f0e9 100644 --- a/apps/sim/lib/knowledge/application/tags.ts +++ b/apps/sim/lib/knowledge/application/tags.ts @@ -355,11 +355,7 @@ export const readKnowledgeTagUsage = defineAuthorizedKnowledgeUseCase({ resolveActiveKnowledgeResourceContext(input, principal), async execute({ context }) { return { - usage: await getTagUsageStats( - context.knowledgeBaseId, - context.organizationId ? context.access : await context.access.get(), - generateRequestId() - ), + usage: await getTagUsageStats(context.knowledgeBaseId, context.access, generateRequestId()), } }, }) @@ -370,11 +366,7 @@ export const readDetailedKnowledgeTagUsage = defineAuthorizedKnowledgeUseCase({ resolveActiveKnowledgeResourceContext(input, principal), async execute({ context }) { return { - usage: await getTagUsage( - context.knowledgeBaseId, - generateRequestId(), - context.organizationId ? context.access : await context.access.get() - ), + usage: await getTagUsage(context.knowledgeBaseId, generateRequestId(), context.access), } }, }) diff --git a/apps/sim/lib/knowledge/documents/document-indexing-usage.test.ts b/apps/sim/lib/knowledge/documents/document-indexing-usage.test.ts index e68b00db64f..437de7bf205 100644 --- a/apps/sim/lib/knowledge/documents/document-indexing-usage.test.ts +++ b/apps/sim/lib/knowledge/documents/document-indexing-usage.test.ts @@ -65,11 +65,13 @@ vi.mock('@/providers/utils', () => ({ })) import * as billingAttribution from '@/lib/billing/core/billing-attribution' +import { resetIngestionUsageGateCache } from '@/lib/billing/core/ingestion-usage-gate' import * as embeddingClient from '@/lib/embeddings/client' import { processDocumentAsync } from '@/lib/knowledge/documents/service' const mockEmbeddingCapacity = vi.fn() beforeEach(() => { + resetIngestionUsageGateCache() vi.spyOn(billingAttribution, 'checkAttributedUsageLimits').mockImplementation( mockCheckAttributedUsageLimits ) diff --git a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts index c3eeff755ca..5b36cbdf3c2 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts @@ -66,6 +66,7 @@ vi.mock('@/lib/uploads/server/metadata', () => ({ })) import * as billingAttribution from '@/lib/billing/core/billing-attribution' +import { resetIngestionUsageGateCache } from '@/lib/billing/core/ingestion-usage-gate' import { env } from '@/lib/core/config/env' import { markInsideTriggerRun, @@ -92,6 +93,7 @@ import { MAX_PROCESSING_ATTEMPTS } from '@/lib/knowledge/documents/types' const mockEmbeddingCapacity = vi.fn() beforeEach(() => { + resetIngestionUsageGateCache() vi.spyOn(billingAttribution, 'checkAttributedUsageLimits').mockImplementation( mockCheckAttributedUsageLimits ) diff --git a/apps/sim/lib/knowledge/documents/service.test.ts b/apps/sim/lib/knowledge/documents/service.test.ts index 974a5e3fdd1..d2cbd4f456c 100644 --- a/apps/sim/lib/knowledge/documents/service.test.ts +++ b/apps/sim/lib/knowledge/documents/service.test.ts @@ -4,7 +4,11 @@ import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' import type { KnowledgeAccessProvider, KnowledgeAccessScope } from '@/lib/knowledge/access/types' -import { getDocuments } from '@/lib/knowledge/documents/service' +import { + getDocuments, + getKnowledgeDocument, + getKnowledgeDocumentById, +} from '@/lib/knowledge/documents/service' beforeEach(() => { vi.clearAllMocks() @@ -52,3 +56,40 @@ describe('getDocuments pagination', () => { expect(result.documents).toMatchObject([{ id: 'newly-readable', filename: 'Visible file' }]) }) }) + +describe('knowledge document detail live authorization', () => { + it.each(['parent', 'canonical'] as const)( + 'authorizes a %s lookup before reading document fields', + async (lookup) => { + const identity: KnowledgeAccessScope = { kind: 'user', userId: 'reader', tokens: [] } + const getForDocuments = vi.fn(async () => { + expect(dbChainMockFns.select).not.toHaveBeenCalled() + return identity + }) + const access: KnowledgeAccessProvider = { + get: async () => identity, + getForConnectors: async () => identity, + getForDocuments, + } + queueTableRows(document, [{ id: 'document-1', filename: 'Readable page' }]) + const result = + lookup === 'parent' + ? await getKnowledgeDocument('knowledge-1', 'document-1', access) + : await getKnowledgeDocumentById('document-1', access) + expect(getForDocuments).toHaveBeenCalledExactlyOnceWith(['document-1']) + expect(result).toMatchObject({ id: 'document-1', filename: 'Readable page' }) + } + ) + + it('does not hydrate a document after source authorization fails', async () => { + const access: KnowledgeAccessProvider = { + get: vi.fn(), + getForConnectors: vi.fn(), + getForDocuments: vi.fn().mockRejectedValue(new Error('reader revoked')), + } + await expect(getKnowledgeDocument('knowledge-1', 'document-1', access)).rejects.toThrow( + 'reader revoked' + ) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 8a555fb1834..baa3a7f4309 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -34,9 +34,9 @@ import { assertBillingAttributionOwner, assertBillingAttributionSnapshot, type BillingAttributionSnapshot, - checkAttributedUsageLimits, toBillingContext, } from '@/lib/billing/core/billing-attribution' +import { checkIngestionUsageLimits } from '@/lib/billing/core/ingestion-usage-gate' import { recordUsage } from '@/lib/billing/core/usage-log' import { applyStorageUsageDeltasInTx, @@ -78,7 +78,6 @@ import { type KnowledgeAccessScope, MAX_KNOWLEDGE_ACCESS_CANDIDATES, SYSTEM_ACCESS_SCOPE, - type SystemAccessScope, } from '@/lib/knowledge/access/types' import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock' import { @@ -1618,7 +1617,7 @@ export async function processDocumentAsync( assertBillingAttributionOwner(billingAttribution, ctx) const documentActorUserId = billingAttribution.actorUserId - const usageGate = await checkAttributedUsageLimits(billingAttribution) + const usageGate = await checkIngestionUsageLimits(billingAttribution) if (usageGate.isExceeded) { logger.warn(`[${documentId}] Usage limit reached — skipping document indexing`) throw new UsageLimitDocumentProcessingError( @@ -2742,8 +2741,9 @@ export type ActiveKnowledgeDocument = typeof document.$inferSelect & { export async function getKnowledgeDocument( knowledgeBaseId: string, documentId: string, - access: KnowledgeAccessScope | SystemAccessScope + access: KnowledgeReadAccess ): Promise { + const scope = 'get' in access ? await access.getForDocuments([documentId]) : access const [row] = await db .select({ ...getTableColumns(document), @@ -2758,7 +2758,7 @@ export async function getKnowledgeDocument( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), - knowledgeAccessCondition(access) + knowledgeAccessCondition(scope) ) ) .limit(1) @@ -2769,8 +2769,9 @@ export async function getKnowledgeDocument( /** Loads one visible document by its canonical ID before any asserted parent is trusted. */ export async function getKnowledgeDocumentById( documentId: string, - access: KnowledgeAccessScope | SystemAccessScope + access: KnowledgeReadAccess ): Promise { + const scope = 'get' in access ? await access.getForDocuments([documentId]) : access const [row] = await db .select({ ...getTableColumns(document), @@ -2784,7 +2785,7 @@ export async function getKnowledgeDocumentById( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), - knowledgeAccessCondition(access) + knowledgeAccessCondition(scope) ) ) .limit(1) diff --git a/apps/sim/lib/knowledge/search/queries-github-discovery.test.ts b/apps/sim/lib/knowledge/search/queries-github-discovery.test.ts new file mode 100644 index 00000000000..977a5997ef7 --- /dev/null +++ b/apps/sim/lib/knowledge/search/queries-github-discovery.test.ts @@ -0,0 +1,162 @@ +/** @vitest-environment node */ +import { + dbChainMockFns, + hasMockCondition, + queueTableRows, + resetDbChainMock, + schemaMock, +} from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' +import { handleTagOnlySearch } from '@/lib/knowledge/search/queries' + +vi.mock('@/lib/knowledge/access/availability', () => ({ + resolveKnowledgeAccessAvailability: async () => ({ memberScoped: true, sourceMirrored: false }), +})) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ checkWorkspaceAccess: vi.fn() })) +vi.mock('@/lib/credentials/managed-oauth', () => ({ resolveManagedOAuthToken: vi.fn() })) +vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: vi.fn() })) +vi.mock('@/lib/oauth/github-installation', () => ({ + parseGitHubInstallationBinding: vi.fn(), + assertGitHubInstallationActive: vi.fn(), + assertGitHubInstallationRepositoryActive: vi.fn(), +})) + +async function createReaderProvider(hasGitHubReader = true) { + queueTableRows(schemaMock.member, [{ id: 'membership-1' }]) + queueTableRows(schemaMock.user, [ + { + credentialId: 'github-reader', + providerId: hasGitHubReader ? 'github-repositories' : 'gmail', + providerSubjectId: '42', + providerTenantId: null, + }, + ]) + const provider = createKnowledgeAccessProvider( + { kind: 'session', userId: 'reader', sessionId: 'session-1' }, + { organizationId: 'org-1', knowledgeBaseIds: ['index-1'] } + ) + const access = await provider.get() + return { provider, access } +} + +describe('GitHub discovery through the request access provider', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it.each([true, false])( + 'skips discovery for explicit Gmail with GitHub reader present: %s', + async (hasGitHubReader) => { + const { provider, access } = await createReaderProvider(hasGitHubReader) + await provider.getForConnectors(['gmail-source']) + expect( + dbChainMockFns.from.mock.calls.filter(([table]) => table === schemaMock.knowledgeConnector) + ).toHaveLength(hasGitHubReader ? 1 : 0) + dbChainMockFns.from.mockClear() + + queueTableRows(schemaMock.embedding, [ + { + id: 'gmail', + documentId: 'gmail-doc', + connectorId: 'gmail-source', + liveAuthorizationSource: false, + }, + ]) + const hydrated = [{ id: 'gmail', content: 'permitted content' }] + queueTableRows(schemaMock.embedding, hydrated) + + expect( + await handleTagOnlySearch({ + knowledgeBaseIds: ['index-1'], + topK: 1, + access, + accessProvider: provider, + filters: { source: 'gmail' }, + structuredFilters: [ + { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'release' }, + ], + }) + ).toEqual(hydrated) + expect(dbChainMockFns.from.mock.calls.map(([table]) => table)).toEqual([ + schemaMock.embedding, + schemaMock.embedding, + ]) + } + ) + + it.each([undefined, '', 'github'])( + 'keeps current discovery for classic GitHub with source: %s', + async (source) => { + const { provider, access } = await createReaderProvider() + queueTableRows(schemaMock.embedding, [ + { + id: 'github', + documentId: 'github-doc', + connectorId: 'classic-github', + liveAuthorizationSource: false, + }, + { + id: 'github-second', + documentId: 'github-doc', + connectorId: 'classic-github', + liveAuthorizationSource: false, + }, + ]) + queueTableRows(schemaMock.embedding, [{ id: 'github', content: 'permitted content' }]) + + await handleTagOnlySearch({ + knowledgeBaseIds: ['index-1'], + topK: 1, + access, + accessProvider: provider, + filters: { source }, + structuredFilters: [ + { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'release' }, + ], + }) + expect( + dbChainMockFns.from.mock.calls.filter(([table]) => table === schemaMock.knowledgeConnector) + ).toHaveLength(1) + expect( + dbChainMockFns.where.mock.calls.some(([condition]) => + hasMockCondition( + condition, + (node) => + node.type === 'inArray' && + node.column === schemaMock.knowledgeConnector.id && + JSON.stringify(node.values) === JSON.stringify(['classic-github']) + ) + ) + ).toBe(true) + } + ) + + it('keeps upload hydration without source discovery', async () => { + const { provider, access } = await createReaderProvider() + dbChainMockFns.from.mockClear() + queueTableRows(schemaMock.embedding, [ + { id: 'upload', documentId: 'upload-doc', connectorId: null, liveAuthorizationSource: false }, + ]) + const hydrated = [{ id: 'upload', content: 'permitted content' }] + queueTableRows(schemaMock.embedding, hydrated) + + expect( + await handleTagOnlySearch({ + knowledgeBaseIds: ['index-1'], + topK: 1, + access, + accessProvider: provider, + filters: { source: 'upload' }, + structuredFilters: [ + { tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'release' }, + ], + }) + ).toEqual(hydrated) + expect(dbChainMockFns.from.mock.calls.map(([table]) => table)).toEqual([ + schemaMock.embedding, + schemaMock.embedding, + ]) + }) +}) diff --git a/apps/sim/lib/knowledge/search/queries.test.ts b/apps/sim/lib/knowledge/search/queries.test.ts index e8edea4491d..bb56e09e2d7 100644 --- a/apps/sim/lib/knowledge/search/queries.test.ts +++ b/apps/sim/lib/knowledge/search/queries.test.ts @@ -497,7 +497,7 @@ describe('live repository authorization follows ranked candidates', () => { id, documentId: `doc-${id}`, connectorId, - installationSource: true, + liveAuthorizationSource: true, distance: 0.1, }) const getForConnectors = vi.fn() @@ -547,7 +547,7 @@ describe('live repository authorization follows ranked candidates', () => { 'id', 'documentId', 'connectorId', - 'installationSource', + 'liveAuthorizationSource', ...(mode === 'keyword' ? ['keywordRank'] : mode === 'tags' ? [] : ['distance']), ].sort() ) @@ -576,6 +576,77 @@ describe('live repository authorization follows ranked candidates', () => { } ) + it.each(['vector', 'tag-vector', 'tags', 'keyword'] as const)( + '%s skips discovery for an explicit non-GitHub source and retains full hydration', + async (mode) => { + getForConnectors.mockResolvedValue(identity) + queueTableRows(schemaMock.embedding, [ + { ...candidate('gmail', 'gmail-source'), installationSource: false }, + ]) + const hydrated = [{ id: 'gmail', content: 'current permitted content' }] + queueTableRows(schemaMock.embedding, hydrated) + const searchParams = { ...params, filters: { source: 'gmail' } } + const rows = + mode === 'vector' + ? await handleVectorOnlySearch(searchParams) + : mode === 'tag-vector' + ? await handleTagAndVectorSearch(searchParams) + : mode === 'tags' + ? await handleTagOnlySearch(searchParams) + : await executeKeywordSearch({ + ...searchParams, + query: 'release', + queryVector: searchParams.queryVector!, + }) + + expect(rows).toEqual(hydrated) + expect(getForConnectors).toHaveBeenCalledExactlyOnceWith([], undefined) + for (const [condition] of dbChainMockFns.where.mock.calls) { + expect(JSON.stringify(condition)).toContain('gmail') + } + const hydration = JSON.stringify(dbChainMockFns.where.mock.calls[1][0]) + expect(hydration).toContain('acl') + expect(hydration).toContain('knowledgeConnectorMember') + expect(dbChainMockFns.select).toHaveBeenCalledTimes(2) + } + ) + + it.each([undefined, '', 'github'])( + 'retains discovery for classic GitHub when source is %s', + async (source) => { + queueTableRows(schemaMock.embedding, [ + { ...candidate('selected', 'allowed-source'), installationSource: false }, + { ...candidate('second', 'allowed-source'), installationSource: false }, + ]) + queueTableRows(schemaMock.embedding, [{ id: 'selected', content: 'verified result' }]) + + expect(await handleTagOnlySearch({ ...params, filters: { source } })).toEqual([ + { id: 'selected', content: 'verified result' }, + ]) + expect(getForConnectors).toHaveBeenCalledExactlyOnceWith(['allowed-source'], undefined) + expect(JSON.stringify(dbChainMockFns.where.mock.calls[1][0])).toContain('github_read_grant') + } + ) + + it('retains every connector in unfiltered mixed pages', async () => { + queueTableRows(schemaMock.embedding, [ + { ...candidate('gmail', 'gmail-source'), installationSource: false }, + { ...candidate('classic', 'classic-source'), installationSource: false }, + candidate('selected', 'allowed-source'), + candidate('selected-second-chunk', 'allowed-source'), + { ...candidate('upload', 'unused'), connectorId: null, installationSource: false }, + ]) + queueTableRows(schemaMock.embedding, [{ id: 'selected', content: 'verified result' }]) + + expect(await handleTagOnlySearch(params)).toEqual([ + { id: 'selected', content: 'verified result' }, + ]) + expect(getForConnectors).toHaveBeenCalledExactlyOnceWith( + ['gmail-source', 'classic-source', 'allowed-source'], + undefined + ) + }) + it('refills after a denied repository instead of letting its matches consume the result limit', async () => { getForConnectors.mockResolvedValueOnce(identity) queueTableRows(schemaMock.embedding, [candidate('denied', 'revoked-source')]) @@ -594,6 +665,43 @@ describe('live repository authorization follows ranked candidates', () => { expect(refillPredicate).toContain('revoked-source') }) + it.each([undefined, 'confluence'])( + 'refills a denied Confluence site under its exact reader proof with source filter %s', + async (source) => { + getForConnectors.mockResolvedValueOnce(identity).mockResolvedValueOnce({ + ...identity, + confluenceSiteGrants: [ + { + connectorId: 'allowed-site', + contentCredentialId: 'crawler', + readerCredentialId: 'confluence-reader', + readerSubjectToken: 's:confluence:-:alice', + domain: 'company.atlassian.net', + cloudId: 'cloud-1', + }, + ], + }) + queueTableRows(schemaMock.embedding, [candidate('denied', 'revoked-site')]) + queueTableRows(schemaMock.embedding, []) + queueTableRows(schemaMock.embedding, [candidate('selected', 'allowed-site')]) + queueTableRows(schemaMock.embedding, [ + { id: 'selected', content: 'authorized Confluence page' }, + ]) + expect(await handleTagOnlySearch({ ...params, filters: { source } })).toEqual([ + { id: 'selected', content: 'authorized Confluence page' }, + ]) + expect(getForConnectors.mock.calls.map(([ids]) => ids)).toEqual([ + ['revoked-site'], + ['allowed-site'], + ]) + expect(dbChainMockFns.offset.mock.calls).toEqual([[0], [0]]) + expect(JSON.stringify(dbChainMockFns.where.mock.calls[2][0])).toContain('revoked-site') + const readPredicate = JSON.stringify(dbChainMockFns.where.mock.calls[3][0]) + expect(readPredicate).toContain('confluence_read_grant') + expect(readPredicate).toContain('confluence-reader') + expect(readPredicate).toContain('company.atlassian.net') + } + ) it('retains a completed authorized result when the next candidate page exhausts its deadline', async () => { vi.useFakeTimers() vi.setSystemTime(new Date(10000)) @@ -612,16 +720,23 @@ describe('live repository authorization follows ranked candidates', () => { expect(getForConnectors).toHaveBeenCalledOnce() }) - it('propagates caller cancellation before content hydration', async () => { - const cancellation = new AbortController() - getForConnectors.mockImplementation(async () => { - cancellation.abort(new Error('Search cancelled')) - return allowed - }) - queueTableRows(schemaMock.embedding, [candidate('selected', 'allowed-source')]) - await expect(handleTagOnlySearch({ ...params, signal: cancellation.signal })).rejects.toThrow( - 'Search cancelled' - ) - expect(dbChainMockFns.select).toHaveBeenCalledOnce() - }) + it.each([undefined, 'gmail'])( + 'propagates caller cancellation before hydration with source %s', + async (source) => { + const cancellation = new AbortController() + getForConnectors.mockImplementation(async () => { + cancellation.abort(new Error('Search cancelled')) + return allowed + }) + queueTableRows(schemaMock.embedding, [candidate('selected', 'allowed-source')]) + await expect( + handleTagOnlySearch({ ...params, filters: { source }, signal: cancellation.signal }) + ).rejects.toThrow('Search cancelled') + expect(getForConnectors).toHaveBeenCalledExactlyOnceWith( + source ? [] : ['allowed-source'], + cancellation.signal + ) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + } + ) }) diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index dbe5d64735b..fc7b69969ab 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -428,7 +428,7 @@ interface SearchReadCandidate { id: string documentId: string connectorId: string | null - installationSource: boolean + liveAuthorizationSource: boolean } /** Only opaque identifiers leave candidate ranking; content stays behind the full read predicate. */ @@ -436,11 +436,15 @@ const SEARCH_READ_CANDIDATE_FIELDS = { id: embedding.id, documentId: document.id, connectorId: document.connectorId, - installationSource: sql`EXISTS ( + liveAuthorizationSource: sql`EXISTS ( SELECT 1 FROM ${knowledgeConnector} WHERE ${knowledgeConnector.id} = ${document.connectorId} - AND ${knowledgeConnector.connectorType} = 'github' - AND ${knowledgeConnector.sourceConfig}::jsonb ? 'githubRepositoryId' + AND ( + (${knowledgeConnector.connectorType} = 'github' + AND ${knowledgeConnector.sourceConfig}::jsonb ? 'githubRepositoryId') + OR (${knowledgeConnector.connectorType} = 'confluence' + AND ${knowledgeConnector.accessMode} = 'admin') + ) )`, } @@ -449,11 +453,12 @@ const LIVE_SEARCH_BUDGET_MS = 8000 /** * Verification follows ranked candidates, never the organization's source order. Denied - * repositories are excluded on refill, so many matches from one revoked source cannot + * sources are excluded on refill, so many matches from one revoked source cannot * consume every result slot. The existing vector tuple budget also bounds candidate work. */ async function selectAuthorizedSearchResults(input: { accessProvider: KnowledgeAccessProvider + filters?: WorkspaceSearchFilters signal?: AbortSignal topK: number selectPage: ( @@ -478,22 +483,33 @@ async function selectAuthorizedSearchResults(input: { const candidates = await input.selectPage(pageSize, offset, [...excludedSources]) if (!candidates.length) break scanned += candidates.length - const connectorIds = [ - ...new Set( - candidates.flatMap((candidate) => (candidate.connectorId ? [candidate.connectorId] : [])) - ), - ] + /** Candidate and hydration queries enforce this source filter; connector types are immutable. */ + const connectorIds = + input.filters?.source && + input.filters.source !== 'github' && + input.filters.source !== 'confluence' + ? [] + : [ + ...new Set( + candidates.flatMap((candidate) => + candidate.connectorId ? [candidate.connectorId] : [] + ) + ), + ] const access = await input.accessProvider.getForConnectors(connectorIds, input.signal) input.signal?.throwIfAborted() const grantedSources = new Set( access.kind === 'user' - ? (access.githubInstallationGrants?.map((grant) => grant.connectorId) ?? []) + ? [ + ...(access.githubInstallationGrants?.map((grant) => grant.connectorId) ?? []), + ...(access.confluenceSiteGrants?.map((grant) => grant.connectorId) ?? []), + ] : [] ) const excludedBefore = excludedSources.size for (const candidate of candidates) { if ( - candidate.installationSource && + candidate.liveAuthorizationSource && candidate.connectorId && !grantedSources.has(candidate.connectorId) ) @@ -578,6 +594,7 @@ export async function handleTagOnlySearch(params: SearchParams): Promise @@ -705,6 +722,7 @@ function selectLiveVectorResults( const conditions = [inArray(embedding.knowledgeBaseId, params.knowledgeBaseIds), ...filters] return selectAuthorizedSearchResults({ accessProvider, + filters: params.filters, signal: params.signal, topK: params.topK, selectPage: (limit, offset, excludedSources) => @@ -818,6 +836,7 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise ] return selectAuthorizedSearchResults({ accessProvider: params.accessProvider, + filters: params.filters, signal: params.signal, topK, selectPage: (limit, offset, excludedSources) => diff --git a/apps/sim/lib/knowledge/service.test.ts b/apps/sim/lib/knowledge/service.test.ts index 8d05b403c38..bd8a8780ac1 100644 --- a/apps/sim/lib/knowledge/service.test.ts +++ b/apps/sim/lib/knowledge/service.test.ts @@ -6,10 +6,13 @@ import { hasMockCondition, permissionsMock, permissionsMockFns, + queueTableRows, resetDbChainMock, schemaMock, } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { KnowledgeAccessProvider } from '@/lib/knowledge/access/types' +import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' const { mockApplyStorageUsageDeltasInTx, @@ -29,8 +32,10 @@ vi.mock('@/lib/billing/storage', () => ({ })) import { + attachKnowledgeBaseConnectors, findActiveKnowledgeBasesByExactName, getActiveKnowledgeBaseReference, + getActiveKnowledgeBaseReferences, getKnowledgeBaseById, getWorkspaceKnowledgeBases, KnowledgeBasePermissionError, @@ -81,6 +86,71 @@ describe('knowledge base references', () => { await expect(getActiveKnowledgeBaseReference('missing')).resolves.toBeNull() }) + it('loads twenty references in one query without changing their projection or input order', async () => { + const ids = Array.from({ length: 20 }, (_, index) => `kb-${index}`) + const references = ids.map((id) => ({ id, chunkingConfig: { maxSize: 512 } })) + queueTableRows(schemaMock.knowledgeBase, [...references].reverse()) + + await expect(getActiveKnowledgeBaseReferences(ids)).resolves.toEqual(references) + + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + expect(dbChainMockFns.from).toHaveBeenCalledOnce() + const projection = dbChainMockFns.select.mock.calls[0][0] + const [condition] = dbChainMockFns.where.mock.calls[0] + expect( + hasMockCondition( + condition, + (node) => node.type === 'isNull' && node.column === schemaMock.knowledgeBase.deletedAt + ) + ).toBe(true) + expect( + hasMockCondition( + condition, + (node) => + node.type === 'inArray' && + node.column === schemaMock.knowledgeBase.id && + JSON.stringify(node.values) === JSON.stringify(ids) + ) + ).toBe(true) + expect(dbChainMockFns.leftJoin).not.toHaveBeenCalled() + expect(dbChainMockFns.groupBy).not.toHaveBeenCalled() + + await getActiveKnowledgeBaseReference(ids[0]) + expect(dbChainMockFns.select.mock.calls[1][0]).toEqual(projection) + }) + + it('preserves duplicate and absent reference positions without querying duplicate ids', async () => { + const reference = { id: 'kb-1', chunkingConfig: {} } + queueTableRows(schemaMock.knowledgeBase, [reference]) + + await expect( + getActiveKnowledgeBaseReferences(['missing', 'kb-1', 'missing', 'kb-1']) + ).resolves.toEqual([null, reference, null, reference]) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + expect( + hasMockCondition( + dbChainMockFns.where.mock.calls[0][0], + (node) => + node.type === 'inArray' && + JSON.stringify(node.values) === JSON.stringify(['missing', 'kb-1']) + ) + ).toBe(true) + }) + + it('does not query an empty reference batch and retains the singleton query shape', async () => { + await expect(getActiveKnowledgeBaseReferences([])).resolves.toEqual([]) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + await expect(getActiveKnowledgeBaseReferences(['missing'])).resolves.toEqual([null]) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + expect(dbChainMockFns.limit).toHaveBeenCalledWith(1) + }) + + it('propagates reference batch database failures', async () => { + const failure = new Error('reference database unavailable') + dbChainMockFns.where.mockRejectedValueOnce(failure) + await expect(getActiveKnowledgeBaseReferences(['kb-1', 'kb-2'])).rejects.toBe(failure) + }) + it('preserves aggregate counts for knowledge-base detail consumers', async () => { dbChainMockFns.limit.mockResolvedValueOnce([ { @@ -444,3 +514,89 @@ describe('updateKnowledgeBase — file ownership binding re-point on workspace c expect(dbChainMockFns.update).toHaveBeenCalledTimes(1) }) }) + +describe('knowledge base counts with live source permissions', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + function reader() { + const scope = { kind: 'user' as const, userId: 'reader', tokens: ['reader-token'] } + const getForConnectors = vi.fn().mockResolvedValue({ + ...scope, + confluenceSiteGrants: [ + { + cloudId: 'cloud-1', + connectorId: 'confluence-source', + contentCredentialId: 'crawler', + readerCredentialId: 'reader-credential', + readerSubjectToken: 'reader-token', + domain: 'team.atlassian.net', + }, + ], + }) + const access: KnowledgeAccessProvider = { + get: async () => scope, + getForConnectors, + getForDocuments: async () => scope, + } + return { access, getForConnectors } + } + + it('sums ordinary and live-authorized documents once in a paginated list', async () => { + const { access, getForConnectors } = reader() + queueTableRows(schemaMock.knowledgeBase, [ + { + id: 'kb-1', + workspaceId: 'ws-1', + chunkingConfig: {}, + docCount: 99, + tokenCount: 999, + createdAt: new Date('2026-01-01'), + }, + ]) + queueTableRows(schemaMock.document, [{ knowledgeBaseId: 'kb-1', docCount: 2, tokenCount: 10 }]) + queueTableRows(schemaMock.document, [{ connectorId: 'confluence-source' }]) + queueTableRows(schemaMock.document, [{ knowledgeBaseId: 'kb-1', docCount: 3, tokenCount: 20 }]) + const result = await getWorkspaceKnowledgeBases('ws-1', 'active', { access, limit: 2 }) + expect(result.data).toHaveLength(1) + expect(result.data[0]).toMatchObject({ docCount: 5, tokenCount: 30 }) + expect(getForConnectors).toHaveBeenCalledExactlyOnceWith(['confluence-source'], undefined) + expect(dbChainMockFns.selectDistinct).toHaveBeenCalledWith({ + connectorId: schemaMock.document.connectorId, + }) + expect( + dbChainMockFns.where.mock.calls.every( + ([condition]) => + hasMockCondition( + condition, + (node) => node.type === 'inArray' && node.column === schemaMock.knowledgeBase.id + ) || + hasMockCondition( + condition, + (node) => node.type === 'eq' && node.left === schemaMock.knowledgeBase.workspaceId + ) || + hasMockCondition( + condition, + (node) => + node.type === 'inArray' && + node.column === schemaMock.knowledgeConnector.knowledgeBaseId + ) + ) + ).toBe(true) + }) + + it('does not retain stale totals when a live source no longer authorizes its documents', async () => { + const { access, getForConnectors } = reader() + queueTableRows(schemaMock.document, []) + queueTableRows(schemaMock.document, [{ connectorId: 'confluence-source' }]) + queueTableRows(schemaMock.document, []) + const base = { id: 'kb-1', docCount: 5, tokenCount: 50 } as KnowledgeBaseWithCounts + await expect(attachKnowledgeBaseConnectors(base, access)).resolves.toMatchObject({ + docCount: 0, + tokenCount: 0, + }) + expect(getForConnectors).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index 146f3c466ab..0cff654809d 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -34,8 +34,9 @@ import { generateRestoreName } from '@/lib/core/utils/restore-name' import { findActiveFolder, resolveRestoredFolderId } from '@/lib/folders/queries' import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' -import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' +import { MAX_KNOWLEDGE_ACCESS_CANDIDATES } from '@/lib/knowledge/access/types' import { mirrorsSourceAcls } from '@/lib/knowledge/connectors/access-modes' +import { type KnowledgeReadAccess, knowledgeReadAccessBatches } from '@/lib/knowledge/read-access' import type { ChunkingConfig, CreateKnowledgeBaseData, @@ -139,7 +140,7 @@ const KNOWLEDGE_BASE_SORTS = { } satisfies Record[]> export interface GetKnowledgeBasesOptions { - access?: KnowledgeAccessScope + access?: KnowledgeReadAccess /** Restrict to one knowledge-base folder; `undefined` lists all and `null` lists the root. */ folderId?: string | null /** Case-insensitive substring match on the knowledge base name. */ @@ -169,7 +170,7 @@ async function readKnowledgeBaseRows( where: SQL | undefined, orderBy: SQL[], limit?: number, - access?: KnowledgeAccessScope + access?: KnowledgeReadAccess ): Promise< Array> > { @@ -200,7 +201,7 @@ async function readKnowledgeBaseRows( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), - access ? knowledgeAccessCondition(access) : undefined + access ? ('get' in access ? sql`false` : knowledgeAccessCondition(access)) : undefined ) ) .where(where) @@ -209,13 +210,64 @@ async function readKnowledgeBaseRows( const rows = limit === undefined ? await query : await query.limit(limit) + const counts = + access && 'get' in access + ? await readKnowledgeBaseDocumentCounts( + rows.map((kb) => kb.id), + access + ) + : undefined return rows.map((kb) => ({ ...kb, chunkingConfig: kb.chunkingConfig as ChunkingConfig, - docCount: Number(kb.docCount), + docCount: counts ? (counts.get(kb.id)?.docCount ?? 0) : Number(kb.docCount), + tokenCount: counts ? (counts.get(kb.id)?.tokenCount ?? 0) : kb.tokenCount, })) } +/** Counts only hydrated access batches, keeping candidate discovery free of document metadata. */ +async function readKnowledgeBaseDocumentCounts( + knowledgeBaseIds: readonly string[], + access: KnowledgeReadAccess +): Promise> { + const counts = new Map() + for ( + let offset = 0; + offset < knowledgeBaseIds.length; + offset += MAX_KNOWLEDGE_ACCESS_CANDIDATES + ) { + const conditions = [ + inArray( + knowledgeBase.id, + knowledgeBaseIds.slice(offset, offset + MAX_KNOWLEDGE_ACCESS_CANDIDATES) + ), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + ] + for await (const accessCondition of knowledgeReadAccessBatches(access, conditions)) { + const rows = await db + .select({ + knowledgeBaseId: document.knowledgeBaseId, + docCount: count(), + tokenCount: sql`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number), + }) + .from(document) + .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id)) + .where(and(...conditions, accessCondition)) + .groupBy(document.knowledgeBaseId) + for (const row of rows) { + const previous = counts.get(row.knowledgeBaseId) + counts.set(row.knowledgeBaseId, { + docCount: (previous?.docCount ?? 0) + Number(row.docCount), + tokenCount: (previous?.tokenCount ?? 0) + Number(row.tokenCount), + }) + } + } + } + return counts +} + async function attachConnectorTypes( knowledgeBases: Array< Omit @@ -875,6 +927,23 @@ export type ActiveKnowledgeBaseReference = Omit< 'tokenCount' | 'docCount' | 'connectorTypes' | 'hasPermissionScopedConnector' > +const ACTIVE_KNOWLEDGE_BASE_REFERENCE_FIELDS = { + id: knowledgeBase.id, + userId: knowledgeBase.userId, + name: knowledgeBase.name, + isSearchIndex: knowledgeBase.isSearchIndex, + description: knowledgeBase.description, + embeddingModel: knowledgeBase.embeddingModel, + embeddingDimension: knowledgeBase.embeddingDimension, + chunkingConfig: knowledgeBase.chunkingConfig, + createdAt: knowledgeBase.createdAt, + updatedAt: knowledgeBase.updatedAt, + deletedAt: knowledgeBase.deletedAt, + workspaceId: knowledgeBase.workspaceId, + organizationId: knowledgeBase.organizationId, + folderId: knowledgeBase.folderId, +} + /** * Canonical identity and configuration for application authorization and retrieval. * Reading a reference never scans the base's documents to compute display counts. @@ -883,22 +952,7 @@ export async function getActiveKnowledgeBaseReference( knowledgeBaseId: string ): Promise { const [row] = await db - .select({ - id: knowledgeBase.id, - userId: knowledgeBase.userId, - name: knowledgeBase.name, - isSearchIndex: knowledgeBase.isSearchIndex, - description: knowledgeBase.description, - embeddingModel: knowledgeBase.embeddingModel, - embeddingDimension: knowledgeBase.embeddingDimension, - chunkingConfig: knowledgeBase.chunkingConfig, - createdAt: knowledgeBase.createdAt, - updatedAt: knowledgeBase.updatedAt, - deletedAt: knowledgeBase.deletedAt, - workspaceId: knowledgeBase.workspaceId, - organizationId: knowledgeBase.organizationId, - folderId: knowledgeBase.folderId, - }) + .select(ACTIVE_KNOWLEDGE_BASE_REFERENCE_FIELDS) .from(knowledgeBase) .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) .limit(1) @@ -906,6 +960,29 @@ export async function getActiveKnowledgeBaseReference( return row ? { ...row, chunkingConfig: row.chunkingConfig as ChunkingConfig } : null } +/** Loads active references in one statement while preserving requested order and missing entries. */ +export async function getActiveKnowledgeBaseReferences( + knowledgeBaseIds: readonly string[] +): Promise> { + if (knowledgeBaseIds.length === 0) return [] + if (knowledgeBaseIds.length === 1) + return [await getActiveKnowledgeBaseReference(knowledgeBaseIds[0])] + + const rows = await db + .select(ACTIVE_KNOWLEDGE_BASE_REFERENCE_FIELDS) + .from(knowledgeBase) + .where( + and( + inArray(knowledgeBase.id, [...new Set(knowledgeBaseIds)]), + isNull(knowledgeBase.deletedAt) + ) + ) + const byId = new Map( + rows.map((row) => [row.id, { ...row, chunkingConfig: row.chunkingConfig as ChunkingConfig }]) + ) + return knowledgeBaseIds.map((id) => byId.get(id) ?? null) +} + /** * Get a single knowledge base by ID */ @@ -938,26 +1015,16 @@ export async function getKnowledgeBaseById( */ export async function attachKnowledgeBaseConnectors( knowledgeBase: KnowledgeBaseWithCounts, - access?: KnowledgeAccessScope + access?: KnowledgeReadAccess ): Promise { let visible = knowledgeBase if (access) { - const [counts] = await db - .select({ - docCount: count(), - tokenCount: sql`COALESCE(SUM(${document.tokenCount}), 0)`.mapWith(Number), - }) - .from(document) - .where( - and( - eq(document.knowledgeBaseId, knowledgeBase.id), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - knowledgeAccessCondition(access) - ) - ) - visible = { ...knowledgeBase, docCount: Number(counts.docCount), tokenCount: counts.tokenCount } + const counts = await readKnowledgeBaseDocumentCounts([knowledgeBase.id], access) + visible = { + ...knowledgeBase, + docCount: counts.get(knowledgeBase.id)?.docCount ?? 0, + tokenCount: counts.get(knowledgeBase.id)?.tokenCount ?? 0, + } } const [withConnectors] = await attachConnectorTypes([visible]) return withConnectors diff --git a/apps/sim/lib/knowledge/tags/filter-resolution.test.ts b/apps/sim/lib/knowledge/tags/filter-resolution.test.ts index c1a4322cf57..7a88bfc8817 100644 --- a/apps/sim/lib/knowledge/tags/filter-resolution.test.ts +++ b/apps/sim/lib/knowledge/tags/filter-resolution.test.ts @@ -3,12 +3,13 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockGetDocumentTagDefinitions } = vi.hoisted(() => ({ +const { mockGetDocumentTagDefinitions, mockGetDocumentTagDefinitionsBatch } = vi.hoisted(() => ({ mockGetDocumentTagDefinitions: vi.fn(), + mockGetDocumentTagDefinitionsBatch: vi.fn(), })) vi.mock('@/lib/knowledge/tags/service', () => ({ - getDocumentTagDefinitions: mockGetDocumentTagDefinitions, + getDocumentTagDefinitionsByKnowledgeBaseIds: mockGetDocumentTagDefinitionsBatch, })) import { @@ -38,6 +39,12 @@ function definition( describe('resolveKnowledgeTagFilters', () => { beforeEach(() => { vi.clearAllMocks() + mockGetDocumentTagDefinitionsBatch.mockImplementation( + async (ids: string[]) => + new Map( + await Promise.all(ids.map(async (id) => [id, await mockGetDocumentTagDefinitions(id)])) + ) + ) }) it('resolves a display name to the slot it is stored in', async () => { diff --git a/apps/sim/lib/knowledge/tags/filter-resolution.ts b/apps/sim/lib/knowledge/tags/filter-resolution.ts index 57b4bde8854..c374105b284 100644 --- a/apps/sim/lib/knowledge/tags/filter-resolution.ts +++ b/apps/sim/lib/knowledge/tags/filter-resolution.ts @@ -2,7 +2,7 @@ import { KNOWLEDGE_TAG_FILTER_OPERATORS_BY_FIELD_TYPE } from '@/lib/api/contract import { OrchestrationError } from '@/lib/core/orchestration/types' import { SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/constants' import type { TagFilterCondition } from '@/lib/knowledge/documents/tag-filter' -import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service' +import { getDocumentTagDefinitionsByKnowledgeBaseIds } from '@/lib/knowledge/tags/service' import type { DocumentTagDefinition } from '@/lib/knowledge/tags/types' import { buildUndefinedTagsError, validateTagValue } from '@/lib/knowledge/tags/utils' import type { StructuredFilter } from '@/lib/knowledge/types' @@ -72,15 +72,11 @@ export async function resolveKnowledgeTagFilters( filters: KnowledgeTagNameFilter[], knowledgeBaseIds: string[] ): Promise { - const definitionEntries = await Promise.all( - knowledgeBaseIds.map( - async (knowledgeBaseId) => - [knowledgeBaseId, await getDocumentTagDefinitions(knowledgeBaseId)] as const - ) - ) - const definitionsByKnowledgeBase = new Map(definitionEntries) + const definitionsByKnowledgeBase = + await getDocumentTagDefinitionsByKnowledgeBaseIds(knowledgeBaseIds) const sharedDefinitions = new Map() - for (const [, definitions] of definitionEntries) { + for (const knowledgeBaseId of knowledgeBaseIds) { + const definitions = definitionsByKnowledgeBase.get(knowledgeBaseId)! const currentByName = new Map( definitions.map((definition) => [ definition.displayName, diff --git a/apps/sim/lib/knowledge/tags/service.test.ts b/apps/sim/lib/knowledge/tags/service.test.ts index 2dc314fd369..92e8c31f515 100644 --- a/apps/sim/lib/knowledge/tags/service.test.ts +++ b/apps/sim/lib/knowledge/tags/service.test.ts @@ -3,7 +3,7 @@ */ import { knowledgeBaseTagDefinitions } from '@sim/db/schema' -import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { dbChainMockFns, hasMockCondition, queueTableRows, resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' vi.mock('@sim/utils/id', () => ({ @@ -14,6 +14,8 @@ vi.mock('@sim/utils/id', () => ({ import { createOrUpdateTagDefinitionsBulk, createTagDefinition, + getDocumentTagDefinitions, + getDocumentTagDefinitionsByKnowledgeBaseIds, updateTagDefinition, } from '@/lib/knowledge/tags/service' @@ -32,6 +34,83 @@ function existingDefinition(overrides: Record) { } } +describe('getDocumentTagDefinitionsByKnowledgeBaseIds', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('loads twenty bases in one query and retains each base in requested order', async () => { + const ids = Array.from({ length: 20 }, (_, index) => `kb-${index}`) + queueTableRows(knowledgeBaseTagDefinitions, [ + existingDefinition({ knowledgeBaseId: 'kb-2', tagSlot: 'number1', fieldType: 'number' }), + existingDefinition({ knowledgeBaseId: 'kb-1', tagSlot: 'tag1' }), + existingDefinition({ knowledgeBaseId: 'kb-2', tagSlot: 'tag1' }), + ]) + + const result = await getDocumentTagDefinitionsByKnowledgeBaseIds(ids) + + expect([...result.keys()]).toEqual(ids) + expect(result.get('kb-0')).toEqual([]) + expect(result.get('kb-1')?.map((definition) => definition.tagSlot)).toEqual(['tag1']) + expect(result.get('kb-2')?.map((definition) => definition.tagSlot)).toEqual(['number1', 'tag1']) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + expect(dbChainMockFns.orderBy).toHaveBeenCalledWith(knowledgeBaseTagDefinitions.tagSlot) + expect( + hasMockCondition( + dbChainMockFns.where.mock.calls[0][0], + (node) => + node.type === 'inArray' && + node.column === knowledgeBaseTagDefinitions.knowledgeBaseId && + JSON.stringify(node.values) === JSON.stringify(ids) + ) + ).toBe(true) + const projection = dbChainMockFns.select.mock.calls[0][0] + await getDocumentTagDefinitions('kb-0') + expect(dbChainMockFns.select.mock.calls[1][0]).toEqual(projection) + }) + + it('keeps empty bases and deduplicates requested ids', async () => { + const result = await getDocumentTagDefinitionsByKnowledgeBaseIds(['kb-2', 'kb-1', 'kb-2']) + expect([...result]).toEqual([ + ['kb-2', []], + ['kb-1', []], + ]) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + expect( + hasMockCondition( + dbChainMockFns.where.mock.calls[0][0], + (node) => + node.type === 'inArray' && + JSON.stringify(node.values) === JSON.stringify(['kb-2', 'kb-1']) + ) + ).toBe(true) + }) + + it('skips an empty batch and preserves the singleton equality predicate', async () => { + expect(await getDocumentTagDefinitionsByKnowledgeBaseIds([])).toEqual(new Map()) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + expect(await getDocumentTagDefinitionsByKnowledgeBaseIds(['kb-1'])).toEqual( + new Map([['kb-1', []]]) + ) + expect(dbChainMockFns.select).toHaveBeenCalledOnce() + expect( + hasMockCondition( + dbChainMockFns.where.mock.calls[0][0], + (node) => node.type === 'eq' && node.right === 'kb-1' + ) + ).toBe(true) + }) + + it('propagates batch database failures', async () => { + const failure = new Error('tag database unavailable') + dbChainMockFns.orderBy.mockRejectedValueOnce(failure) + await expect(getDocumentTagDefinitionsByKnowledgeBaseIds(['kb-1', 'kb-2'])).rejects.toBe( + failure + ) + }) +}) + describe('createOrUpdateTagDefinitionsBulk', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/apps/sim/lib/knowledge/tags/service.ts b/apps/sim/lib/knowledge/tags/service.ts index 8f837de9ccb..7a599fea365 100644 --- a/apps/sim/lib/knowledge/tags/service.ts +++ b/apps/sim/lib/knowledge/tags/service.ts @@ -9,7 +9,7 @@ import { import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { and, eq, isNotNull, isNull, or, sql } from 'drizzle-orm' +import { and, eq, inArray, isNotNull, isNull, or, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { DbOrTx, DbTransaction } from '@/lib/db/types' import { @@ -201,23 +201,23 @@ export async function getNextAvailableSlot( return null // All slots for this field type are used } -/** - * Get all tag definitions for a knowledge base - */ +const DOCUMENT_TAG_DEFINITION_FIELDS = { + id: knowledgeBaseTagDefinitions.id, + knowledgeBaseId: knowledgeBaseTagDefinitions.knowledgeBaseId, + tagSlot: knowledgeBaseTagDefinitions.tagSlot, + displayName: knowledgeBaseTagDefinitions.displayName, + fieldType: knowledgeBaseTagDefinitions.fieldType, + createdAt: knowledgeBaseTagDefinitions.createdAt, + updatedAt: knowledgeBaseTagDefinitions.updatedAt, +} + +/** Get all tag definitions for a knowledge base. */ export async function getDocumentTagDefinitions( knowledgeBaseId: string, txDb?: DbOrTx ): Promise { const definitions = await (txDb ?? db) - .select({ - id: knowledgeBaseTagDefinitions.id, - knowledgeBaseId: knowledgeBaseTagDefinitions.knowledgeBaseId, - tagSlot: knowledgeBaseTagDefinitions.tagSlot, - displayName: knowledgeBaseTagDefinitions.displayName, - fieldType: knowledgeBaseTagDefinitions.fieldType, - createdAt: knowledgeBaseTagDefinitions.createdAt, - updatedAt: knowledgeBaseTagDefinitions.updatedAt, - }) + .select(DOCUMENT_TAG_DEFINITION_FIELDS) .from(knowledgeBaseTagDefinitions) .where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId)) .orderBy(knowledgeBaseTagDefinitions.tagSlot) @@ -228,6 +228,35 @@ export async function getDocumentTagDefinitions( })) } +/** Loads each requested base's slot-ordered definitions in one statement, including empty bases. */ +export async function getDocumentTagDefinitionsByKnowledgeBaseIds( + knowledgeBaseIds: readonly string[] +): Promise> { + const definitionsByKnowledgeBase = new Map( + knowledgeBaseIds.map((id) => [id, []]) + ) + if (definitionsByKnowledgeBase.size === 0) return definitionsByKnowledgeBase + if (definitionsByKnowledgeBase.size === 1) { + const id = knowledgeBaseIds[0] + definitionsByKnowledgeBase.set(id, await getDocumentTagDefinitions(id)) + return definitionsByKnowledgeBase + } + const definitions = await db + .select(DOCUMENT_TAG_DEFINITION_FIELDS) + .from(knowledgeBaseTagDefinitions) + .where( + inArray(knowledgeBaseTagDefinitions.knowledgeBaseId, [...definitionsByKnowledgeBase.keys()]) + ) + .orderBy(knowledgeBaseTagDefinitions.tagSlot) + for (const definition of definitions) { + definitionsByKnowledgeBase.get(definition.knowledgeBaseId)!.push({ + ...definition, + tagSlot: definition.tagSlot as string, + }) + } + return definitionsByKnowledgeBase +} + /** * Get all tag definitions for a knowledge base (alias for compatibility) */ diff --git a/apps/sim/lib/oauth/utils.test.ts b/apps/sim/lib/oauth/utils.test.ts index 37c4320c7c7..830761fc1f6 100644 --- a/apps/sim/lib/oauth/utils.test.ts +++ b/apps/sim/lib/oauth/utils.test.ts @@ -424,6 +424,11 @@ describe('getCanonicalScopesForProvider', () => { }) describe('getScopeDescription', () => { + it('describes Confluence directory access', () => { + expect(getScopeDescription('read:group:confluence', 'confluence')).toBe( + 'View Confluence groups and memberships' + ) + }) it.concurrent('uses provider-specific labels for Bitbucket scope names', () => { expect(getScopeDescription('account', 'bitbucket')).toBe( 'View your Bitbucket account and workspace memberships' diff --git a/apps/sim/lib/oauth/utils.ts b/apps/sim/lib/oauth/utils.ts index c7f72c730dd..39613c6cd32 100644 --- a/apps/sim/lib/oauth/utils.ts +++ b/apps/sim/lib/oauth/utils.ts @@ -109,6 +109,7 @@ export const SCOPE_DESCRIPTIONS: Record = { 'read:hierarchical-content:confluence': 'View page hierarchy (children and ancestors)', 'read:content.metadata:confluence': 'View content metadata (required for ancestors)', 'read:user:confluence': 'View Confluence user profiles', + 'read:group:confluence': 'View Confluence groups and memberships', 'read:confluence-user': 'View Confluence user profiles (v1 API)', 'read:task:confluence': 'View Confluence inline tasks', 'write:task:confluence': 'Update Confluence inline tasks', diff --git a/apps/sim/lib/selectors/manifest.test.ts b/apps/sim/lib/selectors/manifest.test.ts index 76d9c47e0ff..e9cb015bf5f 100644 --- a/apps/sim/lib/selectors/manifest.test.ts +++ b/apps/sim/lib/selectors/manifest.test.ts @@ -9,8 +9,8 @@ describe('selector manifest', () => { const count = (classification: (typeof classifications)[number]) => classifications.filter((value) => value === classification).length - expect(Object.keys(selectorManifest)).toHaveLength(96) - expect(count('provider-server')).toBe(83) + expect(Object.keys(selectorManifest)).toHaveLength(97) + expect(count('provider-server')).toBe(84) expect(count('internal-server')).toBe(12) expect(count('local')).toBe(1) expect(classifications).not.toContain('provider-legacy') @@ -36,7 +36,7 @@ describe('selector manifest', () => { const rawConnectionKeys = providerKeys.filter( (key) => !serverSelectorRegistry[key as keyof typeof serverSelectorRegistry].credential ) - expect(providerKeys).toHaveLength(83) + expect(providerKeys).toHaveLength(84) expect(rawConnectionKeys.sort()).toEqual([ 'cloudwatch.logGroups', 'cloudwatch.logStreams', @@ -110,7 +110,7 @@ describe('selector manifest', () => { } }) - it('preserves credential-use auditing only for the seven legacy-audited selectors', () => { + it('preserves credential-use auditing for the declared selectors', () => { const auditedKeys = Object.entries(serverSelectorRegistry) .flatMap(([key, attachment]) => (attachment.auditCredentialUse ? [key] : [])) .sort() @@ -118,6 +118,7 @@ describe('selector manifest', () => { expect(auditedKeys).toEqual([ 'confluence.pages', 'jira.issues', + 'jira.projectKeys', 'jira.projects', 'managedAgent.agents', 'managedAgent.environments', diff --git a/apps/sim/lib/selectors/manifest.ts b/apps/sim/lib/selectors/manifest.ts index 418e0da162d..b6596873305 100644 --- a/apps/sim/lib/selectors/manifest.ts +++ b/apps/sim/lib/selectors/manifest.ts @@ -298,6 +298,12 @@ export const selectorManifest = { search: true, detail: true, }), + 'jira.projectKeys': providerSelector(['domain'], { + readiness: { all: ['oauthCredential', 'domain'] }, + listMode: 'paginated', + search: true, + detail: true, + }), 'linear.projects': providerSelector(['teamId'], { readiness: { all: ['oauthCredential', 'teamId'] }, listMode: 'paginated', diff --git a/apps/sim/lib/selectors/server/providers/google.test.ts b/apps/sim/lib/selectors/server/providers/google.test.ts index 318a68b1136..0f737ec42fa 100644 --- a/apps/sim/lib/selectors/server/providers/google.test.ts +++ b/apps/sim/lib/selectors/server/providers/google.test.ts @@ -150,6 +150,28 @@ describe('Google server selector adapters', () => { expect(mockFetch).toHaveBeenCalledTimes(2) }) + it.each([ + { files: [{ id: 'folder-1', name: 'Notes' }], nextPageToken: undefined }, + { files: [], nextPageToken: undefined }, + { files: [{ id: 'folder-1', name: 'Notes' }], nextPageToken: 'next' }, + ])('reports incomplete Drive searches without inventing pagination: %j', async (page) => { + mockFetch + .mockResolvedValueOnce(new Response(JSON.stringify({ drives: [] }))) + .mockResolvedValueOnce(new Response(JSON.stringify({ ...page, incompleteSearch: true }))) + const args = listArgs('google.drive') + args.context.mimeType = 'application/vnd.google-apps.folder' + + await expect(googleSelectorAttachments['google.drive'].execute(args)).resolves.toEqual({ + kind: 'list', + items: page.files.map((file) => ({ id: file.id, label: file.name })), + ...(page.nextPageToken ? { nextCursor: `f:${page.nextPageToken}` } : {}), + diagnostics: { truncated: { reason: 'provider-cap' } }, + }) + expect(mockFetch).toHaveBeenCalledTimes(2) + const fileUrl = new URL(String(mockFetch.mock.calls[1]?.[0])) + expect(fileUrl.searchParams.get('fields')).toContain('incompleteSearch') + }) + it('continues real folder pages after the final shared-drive page', async () => { mockFetch .mockResolvedValueOnce( diff --git a/apps/sim/lib/selectors/server/providers/google.ts b/apps/sim/lib/selectors/server/providers/google.ts index 8784433ffaa..37c92ecef0d 100644 --- a/apps/sim/lib/selectors/server/providers/google.ts +++ b/apps/sim/lib/selectors/server/providers/google.ts @@ -68,6 +68,7 @@ interface Sheet { interface GooglePage { items: T[] nextCursor?: string + truncated?: boolean } async function googleAccessToken(args: ExecuteServerSelectorArgs, serviceId: string) { @@ -297,9 +298,13 @@ async function listDriveFiles( url.searchParams.set('supportsAllDrives', 'true') url.searchParams.set('includeItemsFromAllDrives', 'true') url.searchParams.set('pageSize', '100') - url.searchParams.set('fields', 'nextPageToken,files(id,name,mimeType)') + url.searchParams.set('fields', 'nextPageToken,incompleteSearch,files(id,name,mimeType)') if (pageToken) url.searchParams.set('pageToken', pageToken) - const data = await fetchProviderJson<{ files?: DriveFile[]; nextPageToken?: string }>(url, { + const data = await fetchProviderJson<{ + files?: DriveFile[] + nextPageToken?: string + incompleteSearch?: boolean + }>(url, { headers: { Authorization: `Bearer ${accessToken}` }, signal: args.signal, }) @@ -308,6 +313,7 @@ async function listDriveFiles( return { items: [...sharedDrives, ...(data.files ?? [])], ...(nextPageToken ? { nextCursor: driveCursor('files', nextPageToken) } : {}), + ...(data.incompleteSearch === true ? { truncated: true } : {}), } } @@ -371,7 +377,8 @@ async function executeDrive(args: ExecuteServerSelectorArgs) { id: file.id, label: file.name, })), - result.nextCursor + result.nextCursor, + result.truncated ? { truncated: { reason: 'provider-cap' } } : undefined ) } diff --git a/apps/sim/lib/selectors/server/providers/jira.test.ts b/apps/sim/lib/selectors/server/providers/jira.test.ts index 5ddb6bd34f6..d4ea2fcf665 100644 --- a/apps/sim/lib/selectors/server/providers/jira.test.ts +++ b/apps/sim/lib/selectors/server/providers/jira.test.ts @@ -47,52 +47,71 @@ describe('Jira server selector adapter', () => { afterAll(() => vi.unstubAllGlobals()) - it('returns one project page and preserves provider search and continuation', async () => { - mockFetch.mockResolvedValueOnce( - new Response( - JSON.stringify({ - values: Array.from({ length: 50 }, (_, index) => ({ - id: `project-${index + 1}`, - name: `Payments ${index + 1}`, - })), - maxResults: 50, - isLast: false, - }), - { status: 200 } + it.each(['jira.projects', 'jira.projectKeys'] as const)( + 'returns one %s page and preserves provider search and continuation', + async (selectorKey) => { + mockFetch.mockResolvedValueOnce( + new Response( + JSON.stringify({ + values: Array.from({ length: 50 }, (_, index) => ({ + id: `project-${index + 1}`, + key: `PAY${index + 1}`, + name: `Payments ${index + 1}`, + })), + maxResults: 50, + isLast: false, + }), + { status: 200 } + ) ) - ) - await expect(jiraSelectorAttachments['jira.projects'].execute(args())).resolves.toEqual({ - kind: 'list', - items: Array.from({ length: 50 }, (_, index) => ({ - id: `project-${index + 1}`, - label: `Payments ${index + 1}`, - })), - nextCursor: '100', - }) - const url = new URL(String(mockFetch.mock.calls[0]?.[0])) - expect(url.searchParams.get('query')).toBe('payments') - expect(url.searchParams.get('startAt')).toBe('50') - expect(url.searchParams.get('maxResults')).toBe('50') - expect(mockFetch).toHaveBeenCalledTimes(1) - expect(mockResolveSelectorCredentialBundle).toHaveBeenCalledWith( - expect.objectContaining({ scopes: ['read:jira-work'] }) - ) - }) + await expect( + jiraSelectorAttachments[selectorKey].execute({ ...args(), selectorKey }) + ).resolves.toEqual({ + kind: 'list', + items: Array.from({ length: 50 }, (_, index) => ({ + id: selectorKey === 'jira.projectKeys' ? `PAY${index + 1}` : `project-${index + 1}`, + label: `Payments ${index + 1}`, + })), + nextCursor: '100', + }) + const url = new URL(String(mockFetch.mock.calls[0]?.[0])) + expect(url.searchParams.get('query')).toBe('payments') + expect(url.searchParams.get('startAt')).toBe('50') + expect(url.searchParams.get('maxResults')).toBe('50') + expect(mockFetch).toHaveBeenCalledTimes(1) + expect(mockResolveSelectorCredentialBundle).toHaveBeenCalledWith( + expect.objectContaining({ scopes: ['read:jira-work'] }) + ) + } + ) - it('preserves a requested project key when hydrating its label', async () => { - mockFetch.mockResolvedValueOnce( - new Response(JSON.stringify({ id: '10001', name: 'Engineering' }), { status: 200 }) - ) + it.each([ + ['jira.projects', 'ENG'], + ['jira.projectKeys', 'ENG'], + ['jira.projectKeys', '10001'], + ] as const)( + 'hydrates %s selection %s without replacing its saved value', + async (selectorKey, id) => { + mockFetch.mockResolvedValueOnce( + new Response(JSON.stringify({ id: '10001', key: 'ENG', name: 'Engineering' }), { + status: 200, + }) + ) - await expect( - jiraSelectorAttachments['jira.projects'].execute({ - ...args(), - request: { kind: 'detail', id: 'ENG' }, + await expect( + jiraSelectorAttachments[selectorKey].execute({ + ...args(), + selectorKey, + request: { kind: 'detail', id }, + }) + ).resolves.toEqual({ + kind: 'detail', + item: { id, label: 'Engineering' }, }) - ).resolves.toEqual({ - kind: 'detail', - item: { id: 'ENG', label: 'Engineering' }, - }) - }) + expect(String(mockFetch.mock.calls[0]?.[0])).toBe( + `https://api.atlassian.com/ex/jira/cloud-1/rest/api/3/project/${id}` + ) + } + ) }) diff --git a/apps/sim/lib/selectors/server/providers/jira.ts b/apps/sim/lib/selectors/server/providers/jira.ts index f6892a69517..bf5e028837a 100644 --- a/apps/sim/lib/selectors/server/providers/jira.ts +++ b/apps/sim/lib/selectors/server/providers/jira.ts @@ -15,7 +15,10 @@ import { type ServerSelectorAttachmentMap, } from '@/lib/selectors/server/types' -type JiraSelectorKey = Extract +type JiraSelectorKey = Extract< + ServerSelectorKey, + 'jira.projects' | 'jira.projectKeys' | 'jira.issues' +> const JIRA_SCOPES = ['read:jira-work'] const JIRA_PROJECTS_PAGE_SIZE = 50 @@ -23,6 +26,7 @@ const JIRA_ISSUES_LIMIT = 25 const jiraProjectSchema = z.object({ id: z.string().min(1).max(100), + key: z.string().min(1).max(100), name: z.string().min(1).max(1_000), }) @@ -125,7 +129,10 @@ async function listProjects(args: ExecuteServerSelectorArgs) { parsed.data.isLast === false || (parsed.data.isLast === undefined && values.length >= pageSize) return { - items: values.map((project) => ({ id: project.id, label: project.name })), + items: values.map((project) => ({ + id: args.selectorKey === 'jira.projectKeys' ? project.key : project.id, + label: project.name, + })), nextCursor: hasMore ? String(nextStartAt) : undefined, } } @@ -206,6 +213,12 @@ async function executeIssues(args: ExecuteServerSelectorArgs) { const credential = { kind: 'stored', field: 'oauthCredential', serviceIds: ['jira'] } as const export const jiraSelectorAttachments = { + 'jira.projectKeys': { + credential, + destination: 'fixed', + auditCredentialUse: true, + execute: executeProjects, + }, 'jira.projects': { credential, destination: 'fixed', diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts index c9fe94f0725..49ac99226c9 100644 --- a/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts @@ -3,6 +3,7 @@ */ import { redisConfigMockFns, resetRedisConfigMock } from '@sim/testing' import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import type { KnowledgeAccessProvider } from '@/lib/knowledge/access/types' import { cleanupExecutionBase64Cache, hydrateUserFilesWithBase64, @@ -14,7 +15,7 @@ const { mockDownloadServableFileFromStorage, mockRedis, mockVerifyFileAccess, - mockResolveKnowledgeAccessScope, + mockCreateKnowledgeAccessProvider, } = vi.hoisted(() => { const mockRedis = { get: vi.fn(), @@ -32,7 +33,7 @@ const { mockDownloadServableFileFromStorage: vi.fn(), mockRedis, mockVerifyFileAccess: vi.fn(), - mockResolveKnowledgeAccessScope: vi.fn(), + mockCreateKnowledgeAccessProvider: vi.fn(), } }) @@ -60,12 +61,14 @@ vi.mock('@/app/api/files/authorization', () => ({ })) vi.mock('@/lib/knowledge/access/scope', () => ({ - resolveKnowledgeAccessScope: mockResolveKnowledgeAccessScope, + createKnowledgeAccessProvider: mockCreateKnowledgeAccessProvider, })) describe('hydrateUserFilesWithBase64', () => { beforeEach(() => { vi.clearAllMocks() + mockDownloadFile.mockReset() + mockDownloadServableFileFromStorage.mockReset() mockGetRedisClient.mockReturnValue(null) mockRedis.get.mockResolvedValue(null) mockRedis.set.mockResolvedValue('OK') @@ -250,45 +253,59 @@ describe('hydrateUserFilesWithBase64', () => { expect(hydrated.file).not.toHaveProperty('base64') }) - it('reads a knowledge-base file as the principal behind the run', async () => { - mockDownloadFile.mockResolvedValueOnce(Buffer.from('hello', 'utf8')) - const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } - const scope = { kind: 'user' as const, tokens: ['user:user-1'] } - mockResolveKnowledgeAccessScope.mockResolvedValue(scope) - const file: UserFile = { - id: 'file-1', - name: 'shared.txt', - key: 'kb/workspace/shared.txt', - url: '/api/files/serve/kb/workspace/shared.txt?context=knowledge-base', - size: 5, - type: 'text/plain', - context: 'knowledge-base', - } - - const hydrated = await hydrateUserFilesWithBase64( - { file }, - { - workspaceId: 'workspace', - workflowId: 'workflow', - userId: 'user-1', - principal, - maxBytes: 10, + it.each([true, false])( + 'retains the run principal and knowledge reader when file access is %s', + async (allowed) => { + mockDownloadFile.mockResolvedValueOnce(Buffer.from('hello', 'utf8')) + const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } + const scope = { kind: 'user' as const, userId: 'user-1', tokens: ['user:user-1'] } + const access: KnowledgeAccessProvider = { + get: vi.fn().mockResolvedValue(scope), + getForConnectors: vi.fn().mockResolvedValue(scope), + getForDocuments: vi.fn().mockResolvedValue(scope), + } + mockCreateKnowledgeAccessProvider.mockReturnValue(access) + mockVerifyFileAccess.mockResolvedValue(allowed) + const file: UserFile = { + id: 'file-1', + name: 'shared.txt', + key: 'kb/workspace/shared.txt', + url: '/api/files/serve/kb/workspace/shared.txt?context=knowledge-base', + size: 5, + type: 'text/plain', + context: 'knowledge-base', } - ) - expect(hydrated.file.base64).toBe(Buffer.from('hello').toString('base64')) - expect(mockResolveKnowledgeAccessScope).toHaveBeenCalledWith(principal, { - workspaceId: 'workspace', - }) - expect(mockVerifyFileAccess).toHaveBeenCalledWith( - file.key, - 'user-1', - undefined, - 'knowledge-base', - false, - { knowledgeAccess: scope } - ) - }) + const hydrated = await hydrateUserFilesWithBase64( + { file }, + { + workspaceId: 'workspace', + workflowId: 'workflow', + userId: 'user-1', + principal, + maxBytes: 10, + } + ) + + if (allowed) { + expect(hydrated.file.base64).toBe(Buffer.from('hello').toString('base64')) + } else { + expect(hydrated.file).not.toHaveProperty('base64') + expect(mockDownloadFile).not.toHaveBeenCalled() + } + expect(mockCreateKnowledgeAccessProvider).toHaveBeenCalledWith(principal, { + workspaceId: 'workspace', + }) + expect(mockVerifyFileAccess).toHaveBeenCalledWith( + file.key, + 'user-1', + undefined, + 'knowledge-base', + false, + { knowledgeAccess: access } + ) + } + ) it('hydrates prior-execution files when workflow-scoped reads are enabled', async () => { mockDownloadFile.mockResolvedValueOnce(Buffer.from('hello', 'utf8')) diff --git a/apps/sim/vitest.config.ts b/apps/sim/vitest.config.ts index 6c3930cda1e..c9ecd52838f 100644 --- a/apps/sim/vitest.config.ts +++ b/apps/sim/vitest.config.ts @@ -23,7 +23,13 @@ export default defineConfig(({ mode }) => { globals: true, environment: 'node', include: integration ? ['**/*.integration.ts'] : ['**/*.test.{ts,tsx}'], - exclude: [...configDefaults.exclude, '**/node_modules/**', '**/dist/**'], + exclude: [ + ...configDefaults.exclude, + '**/node_modules/**', + '**/dist/**', + /** Workspace suites require their dedicated database and realtime setup. */ + ...(integration ? ['lib/workspaces/__integration__/*.integration.ts'] : []), + ], setupFiles: integration ? ['./lib/knowledge/__integration__/setup.ts'] : ['./vitest.setup.ts'], diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index 4d5eec6bb50..64ae6b86cec 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -4748,6 +4748,7 @@ export type GetSelectorBody = { | 'wealthbox.contacts' | 'jira.issues' | 'jira.projects' + | 'jira.projectKeys' | 'linear.projects' | 'linear.teams' | 'monday.boards' @@ -6798,6 +6799,7 @@ export type ListSelectorBody = { | 'wealthbox.contacts' | 'jira.issues' | 'jira.projects' + | 'jira.projectKeys' | 'linear.projects' | 'linear.teams' | 'monday.boards' @@ -13335,6 +13337,7 @@ export const V2_OPERATIONS = { 'wealthbox.contacts', 'jira.issues', 'jira.projects', + 'jira.projectKeys', 'linear.projects', 'linear.teams', 'monday.boards', @@ -14868,6 +14871,7 @@ export const V2_OPERATIONS = { 'wealthbox.contacts', 'jira.issues', 'jira.projects', + 'jira.projectKeys', 'linear.projects', 'linear.teams', 'monday.boards', diff --git a/scripts/create-single-release.test.ts b/scripts/create-single-release.test.ts new file mode 100644 index 00000000000..651f3677886 --- /dev/null +++ b/scripts/create-single-release.test.ts @@ -0,0 +1,152 @@ +import { execFileSync } from 'node:child_process' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +const SCRIPT = path.resolve(import.meta.dirname, 'create-single-release.ts') + +interface VersionCommit { + hash: string + version: string + title: string + date: string + author: string +} + +interface ReleaseLookup { + current: VersionCommit | null + previous: VersionCommit | null +} + +describe('release commit lookup', () => { + let directory: string + let tree: string + let head: string + + function git(args: string[], input?: string): string { + return execFileSync('git', args, { + cwd: directory, + encoding: 'utf8', + input, + env: { + ...process.env, + GIT_AUTHOR_NAME: 'Release Author', + GIT_AUTHOR_EMAIL: 'release@example.com', + GIT_COMMITTER_NAME: 'Release Author', + GIT_COMMITTER_EMAIL: 'release@example.com', + }, + }).trim() + } + + function commit(message: string, parents = head ? [head] : []): string { + head = git(['commit-tree', tree, ...parents.flatMap((parent) => ['-p', parent])], message) + git(['update-ref', 'refs/heads/main', head]) + return head + } + + function lookup(version: string, commitSha = ''): ReleaseLookup { + const output = execFileSync( + 'bun', + [ + '--no-env-file', + '--eval', + `import { findVersionCommit, findPreviousVersionCommit } from ${JSON.stringify(SCRIPT)}; + const current = findVersionCommit(${JSON.stringify(version)}); + const previous = current ? findPreviousVersionCommit(current) : null; + process.stdout.write(JSON.stringify({ current, previous }));`, + ], + { + cwd: directory, + encoding: 'utf8', + env: { ...process.env, GH_PAT: '', GITHUB_SHA: commitSha, LOG_LEVEL: 'ERROR' }, + } + ) + return JSON.parse(output) + } + + beforeEach(() => { + directory = mkdtempSync(path.join(tmpdir(), 'sim-release-test-')) + head = '' + git(['init', '--initial-branch=main', '--quiet']) + tree = git(['mktree'], '') + }) + + afterEach(() => { + rmSync(directory, { recursive: true, force: true }) + }) + + it('finds the release boundaries when older history exceeds the subprocess buffer', () => { + for (let index = 0; index < 40; index++) { + commit(`chore: historical change ${index} ${'x'.repeat(32_000)}`) + } + const previous = commit('v0.8.30: previous release') + commit('fix(search): improve indexing (#7720)') + const current = commit('v0.8.31: current release') + + expect(() => git(['log', '--format=%H|%s|%ai|%an', 'main'])).toThrow(/ENOBUFS/) + expect(lookup('v0.8.31')).toMatchObject({ + current: { hash: current, version: 'v0.8.31' }, + previous: { hash: previous, version: 'v0.8.30' }, + }) + }) + + it('uses the CI commit when main has advanced and HEAD is detached', () => { + const previous = commit('v0.8.30: previous release') + const current = commit('v0.8.31: current release') + commit('v0.8.32: later release') + git(['checkout', '--detach', '--quiet', current]) + git(['branch', '-D', 'main']) + + expect(lookup('v0.8.31', current)).toMatchObject({ + current: { hash: current }, + previous: { hash: previous }, + }) + }) + + it('rejects a CI commit whose version differs from the requested release', () => { + commit('v0.8.30: previous release') + const current = commit('v0.8.31: current release') + + expect(lookup('v0.8.30', current)).toEqual({ current: null, previous: null }) + }) + + it('supports looking up an older release on main', () => { + const previous = commit('v0.8.30: previous release') + const current = commit('v0.8.31: current release') + commit('v0.8.32: later release') + + expect(lookup('v0.8.31')).toMatchObject({ + current: { hash: current }, + previous: { hash: previous }, + }) + }) + + it('ignores release-like commit bodies and releases merged from another branch', () => { + const previous = commit('v0.8.30: previous release') + const sideRelease = commit('v9.0.0: release on staging', [previous]) + const mainCommit = commit('chore: mention a version\n\nv8.0.0: not a release', [previous]) + const current = commit('v0.8.31: current release', [mainCommit, sideRelease]) + commit('v0.8.32: later release\n\nv0.8.31: mentioned in the body') + + expect(lookup('v0.8.31')).toMatchObject({ + current: { hash: current }, + previous: { hash: previous }, + }) + }) + + it('preserves pipe characters in release titles', () => { + const current = commit('v0.8.31: parsers | search improvements') + + expect(lookup('v0.8.31')).toMatchObject({ + current: { hash: current, title: 'parsers | search improvements', author: 'Release Author' }, + previous: null, + }) + }) + + it('returns no match for a missing version or a similar version number', () => { + commit('v0.8.310: a different version') + + expect(lookup('v0.8.31')).toEqual({ current: null, previous: null }) + }) +}) diff --git a/scripts/create-single-release.ts b/scripts/create-single-release.ts index 9d98b930593..162a6966196 100755 --- a/scripts/create-single-release.ts +++ b/scripts/create-single-release.ts @@ -1,24 +1,16 @@ #!/usr/bin/env bun -import { execSync } from 'node:child_process' +import { execFileSync } from 'node:child_process' import { Octokit } from '@octokit/rest' +import { createLogger } from '@sim/logger' import { sleep } from '@sim/utils/helpers' +const logger = createLogger('CreateRelease') const GITHUB_TOKEN = process.env.GH_PAT const REPO_OWNER = 'simstudioai' const REPO_NAME = 'sim' -if (!GITHUB_TOKEN) { - console.error('❌ GH_PAT environment variable is required') - process.exit(1) -} - const targetVersion = process.argv[2] -if (!targetVersion) { - console.error('❌ Version argument is required') - console.error('Usage: bun run scripts/create-single-release.ts v0.3.XX') - process.exit(1) -} const octokit = new Octokit({ auth: GITHUB_TOKEN, @@ -40,70 +32,79 @@ interface CommitDetail { prNumber?: string } -function execCommand(command: string): string { +function execGit(args: string[]): string { try { - return execSync(command, { encoding: 'utf8' }).trim() + return execFileSync('git', args, { encoding: 'utf8' }).trim() } catch (error) { - console.error(`❌ Command failed: ${command}`) + logger.error('Git command failed', { args }) throw error } } -function findVersionCommit(version: string): VersionCommit | null { - console.log(`🔍 Finding commit for version ${version}...`) +const VERSION_COMMIT_FORMAT = '--format=%H%x00%s%x00%aI%x00%an' - const gitLog = execCommand('git log --oneline --format="%H|%s|%ai|%an" main') - const lines = gitLog.split('\n').filter((line) => line.trim()) +function parseVersionCommit(line: string): VersionCommit | null { + if (!line) return null - for (const line of lines) { - const [hash, message, date, author] = line.split('|') + const [hash, message, date, author] = line.split('\0') + const versionMatch = message.match(/^\s*(v\d+\.\d+\.?\d*):\s*(.+)$/) + if (!versionMatch) return null - const versionMatch = message.match(/^\s*(v\d+\.\d+\.?\d*):\s*(.+)$/) - if (versionMatch && versionMatch[1] === version) { - return { - hash, - version, - title: versionMatch[2], - date: new Date(date).toISOString(), - author, - } - } + return { + hash, + version: versionMatch[1], + title: versionMatch[2], + date: new Date(date).toISOString(), + author, } - - return null } -function findPreviousVersionCommit(currentVersion: string): VersionCommit | null { - console.log(`🔍 Finding previous version before ${currentVersion}...`) - - const gitLog = execCommand('git log --oneline --format="%H|%s|%ai|%an" main') - const lines = gitLog.split('\n').filter((line) => line.trim()) - - let foundCurrent = false - - for (const line of lines) { - const [hash, message, date, author] = line.split('|') +/** Reads one release candidate at a time, stopping at the first matching subject. */ +function findReleaseCommit(ref: string, version?: string, skip = 0): VersionCommit | null { + const versionPattern = version ? version.replaceAll('.', '[.]') : 'v[0-9]+[.][0-9]+[.]?[0-9]*' + + while (true) { + const line = execGit([ + 'log', + '--first-parent', + '--max-count=1', + `--skip=${skip}`, + '--extended-regexp', + `--grep=^[[:space:]]*${versionPattern}:[[:space:]]*.+`, + VERSION_COMMIT_FORMAT, + ref, + '--', + ]) + if (!line) return null + + const commit = parseVersionCommit(line) + if (commit && (!version || commit.version === version)) return commit + + /** Git's grep also matches commit bodies; only release subjects are boundaries. */ + skip++ + } +} - const versionMatch = message.match(/^\s*(v\d+\.\d+\.?\d*):\s*(.+)$/) - if (versionMatch) { - if (versionMatch[1] === currentVersion) { - foundCurrent = true - continue - } +export function findVersionCommit( + version: string, + commitSha = process.env.GITHUB_SHA +): VersionCommit | null { + logger.info(`Finding commit for version ${version}`) + if (!/^v\d+\.\d+\.?\d*$/.test(version)) return null - if (foundCurrent) { - return { - hash, - version: versionMatch[1], - title: versionMatch[2], - date: new Date(date).toISOString(), - author, - } - } - } + if (commitSha) { + const commit = parseVersionCommit( + execGit(['log', '-1', VERSION_COMMIT_FORMAT, commitSha, '--']) + ) + return commit?.version === version ? commit : null } - return null + return findReleaseCommit('main', version) +} + +export function findPreviousVersionCommit(currentCommit: VersionCommit): VersionCommit | null { + logger.info(`Finding previous version before ${currentCommit.version}`) + return findReleaseCommit(currentCommit.hash, undefined, 1) } async function fetchGitHubCommitDetails( @@ -145,7 +146,7 @@ async function fetchGitHubCommitDetails( console.warn(`⚠️ Could not fetch commit ${hash.substring(0, 7)}: ${error?.message || error}`) try { - const gitData = execCommand(`git log --format="%s|%an" -1 ${hash}`).split('|') + const gitData = execGit(['log', '--format=%s|%an', '-1', hash, '--']).split('|') let message = gitData[0] || 'Unknown commit' const prMatch = message.match(/\(#(\d+)\)/) @@ -188,7 +189,7 @@ async function getCommitsBetweenVersions( console.log(`🔍 Getting commits before first version ${currentCommit.version}`) } - const gitLog = execCommand(`git log --oneline --format="%H|%s" ${range}`) + const gitLog = execGit(['log', '--format=%H|%s', range, '--']) if (!gitLog.trim()) { console.log(`⚠️ No commits found in range ${range}`) @@ -350,6 +351,17 @@ async function generateReleaseBody( } async function main() { + if (!GITHUB_TOKEN) { + logger.error('GH_PAT environment variable is required') + process.exit(1) + } + if (!targetVersion) { + logger.error( + 'Version argument is required. Usage: bun run scripts/create-single-release.ts vX.Y.Z' + ) + process.exit(1) + } + try { console.log(`🚀 Creating single release for ${targetVersion}...`) @@ -363,7 +375,7 @@ async function main() { `✅ Found version commit: ${versionCommit.hash.substring(0, 7)} - ${versionCommit.title}` ) - const previousCommit = findPreviousVersionCommit(targetVersion) + const previousCommit = findPreviousVersionCommit(versionCommit) if (previousCommit) { console.log(`✅ Found previous version: ${previousCommit.version}`) } else { @@ -414,4 +426,6 @@ async function main() { } } -main() +if (import.meta.main) { + main() +}