diff --git a/.github/scripts/http-e2e.sh b/.github/scripts/http-e2e.sh index 8c5f185e7cd..83ff385d768 100755 --- a/.github/scripts/http-e2e.sh +++ b/.github/scripts/http-e2e.sh @@ -259,6 +259,9 @@ case "$group" in bun --no-env-file scripts/test-assistant-actions-e2e.ts ;; organization-search) + GITLAB_CSV_E2E_BASE_URL="$local_app_url" \ + GITLAB_CSV_E2E_REPORT_PATH="$report_dir/gitlab-csv-permissions-http-report.json" \ + bun --no-env-file scripts/test-gitlab-csv-permissions-e2e.ts ORGANIZATION_SEARCH_E2E_BASE_URL="$local_app_url" \ ORGANIZATION_SEARCH_E2E_DATABASE_URL="$DATABASE_URL" \ ORGANIZATION_SEARCH_E2E_AUTH_SECRET="$BETTER_AUTH_SECRET" \ diff --git a/apps/docs/content/docs/cli/organizations.mdx b/apps/docs/content/docs/cli/organizations.mdx index d339914d780..949884d5180 100644 --- a/apps/docs/content/docs/cli/organizations.mdx +++ b/apps/docs/content/docs/cli/organizations.mdx @@ -229,6 +229,74 @@ sim organizations members remove [options] API request and response schemas: [removeOrganizationMember](/api-reference/organizations/removeOrganizationMember). +## GitLab Search permissions + +### Get GitLab search permission revision + +OAuth login or personal API key required. + +```bash +sim organizations gitlab-permissions get [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `sourceId` | Yes | Identifier of the configured GitLab Search source. | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--organization ` | Yes | Organization identifier. | + + + +API request and response schemas: [getGitLabSearchPermissions](/api-reference/organizations/getGitLabSearchPermissions). + +### Replace GitLab search permission CSVs + +OAuth login or personal API key required. + +This replaces both GitLab permission files. Readers omitted from the new grants lose access. + +```bash +sim organizations gitlab-permissions replace [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `sourceId` | Yes | Identifier of the configured GitLab Search source. | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--organization ` | Yes | Organization identifier. | +| `--expected-revision ` | Yes | Current permission revision obtained before generating this snapshot. | +| `--user-mapping ` | Yes | GitLab user IDs and their verified Sim email addresses. (JSON, or @path / @- to read a file or stdin). | +| `--project-permissions ` | Yes | GitLab project paths and the IDs of users granted access. (JSON, or @path / @- to read a file or stdin). | +| `-y, --yes` | Yes | Confirm this operation. | + + + +API request and response schemas: [replaceGitLabSearchPermissions](/api-reference/organizations/replaceGitLabSearchPermissions). + ## Invitations ### List organization invitations diff --git a/apps/docs/content/docs/cli/reference.mdx b/apps/docs/content/docs/cli/reference.mdx index 75c80cfc054..1b1a2bc7090 100644 --- a/apps/docs/content/docs/cli/reference.mdx +++ b/apps/docs/content/docs/cli/reference.mdx @@ -4288,6 +4288,72 @@ sim organizations sso policy update [options] API request and response schemas: [updateSsoPolicy](/api-reference/organizations/updateSsoPolicy). +### sim organizations gitlab-permissions get + +Get GitLab search permission revision (OAuth login or personal API key required) + +```bash +sim organizations gitlab-permissions get [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `sourceId` | Yes | Identifier of the configured GitLab Search source. | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--organization ` | Yes | Organization identifier. | + + + +API request and response schemas: [getGitLabSearchPermissions](/api-reference/organizations/getGitLabSearchPermissions). + +### sim organizations gitlab-permissions replace + +Replace GitLab search permission CSVs (OAuth login or personal API key required) + +This replaces both GitLab permission files. Readers omitted from the new grants lose access. + +```bash +sim organizations gitlab-permissions replace [options] +``` + +**Arguments** + + + +| Argument | Required | Description | +| --- | --- | --- | +| `sourceId` | Yes | Identifier of the configured GitLab Search source. | + + + +**Options** + + + +| Option | Required | Description | +| --- | --- | --- | +| `--organization ` | Yes | Organization identifier. | +| `--expected-revision ` | Yes | Current permission revision obtained before generating this snapshot. | +| `--user-mapping ` | Yes | GitLab user IDs and their verified Sim email addresses. (JSON, or @path / @- to read a file or stdin). | +| `--project-permissions ` | Yes | GitLab project paths and the IDs of users granted access. (JSON, or @path / @- to read a file or stdin). | +| `-y, --yes` | Yes | Confirm this operation. | + + + +API request and response schemas: [replaceGitLabSearchPermissions](/api-reference/organizations/replaceGitLabSearchPermissions). + ### sim organizations get Get organization (OAuth login or personal API key required) diff --git a/apps/docs/content/docs/search/gitlab.mdx b/apps/docs/content/docs/search/gitlab.mdx index 03a9eeb9634..3317022616a 100644 --- a/apps/docs/content/docs/search/gitlab.mdx +++ b/apps/docs/content/docs/search/gitlab.mdx @@ -11,8 +11,8 @@ GitLab Search uses **Service account** mode, with one self-managed project per s | --- | --- | --- | | Token owner | Active instance administrator | Account that can read the project's content | | PAT scopes | `read_api`; `admin_mode` when Admin Mode is enabled | `read_api` | -| Reader authorization | Current GitLab identity, membership, roles, and supported restrictions | Admin-uploaded email mapping and project grants | -| Membership changes | Checked during the request | Admin updates the CSV files | +| Reader authorization | Current GitLab identity, membership, roles, and supported restrictions | Email mapping and project grants supplied by your organization | +| Membership changes | Checked during the request | After the next successful CSV update | | Confidential issues | Only when supported permissions allow the reader | Always excluded | The administrator path requires self-managed GitLab 17.4 or later and its administrator identity/settings APIs. A project Maintainer or group Owner is not an instance administrator. Non-admin mode does not require those administrator APIs. @@ -69,7 +69,9 @@ Use the full namespace path even if **Project** contains a numeric ID. Include i - Use UTF-8 CSV, at most 4 MiB and 100,000 data rows per file. The full request must fit 10 MiB. - Identical rows are deduplicated. Conflicting user-ID/email mappings and malformed rows are rejected. -These grants are admin-managed authorization. A granted reader can search allowed non-confidential content; Sim does not infer that person's actual GitLab role, restrictions, or membership from the non-admin token. Update both files when access changes. +In this example, Alice can search `engineering/platform` because the files link GitLab user `123` to both that project and `alice@example.com`. Sim checks that Alice is a current organization member and that her verified Sim email uniquely matches the grant. It checks again when she opens a result. GitLab receives the request under the source's read-only service account; Alice does not sign in to GitLab or authorize Sim individually. + +Saving the CSVs grants the access listed in them. Changing a membership in GitLab alone does not change these grants. Replace the CSVs to add or revoke access; the change applies to the next search or read without re-indexing. ## Access checks @@ -77,7 +79,7 @@ The source token retrieves content. Administrator mode proves the reader's curre Non-admin mode uses saved source-local CSV grants and revalidates the token's current host/project identity. Confidential issues are excluded, including issues that became confidential after search. -Reads recheck current settings, permissions, and metadata. A reference from another member, account, organization, project, or host cannot authorize a read. Live Search does not use a stored content or permission index. +Reads recheck current settings, permissions, and metadata. A reference from another member, account, organization, project, or host cannot authorize a read. Sim fetches content live from GitLab. In CSV mode, Sim stores the uploaded mappings and resulting project grants. ## Update permissions or rotate the token @@ -85,6 +87,40 @@ In project settings, replace either CSV file, retaining the other when appropria To rotate the PAT, supply its replacement in source settings. A blank token retains the saved token. Changing host, project, or permission strategy requires valid configuration for the new identity. +### Automate updates with GitLab CI + +After an admin configures a source in **Non-admin token** mode, a job can [read its revision](/api-reference/organizations/getGitLabSearchPermissions) and [replace both CSVs](/api-reference/organizations/replaceGitLabSearchPermissions). Copy `SIM_ORG_ID` and `SIM_SOURCE_ID` from the source settings URL: `/o//settings/integrations/sources/`. Set `SIM_BASE_URL` to your Sim origin (for hosted Sim, `https://www.sim.ai`). + +Store an organization admin's [personal Sim API key](/api-reference/authentication) as a [masked, protected GitLab CI variable](https://docs.gitlab.com/ci/variables/) named `SIM_API_KEY`. A personal key inherits its owner's access. This endpoint can only read the revision or replace both CSVs; it cannot change the GitLab PAT, host, or project. + +This example runs on a schedule against a protected branch. Replace `./ci/export-gitlab-permissions` with your existing export command. It must write a fresh, complete `users.csv` and `project-permissions.csv` and exit nonzero if it cannot finish. + +```yaml title=".gitlab-ci.yml" +sync_gitlab_permissions: + image: alpine:3.22 + resource_group: "sim-permissions-$SIM_SOURCE_ID" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_PROTECTED == "true"' + before_script: + - apk add --no-cache curl jq + script: + - | + set -eu + url="${SIM_BASE_URL%/}/api/v2/organizations/$SIM_ORG_ID/search/sources/$SIM_SOURCE_ID/gitlab-permissions" + auth="X-API-Key: $SIM_API_KEY" + current=$(curl --fail --silent --show-error -H "$auth" "$url") + revision=$(printf '%s' "$current" | jq -er '.data.revision') + ./ci/export-gitlab-permissions + jq -n --argjson rev "$revision" \ + --rawfile users users.csv --rawfile grants project-permissions.csv \ + '{expectedRevision: $rev, userMapping: {filename: "users.csv", content: $users}, + projectPermissions: {filename: "project-permissions.csv", content: $grants}}' > update.json + curl --fail --silent --show-error --request PUT -H "$auth" \ + -H 'Content-Type: application/json' --data-binary @update.json "$url" +``` + +A successful update returns the new revision. A `409 Conflict` means someone updated the source after the job read its revision. Rerun the job from the beginning so it exports current grants. An invalid or header-only export leaves the previous grants in place. Alert on failed jobs so revocations aren't missed. If the project should have no readers, choose **Pause search** in the source settings. You can update the grants while the source is paused, then resume search when the grants are correct. + ## Troubleshooting | Problem | Check | diff --git a/apps/docs/openapi-v2-resources.json b/apps/docs/openapi-v2-resources.json index 17f01a09649..c01d2fa7365 100644 --- a/apps/docs/openapi-v2-resources.json +++ b/apps/docs/openapi-v2-resources.json @@ -9643,6 +9643,234 @@ } } }, + "/api/v2/organizations/{organizationId}/search/sources/{sourceId}/gitlab-permissions": { + "get": { + "operationId": "getGitLabSearchPermissions", + "summary": "Get GitLab Search permission revision", + "description": "Read the current revision of an existing GitLab Search source in CSV permission mode. Requires organization Admin access. Workspace API keys return `403`; use a personal API key or scoped OAuth token.\n\nOAuth scope: `api:read`.", + "x-sim-operation": "knowledge.search.gitlab_permissions.read", + "x-oauth-scope": "api:read", + "tags": ["Organizations"], + "security": [ + { + "apiKey": [] + }, + { + "oauthBearer": ["api:read"] + } + ], + "parameters": [ + { + "name": "organizationId", + "in": "path", + "required": true, + "description": "Organization identifier.", + "schema": { + "type": "string", + "minLength": 1, + "description": "Organization identifier." + } + }, + { + "name": "sourceId", + "in": "path", + "required": true, + "description": "Identifier of the configured GitLab Search source.", + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "Identifier of the configured GitLab Search source." + } + } + ], + "responses": { + "200": { + "description": "The current permission revision.", + "headers": { + "X-RateLimit-Limit": { + "$ref": "#/components/headers/X-RateLimit-Limit" + }, + "X-RateLimit-Remaining": { + "$ref": "#/components/headers/X-RateLimit-Remaining" + }, + "X-RateLimit-Reset": { + "$ref": "#/components/headers/X-RateLimit-Reset" + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GitLabSearchPermissionResponse" + }, + "examples": { + "example1": { + "summary": "Response example 1", + "value": { + "data": { + "revision": 3 + } + } + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "403": { + "$ref": "#/components/responses/Forbidden" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "429": { + "$ref": "#/components/responses/RateLimited" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "$ref": "#/components/responses/ServiceUnavailable" + } + } + }, + "put": { + "operationId": "replaceGitLabSearchPermissions", + "summary": "Replace GitLab Search permission CSVs", + "description": "Replace both permission CSVs for an existing GitLab Search source in CSV mode. Requires organization Admin access. Supply the current revision. A concurrent change returns 409 and leaves both files untouched. Each file accepts up to 4 MiB and 100,000 data rows. Workspace API keys return `403`; use a personal API key or scoped OAuth token.\n\nOAuth scope: `api:write`.", + "x-sim-operation": "knowledge.search.gitlab_permissions.replace", + "x-oauth-scope": "api:write", + "tags": ["Organizations"], + "security": [ + { + "apiKey": [] + }, + { + "oauthBearer": ["api:write"] + } + ], + "parameters": [ + { + "name": "organizationId", + "in": "path", + "required": true, + "description": "Organization identifier.", + "schema": { + "type": "string", + "minLength": 1, + "description": "Organization identifier." + } + }, + { + "name": "sourceId", + "in": "path", + "required": true, + "description": "Identifier of the configured GitLab Search source.", + "schema": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "Identifier of the configured GitLab Search source." + } + } + ], + "requestBody": { + "required": true, + "description": "The expected revision and complete contents of both UTF-8 CSV files.", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReplaceGitLabSearchPermissionsBody" + }, + "examples": { + "example1": { + "summary": "Request example 1", + "value": { + "expectedRevision": 2, + "userMapping": { + "filename": "users.csv", + "content": "user_id,email\n42,reader@example.com\n" + }, + "projectPermissions": { + "filename": "projects.csv", + "content": "project_path,user_id\nteam/project,42\n" + } + } + } + } + } + } + }, + "responses": { + "200": { + "description": "The new permission revision.", + "headers": { + "X-RateLimit-Limit": { + "$ref": "#/components/headers/X-RateLimit-Limit" + }, + "X-RateLimit-Remaining": { + "$ref": "#/components/headers/X-RateLimit-Remaining" + }, + "X-RateLimit-Reset": { + "$ref": "#/components/headers/X-RateLimit-Reset" + } + }, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GitLabSearchPermissionResponse" + }, + "examples": { + "example1": { + "summary": "Response example 1", + "value": { + "data": { + "revision": 3 + } + } + } + } + } + } + }, + "400": { + "$ref": "#/components/responses/BadRequest" + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "403": { + "$ref": "#/components/responses/Forbidden" + }, + "404": { + "$ref": "#/components/responses/NotFound" + }, + "409": { + "$ref": "#/components/responses/Conflict" + }, + "413": { + "$ref": "#/components/responses/PayloadTooLarge" + }, + "415": { + "$ref": "#/components/responses/UnsupportedMediaType" + }, + "429": { + "$ref": "#/components/responses/RateLimited" + }, + "500": { + "$ref": "#/components/responses/InternalError" + }, + "503": { + "$ref": "#/components/responses/ServiceUnavailable" + } + } + } + }, "/api/v2/organizations/{organizationId}/sso/providers": { "get": { "operationId": "listSsoProviders", @@ -23298,6 +23526,109 @@ } ] }, + "GitLabSearchPermissionRevision": { + "type": "object", + "properties": { + "revision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991, + "description": "Current revision of the stored permissions." + } + }, + "required": ["revision"], + "additionalProperties": false, + "title": "GitLab Search permission revision", + "description": "Revision of the permission CSVs for this source." + }, + "GitLabSearchPermissionResponse": { + "type": "object", + "properties": { + "data": { + "description": "Response data.", + "$ref": "#/components/schemas/GitLabSearchPermissionRevision" + } + }, + "required": ["data"], + "additionalProperties": false, + "title": "GitLab Search permission response", + "description": "Current revision of the permission CSVs.", + "examples": [ + { + "data": { + "revision": 3 + } + } + ] + }, + "ReplaceGitLabSearchPermissionsBody": { + "type": "object", + "properties": { + "expectedRevision": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991, + "description": "Current permission revision obtained before generating this snapshot." + }, + "userMapping": { + "type": "object", + "properties": { + "filename": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "CSV filename without path separators." + }, + "content": { + "type": "string", + "minLength": 1, + "maxLength": 4194304, + "description": "UTF-8 CSV contents, up to 4 MiB." + } + }, + "required": ["filename", "content"], + "additionalProperties": false, + "description": "GitLab user IDs and their verified Sim email addresses." + }, + "projectPermissions": { + "type": "object", + "properties": { + "filename": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "CSV filename without path separators." + }, + "content": { + "type": "string", + "minLength": 1, + "maxLength": 4194304, + "description": "UTF-8 CSV contents, up to 4 MiB." + } + }, + "required": ["filename", "content"], + "additionalProperties": false, + "description": "GitLab project paths and the IDs of users granted access." + } + }, + "required": ["expectedRevision", "userMapping", "projectPermissions"], + "additionalProperties": false, + "title": "GitLab Search permission CSVs", + "description": "The expected revision and complete contents of both UTF-8 CSV files.", + "examples": [ + { + "expectedRevision": 2, + "userMapping": { + "filename": "users.csv", + "content": "user_id,email\n42,reader@example.com\n" + }, + "projectPermissions": { + "filename": "projects.csv", + "content": "project_path,user_id\nteam/project,42\n" + } + } + ] + }, "ListSsoProvidersResponse": { "type": "object", "properties": { diff --git a/apps/docs/scripts/fixtures/content-baseline.json b/apps/docs/scripts/fixtures/content-baseline.json index 80b2cab120d..9b596d5ae9b 100644 --- a/apps/docs/scripts/fixtures/content-baseline.json +++ b/apps/docs/scripts/fixtures/content-baseline.json @@ -6746,6 +6746,20 @@ "bytes": 51, "line": 201 }, + { + "lang": "bash", + "meta": null, + "sha256": "f57b9272801ee426bbcac1ba0a76951bd50f0d66de23916fe7beffd36a8078d7", + "bytes": 61, + "line": 238 + }, + { + "lang": "bash", + "meta": null, + "sha256": "d28646b150b37f8c75a2ae8ab0ca07b698279c3b044530810a2f9096079d5ff9", + "bytes": 65, + "line": 270 + }, { "lang": "bash", "meta": null, @@ -12595,6 +12609,20 @@ "bytes": 45, "line": 4259 }, + { + "lang": "bash", + "meta": null, + "sha256": "f57b9272801ee426bbcac1ba0a76951bd50f0d66de23916fe7beffd36a8078d7", + "bytes": 61, + "line": 4295 + }, + { + "lang": "bash", + "meta": null, + "sha256": "d28646b150b37f8c75a2ae8ab0ca07b698279c3b044530810a2f9096079d5ff9", + "bytes": 65, + "line": 4327 + }, { "lang": "bash", "meta": null, @@ -177603,6 +177631,13 @@ "sha256": "5c2ebfcc044ebc01c17a4af2664a4fbc667ae4916ef21555d90cfd9f245394bd", "bytes": 70, "line": 56 + }, + { + "lang": "yaml", + "meta": "title=\".gitlab-ci.yml\"", + "sha256": "cbd29bdea80fc9e1c908abaa8e708dd21435267b66be0583a033686a199b8882", + "bytes": 1046, + "line": 98 } ], "links": [ diff --git a/apps/sim/app/api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions/route.ts b/apps/sim/app/api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions/route.ts new file mode 100644 index 00000000000..02ff63f5f57 --- /dev/null +++ b/apps/sim/app/api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions/route.ts @@ -0,0 +1,33 @@ +import { + v2GetGitLabSearchPermissionsContract, + v2ReplaceGitLabSearchPermissionsContract, +} from '@/lib/api/contracts/v2/organizations' +import { defineV2JsonRoute, v2ApiKeyAuth, v2RateLimits } from '@/lib/api/server/routes' +import { v2OrganizationErrorPolicy } from '@/lib/api/server/routes/organizations' +import { + readGitLabSearchPermissions, + replaceGitLabSearchPermissions, +} from '@/lib/knowledge/application/gitlab-search-permissions' +import { gitLabSearchPermissionOperations } from '@/lib/knowledge/application/operations' + +export const GET = defineV2JsonRoute({ + contract: v2GetGitLabSearchPermissionsContract, + operation: gitLabSearchPermissionOperations.read, + auth: v2ApiKeyAuth, + rateLimit: v2RateLimits.publicApi, + errorPolicy: v2OrganizationErrorPolicy, + mapInput: ({ params }) => params, + useCase: readGitLabSearchPermissions, + present: (data) => ({ data }), +}) + +export const PUT = defineV2JsonRoute({ + contract: v2ReplaceGitLabSearchPermissionsContract, + operation: gitLabSearchPermissionOperations.replace, + auth: v2ApiKeyAuth, + rateLimit: v2RateLimits.publicApi, + errorPolicy: v2OrganizationErrorPolicy, + mapInput: ({ params, body }) => ({ ...params, ...body }), + useCase: replaceGitLabSearchPermissions, + present: (data) => ({ data }), +}) diff --git a/apps/sim/connectors/gitlab/permission-config/prepare.ts b/apps/sim/connectors/gitlab/permission-config/prepare.ts index 7147ed900a5..b695928cb3d 100644 --- a/apps/sim/connectors/gitlab/permission-config/prepare.ts +++ b/apps/sim/connectors/gitlab/permission-config/prepare.ts @@ -1,7 +1,6 @@ import { getErrorMessage } from '@sim/utils/errors' import { decryptApiKey, encryptApiKey } from '@/lib/api-key/crypto' import { OrchestrationError } from '@/lib/core/orchestration/types' -import { GitLabCsvValidationError } from '@/connectors/gitlab/permission-config/parser' import { loadGitLabPermissionSnapshot, type PreparedGitLabPermissions, @@ -76,21 +75,10 @@ export async function prepareGitLabPermissionChange(input: { getErrorMessage(error, 'Unable to validate the GitLab token and project.') ) }) - try { - return { - permissions: prepareGitLabPermissions( - permissions, - project, - existing, - Boolean(input.existing) - ), - ...(input.existing && input.apiKey - ? { encryptedApiKey: (await encryptApiKey(input.apiKey)).encrypted } - : {}), - } - } catch (error) { - if (error instanceof GitLabCsvValidationError) - throw new OrchestrationError('validation', error.message) - throw error + return { + permissions: prepareGitLabPermissions(permissions, project, existing, Boolean(input.existing)), + ...(input.existing && input.apiKey + ? { encryptedApiKey: (await encryptApiKey(input.apiKey)).encrypted } + : {}), } } diff --git a/apps/sim/connectors/gitlab/permission-config/repository.ts b/apps/sim/connectors/gitlab/permission-config/repository.ts index bb012b5c4e8..74f37054e56 100644 --- a/apps/sim/connectors/gitlab/permission-config/repository.ts +++ b/apps/sim/connectors/gitlab/permission-config/repository.ts @@ -8,7 +8,11 @@ import { readConnectorPermissionMetadata, writeConnectorPermissions, } from '@/lib/knowledge/connectors/permission-store' -import { gitLabCsvSubjects, parseGitLabCsv } from '@/connectors/gitlab/permission-config/parser' +import { + GitLabCsvValidationError, + gitLabCsvSubjects, + parseGitLabCsv, +} from '@/connectors/gitlab/permission-config/parser' import { GITLAB_CSV_MAX_ROWS, type GitLabPermissionInput, @@ -23,16 +27,16 @@ const metadataSchema = gitLabPermissionSummarySchema projectId: z.number().int().positive(), projectPath: z.string().min(1), }) -type Snapshot = z.output & { - connectorId: string - revision: number +type RevisionMetadata = z.output & { revision: number } +interface StoredRows { userMappingRows: [string, string][] | null projectPermissionRows: [string, string][] | null } +type Snapshot = RevisionMetadata & StoredRows export interface PreparedGitLabPermissions { expectedRevision: number - snapshot: Omit + snapshot: Snapshot subjects: string[] rewriteAccess: boolean } @@ -41,7 +45,8 @@ function invalidSnapshot(): never { throw new OrchestrationError('internal', 'Could not read the stored GitLab permissions.') } -function readMetadata(value: unknown) { +/** Validates stored file summaries and the canonical project without loading CSV rows. */ +export function readGitLabPermissionMetadata(value: unknown) { const parsed = metadataSchema.safeParse(value) if (!parsed.success) return invalidSnapshot() const result = parsed.data @@ -68,7 +73,7 @@ function readRows(value: unknown): [string, string][] | null { export async function loadGitLabPermissionSnapshot(connectorId: string): Promise { const row = await loadConnectorPermissionSnapshot(connectorId) if (!row) return null - const metadata = readMetadata(row.metadata) + const metadata = readGitLabPermissionMetadata(row.metadata) if (!isPlainRecord(row.payload)) return invalidSnapshot() const userMappingRows = readRows(row.payload.userMappingRows) const projectPermissionRows = readRows(row.payload.projectPermissionRows) @@ -82,7 +87,6 @@ export async function loadGitLabPermissionSnapshot(connectorId: string): Promise ) return invalidSnapshot() return { - connectorId, revision: row.revision, ...metadata, userMappingRows, @@ -93,7 +97,7 @@ export async function loadGitLabPermissionSnapshot(connectorId: string): Promise export function prepareGitLabPermissions( input: GitLabPermissionInput, project: { host: string; projectId: number; projectPath: string }, - existing: Snapshot | null, + existing: (RevisionMetadata & Partial) | null, replacing: boolean ): PreparedGitLabPermissions { const revision = existing?.revision ?? 0 @@ -115,7 +119,17 @@ export function prepareGitLabPermissions( kind === 'userMapping' ? existing?.userMappingRows : existing?.projectPermissionRows return metadata && rows ? { ...metadata, rows } : null } - return { filename: upload.filename.trim(), uploadedAt: now, rows: parseGitLabCsv(upload, kind) } + try { + return { + filename: upload.filename.trim(), + uploadedAt: now, + rows: parseGitLabCsv(upload, kind), + } + } catch (error) { + if (error instanceof GitLabCsvValidationError) + throw new OrchestrationError('validation', error.message) + throw error + } } const userMapping = input.mode === 'csv' ? file('userMapping') : null const projectPermissions = input.mode === 'csv' ? file('projectPermissions') : null @@ -125,7 +139,9 @@ export function prepareGitLabPermissions( return { expectedRevision: revision, snapshot: { - ...project, + host: project.host, + projectId: project.projectId, + projectPath: project.projectPath, mode: input.mode, revision: revision + 1, userMapping: userMapping @@ -183,7 +199,7 @@ export async function readGitLabPermissionSummaries( ): Promise> { const result = new Map() for (const row of await readConnectorPermissionMetadata(connectorIds)) { - const { mode, userMapping, projectPermissions } = readMetadata(row.metadata) + const { mode, userMapping, projectPermissions } = readGitLabPermissionMetadata(row.metadata) result.set(row.connectorId, { mode, revision: row.revision, userMapping, projectPermissions }) } return result @@ -196,6 +212,6 @@ export async function seedGitLabCsvContext( ): Promise { const [row] = await readConnectorPermissionMetadata([connectorId]) if (!row) return - const { mode, host, projectId, projectPath } = readMetadata(row.metadata) + const { mode, host, projectId, projectPath } = readGitLabPermissionMetadata(row.metadata) if (mode === 'csv') setGitLabCsvContext(context, { connectorId, host, projectId, projectPath }) } diff --git a/apps/sim/lib/api/contracts/knowledge/gitlab-permissions.ts b/apps/sim/lib/api/contracts/knowledge/gitlab-permissions.ts index f91d6f6aaa6..1c37329ef80 100644 --- a/apps/sim/lib/api/contracts/knowledge/gitlab-permissions.ts +++ b/apps/sim/lib/api/contracts/knowledge/gitlab-permissions.ts @@ -6,17 +6,30 @@ import { const gitLabCsvUploadSchema = z .object({ - filename: z.string().trim().min(1).max(255), - content: z.string().min(1).max(GITLAB_CSV_MAX_BYTES), + filename: z.string().trim().min(1).max(255).describe('CSV filename without path separators.'), + content: z + .string() + .min(1) + .max(GITLAB_CSV_MAX_BYTES) + .describe('UTF-8 CSV contents, up to 4 MiB.'), }) .strict() export const gitLabPermissionInputSchema = z .object({ provider: z.literal('gitlab'), mode: z.enum(GITLAB_PERMISSION_MODES), - expectedRevision: z.number().int().nonnegative().optional(), - userMapping: gitLabCsvUploadSchema.optional(), - projectPermissions: gitLabCsvUploadSchema.optional(), + expectedRevision: z + .number() + .int() + .nonnegative() + .describe('Current permission revision obtained before generating this snapshot.') + .optional(), + userMapping: gitLabCsvUploadSchema + .describe('GitLab user IDs and their verified Sim email addresses.') + .optional(), + projectPermissions: gitLabCsvUploadSchema + .describe('GitLab project paths and the IDs of users granted access.') + .optional(), }) .strict() export type GitLabPermissionUploadInput = z.input @@ -28,7 +41,7 @@ const gitLabCsvFileSummarySchema = z.object({ export const gitLabPermissionSummarySchema = z.object({ provider: z.literal('gitlab'), mode: z.enum(GITLAB_PERMISSION_MODES), - revision: z.number().int().nonnegative(), + revision: z.number().int().nonnegative().describe('Current revision of the stored permissions.'), userMapping: gitLabCsvFileSummarySchema.nullable(), projectPermissions: gitLabCsvFileSummarySchema.nullable(), }) diff --git a/apps/sim/lib/api/contracts/v2/openapi/organizations.ts b/apps/sim/lib/api/contracts/v2/openapi/organizations.ts index 2be982da7cd..2c0a71310dd 100644 --- a/apps/sim/lib/api/contracts/v2/openapi/organizations.ts +++ b/apps/sim/lib/api/contracts/v2/openapi/organizations.ts @@ -7,6 +7,7 @@ import { } from '@/lib/api/contracts/v2/openapi/shared' import { v2CreateOrganizationInvitationContract, + v2GetGitLabSearchPermissionsContract, v2GetOrganizationContract, v2GetOrganizationInvitationContract, v2ListOrganizationInvitationsContract, @@ -15,16 +16,32 @@ import { v2ListOrganizationsContract, v2ListOrganizationWorkspacesContract, v2RemoveOrganizationMemberContract, + v2ReplaceGitLabSearchPermissionsContract, v2ResendOrganizationInvitationContract, v2RevokeOrganizationInvitationContract, v2UpdateOrganizationMemberContract, } from '@/lib/api/contracts/v2/organizations' import { defineOpenApiRoute } from '@/lib/api/openapi/types' import { invitationOperations } from '@/lib/invitations/application/operations' +import { gitLabSearchPermissionOperations } from '@/lib/knowledge/application/operations' import { organizationOperations } from '@/lib/organizations/application/operations' const TIMESTAMP = '2026-06-01T09:00:00.000Z' +const gitLabSearchPermissionParams = documentedSchema( + v2GetGitLabSearchPermissionsContract.params, + 'GitLabSearchPermissionParams', + 'GitLab Search permission parameters', + 'Organization and Search source identifiers.' +) +const gitLabSearchPermissionResponse = documentedSchema( + v2GetGitLabSearchPermissionsContract.response.schema, + 'GitLabSearchPermissionResponse', + 'GitLab Search permission response', + 'Current revision of the permission CSVs.', + [{ data: { revision: 3 } }] +) + export const organizationOpenApiRoutes = [ defineOpenApiRoute( v2ListOrganizationsContract, @@ -552,4 +569,57 @@ export const organizationOpenApiRoutes = [ ), } ), + defineOpenApiRoute( + v2GetGitLabSearchPermissionsContract, + { + applicationOperation: gitLabSearchPermissionOperations.read, + operationId: 'getGitLabSearchPermissions', + summary: 'Get GitLab Search permission revision', + description: `Read the current revision of an existing GitLab Search source in CSV permission mode. Requires organization Admin access. ${WORKSPACE_API_KEY_DENIED}`, + tags: ['Organizations'], + errors: RESOURCE_ERRORS, + success: { description: 'The current permission revision.', headers: RATE_LIMIT_HEADERS }, + }, + { + params: gitLabSearchPermissionParams, + query: v2GetGitLabSearchPermissionsContract.query, + response: gitLabSearchPermissionResponse, + } + ), + defineOpenApiRoute( + v2ReplaceGitLabSearchPermissionsContract, + { + applicationOperation: gitLabSearchPermissionOperations.replace, + operationId: 'replaceGitLabSearchPermissions', + summary: 'Replace GitLab Search permission CSVs', + description: `Replace both permission CSVs for an existing GitLab Search source in CSV mode. Requires organization Admin access. Supply the current revision. A concurrent change returns 409 and leaves both files untouched. Each file accepts up to 4 MiB and 100,000 data rows. ${WORKSPACE_API_KEY_DENIED}`, + tags: ['Organizations'], + errors: RESOURCE_CONFLICT_ERRORS, + success: { description: 'The new permission revision.', headers: RATE_LIMIT_HEADERS }, + }, + { + params: gitLabSearchPermissionParams, + query: v2ReplaceGitLabSearchPermissionsContract.query, + body: documentedSchema( + v2ReplaceGitLabSearchPermissionsContract.body, + 'ReplaceGitLabSearchPermissionsBody', + 'GitLab Search permission CSVs', + 'The expected revision and complete contents of both UTF-8 CSV files.', + [ + { + expectedRevision: 2, + userMapping: { + filename: 'users.csv', + content: 'user_id,email\n42,reader@example.com\n', + }, + projectPermissions: { + filename: 'projects.csv', + content: 'project_path,user_id\nteam/project,42\n', + }, + }, + ] + ), + response: gitLabSearchPermissionResponse, + } + ), ] as const diff --git a/apps/sim/lib/api/contracts/v2/organizations.ts b/apps/sim/lib/api/contracts/v2/organizations.ts index 160dd9d30f7..c10244be6bd 100644 --- a/apps/sim/lib/api/contracts/v2/organizations.ts +++ b/apps/sim/lib/api/contracts/v2/organizations.ts @@ -1,4 +1,8 @@ import { z } from 'zod' +import { + gitLabPermissionInputSchema, + gitLabPermissionSummarySchema, +} from '@/lib/api/contracts/knowledge/gitlab-permissions' import { noInputSchema, nonEmptyIdSchema, @@ -318,3 +322,35 @@ export const v2ResendOrganizationInvitationContract = defineRouteContract({ body: noInputSchema.optional().default({}), response: { mode: 'json', schema: v2DataResponse(v2OrganizationInvitationSchema) }, }) + +const gitLabSearchSourceParamsSchema = v2OrganizationParamsSchema.extend({ + sourceId: nonEmptyIdSchema + .max(255) + .describe('Identifier of the configured GitLab Search source.'), +}) +const gitLabSearchRevisionResponse = v2DataResponse( + gitLabPermissionSummarySchema.pick({ revision: true }).meta({ + id: 'GitLabSearchPermissionRevision', + title: 'GitLab Search permission revision', + description: 'Revision of the permission CSVs for this source.', + }) +) + +export const v2GetGitLabSearchPermissionsContract = defineRouteContract({ + method: 'GET', + path: '/api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions', + params: gitLabSearchSourceParamsSchema, + query: noInputSchema, + response: { mode: 'json', schema: gitLabSearchRevisionResponse }, +}) + +export const v2ReplaceGitLabSearchPermissionsContract = defineRouteContract({ + method: 'PUT', + path: '/api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions', + params: gitLabSearchSourceParamsSchema, + query: noInputSchema, + body: gitLabPermissionInputSchema + .pick({ expectedRevision: true, userMapping: true, projectPermissions: true }) + .required(), + response: { mode: 'json', schema: gitLabSearchRevisionResponse }, +}) diff --git a/apps/sim/lib/api/mcp/generated/v2-operations.ts b/apps/sim/lib/api/mcp/generated/v2-operations.ts index 37f3dca65ef..208583dcb47 100644 --- a/apps/sim/lib/api/mcp/generated/v2-operations.ts +++ b/apps/sim/lib/api/mcp/generated/v2-operations.ts @@ -167,6 +167,7 @@ import { } from '@/lib/api/contracts/v2/organization-usage' import { v2CreateOrganizationInvitationContract, + v2GetGitLabSearchPermissionsContract, v2GetOrganizationContract, v2GetOrganizationInvitationContract, v2ListOrganizationInvitationsContract, @@ -175,6 +176,7 @@ import { v2ListOrganizationsContract, v2ListOrganizationWorkspacesContract, v2RemoveOrganizationMemberContract, + v2ReplaceGitLabSearchPermissionsContract, v2ResendOrganizationInvitationContract, v2RevokeOrganizationInvitationContract, v2UpdateOrganizationMemberContract, @@ -1326,6 +1328,17 @@ export const V2_MCP_OPERATIONS = { handler: () => import('@/app/api/v2/files/[fileId]/versions/[version]/route').then((route) => route.GET), }, + getGitLabSearchPermissions: { + contract: v2GetGitLabSearchPermissionsContract, + summary: 'Get GitLab Search permission revision', + description: + 'Read the current revision of an existing GitLab Search source in CSV permission mode. Requires organization Admin access. Workspace API keys return `403`; use a personal API key or scoped OAuth token.\n\nOAuth scope: `api:read`.', + workspaceKeyUnsupported: true, + handler: () => + import( + '@/app/api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions/route' + ).then((route) => route.GET), + }, getKnowledgeBase: { contract: v2GetKnowledgeBaseContract, summary: 'Get knowledge base', @@ -2438,6 +2451,17 @@ export const V2_MCP_OPERATIONS = { 'Rename a workspace file without changing its containing folder.\n\nOAuth scope: `api:write`.', handler: () => import('@/app/api/v2/files/[fileId]/route').then((route) => route.PATCH), }, + replaceGitLabSearchPermissions: { + contract: v2ReplaceGitLabSearchPermissionsContract, + summary: 'Replace GitLab Search permission CSVs', + description: + 'Replace both permission CSVs for an existing GitLab Search source in CSV mode. Requires organization Admin access. Supply the current revision. A concurrent change returns 409 and leaves both files untouched. Each file accepts up to 4 MiB and 100,000 data rows. Workspace API keys return `403`; use a personal API key or scoped OAuth token.\n\nOAuth scope: `api:write`.', + workspaceKeyUnsupported: true, + handler: () => + import( + '@/app/api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions/route' + ).then((route) => route.PUT), + }, replaceWorkflowChatDeployment: { contract: v2ReplaceWorkflowChatDeploymentContract, summary: 'Create or replace workflow chat deployment', diff --git a/apps/sim/lib/api/server/routes/copilot-refused-routes.json b/apps/sim/lib/api/server/routes/copilot-refused-routes.json index 0720d6ac7fa..4e35fa54e2e 100644 --- a/apps/sim/lib/api/server/routes/copilot-refused-routes.json +++ b/apps/sim/lib/api/server/routes/copilot-refused-routes.json @@ -174,6 +174,16 @@ "path": "/api/v2/organizations/[organizationId]/permission-groups/[groupId]/members/bulk", "operation": "permission_groups.members.bulk_add" }, + { + "method": "GET", + "path": "/api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions", + "operation": "knowledge.search.gitlab_permissions.read" + }, + { + "method": "PUT", + "path": "/api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions", + "operation": "knowledge.search.gitlab_permissions.replace" + }, { "method": "GET", "path": "/api/v2/organizations/[organizationId]/sso/policy", diff --git a/apps/sim/lib/api/server/routes/v2-route-table.generated.ts b/apps/sim/lib/api/server/routes/v2-route-table.generated.ts index 97242c2b333..ecc078072df 100644 --- a/apps/sim/lib/api/server/routes/v2-route-table.generated.ts +++ b/apps/sim/lib/api/server/routes/v2-route-table.generated.ts @@ -446,6 +446,13 @@ export const V2_ROUTES: readonly V2RouteEntry[] = [ '@/app/api/v2/organizations/[organizationId]/permission-groups/[groupId]/members/bulk/route' ), }, + { + pattern: '/api/v2/organizations/{organizationId}/search/sources/{sourceId}/gitlab-permissions', + load: () => + import( + '@/app/api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions/route' + ), + }, { pattern: '/api/v2/organizations/{organizationId}/sso/policy', load: () => import('@/app/api/v2/organizations/[organizationId]/sso/policy/route'), diff --git a/apps/sim/lib/knowledge/application/gitlab-search-permissions.ts b/apps/sim/lib/knowledge/application/gitlab-search-permissions.ts new file mode 100644 index 00000000000..baaffdcc3c8 --- /dev/null +++ b/apps/sim/lib/knowledge/application/gitlab-search-permissions.ts @@ -0,0 +1,143 @@ +import { AuditAction, AuditResourceType } from '@sim/audit' +import { db } from '@sim/db' +import { + knowledgeBase, + knowledgeConnector, + knowledgeConnectorPermissionSnapshot, +} from '@sim/db/schema' +import { and, eq, inArray, isNull } from 'drizzle-orm' +import { + defineAuthorizedOrganizationUseCase, + type OrganizationUseCaseContext, +} from '@/lib/core/application/authorized-organization-use-case' +import { authorizeOrganizationOperation } from '@/lib/core/application/organization-authorization' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { resourceScopeCondition } from '@/lib/core/resource-scope.server' +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import { gitLabSearchPermissionOperations } from '@/lib/knowledge/application/operations' +import { connectorIsLive } from '@/lib/knowledge/connectors/sync-lock' +import { + lockOrganizationSearchApproval, + searchIntegrationAccessCondition, +} from '@/lib/knowledge/search/integration-policy' +import { + prepareGitLabPermissions, + readGitLabPermissionMetadata, + writeGitLabPermissions, +} from '@/connectors/gitlab/permission-config/repository' +import type { GitLabCsvUpload } from '@/connectors/gitlab/permission-config/types' + +interface SourceInput { + organizationId: string + sourceId: string +} +interface ReplacementInput extends SourceInput { + expectedRevision: number + userMapping: GitLabCsvUpload + projectPermissions: GitLabCsvUpload +} + +function sourceNotFound(): never { + throw new OrchestrationError('not_found', 'GitLab CSV Search source not found') +} + +function searchIndex(organizationId: string) { + return and( + resourceScopeCondition(knowledgeBase, { kind: 'organization', organizationId }), + eq(knowledgeBase.isSearchIndex, true), + isNull(knowledgeBase.deletedAt) + ) +} + +function gitLabSource(sourceId: string) { + return and( + eq(knowledgeConnector.id, sourceId), + eq(knowledgeConnector.connectorType, 'gitlab'), + eq(knowledgeConnector.accessMode, 'admin'), + inArray(knowledgeConnector.status, ['active', 'paused', 'pending', 'syncing', 'error']), + connectorIsLive(), + searchIntegrationAccessCondition() + ) +} + +async function readCsvSource({ organizationId, sourceId }: SourceInput) { + if (!(await isKnowledgeMemberAccessAvailable({ organizationId }))) return sourceNotFound() + const snapshot = knowledgeConnectorPermissionSnapshot + const [source] = await db + .select({ + knowledgeBaseId: knowledgeBase.id, + revision: snapshot.revision, + metadata: snapshot.metadata, + }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .innerJoin(snapshot, eq(snapshot.connectorId, knowledgeConnector.id)) + .where(and(searchIndex(organizationId), gitLabSource(sourceId))) + .limit(1) + if (!source) return sourceNotFound() + const metadata = readGitLabPermissionMetadata(source.metadata) + if (metadata.mode !== 'csv') return sourceNotFound() + return { + knowledgeBaseId: source.knowledgeBaseId, + permissions: { ...metadata, revision: source.revision }, + } +} + +/** Allows CI to pin the revision without disclosing uploaded identities or project memberships. */ +export const readGitLabSearchPermissions = defineAuthorizedOrganizationUseCase({ + operation: gitLabSearchPermissionOperations.read, + async execute({ input }: OrganizationUseCaseContext) { + const { permissions } = await readCsvSource(input) + return { revision: permissions.revision } + }, +}) + +/** Replaces both permission files and the grants they produce in one revision. */ +export const replaceGitLabSearchPermissions = defineAuthorizedOrganizationUseCase({ + operation: gitLabSearchPermissionOperations.replace, + async execute({ principal, input }: OrganizationUseCaseContext) { + const { organizationId, sourceId, expectedRevision, userMapping, projectPermissions } = input + const { knowledgeBaseId, permissions } = await readCsvSource(input) + const prepared = prepareGitLabPermissions( + { mode: 'csv', expectedRevision, userMapping, projectPermissions }, + permissions, + permissions, + true + ) + await db.transaction(async (tx) => { + await lockOrganizationSearchApproval(tx, organizationId) + await authorizeOrganizationOperation( + principal, + gitLabSearchPermissionOperations.replace, + input, + { + executor: tx, + forShare: true, + } + ) + /** Preserve the parent-then-source lock order shared with source removal. */ + const [index] = await tx + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where(and(eq(knowledgeBase.id, knowledgeBaseId), searchIndex(organizationId))) + .for('share') + if (!index) return sourceNotFound() + const [source] = await tx + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .where(and(eq(knowledgeConnector.knowledgeBaseId, index.id), gitLabSource(sourceId))) + .for('update') + if (!source) return sourceNotFound() + await writeGitLabPermissions(tx, source.id, prepared) + }) + return { revision: prepared.snapshot.revision } + }, + projectAudit: ({ input, result }) => ({ + action: AuditAction.CONNECTOR_UPDATED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: input.sourceId, + resourceName: 'gitlab', + description: 'Replaced GitLab Search permission CSVs', + metadata: { revision: result.revision }, + }), +}) diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index d15f9f60fca..2b81f1f0f47 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -1031,6 +1031,23 @@ function defineKnowledgeSessionOperation( return Object.freeze(operation) } +export const gitLabSearchPermissionOperations = { + read: defineOrganizationOperation({ + minimumRole: 'admin', + principalKinds: ['personal_api_key', 'oauth_access_token'], + capability: 'knowledge.use', + id: 'knowledge.search.gitlab_permissions.read', + oauthScope: 'api:read', + }), + replace: defineOrganizationOperation({ + minimumRole: 'admin', + principalKinds: ['personal_api_key', 'oauth_access_token'], + capability: 'knowledge.use', + id: 'knowledge.search.gitlab_permissions.replace', + oauthScope: 'api:write', + }), +} as const + export const knowledgeSessionOperations = { // permission-group-exempt: delegates to knowledgeOperations.list, which carries knowledge.use list: defineKnowledgeSessionOperation({ id: 'knowledge.session.list', capability: 'none' }), diff --git a/apps/sim/scripts/test-gitlab-csv-permissions-e2e.ts b/apps/sim/scripts/test-gitlab-csv-permissions-e2e.ts new file mode 100644 index 00000000000..ae639a9595a --- /dev/null +++ b/apps/sim/scripts/test-gitlab-csv-permissions-e2e.ts @@ -0,0 +1,210 @@ +import assert from 'node:assert/strict' +import { writeFile } from 'node:fs/promises' +import { assertDisposableTestDatabaseUrl } from '@sim/db/testing/test-infrastructure' +import { sha256Hex } from '@sim/security/hash' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import postgres from 'postgres' + +/** Exercises the public CI endpoint against a running Sim and preconfigured CSV source. */ +const base = new URL(process.env.GITLAB_CSV_E2E_BASE_URL ?? '') +assert.equal(base.protocol, 'http:') +assert(['localhost', '127.0.0.1', '[::1]'].includes(base.hostname)) +const sql = postgres(assertDisposableTestDatabaseUrl(process.env.DATABASE_URL ?? '').href) +const reportPath = process.env.GITLAB_CSV_E2E_REPORT_PATH +assert(reportPath, 'Set GITLAB_CSV_E2E_REPORT_PATH') +const checks: string[] = [] +const org = generateId() +const otherOrg = generateId() +const admin = generateId() +const member = generateId() +const otherAdmin = generateId() +const workspace = generateId() +const kb = generateId() +const source = generateId() +const key = () => `sk-sim-fixture-${generateId()}` +const adminKey = key() +const memberKey = key() +const otherAdminKey = key() +const workspaceKey = key() +const project = 'engineering/pilot' +const initialEmail = `${admin}@fixture.test` +const nextEmail = `${member}@fixture.test` +const pat = `fixture-preserved-${generateId()}` +const file = (filename: string, content: string) => ({ filename, content }) +const labels = { filename: 'initial.csv', rowCount: 1, uploadedAt: new Date().toISOString() } +const csv = (expectedRevision: number, id = 2) => ({ + expectedRevision, + userMapping: file( + 'users.csv', + `user_id,email\n1,${initialEmail}\n2,${nextEmail.toUpperCase()}\n` + ), + projectPermissions: file( + 'projects.csv', + `project_path,user_id\n${project},${id}\nelsewhere/team,1\n` + ), +}) +const path = (organizationId = org) => + `/api/v2/organizations/${organizationId}/search/sources/${source}/gitlab-permissions` + +async function request(token: string, body?: unknown, organizationId = org) { + return fetch(new URL(path(organizationId), base), { + method: body === undefined ? 'GET' : 'PUT', + headers: { 'X-API-Key': token, 'Content-Type': 'application/json' }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + signal: AbortSignal.timeout(180_000), + redirect: 'error', + }) +} + +async function checkRevision(response: Response, revision: number) { + assert.equal(response.status, 200, `Expected revision ${revision}, HTTP ${response.status}`) + assert.deepEqual(await response.json(), { data: { revision } }) +} + +async function grants() { + const rows = await sql`select subject_token from knowledge_connector_permission_grant + where connector_id = ${source} and group_key = 'project' order by subject_token` + return rows.map((row) => row.subject_token) +} + +let failure: string | undefined +try { + await sql.begin(async (tx) => { + for (const id of [admin, member, otherAdmin]) { + await tx`insert into "user" (id, name, email, email_verified, created_at, updated_at) + values (${id}, 'GitLab CSV fixture', ${`${id}@fixture.test`}, true, now(), now())` + } + for (const id of [org, otherOrg]) { + await tx`insert into organization (id, name, slug) values (${id}, 'GitLab CSV fixture', ${id})` + await tx`insert into member (id, organization_id, user_id, role) + values (${generateId()}, ${id}, ${id === org ? admin : otherAdmin}, 'admin')` + } + await tx`insert into member (id, organization_id, user_id, role) + values (${generateId()}, ${org}, ${member}, 'member')` + await tx`insert into project (id, name, owner_id) values (${workspace}, 'GitLab CSV fixture', ${admin})` + await tx`insert into workspace (id, project_id, name, owner_id, billed_account_user_id) + values (${workspace}, ${workspace}, 'GitLab CSV fixture', ${admin}, ${admin})` + for (const [id, token, scope] of [ + [admin, adminKey, null], + [member, memberKey, null], + [otherAdmin, otherAdminKey, null], + [admin, workspaceKey, workspace], + ] as const) { + await tx`insert into api_key (id, user_id, workspace_id, name, key, key_hash, type) values + (${generateId()}, ${id}, ${scope}, 'GitLab CSV fixture', ${token}, ${sha256Hex(token)}, ${scope ? 'workspace' : 'personal'})` + } + await tx`insert into knowledge_base (id, name, user_id, organization_id, is_search_index) + values (${kb}, 'GitLab CSV fixture', ${admin}, ${org}, true)` + await tx`insert into knowledge_connector + (id, knowledge_base_id, connector_type, access_mode, encrypted_api_key, source_config) + values (${source}, ${kb}, 'gitlab', 'admin', ${pat}, ${tx.json({ host: 'gitlab.fixture.invalid', project })})` + await tx`insert into knowledge_connector_permission_snapshot (connector_id, revision, metadata, payload) + values (${source}, 1, ${tx.json({ + host: 'gitlab.fixture.invalid', + projectId: 10, + projectPath: project, + mode: 'csv', + userMapping: labels, + projectPermissions: labels, + })}, + ${tx.json({ userMappingRows: [['1', initialEmail]], projectPermissionRows: [[project, '1']] })})` + await tx`insert into knowledge_connector_permission_grant (connector_id, group_key, subject_token) + values (${source}, 'project', ${`u:${initialEmail}`})` + }) + await checkRevision(await request(adminKey), 1) + checks.push( + 'An organization admin can read the revision without exposing the CSV contents or PAT' + ) + + for (const token of [memberKey, workspaceKey]) { + for (const body of [undefined, csv(1)]) assert.equal((await request(token, body)).status, 403) + } + assert.equal((await request(otherAdminKey, csv(1), otherOrg)).status, 404) + assert.equal( + (await request(adminKey, { expectedRevision: 1, userMapping: csv(1).userMapping })).status, + 400 + ) + assert.equal((await request(adminKey, { ...csv(1), apiKey: 'unexpected' })).status, 400) + assert.equal( + ( + await request(adminKey, { + ...csv(1), + userMapping: file('users.csv', 'x'.repeat(10 * 1024 * 1024)), + }) + ).status, + 413 + ) + assert.equal( + (await request(adminKey, { ...csv(1), userMapping: file('users.csv', '1,invalid') })).status, + 400 + ) + assert.deepEqual(await grants(), [`u:${initialEmail}`]) + await checkRevision(await request(adminKey), 1) + checks.push( + 'Unauthorized, cross-organization and invalid replacements preserve the existing grant and revision' + ) + + await sql`update knowledge_connector set connector_type = 'github' where id = ${source}` + assert.equal((await request(adminKey, csv(1))).status, 404) + await sql`update knowledge_connector set connector_type = 'gitlab' where id = ${source}` + const [original] = + await sql`select metadata from knowledge_connector_permission_snapshot where connector_id = ${source}` + assert(original) + await sql`update knowledge_connector_permission_snapshot + set metadata = metadata || '{"mode":"administrator","userMapping":null,"projectPermissions":null}'::jsonb + where connector_id = ${source}` + assert.equal((await request(adminKey, csv(1))).status, 404) + await sql`update knowledge_connector_permission_snapshot set metadata = ${sql.json(original.metadata)} where connector_id = ${source}` + assert.deepEqual(await grants(), [`u:${initialEmail}`]) + checks.push('Only an existing GitLab CSV source accepts replacements') + + const race = await Promise.all([request(adminKey, csv(1)), request(adminKey, csv(1))]) + assert.deepEqual(race.map((r) => r.status).sort(), [200, 409]) + const winner = race.find((r) => r.ok) + assert(winner) + await checkRevision(winner, 2) + assert.deepEqual(await grants(), [`u:${nextEmail}`]) + assert.equal((await request(adminKey, csv(1, 1))).status, 409) + assert.deepEqual(await grants(), [`u:${nextEmail}`]) + checks.push( + 'Concurrent updates grant only the mapped member for this project and stale retries cannot restore removed access' + ) + + await sql`update knowledge_connector set status = 'paused' where id = ${source}` + await checkRevision(await request(adminKey, csv(2, 1)), 3) + await checkRevision(await request(adminKey), 3) + assert.deepEqual(await grants(), [`u:${initialEmail}`]) + const [connector] = + await sql`select status, encrypted_api_key, source_config, next_sync_at, access_rewrite_pending + from knowledge_connector where id = ${source}` + assert.deepEqual(connector?.source_config, { host: 'gitlab.fixture.invalid', project }) + assert.equal(connector?.status, 'paused') + assert.equal(connector?.encrypted_api_key, pat) + assert.equal(connector?.next_sync_at, null) + assert.equal(connector?.access_rewrite_pending, false) + checks.push( + 'A paused source can revoke a prior grant while preserving its status, configuration, PAT and schedule' + ) +} catch (error) { + failure = getErrorMessage(error) +} finally { + try { + await sql.begin(async (tx) => { + await tx`delete from workspace where id = ${workspace}` + await tx`delete from project where id = ${workspace}` + await tx`delete from organization where id in (${org}, ${otherOrg})` + await tx`delete from "user" where id in (${admin}, ${member}, ${otherAdmin})` + }) + } catch (error) { + failure ??= getErrorMessage(error) + } finally { + await sql.end() + await writeFile( + reportPath, + JSON.stringify({ status: failure ? 'failed' : 'passed', checks, failure }, null, 2) + ) + } +} + +if (failure) throw new Error(failure) diff --git a/apps/sim/turbo.json b/apps/sim/turbo.json index 5e69d34014e..3c133ac8651 100644 --- a/apps/sim/turbo.json +++ b/apps/sim/turbo.json @@ -28,6 +28,7 @@ "!scripts/test-desktop-inbox-e2e.ts", "!scripts/test-assistant-actions-e2e.ts", "!scripts/test-organization-search-e2e.ts", + "!scripts/test-gitlab-csv-permissions-e2e.ts", "!scripts/test-docs-search-provider-e2e.ts", "!scripts/test-file-*-e2e.ts", "!scripts/test-landing-seo-e2e.ts", @@ -132,7 +133,10 @@ }, "ci:e2e-organization-search": { "dependsOn": ["ci:server", "ci:ui", "@sim/realtime#ci:runtime"], - "inputs": ["scripts/test-organization-search-e2e.ts"], + "inputs": [ + "scripts/test-organization-search-e2e.ts", + "scripts/test-gitlab-csv-permissions-e2e.ts" + ], "cache": false }, "ci:e2e-docs-platform": { @@ -177,6 +181,7 @@ "!scripts/test-desktop-inbox-e2e.ts", "!scripts/test-assistant-actions-e2e.ts", "!scripts/test-organization-search-e2e.ts", + "!scripts/test-gitlab-csv-permissions-e2e.ts", "!scripts/test-docs-search-provider-e2e.ts", "!scripts/test-file-*-e2e.ts", "!scripts/test-landing-seo-e2e.ts", diff --git a/packages/sim-cli/src/contract/commands.ts b/packages/sim-cli/src/contract/commands.ts index f62fae24641..78baeda7163 100644 --- a/packages/sim-cli/src/contract/commands.ts +++ b/packages/sim-cli/src/contract/commands.ts @@ -1238,6 +1238,16 @@ export const CLI_CONTRACT: CliContract = { columns: [{ header: 'id' }, { header: 'name' }, { header: 'role' }], }, getOrganization: { command: 'organizations get' }, + getGitLabSearchPermissions: { + command: 'organizations gitlab-permissions get', + pathFlags: ORGANIZATION_FLAG, + }, + replaceGitLabSearchPermissions: { + command: 'organizations gitlab-permissions replace', + pathFlags: ORGANIZATION_FLAG, + confirm: + 'This replaces both GitLab permission files. Readers omitted from the new grants lose access.', + }, listSsoProviders: { command: 'organizations sso providers list', pathFlags: ORGANIZATION_FLAG, diff --git a/packages/sim-cli/src/generated/v2-api.ts b/packages/sim-cli/src/generated/v2-api.ts index d874e376b3d..12f01c3613a 100644 --- a/packages/sim-cli/src/generated/v2-api.ts +++ b/packages/sim-cli/src/generated/v2-api.ts @@ -6221,6 +6221,22 @@ export type GetFileVersionResponse = { data: GetFileVersionResponseRef0 } +/** `GET /api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions` */ +export type GetGitLabSearchPermissionsParams = { + organizationId: string + sourceId: string +} + +export type GetGitLabSearchPermissionsQuery = Record + +type GetGitLabSearchPermissionsResponseRef0 = { + revision: number +} + +export type GetGitLabSearchPermissionsResponse = { + data: GetGitLabSearchPermissionsResponseRef0 +} + /** `GET /api/v2/knowledge/[knowledgeBaseId]` */ export type GetKnowledgeBaseParams = { knowledgeBaseId: string @@ -12725,6 +12741,34 @@ export type RenameProjectFileResponse = { data: RenameProjectFileResponseRef0 } +/** `PUT /api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions` */ +export type ReplaceGitLabSearchPermissionsParams = { + organizationId: string + sourceId: string +} + +export type ReplaceGitLabSearchPermissionsQuery = Record + +export type ReplaceGitLabSearchPermissionsBody = { + expectedRevision: number + userMapping: { + filename: string + content: string + } + projectPermissions: { + filename: string + content: string + } +} + +type ReplaceGitLabSearchPermissionsResponseRef0 = { + revision: number +} + +export type ReplaceGitLabSearchPermissionsResponse = { + data: ReplaceGitLabSearchPermissionsResponseRef0 +} + /** `PUT /api/v2/workflows/[workflowId]/deployments/chat` */ export type ReplaceWorkflowChatDeploymentParams = { workflowId: string @@ -18793,6 +18837,18 @@ export const V2_OPERATIONS = { workspaceId: { kind: 'string', required: true, describe: 'Workspace that owns the file.' }, }, }, + getGitLabSearchPermissions: { + method: 'GET', + path: '/api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions', + pathParams: ['organizationId', 'sourceId'] as const, + pathParamDocs: { + organizationId: 'Organization identifier.', + sourceId: 'Identifier of the configured GitLab Search source.', + }, + responseMode: 'json', + summary: 'Get GitLab Search permission revision', + workspaceKeyUnsupported: true, + }, getKnowledgeBase: { method: 'GET', path: '/api/v2/knowledge/[knowledgeBaseId]', @@ -23073,6 +23129,35 @@ export const V2_OPERATIONS = { name: { kind: 'string', required: true, describe: 'New file name, including its extension.' }, }, }, + replaceGitLabSearchPermissions: { + method: 'PUT', + path: '/api/v2/organizations/[organizationId]/search/sources/[sourceId]/gitlab-permissions', + pathParams: ['organizationId', 'sourceId'] as const, + pathParamDocs: { + organizationId: 'Organization identifier.', + sourceId: 'Identifier of the configured GitLab Search source.', + }, + responseMode: 'json', + summary: 'Replace GitLab Search permission CSVs', + workspaceKeyUnsupported: true, + body: { + expectedRevision: { + kind: 'integer', + required: true, + describe: 'Current permission revision obtained before generating this snapshot.', + }, + userMapping: { + kind: 'object', + required: true, + describe: 'GitLab user IDs and their verified Sim email addresses.', + }, + projectPermissions: { + kind: 'object', + required: true, + describe: 'GitLab project paths and the IDs of users granted access.', + }, + }, + }, replaceWorkflowChatDeployment: { method: 'PUT', path: '/api/v2/workflows/[workflowId]/deployments/chat', diff --git a/packages/sim-cli/src/http/client.test.ts b/packages/sim-cli/src/http/client.test.ts index 73d59220211..91c9f17ba31 100644 --- a/packages/sim-cli/src/http/client.test.ts +++ b/packages/sim-cli/src/http/client.test.ts @@ -407,6 +407,7 @@ describe('destructive operations are gated', () => { 'bulkDeleteTables', 'bulkUpdateKnowledgeChunks', 'replaceWorkflowChatDeployment', + 'replaceGitLabSearchPermissions', 'replaceWorkflowState', 'revertWorkflowVersion', 'updateRowsByFilter', diff --git a/scripts/generate-cli-docs.ts b/scripts/generate-cli-docs.ts index 0eab47add57..762d961e453 100644 --- a/scripts/generate-cli-docs.ts +++ b/scripts/generate-cli-docs.ts @@ -308,6 +308,7 @@ const GROUP_SECTIONS: Record = { organizations: [ { title: 'Organizations and workspaces', prefixes: ['list', 'get', 'workspaces'] }, { title: 'Members and credit limits', prefixes: ['members'] }, + { title: 'GitLab Search permissions', prefixes: ['gitlab-permissions'] }, { title: 'Invitations', prefixes: ['invitations'] }, { title: 'Usage', prefixes: ['usage summary', 'usage breakdown', 'usage events'] }, {