fix(mothership): remove eager catalogs and bulky admission saves

Author: Sg312

apps/sim/app/api/mothership/execute/route.test.ts

@@ -109,8 +109,7 @@ describe('buildExecuteResponsePayload', () => {
   it('still admits integration and mcp tool calls, and still drops other server tools', () => {
     const payload = buildExecuteResponsePayload(
       resultWithToolCalls(['gmail_send', 'mcp-notion-create', 'read', 'edit_workflow']),
-      'chat-1',
-      [{ name: 'gmail_send' }]
+      'chat-1'
     )
 
     const names = payload.toolCalls.map((tc: { name: string }) => tc.name)
@@ -422,6 +421,17 @@ describe('mothership private trace provenance transport', () => {
         'POST',
         {
           ...requestBody,
+          mcpTools: [
+            {
+              type: 'mcp',
+              params: { serverId: 'mcp-cg-123456789012345678901', toolName: 'search_transcripts' },
+            },
+            {
+              type: 'mcp',
+              usageControl: 'none',
+              params: { serverId: 'mcp-cg-123456789012345678901', toolName: 'delete_transcripts' },
+            },
+          ],
           contexts: [
             { kind: 'mcp', label: 'MCP 123', serverId: '123', path: '123' },
             {
@@ -445,12 +455,21 @@ describe('mothership private trace provenance transport', () => {
     )
 
     expect(response.status).toBe(200)
-    expect(mockBuildTaggedMcpToolSchemas).toHaveBeenCalledWith(
-      'user-1',
-      'workspace-1',
-      ['123'],
-      expect.objectContaining({ mcpBlockId: 'block-1' })
-    )
+    expect(mockRunHeadlessCopilotLifecycle.mock.calls[0]?.[0]).toMatchObject({
+      integrationCatalog: {
+        mcpServerIds: ['123'],
+        mcpToolIds: ['mcp-cg-123456789012345678901-search_transcripts'],
+        mcpExecution: {
+          workflowId: 'workflow-1',
+          executionId: 'execution-1',
+          mcpBlockId: 'block-1',
+          subjectUserId: 'user-1',
+        },
+      },
+    })
+    expect(mockBuildIntegrationToolSchemas).not.toHaveBeenCalled()
+    expect(mockBuildTaggedMcpToolSchemas).not.toHaveBeenCalled()
+    expect(mockBuildSelectedMcpToolSchemas).not.toHaveBeenCalled()
     expect(mockProcessContextsServer).toHaveBeenCalledWith(
       [
         {
@@ -638,7 +657,7 @@ describe('mothership private trace provenance transport', () => {
     expect(mockGetPersonalAndWorkspaceEnv).toHaveBeenCalledTimes(1)
   })
 
-  it('keeps discovered MCP schemas raw without activating matching configured secrets', async () => {
+  it('sends enabled MCP IDs without schemas or activating matching configured secrets', async () => {
     mockBuildTaggedMcpToolSchemas.mockResolvedValueOnce([
       { name: 'mcp-docs', description: 'Uses secret-value' },
     ])
@@ -653,9 +672,18 @@ describe('mothership private trace provenance transport', () => {
           entries: [],
           scope: { userId: 'user-1', workspaceId: 'workspace-1' },
         })
-        expect(payload.mothershipTools).toEqual([
-          { name: 'mcp-docs', description: 'Uses secret-value' },
-        ])
+        expect(payload).not.toHaveProperty('mothershipTools')
+        expect(payload).not.toHaveProperty('integrationTools')
+        expect(payload.integrationCatalog).toEqual({
+          mcpServerIds: ['server-1'],
+          mcpToolIds: [],
+          mcpExecution: {
+            workflowId: 'workflow-1',
+            executionId: 'execution-1',
+            mcpBlockId: 'block-1',
+            subjectUserId: 'user-1',
+          },
+        })
         expect(JSON.stringify(payload.messages)).toContain('search_integration_tools')
         expect(JSON.stringify(payload.messages)).toContain('call_integration_tool')
         expect(JSON.stringify(payload.messages)).not.toContain('callable directly')

apps/sim/app/api/mothership/execute/route.ts

@@ -1,3 +1,4 @@
+import { serializePrincipal } from '@sim/auth/principal'
 import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
@@ -18,18 +19,19 @@ import {
 } from '@/lib/execution/private-tool-metadata'
 import { createExecutorPrincipalFromExecutionContext } from '@/lib/internal/principals/executor'
 import { MCP_SERVER_DELEGATION_AUDIENCE } from '@/lib/mcp/application/authorization'
-import { buildIntegrationToolSchemas } from '@/lib/mothership/chat/payload'
+import { resolveMcpToolBinding } from '@/lib/mcp/tool-binding'
+import { createMcpToolId } from '@/lib/mcp/utils'
 import { processContextsServer } from '@/lib/mothership/chat/process-contents'
 import {
   type CopilotEnvironmentContext,
   createCopilotEnvironmentContext,
 } from '@/lib/mothership/environment-context'
+import { IntegrationCatalogMcpExecution } from '@/lib/mothership/generated/integration-catalog'
 import {
   MothershipStreamV1EventType,
   MothershipStreamV1TextChannel,
 } from '@/lib/mothership/generated/mothership-stream-v1'
 import { PROTOCOL_VERSION } from '@/lib/mothership/generated/protocol'
-import { buildSelectedMcpToolSchemas, buildTaggedMcpToolSchemas } from '@/lib/mothership/mcp-tools'
 import { runHeadlessCopilotLifecycle } from '@/lib/mothership/request/lifecycle/headless'
 import { requestExplicitStreamAbort } from '@/lib/mothership/request/session/explicit-abort'
 import type { StreamEvent } from '@/lib/mothership/request/types'
@@ -43,6 +45,7 @@ import {
   type ResolvedSecretTraceRegistry,
 } from '@/executor/utils/resolved-secret-trace-registry'
 import type { ChatContext } from '@/stores/panel'
+import { hasToolId } from '@/tools/tool-ids'
 
 export const maxDuration = 3600
 
@@ -94,12 +97,10 @@ function encodeNdjson(value: unknown): Uint8Array {
 
 export function buildExecuteResponsePayload(
   result: Awaited<ReturnType<typeof runHeadlessCopilotLifecycle>>,
-  effectiveChatId: string,
-  integrationTools: Array<{ name: string }>
+  effectiveChatId: string
 ) {
-  const clientToolNames = new Set(integrationTools.map((t) => t.name))
   const clientToolCalls = (result.toolCalls || []).filter(
-    (tc: { name: string }) => clientToolNames.has(tc.name) || tc.name.startsWith('mcp-')
+    (tc: { name: string }) => hasToolId(tc.name) || tc.name.startsWith('mcp-')
   )
 
   return {
@@ -246,34 +247,25 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
     )
     const nonMcpAgentMentions = agentMentions?.filter((context) => context.kind !== 'mcp')
     const userPermission = workspaceAccess.permission
-    const mothershipToolsPromise = Promise.allSettled([
-      buildSelectedMcpToolSchemas(userId, workspaceId, mcpTools ?? [], mcpContext),
-      buildTaggedMcpToolSchemas(userId, workspaceId, taggedMcpServerIds, mcpContext),
-    ]).then((results) => {
-      const groups = results.map((result) => {
-        if (result.status === 'rejected') throw result.reason
-        return result.value
+    const selectedMcpToolIds = (mcpTools ?? [])
+      .filter((tool) => tool.type === 'mcp' && (tool.usageControl || 'auto') !== 'none')
+      .map((tool) => {
+        const { serverId, toolName } = resolveMcpToolBinding(tool)
+        return createMcpToolId(serverId, toolName)
       })
-      const byName = new Map(groups.flat().map((tool) => [tool.name, tool]))
-      return [...byName.values()]
+    const agentContexts = await processContextsServer(
+      nonMcpAgentMentions,
+      userId,
+      lastUserMessage,
+      workspaceId,
+      effectiveChatId,
+      activeResolvedSecretTraceRegistry
+    ).catch((error) => {
+      reqLogger.warn('Failed to resolve agent contexts for execution', {
+        error: toError(error).message,
+      })
+      return []
     })
-    const [integrationTools, mothershipTools, agentContexts] = await Promise.all([
-      buildIntegrationToolSchemas(userId, undefined, workspaceId),
-      mothershipToolsPromise,
-      processContextsServer(
-        nonMcpAgentMentions,
-        userId,
-        lastUserMessage,
-        workspaceId,
-        effectiveChatId,
-        activeResolvedSecretTraceRegistry
-      ).catch((error) => {
-        reqLogger.warn('Failed to resolve agent contexts for execution', {
-          error: toError(error).message,
-        })
-        return []
-      }),
-    ])
     /**
      * The wire payload IS the shared ExecuteRequest contract. Caller-side context —
      * resolved mentions and the MCP-enablement notice — folds into the message array
@@ -285,12 +277,13 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
         const c = ctx as { type?: string; content?: string }
         return `[Attached ${c.type ?? 'context'}]\n${c.content ?? ''}`
       }),
-      ...(mothershipTools.length > 0
+      ...(taggedMcpServerIds.length || selectedMcpToolIds.length
         ? [
             [
-              'The following MCP operations are enabled. Find their input schemas with search_integration_tools using the exact toolId shown, then invoke them through call_integration_tool.',
+              'MCP operations are enabled. Discover their current input schemas with search_integration_tools, then invoke them through call_integration_tool.',
               'Do not narrate discovery, tool-name selection, or retries. Call the tool first, then respond once with the result. Never claim the server works before a successful tool result. Do not automatically retry a timed-out or abandoned MCP call.',
-              ...mothershipTools.map((tool) => `- ${tool.name}: ${tool.description || tool.name}`),
+              ...taggedMcpServerIds.map((id) => `Enabled MCP server: ${id}`),
+              ...selectedMcpToolIds.map((id) => `Enabled MCP operation: ${id}`),
             ].join('\n'),
           ]
         : []),
@@ -310,8 +303,18 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
       workspaceId,
       chatId: effectiveChatId,
       messageId,
-      ...(integrationTools.length > 0 ? { integrationTools } : {}),
-      ...(mothershipTools.length > 0 ? { mothershipTools } : {}),
+      integrationCatalog: {
+        mcpServerIds: taggedMcpServerIds,
+        mcpToolIds: selectedMcpToolIds,
+        mcpExecution: IntegrationCatalogMcpExecution.parse({
+          workflowId: delegation.workflowId,
+          executionId: delegation.executionId,
+          mcpBlockId: delegation.mcpBlockId,
+          subjectUserId: delegation.subjectUserId,
+          ...(delegation.principal ? { principal: serializePrincipal(delegation.principal) } : {}),
+          ...(delegation.currentWorkflow ? { currentWorkflow: delegation.currentWorkflow } : {}),
+        }),
+      },
     }
 
     let allowExplicitAbort = true
@@ -491,7 +494,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
               send({
                 type: 'final',
                 data: withPrivateProvenance(
-                  buildExecuteResponsePayload(result, effectiveChatId, integrationTools),
+                  buildExecuteResponsePayload(result, effectiveChatId),
                   resolvedSecretTraceRegistry,
                   includePrivateProvenance
                 ),
@@ -619,7 +622,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => {
 
       return NextResponse.json(
         withPrivateProvenance(
-          buildExecuteResponsePayload(result, effectiveChatId, integrationTools),
+          buildExecuteResponsePayload(result, effectiveChatId),
           resolvedSecretTraceRegistry,
           includePrivateProvenance
         ),

apps/sim/app/api/mothership/integrations/catalog/route.test.ts

@@ -0,0 +1,103 @@
+/** @vitest-environment node */
+import { NextRequest } from 'next/server'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const mocks = vi.hoisted(() => ({ build: vi.fn(), workspace: vi.fn(), permission: vi.fn() }))
+vi.unmock('@/lib/mothership/request/http')
+vi.mock('@/lib/mothership/chat/payload', () => ({ buildIntegrationToolSchemas: mocks.build }))
+vi.mock('@/lib/mothership/mcp-tools', () => ({ buildTaggedMcpToolSchemas: vi.fn() }))
+vi.mock('@/lib/mcp/application/use-cases', () => ({ listMcpServersUseCase: { execute: vi.fn() } }))
+vi.mock('@/lib/workspaces/application/workspace-context', () => ({
+  resolveActiveWorkspaceApplicationContext: mocks.workspace,
+}))
+vi.mock('@sim/platform-authz/workspace', () => ({
+  resolveEffectiveWorkspacePermission: mocks.permission,
+  permissionSatisfies: (actual: string | null, needed: string) =>
+    actual === 'admin' || actual === 'write' || actual === needed,
+}))
+vi.mock('@/lib/mothership/chat/application/workspace-context', () => ({
+  readWorkspaceContext: { execute: vi.fn() },
+}))
+vi.mock('@/lib/mothership/request/application/read-control', () => ({
+  RUN_CONTROL_AUDIENCE: 'control',
+  readRunControl: { execute: vi.fn() },
+}))
+vi.mock('@/lib/mothership/tasks/application/read-workflow-status', () => ({
+  readWatchedWorkflowStatus: { execute: vi.fn() },
+}))
+vi.mock('@/lib/mothership/tasks/application/prepare-wake', () => ({
+  prepareTaskWake: { execute: vi.fn() },
+}))
+vi.mock('@/lib/mothership/tasks/application/context', () => ({ TASK_DELEGATION_AUDIENCE: 'tasks' }))
+vi.mock('@/lib/mothership/tasks/wake', () => ({ runWakeTurn: vi.fn() }))
+
+import { env } from '@/lib/core/config/env'
+import { executeSimControl } from '@/lib/mothership/transport/control'
+import { POST } from '@/app/api/mothership/integrations/catalog/route'
+
+const scope = {
+  userId: 'actor',
+  workspaceId: '11111111-1111-4111-8111-111111111111',
+  chatId: '33333333-3333-4333-8333-333333333333',
+}
+const input = { mode: 'agent' as const, mcpServerIds: [], toolId: 'gmail_send', limit: 1 }
+function request(key = env.INTERNAL_API_SECRET ?? '') {
+  return new NextRequest('http://localhost/api/mothership/integrations/catalog', {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      'x-api-key': key,
+      'x-mothership-user-id': scope.userId,
+      'x-mothership-workspace-id': scope.workspaceId,
+      'x-mothership-chat-id': scope.chatId,
+    },
+    body: JSON.stringify(input),
+  })
+}
+function checkpoint() {
+  return executeSimControl({
+    id: 'catalog-control',
+    scope,
+    expiresAt: Date.now() + 5000,
+    operation: { kind: 'integration_catalog', input },
+  })
+}
+beforeEach(() => {
+  vi.clearAllMocks()
+  mocks.permission.mockResolvedValue('read')
+  mocks.workspace.mockResolvedValue({
+    workspaceId: scope.workspaceId,
+    workspaceOrganizationId: null,
+    allowPersonalApiKeys: true,
+  })
+  mocks.build.mockResolvedValue([
+    {
+      name: 'gmail_send',
+      description: 'Send email',
+      service: 'gmail',
+      input_schema: { type: 'object' },
+    },
+  ])
+})
+describe('direct and checkpoint catalog authorization parity', () => {
+  it('returns identical authorized schema responses through HTTP and outbound control', async () => {
+    const direct = await POST(request())
+    const outbound = await checkpoint()
+    expect(direct.status).toBe(200)
+    expect(outbound.status).toBe(200)
+    expect(await direct.json()).toEqual(JSON.parse(outbound.body))
+    expect(mocks.permission).toHaveBeenCalledTimes(2)
+    expect(mocks.build).toHaveBeenCalledTimes(2)
+  })
+  it('rejects revoked workspace membership on both transports before catalog loading', async () => {
+    mocks.permission.mockResolvedValue(null)
+    expect((await POST(request())).status).toBe(403)
+    expect((await checkpoint()).status).toBe(403)
+    expect(mocks.build).not.toHaveBeenCalled()
+  })
+  it('rejects untrusted HTTP keys before canonical context loading', async () => {
+    expect((await POST(request('browser-key'))).status).toBe(401)
+    expect(mocks.workspace).not.toHaveBeenCalled()
+    expect(mocks.build).not.toHaveBeenCalled()
+  })
+})

apps/sim/app/api/mothership/integrations/catalog/route.ts

@@ -0,0 +1,24 @@
+import { readIntegrationCatalogContract } from '@/lib/api/contracts/mothership-integrations'
+import {
+  defineInternalJsonRoute,
+  internalOrchestrationErrorPolicy,
+  internalRateLimits,
+} from '@/lib/api/server/routes'
+import { internalCopilotAuth } from '@/lib/mothership/auth/internal'
+import {
+  INTEGRATION_CATALOG_AUDIENCE,
+  readIntegrationCatalog,
+  readIntegrationCatalogOperation,
+} from '@/lib/mothership/integrations/application/catalog'
+
+export const POST = defineInternalJsonRoute({
+  contract: readIntegrationCatalogContract,
+  auth: internalCopilotAuth(INTEGRATION_CATALOG_AUDIENCE, { organization: true }),
+  operation: readIntegrationCatalogOperation,
+  rateLimit: internalRateLimits.none({
+    reason: 'Private bounded catalog discovery rechecks current authorization on every request.',
+  }),
+  errorPolicy: internalOrchestrationErrorPolicy,
+  mapInput: ({ body }) => body,
+  useCase: readIntegrationCatalog,
+})