memory: improving knowledge extraction, memory service integration, and RPC option handling

.gitignore

@@ -452,6 +452,7 @@ FodyWeavers.xsd
 
 /ts/packages/cli/.collection-backups
 /ts/packages/cli/.import-states
+/ts/packages/agentServer/server/.import-states
 
 # agent files
 .claude/

ts/packages/agentRpc/src/client.ts

@@ -206,10 +206,18 @@ function createOptionsRpc(
                             fn = options[name];
                         } else {
                             const funcName = names.pop();
-                            thisObject = getObjectProperty(options, name);
+                            thisObject = getObjectProperty(
+                                options,
+                                names.join("."),
+                            );
                             fn = thisObject[funcName!];
                         }
                     }
+                    if (typeof fn !== "function") {
+                        throw new Error(
+                            `Options callback '${name}' for object ${param.id} is not a function`,
+                        );
+                    }
                     return fn.call(thisObject, ...param.args);
                 },
             },

ts/packages/agentRpc/test/actionContext.spec.ts

@@ -18,6 +18,75 @@ import {
 } from "@typeagent/agent-sdk/helpers/action";
 
 describe("agent action context RPC", () => {
+    test("invokes nested initialization option methods with their receiver", async () => {
+        let clientProvider: ChannelProviderAdapter;
+        let serverProvider: ChannelProviderAdapter;
+        clientProvider = createChannelProviderAdapter(
+            "options-client",
+            (message, callback) => {
+                queueMicrotask(() =>
+                    serverProvider.notifyMessage(
+                        JSON.parse(JSON.stringify(message)),
+                    ),
+                );
+                callback?.(null);
+            },
+        );
+        serverProvider = createChannelProviderAdapter(
+            "options-server",
+            (message, callback) => {
+                queueMicrotask(() =>
+                    clientProvider.notifyMessage(
+                        JSON.parse(JSON.stringify(message)),
+                    ),
+                );
+                callback?.(null);
+            },
+        );
+
+        let result: string | undefined;
+        const serverAgent: AppAgent = {
+            initializeAgentContext: async (settings) => {
+                const options = settings?.options as {
+                    service: {
+                        prefix: string;
+                        getValue(value: string): string;
+                    };
+                };
+                result = await options.service.getValue("value");
+                return {};
+            },
+        };
+        const server = createAgentRpcServer(
+            "nested-options",
+            serverAgent,
+            serverProvider,
+        );
+        const client = await createAgentRpcClient(
+            "nested-options",
+            clientProvider,
+            server.agentInterface,
+        );
+
+        try {
+            await client.initializeAgentContext?.({
+                options: {
+                    service: {
+                        prefix: "nested",
+                        getValue(value: string) {
+                            return `${this.prefix}:${value}`;
+                        },
+                    },
+                },
+            });
+            expect(result).toBe("nested:value");
+        } finally {
+            server.closeFn();
+            clientProvider.notifyDisconnected();
+            serverProvider.notifyDisconnected();
+        }
+    });
+
     test("cancels a real SDK choice over agent RPC without invoking its callback", async () => {
         let clientProvider: ChannelProviderAdapter;
         let serverProvider: ChannelProviderAdapter;

ts/packages/agentServer/server/src/server.ts

@@ -43,8 +43,13 @@ import { DefaultAzureCredential } from "@azure/identity";
 import { otel } from "@typeagent/telemetry";
 import { MacroManager } from "@typeagent/copilot-macros";
 import { MemoryServiceHost } from "@typeagent/memory-mcp-server";
-import { FileMemoryService } from "@typeagent/memory-service";
-import { InProcessMemoryServiceClient } from "@typeagent/memory-client";
+import {
+    FileMemoryService,
+    createKnowProCorpusIndex,
+} from "@typeagent/memory-service";
+import { createMemoryServiceRpcFacade } from "@typeagent/memory-service/rpc";
+import { createDocMemorySettings } from "@typeagent/conversation-memory";
+import { openai } from "@typeagent/aiclient";
 
 // Exit code the worker uses to ask the supervisor to relaunch it in place.
 const RESTART_EXIT_CODE = 42;
@@ -334,6 +339,21 @@ async function main() {
     debugStartup("starting instance memory service");
     const memoryService = new FileMemoryService(
         path.join(instanceDir, "memory"),
+        {
+            indexFactory: (corpusId, indexDirectory) =>
+                createKnowProCorpusIndex(corpusId, indexDirectory, () =>
+                    createDocMemorySettings(
+                        64,
+                        undefined,
+                        openai.createChatModel(
+                            openai.GPT_5_6_LUNA,
+                            undefined,
+                            undefined,
+                            ["website-knowledge", "durable-index"],
+                        ),
+                    ),
+                ),
+        },
     );
     const memoryServiceHost = await MemoryServiceHost.start(memoryService, {
         onError: (error) =>
@@ -425,9 +445,8 @@ async function main() {
                 allowSharedLocalView: ["browser"],
                 agentInitOptions: {
                     browser: {
-                        memoryServiceClient: new InProcessMemoryServiceClient(
-                            memoryService,
-                        ),
+                        memoryServiceClient:
+                            createMemoryServiceRpcFacade(memoryService),
                     },
                 },
             },

ts/packages/agents/browser/package.json

@@ -17,8 +17,7 @@
   "type": "module",
   "exports": {
     "./agent/manifest": "./src/agent/manifest.json",
-    "./agent/handlers": "./dist/agent/browserActionHandler.mjs",
-    "./agent/indexing": "./dist/agent/indexing/browserIndexingService.js"
+    "./agent/handlers": "./dist/agent/browserActionHandler.mjs"
   },
   "main": "index.js",
   "files": [

ts/packages/agents/browser/src/agent/agentServiceHandlers.mts

@@ -3,7 +3,10 @@
 
 import { SessionContext } from "@typeagent/agent-sdk";
 import type { BrowserAgentInvokeFunctions } from "@typeagent/browser-control-rpc/serviceTypes";
-import type { BrowserActionContext } from "./browserActions.mjs";
+import {
+    getSessionBrowserControl,
+    type BrowserActionContext,
+} from "./browserActions.mjs";
 import { handleKnowledgeAction } from "./knowledge/actions/knowledgeActionRouter.mjs";
 import { handleSchemaDiscoveryAction } from "./discovery/actionHandler.mjs";
 import {
@@ -24,6 +27,27 @@ export function createAgentInvokeHandlers(
         return handleKnowledgeAction(method, params, context);
     }
 
+    async function extractionHandler(params: any): Promise<any> {
+        if (Array.isArray(params.htmlFragments)) {
+            return knowledgeHandler("extractKnowledgeFromPage", params);
+        }
+        const browserControl = getSessionBrowserControl(context);
+        const url = params.url ?? (await browserControl.getPageUrl());
+        const htmlFragments = await browserControl.getHtmlFragments(
+            false,
+            "knowledgeExtraction",
+        );
+        return knowledgeHandler("extractKnowledgeFromPage", {
+            ...params,
+            url,
+            title: params.title ?? url,
+            htmlFragments,
+            extractEntities: params.extractEntities ?? true,
+            extractRelationships: params.extractRelationships ?? true,
+            suggestQuestions: params.suggestQuestions ?? true,
+        });
+    }
+
     async function discoveryHandler(method: string, params: any): Promise<any> {
         const result = await handleSchemaDiscoveryAction(
             { actionName: method as any, parameters: params },
@@ -38,8 +62,7 @@ export function createAgentInvokeHandlers(
 
     const handlers: BrowserAgentInvokeFunctions = {
         // Knowledge extraction
-        extractKnowledgeFromPage: (params: any) =>
-            knowledgeHandler("extractKnowledgeFromPage", params),
+        extractKnowledgeFromPage: extractionHandler,
         // Knowledge queries
         searchWebMemories: (params: any) =>
             websiteHandler("searchWebMemories", params),