Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 32 additions & 23 deletions packages/acp/tests/agent-certification.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { captureTestJournal } from "@executablemd/test-support/failure-journal";
import { describe, it } from "@executablemd/test-support/bdd";
import { expect } from "@executablemd/test-support/expect";
import { ensure, scoped, spawn, withResolvers } from "effection";
Expand Down Expand Up @@ -208,31 +209,39 @@ describe(
});
for (const setting of ["model", "effort"]) {
for (const kind of ["absent", "mismatch"]) {
it(`reports exact ${setting} ${kind} evidence before any Prompt`, function* () {
const name = `reports exact ${setting} ${kind} evidence before any Prompt`;
it(name, function* () {
yield* withCertificationFixture(function* (f) {
const mode = kind + "-" + setting;
const report = reportFor(f.command(mode));
const failures: unknown[] = [];
const value = setting === "model" ? "selected" : "high";
const result = yield* document(
f,
mode,
`<Session name="target" ${setting}="${value}"><Prompt text="must not send" /></Session>`,
report,
failures,
);
expect(result.ok).toBe(false);
expect(failures).toEqual([
{
reason: kind === "absent" ? "selector-unavailable" : "configuration-mismatch",
setting,
yield* captureTestJournal(
{ owner: "packages/acp/tests/agent-certification.test.ts", name },
join(f.dir, "document.md"),
function* (stream) {
const mode = kind + "-" + setting;
const report = reportFor(f.command(mode));
const failures: unknown[] = [];
const value = setting === "model" ? "selected" : "high";
const result = yield* document(
f,
mode,
`<Session name="target" ${setting}="${value}"><Prompt text="must not send" /></Session>`,
report,
failures,
stream,
);
expect(result.ok).toBe(false);
expect(failures).toEqual([
{
reason: kind === "absent" ? "selector-unavailable" : "configuration-mismatch",
setting,
},
]);
const row =
setting === "model" ? report.capabilities.model : report.capabilities.effort;
expect(row.outcome).toBe(kind === "absent" ? "unsupported" : "failure");
expect((yield* f.methods()).includes("session/prompt")).toBe(false);
expect(JSON.stringify(report)).not.toContain("must not send");
},
]);
const row =
setting === "model" ? report.capabilities.model : report.capabilities.effort;
expect(row.outcome).toBe(kind === "absent" ? "unsupported" : "failure");
expect((yield* f.methods()).includes("session/prompt")).toBe(false);
expect(JSON.stringify(report)).not.toContain("must not send");
);
});
});
}
Expand Down
64 changes: 41 additions & 23 deletions packages/agent/tests/package-boundary.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { captureTestJournal } from "@executablemd/test-support/failure-journal";
/**
* Tier PB — the boundary between Core and the Agent package.
*
Expand All @@ -18,7 +19,6 @@ import { randomUUID } from "node:crypto";
import * as os from "node:os";
import * as path from "node:path";
import { fileURLToPath } from "node:url";
import { InMemoryStream } from "@executablemd/durable-streams";
import { execute } from "@executablemd/core";
import type { DocumentExecution, Json } from "@executablemd/core";
import { Agent } from "@executablemd/agent/api";
Expand Down Expand Up @@ -84,10 +84,10 @@ const echo: AgentProviderFactory = function* (options) {
};

function* run(
dir: string,
doc: string,
start: (docPath: string) => Operation<DocumentExecution>,
): Operation<{ output: string; result: Result<Json> }> {
const dir = path.join(os.tmpdir(), `xmd-pb-${randomUUID()}`);
yield* ensureDir(dir);
return yield* scoped(function* () {
yield* ensure(() => rm(dir, { recursive: true, force: true }));
Expand All @@ -103,6 +103,8 @@ function* run(
});
}

const OWNER = "packages/agent/tests/package-boundary.test.ts";

const DOCUMENT = '<Session name="review">\n<Prompt text="hello" />\n</Session>\n';

describe("Tier PB — Core resolves no Agent package", () => {
Expand Down Expand Up @@ -135,34 +137,50 @@ describe("Tier PB — Core resolves no Agent package", () => {
}
});

it("PB4: a Core-only execution cannot resolve the Agent vocabulary", function* () {
const vocabulary = "PB4: a Core-only execution cannot resolve the Agent vocabulary";
it(vocabulary, function* () {
for (const [doc, name] of [
[DOCUMENT, "Session"],
['<Prompt text="hello" />\n', "Prompt"],
] as const) {
const { output, result } = yield* run(doc, (docPath) =>
execute({ path: docPath, stream: new InMemoryStream() }),
const dir = path.join(os.tmpdir(), `xmd-pb-${randomUUID()}`);
yield* captureTestJournal(
{ owner: OWNER, name: vocabulary },
path.join(dir, "doc.md"),
function* (stream) {
const { output, result } = yield* run(dir, doc, (docPath) =>
execute({ path: docPath, stream }),
);
expect(output).not.toContain("[hello]");
expect(!result.ok && result.error.message).toContain(`Cannot resolve component: ${name}`);
},
);
expect(output).not.toContain("[hello]");
expect(!result.ok && result.error.message).toContain(`Cannot resolve component: ${name}`);
}
});

it("PB5: the Agent package installed over Core answers the same document", function* () {
const { output, result } = yield* run(DOCUMENT, function* (docPath) {
yield* installAgentComponents({
rootProvider: {
factory: echo,
options: { defaultAgent: "pb", permissionMode: "deny-all" },
},
});
return yield* executeInstalled({ path: docPath, stream: new InMemoryStream() }, [
{ components: agentIdentityComponents() },
]);
});
if (!result.ok) {
throw result.error;
}
expect(output).toContain("[hello]");
const positiveCase = "PB5: the Agent package installed over Core answers the same document";
it(positiveCase, function* () {
const dir = path.join(os.tmpdir(), `xmd-pb-${randomUUID()}`);
yield* captureTestJournal(
{ owner: OWNER, name: positiveCase },
path.join(dir, "doc.md"),
function* (stream) {
const { output, result } = yield* run(dir, DOCUMENT, function* (docPath) {
yield* installAgentComponents({
rootProvider: {
factory: echo,
options: { defaultAgent: "pb", permissionMode: "deny-all" },
},
});
return yield* executeInstalled({ path: docPath, stream }, [
{ components: agentIdentityComponents() },
]);
});
if (!result.ok) {
throw result.error;
}
expect(output).toContain("[hello]");
},
);
});
});
72 changes: 46 additions & 26 deletions packages/cli/tests/plan-command-document.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { journalBeforeEach, useJournalStream } from "@executablemd/test-support/failure-journal";
/**
* The packaged plan command document, executed as itself.
*
Expand All @@ -18,7 +19,7 @@
* so what a row observes is the channel a person actually watches rather than a
* transcript assembled afterwards.
*/
import { describe, it } from "@executablemd/test-support/bdd";
import { beforeEach, describe, it } from "@executablemd/test-support/bdd";
import { expect } from "@executablemd/test-support/expect";
import { ensure, scoped, sleep, spawn, useScope, withResolvers } from "effection";
import type { Operation } from "effection";
Expand Down Expand Up @@ -175,7 +176,10 @@ interface RunOptions {
symbols?: SyntaxSymbols;
}

function* runDocument(options: RunOptions = {}): Operation<CommandRun> {
function* runDocument(
options: RunOptions = {},
stream: InMemoryStream | undefined = undefined,
): Operation<CommandRun> {
const source = yield* readPackagedDocument(PLAN_COMMAND_DOCUMENT);

let value: Json | undefined;
Expand Down Expand Up @@ -235,7 +239,7 @@ function* runDocument(options: RunOptions = {}): Operation<CommandRun> {
const execution = yield* executeInstalled(
{
...retainedSource(PLAN_COMMAND_IDENTITY, source),
stream: new InMemoryStream(),
stream: stream ?? (yield* useJournalStream(PLAN_COMMAND_IDENTITY)),
includes: [],
secretDetection: true,
props: {
Expand Down Expand Up @@ -340,7 +344,10 @@ function unattributed(chunks: readonly string[]): string[] {
return kept;
}

const OWNER = "packages/cli/tests/plan-command-document.test.ts";

describe("the packaged plan command document", () => {
beforeEach(journalBeforeEach(OWNER));
it("C2: returns the approved candidate's exact bytes and never reaches exhaustion", function* () {
const run = yield* useWorkingDirectory(function* () {
return yield* runDocument();
Expand Down Expand Up @@ -744,6 +751,7 @@ describe("the packaged plan command document", () => {
* through the captured operation — there is no other way to reach it.
*/
describe("Tier PI — read-only information requests", () => {
beforeEach(journalBeforeEach(OWNER));
/** One request that binds all three reads and renders a chosen object. */
const COMPOSED = [
'<Glob include={["notes.md"]} as="paths" />',
Expand Down Expand Up @@ -1015,13 +1023,16 @@ describe("Tier PI — read-only information requests", () => {
});

it("PI1: named documentation comes back, then an ordinary Plan", function* () {
const run = yield* runDocument({
// Two of the three names this case's vocabulary has, asked for together.
// The third is the control: a selection that returned it would be a
// catalog injection wearing a selection's clothes.
turns: [{ reply: '<Syntax names={["File", "Elicit"]} />\n' }, { reply: CANDIDATE }],
symbols: SELECTABLE,
});
const run = yield* runDocument(
{
// Two of the three names this case's vocabulary has, asked for together.
// The third is the control: a selection that returned it would be a
// catalog injection wearing a selection's clothes.
turns: [{ reply: '<Syntax names={["File", "Elicit"]} />\n' }, { reply: CANDIDATE }],
symbols: SELECTABLE,
},
undefined,
);

expect(run.failure).toBe(undefined);
const asked = run.prompts[0] ?? "";
Expand All @@ -1040,9 +1051,12 @@ describe("Tier PI — read-only information requests", () => {
});

it("PI10: an unknown documented name is one retry, not a stopped invocation", function* () {
const run = yield* runDocument({
turns: [{ reply: '<Syntax names={["NoSuchComponent"]} />\n' }, { reply: CANDIDATE }],
});
const run = yield* runDocument(
{
turns: [{ reply: '<Syntax names={["NoSuchComponent"]} />\n' }, { reply: CANDIDATE }],
},
undefined,
);

// Recoverable: the workflow asked again rather than ending, and the reason
// quotes only the name the candidate itself wrote.
Expand All @@ -1055,14 +1069,17 @@ describe("Tier PI — read-only information requests", () => {

it("PI2: an empty match renders as an empty array", function* () {
const files = recordedFiles({ "notes.md": "the retained note\n" });
const run = yield* runDocument({
turns: [
{ reply: '<Glob include={["absent.md"]} as="paths" />\n<Json value={paths} />\n' },
{ reply: CANDIDATE },
],
generated: { File: ["self-closing"], Glob: ["self-closing"], Syntax: ["self-closing"] },
files,
});
const run = yield* runDocument(
{
turns: [
{ reply: '<Glob include={["absent.md"]} as="paths" />\n<Json value={paths} />\n' },
{ reply: CANDIDATE },
],
generated: { File: ["self-closing"], Glob: ["self-closing"], Syntax: ["self-closing"] },
files,
},
undefined,
);

expect(run.failure).toBe(undefined);
// The search ran and found nothing, and nothing is a result rather than a
Expand Down Expand Up @@ -1316,11 +1333,14 @@ describe("Tier PI — read-only information requests", () => {

it("PI3: a titled draft is never evaluated", function* () {
const files = recordedFiles({ "notes.md": "the retained note\n" });
const run = yield* runDocument({
turns: [{ reply: CANDIDATE }],
generated: { File: ["self-closing"], Glob: ["self-closing"], Syntax: ["self-closing"] },
files,
});
const run = yield* runDocument(
{
turns: [{ reply: CANDIDATE }],
generated: { File: ["self-closing"], Glob: ["self-closing"], Syntax: ["self-closing"] },
files,
},
undefined,
);

expect(run.failure).toBe(undefined);
// One turn, no evaluation, no information phase at all.
Expand Down
Loading
Loading