Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions src/codealmanac/workflows/run_queue/service.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from pathlib import Path

from codealmanac.core.errors import error_summary
from codealmanac.core.errors import ExecutionFailed, error_summary
from codealmanac.services.repositories.service import RepositoriesService
from codealmanac.services.runs.models import (
RunCancelResult,
Expand Down Expand Up @@ -86,7 +86,7 @@ def queue_build(self, request: BuildRequest) -> RunRecord:

def start_build(self, request: BuildRequest) -> RunQueueStartResult:
run = self.queue_build(request)
worker = self.spawn_worker(request.path)
worker = self._spawn_worker_safely(run, request.path)
return self.start_result(run, worker)

def queue_ingest(self, request: IngestRequest) -> RunRecord:
Expand All @@ -104,7 +104,7 @@ def queue_ingest(self, request: IngestRequest) -> RunRecord:

def start_ingest(self, request: IngestRequest) -> RunQueueStartResult:
run = self.queue_ingest(request)
worker = self.spawn_worker(request.cwd)
worker = self._spawn_worker_safely(run, request.cwd)
return self.start_result(run, worker)

def queue_garden(self, request: GardenRequest) -> RunRecord:
Expand All @@ -122,7 +122,7 @@ def queue_garden(self, request: GardenRequest) -> RunRecord:

def start_garden(self, request: GardenRequest) -> RunQueueStartResult:
run = self.queue_garden(request)
worker = self.spawn_worker(request.cwd)
worker = self._spawn_worker_safely(run, request.cwd)
return self.start_result(run, worker)

def start_scheduled_garden(
Expand Down Expand Up @@ -166,6 +166,17 @@ def start_scheduled_garden(
def spawn_worker(self, cwd: Path) -> RunWorkerSpawnResult:
return self.spawner.spawn(SpawnRunWorkerRequest(cwd=cwd))

def _spawn_worker_safely(self, run: RunRecord, cwd: Path) -> RunWorkerSpawnResult:
try:
return self.spawn_worker(cwd)
except Exception as error:
raise ExecutionFailed(
f"Run queued as {run.run_id} but worker failed to start: {error}. "
f"Use 'codealmanac jobs attach {run.run_id}' "
"once a worker is available."
) from error


def start_result(
self,
run: RunRecord,
Expand Down
35 changes: 35 additions & 0 deletions tests/test_run_queue_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,41 @@ def test_run_queue_start_persists_spec_and_spawns_worker(
assert not (repo / "almanac/jobs").exists()


class FailingWorkerSpawner:
def spawn(self, request: SpawnRunWorkerRequest) -> RunWorkerSpawnResult:
raise OSError("fake spawn failure")


def test_run_queue_start_handles_worker_spawn_failure(
tmp_path: Path,
isolated_home: Path,
):
repo = tmp_path / "repo"
repo.mkdir()
(repo / "note.md").write_text("queue design note\n", encoding="utf-8")
spawner = FailingWorkerSpawner()
app = create_app(
AppConfig(database_path=isolated_home / ".codealmanac/codealmanac.db"),
harness_adapters=(QueueWritingHarnessAdapter(),),
worker_spawner=spawner,
)
initialize_repository(app, repo)

with pytest.raises(ExecutionFailed, match="fake spawn failure"):
app.workflows.queue.start_ingest(
IngestRequest(
cwd=repo,
inputs=("note.md",),
harness=HarnessKind.CODEX,
model="gpt-5.5",
)
)

runs = app.runs.list(ListRunsRequest(repository_name=repo.name))
assert len(runs) == 1
assert runs[0].status == RunStatus.QUEUED


def test_run_queue_drains_persisted_ingest_spec(
tmp_path: Path,
isolated_home: Path,
Expand Down
Loading