Don't spam worktrees! Disk Space is Precious

Author: dwash96

cecli/helpers/agents/service.py

@@ -826,6 +826,7 @@ async def _create_sub_agent_coder(
                     source=metadata.get("worktree_source") or parent_root,
                     base_ref=base_ref,
                     base_root=parent_root,
+                    reuse_existing=True,
                 )
             except Exception as e:
                 raise RuntimeError(f"Failed to create worktree for sub-agent '{name}': {e}") from e

cecli/helpers/worktrees/manager.py

@@ -119,38 +119,53 @@ def create(
         source: str | Path,
         base_ref: str | None = None,
         base_root: str | Path | None = None,
+        reuse_existing: bool = True,
     ) -> WorktreeRecord:
         """Create a new worktree for *agent_name* and return its record.
 
         *source* may be a local repository path or a remote URL.  Remotes are
         cloned into a shared bare cache and fetched on reuse.  The default base
         branch is resolved from ``origin/HEAD``, ``main``, ``master``, or the
         current ``HEAD`` in that order.
+
+        By default an already-recorded checkout for the same agent and
+        repository is adopted (its owner is refreshed) instead of creating a
+        duplicate; workflows never run the same pair concurrently.  Pass
+        ``reuse_existing=False`` only when deliberately creating a second
+        independent checkout for the same agent.
         """
         base = cls._base()
         repo = cls._prepare_source(source, base_root=base_root)
-        base_ref_name, base_sha = cls._resolve_base(repo, base_ref)
-        branch = paths.branch_name(agent_name, uuid)
-        path = paths.worktree_path(base, repo, agent_name, uuid)
-        path.parent.mkdir(parents=True, exist_ok=True)
 
+        # Reuse and creation are serialized per repository so concurrent runs of
+        # the same workflow cannot each create their own checkout.
         with cls._repo_lock(base, repo):
+            if reuse_existing:
+                existing = cls._find_reusable(repo, agent_name)
+
+                if existing is not None:
+                    return cls._adopt(existing)
+
+            base_ref_name, base_sha = cls._resolve_base(repo, base_ref)
+            branch = paths.branch_name(agent_name, uuid)
+            path = paths.worktree_path(base, repo, agent_name, uuid)
+            path.parent.mkdir(parents=True, exist_ok=True)
             branch, path = cls._add_worktree(repo, branch, path, base_ref_name, base_sha)
 
-        record = WorktreeRecord(
-            source=str(source),
-            repo=str(Path(repo).resolve()),
-            path=str(path.resolve()),
-            branch=branch,
-            base_ref=base_ref_name,
-            base_sha=base_sha,
-            agent=str(agent_name),
-            uuid=str(uuid),
-            pid=os.getpid(),
-            host=socket.gethostname(),
-            created_at=_now(),
-        )
-        cls._save_record(record)
+            record = WorktreeRecord(
+                source=str(source),
+                repo=str(Path(repo).resolve()),
+                path=str(path.resolve()),
+                branch=branch,
+                base_ref=base_ref_name,
+                base_sha=base_sha,
+                agent=str(agent_name),
+                uuid=str(uuid),
+                pid=os.getpid(),
+                host=socket.gethostname(),
+                created_at=_now(),
+            )
+            cls._save_record(record)
 
         return record
 
@@ -195,6 +210,16 @@ def recreate(
             target_path.parent.mkdir(parents=True, exist_ok=True)
 
             with cls._repo_lock(base, repo):
+                reusable = cls._find_reusable(repo, new_agent) if path is None else None
+
+                if reusable is not None:
+                    adopted = cls._adopt(reusable)
+
+                    if original.uuid != adopted.uuid:
+                        cls._discard_record(original.uuid)
+
+                    return adopted
+
                 if branch_exists(branch, repo):
                     try:
                         run_git("worktree", "add", str(target_path), branch, cwd=repo)
@@ -493,36 +518,47 @@ def _add_worktree(
         base_sha: str,
     ) -> tuple[str, Path]:
         start = base_ref if rev_parse(base_ref, cwd=repo, check=False) else base_sha
+
+        if path.exists():
+            existing = cls._existing_branch(repo, path, branch)
+
+            if existing is not None:
+                return existing, path
+
+            raise WorktreeError(f"worktree path already exists: {path}")
+
         last_error: WorktreeError | None = None
 
         for attempt in range(cls.MAX_NAME_ATTEMPTS):
             candidate_branch = branch if attempt == 0 else f"{branch}-{attempt + 1}"
-            candidate_path = path if attempt == 0 else Path(f"{path}-{attempt + 1}")
-
-            if candidate_path.exists():
-                last_error = WorktreeError(f"worktree path already exists: {candidate_path}")
-                continue
-
-            candidate_path.parent.mkdir(parents=True, exist_ok=True)
 
             try:
                 if branch_exists(candidate_branch, repo):
-                    run_git("worktree", "add", str(candidate_path), candidate_branch, cwd=repo)
+                    run_git("worktree", "add", str(path), candidate_branch, cwd=repo)
                 else:
                     run_git(
                         "worktree",
                         "add",
                         "-b",
                         candidate_branch,
-                        str(candidate_path),
+                        str(path),
                         start,
                         cwd=repo,
                     )
             except WorktreeError as exc:
                 last_error = exc
+
+                if path.exists():
+                    existing = cls._existing_branch(repo, path, candidate_branch)
+
+                    if existing is not None:
+                        return existing, path
+
+                    break
+
                 continue
 
-            return candidate_branch, candidate_path
+            return candidate_branch, path
 
         raise WorktreeError(
             f"could not create worktree for branch {branch!r} in {repo}: {last_error}"
@@ -754,6 +790,51 @@ def _repo_lock(cls, base: Path, repo: str | Path) -> Iterator[None]:
         with _file_lock(paths.lock_path(base, f"repo-{digest}")):
             yield
 
+    @classmethod
+    def _find_reusable(cls, repo: str | Path, agent_name: str) -> WorktreeRecord | None:
+        """Return a recorded checkout for *agent_name* in *repo*, if usable."""
+        resolved_repo = str(Path(repo).resolve())
+
+        for record in sorted(cls._records(), key=lambda item: item.created_at, reverse=True):
+            if record.agent != agent_name:
+                continue
+
+            if str(Path(record.repo).resolve()) != resolved_repo:
+                continue
+
+            if cls._is_worktree(Path(record.path), Path(repo)):
+                return record
+
+        return None
+
+    @classmethod
+    def _adopt(cls, record: WorktreeRecord) -> WorktreeRecord:
+        """Refresh *record* ownership and persist it."""
+        adopted = WorktreeRecord(
+            source=record.source,
+            repo=record.repo,
+            path=record.path,
+            branch=record.branch,
+            base_ref=record.base_ref,
+            base_sha=record.base_sha,
+            agent=record.agent,
+            uuid=record.uuid,
+            pid=os.getpid(),
+            host=socket.gethostname(),
+            created_at=record.created_at,
+        )
+        cls._save_record(adopted)
+
+        return adopted
+
+    @classmethod
+    def _existing_branch(cls, repo: str | Path, path: Path, fallback: str) -> str | None:
+        """Return the branch checked out at an existing managed worktree."""
+        if not cls._is_worktree(path, Path(repo)):
+            return None
+
+        return current_branch(path) or fallback
+
 
 @contextmanager
 def _file_lock(path: Path) -> Iterator[None]:

cecli/website/docs/config/worktrees.md

@@ -17,7 +17,7 @@ Every worktree gets a generated branch named `cecli/<agent>-<uuid>`. Agent names
 
 ## Lifecycle
 
-1. **Create** — a worktree is created from the source repository on the generated branch. Remote sources are cached as bare repositories first.
+1. **Create** — a worktree is created from the source repository on the generated branch. An existing checkout for the same agent and repository is reused (ownership is refreshed) instead of creating a duplicate, so repeated or concurrent workflow runs share one checkout. Remote sources are cached as bare repositories first.
 2. **Run** — the agent works inside the worktree; its `root` points at the checkout.
 3. **Release** — on cleanup, a dirty worktree is committed with the subject `Unfinished Dirty Worktree from agent <uuid>` so uncommitted work survives. Branches with unmerged commits are kept; empty or already-merged branches are deleted.
 4. **Remove** — the worktree checkout under `~/.cecli/worktrees/` is removed after the commit/release step. Kept branches remain in the source repository.
@@ -34,7 +34,7 @@ A workflow or delegated agent restored from a saved session keeps its worktree r
 ~/.cecli/worktrees/
   _sources/<repo-slug>-<sha8>/       # cached bare clones for remote sources
   <repo-slug>/
-    <agent-slug>-<uuid>/             # one linked worktree checkout per run
+    <agent-slug>-<uuid>/             # one linked worktree checkout per agent
 ```
 
 Local `repo:` paths are used in place as the source repository; the checkout itself still lives under the worktrees base directory. Removing a checkout or the whole base directory is safe once a run has finished: commits are on the kept branch, and empty branches are disposable.

tests/helpers/worktrees/test_manager.py

@@ -7,6 +7,7 @@
 import shutil
 import subprocess
 import sys
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import replace
 from pathlib import Path
 
@@ -125,15 +126,50 @@ def test_branch_and_path_naming_are_sanitized(tmp_path):
 def test_two_creates_produce_distinct_paths_and_branches(tmp_path):
     repo = _init_repo(tmp_path / "repo")
 
-    first = _create(repo, uuid="aaaa1111")
-    second = _create(repo, uuid="bbbb2222")
+    first = _create(repo, uuid="aaaa1111", reuse_existing=False)
+    second = _create(repo, uuid="bbbb2222", reuse_existing=False)
 
     assert first.path != second.path
     assert first.branch != second.branch
     assert Path(first.path).is_dir()
     assert Path(second.path).is_dir()
 
 
+def test_create_reuses_existing_worktree_for_same_agent(tmp_path):
+    repo = _init_repo(tmp_path / "repo")
+
+    first = _create(repo, uuid="aaaa1111")
+    second = _create(repo, uuid="bbbb2222")
+
+    assert first.uuid == second.uuid
+    assert first.path == second.path
+    assert first.branch == second.branch
+    assert len(list((tmp_path / "worktrees" / "repo").iterdir())) == 1
+
+
+def test_create_same_uuid_does_not_create_suffixed_copies(tmp_path):
+    repo = _init_repo(tmp_path / "repo")
+
+    first = _create(repo, uuid="abcd1234", reuse_existing=False)
+    second = _create(repo, uuid="abcd1234", reuse_existing=False)
+
+    assert first.path == second.path
+    names = sorted(item.name for item in (tmp_path / "worktrees" / "repo").iterdir())
+    assert names == ["wf-nightly-tidy-abcd1234"]
+
+
+def test_concurrent_creates_reuse_a_single_worktree(tmp_path):
+    repo = _init_repo(tmp_path / "repo")
+    uuids = [f"aaaa{index:04d}" for index in range(8)]
+
+    with ThreadPoolExecutor(max_workers=len(uuids)) as pool:
+        records = list(pool.map(lambda value: _create(repo, uuid=value), uuids))
+
+    assert {record.path for record in records} == {records[0].path}
+    assert {record.uuid for record in records} == {records[0].uuid}
+    assert len(list((tmp_path / "worktrees" / "repo").iterdir())) == 1
+
+
 def test_clean_release_deletes_branch(tmp_path):
     repo = _init_repo(tmp_path / "repo")
     record = _create(repo)
@@ -314,6 +350,19 @@ def test_recreate_with_new_uuid_attaches_saved_branch(tmp_path):
     assert WorktreeManager.get("new99999") == recreated
 
 
+def test_recreate_reuses_another_checkout_for_same_agent(tmp_path):
+    repo = _init_repo(tmp_path / "repo")
+    missing = _create(repo, uuid="aaaa1111", reuse_existing=False)
+    _git(repo, "worktree", "remove", "--force", missing.path)
+    live = _create(repo, uuid="bbbb2222", reuse_existing=False)
+
+    recreated = WorktreeManager.recreate(missing, uuid="cccc3333")
+
+    assert recreated.path == live.path
+    assert recreated.uuid == live.uuid
+    assert WorktreeManager.get(missing.uuid) is None
+
+
 def test_release_keeps_saved_branch_when_agent_switched_branches(tmp_path):
     repo = _init_repo(tmp_path / "repo")
     record = _create(repo)
@@ -352,8 +401,8 @@ def test_get_unknown_uuid_returns_none():
 
 def test_release_all_releases_matching_pid(tmp_path):
     repo = _init_repo(tmp_path / "repo")
-    first = _create(repo, uuid="aaaa1111")
-    second = _create(repo, uuid="bbbb2222")
+    first = _create(repo, uuid="aaaa1111", reuse_existing=False)
+    second = _create(repo, uuid="bbbb2222", reuse_existing=False)
 
     results = WorktreeManager.release_all()
 
@@ -431,8 +480,8 @@ def test_create_from_file_url_remote(tmp_path, isolated_base):
     repo = _init_repo(tmp_path / "repo")
     source = repo.as_uri()
 
-    first = _create(repo, uuid="aaaa1111", source=source)
-    second = _create(repo, uuid="bbbb2222", source=source)
+    first = _create(repo, uuid="aaaa1111", source=source, reuse_existing=False)
+    second = _create(repo, uuid="bbbb2222", source=source, reuse_existing=False)
 
     assert first.repo == second.repo
     assert Path(first.repo).is_dir()