From 467bcf6026092a5414ca956e871367eb07a993e9 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sun, 1 Nov 2026 00:59:59 +0000 Subject: [PATCH 1/7] feat: add Argo CD and Flux health rules for CoderWorkspacePool and CoderWorkspaceClaim (#153) Change-Id: Id132668b4604dcd75c7dbcff651c3f25512c0e0f Signed-off-by: Thomas Kosiewski --- .../argocd-health-coderworkspaceclaim.lua | 40 +++++ .../argocd-health-coderworkspacepool.lua | 35 +++++ .../flux-healthcheck-coderworkspaceclaim.yaml | 9 ++ .../flux-healthcheck-coderworkspacepool.yaml | 9 ++ config/gitops/health_test.go | 114 +++++++++++++- docs/how-to/gitops.md | 139 +++++++++++++++++- 6 files changed, 339 insertions(+), 7 deletions(-) create mode 100644 config/gitops/argocd-health-coderworkspaceclaim.lua create mode 100644 config/gitops/argocd-health-coderworkspacepool.lua create mode 100644 config/gitops/flux-healthcheck-coderworkspaceclaim.yaml create mode 100644 config/gitops/flux-healthcheck-coderworkspacepool.yaml diff --git a/config/gitops/argocd-health-coderworkspaceclaim.lua b/config/gitops/argocd-health-coderworkspaceclaim.lua new file mode 100644 index 00000000..baae2ca8 --- /dev/null +++ b/config/gitops/argocd-health-coderworkspaceclaim.lua @@ -0,0 +1,40 @@ +-- Argo CD health check for CoderWorkspaceClaim (coder.com/v1alpha1). +-- Install it as the argocd-cm key +-- resource.customizations.health.coder.com_CoderWorkspaceClaim. +-- docs/how-to/gitops.md copies this file. A test keeps the copy equal. +local hs = { status = "Progressing", message = "Waiting for the controller" } +if obj.metadata.deletionTimestamp ~= nil then + -- Argo CD shows deletionMessage, not message, for a resource being deleted. + hs.deletionMessage = "Deleting the workspace claim" + hs.message = hs.deletionMessage + return hs +end +local st = obj.status +if st == nil then + return hs +end +-- A claim has no status.observedGeneration, so only the conditions tell +-- whether the controller saw the current spec. +local conditions = {} +for _, c in ipairs(st.conditions or {}) do + if c.observedGeneration == obj.metadata.generation then + table.insert(conditions, c) + end +end +for _, c in ipairs(conditions) do + if c.type == "Ready" and c.reason ~= nil and c.message ~= nil then + hs.message = c.reason .. ": " .. c.message + end +end +for _, c in ipairs(conditions) do + if c.type == "Ready" and c.status == "True" then + hs.status = "Healthy" + elseif c.type == "Stalled" and c.status == "True" then + hs.status = "Degraded" + end +end +-- Lost is final: the claim never binds again. +if st.phase == "Lost" then + hs.status = "Degraded" +end +return hs diff --git a/config/gitops/argocd-health-coderworkspacepool.lua b/config/gitops/argocd-health-coderworkspacepool.lua new file mode 100644 index 00000000..3c7c6c32 --- /dev/null +++ b/config/gitops/argocd-health-coderworkspacepool.lua @@ -0,0 +1,35 @@ +-- Argo CD health check for CoderWorkspacePool (coder.com/v1alpha1). +-- Install it as the argocd-cm key +-- resource.customizations.health.coder.com_CoderWorkspacePool. +-- docs/how-to/gitops.md copies this file. A test keeps the copy equal. +local hs = { status = "Progressing", message = "Waiting for the controller" } +if obj.metadata.deletionTimestamp ~= nil then + -- Argo CD shows deletionMessage, not message, for a resource being deleted. + hs.deletionMessage = "Deleting the workspace pool" + hs.message = hs.deletionMessage + return hs +end +local st = obj.status +if st == nil or st.observedGeneration == nil or st.observedGeneration ~= obj.metadata.generation then + return hs +end +-- A condition from an older generation does not describe the current spec. +local conditions = {} +for _, c in ipairs(st.conditions or {}) do + if c.observedGeneration == obj.metadata.generation then + table.insert(conditions, c) + end +end +for _, c in ipairs(conditions) do + if c.type == "Ready" and c.reason ~= nil and c.message ~= nil then + hs.message = c.reason .. ": " .. c.message + end +end +for _, c in ipairs(conditions) do + if c.type == "Ready" and c.status == "True" then + hs.status = "Healthy" + elseif c.type == "Stalled" and c.status == "True" then + hs.status = "Degraded" + end +end +return hs diff --git a/config/gitops/flux-healthcheck-coderworkspaceclaim.yaml b/config/gitops/flux-healthcheck-coderworkspaceclaim.yaml new file mode 100644 index 00000000..d837d5b7 --- /dev/null +++ b/config/gitops/flux-healthcheck-coderworkspaceclaim.yaml @@ -0,0 +1,9 @@ +# Flux health check expressions for CoderWorkspaceClaim (coder.com/v1alpha1). +# Add this entry to spec.healthCheckExprs of a Flux v2.5+ Kustomization +# with spec.wait: true. +# docs/how-to/gitops.md copies this file. A test keeps the copy equal. +- apiVersion: coder.com/v1alpha1 + kind: CoderWorkspaceClaim + inProgress: "(!has(status.phase) || status.phase != 'Lost') && (!has(status.conditions) || !status.conditions.exists(c, c.type == 'Ready' && has(c.observedGeneration) && c.observedGeneration == metadata.generation) || status.conditions.exists(c, c.type == 'Reconciling' && c.status == 'True'))" + failed: "(has(status.phase) && status.phase == 'Lost') || status.conditions.exists(c, c.type == 'Stalled' && c.status == 'True')" + current: "status.conditions.exists(c, c.type == 'Ready' && c.status == 'True')" diff --git a/config/gitops/flux-healthcheck-coderworkspacepool.yaml b/config/gitops/flux-healthcheck-coderworkspacepool.yaml new file mode 100644 index 00000000..5ab75703 --- /dev/null +++ b/config/gitops/flux-healthcheck-coderworkspacepool.yaml @@ -0,0 +1,9 @@ +# Flux health check expressions for CoderWorkspacePool (coder.com/v1alpha1). +# Add this entry to spec.healthCheckExprs of a Flux v2.5+ Kustomization +# with spec.wait: true. +# docs/how-to/gitops.md copies this file. A test keeps the copy equal. +- apiVersion: coder.com/v1alpha1 + kind: CoderWorkspacePool + inProgress: "!has(status.observedGeneration) || status.observedGeneration != metadata.generation || !has(status.conditions) || !status.conditions.exists(c, c.type == 'Ready' && has(c.observedGeneration) && c.observedGeneration == metadata.generation) || status.conditions.exists(c, c.type == 'Reconciling' && c.status == 'True')" + failed: "status.conditions.exists(c, c.type == 'Stalled' && c.status == 'True')" + current: "status.conditions.exists(c, c.type == 'Ready' && c.status == 'True')" diff --git a/config/gitops/health_test.go b/config/gitops/health_test.go index ed6d47d9..605ad9b2 100644 --- a/config/gitops/health_test.go +++ b/config/gitops/health_test.go @@ -1,6 +1,7 @@ // Package gitops_test checks the GitOps health rules for CoderTemplateTest, -// CoderMCPServer and CoderAIProvider. The rules are data files, so this -// package has no production code. +// CoderMCPServer, CoderAIProvider, CoderWorkspacePool and +// CoderWorkspaceClaim. The rules are data files, so this package has no +// production code. package gitops_test import ( @@ -30,6 +31,8 @@ var ruleSets = []ruleSet{ {"CoderTemplateTest", "argocd-health-codertemplatetest.lua", "flux-healthcheck-codertemplatetest.yaml", "../../docs/how-to/gitops.md", healthCases}, {"CoderMCPServer", "argocd-health-codermcpserver.lua", "flux-healthcheck-codermcpserver.yaml", "../../docs/how-to/manage-mcp-servers.md", mcpServerCases}, {"CoderAIProvider", "argocd-health-coderaiprovider.lua", "flux-healthcheck-coderaiprovider.yaml", "../../docs/how-to/gitops.md", aiProviderCases}, + {"CoderWorkspacePool", "argocd-health-coderworkspacepool.lua", "flux-healthcheck-coderworkspacepool.yaml", "../../docs/how-to/gitops.md", workspacePoolCases}, + {"CoderWorkspaceClaim", "argocd-health-coderworkspaceclaim.lua", "flux-healthcheck-coderworkspaceclaim.yaml", "../../docs/how-to/gitops.md", workspaceClaimCases}, } // fluxHealthCheck is one entry of a Flux Kustomization spec.healthCheckExprs. @@ -129,9 +132,10 @@ func mcpServerObject(status map[string]any) map[string]any { return obj } -// readyReconcilingStalled returns the three conditions that the CoderMCPServer -// and CoderAIProvider controllers always write together, with one reason and -// message: ready, reconciling (neither ready nor stalled), or stalled. +// readyReconcilingStalled returns the three conditions that the CoderMCPServer, +// CoderAIProvider, CoderWorkspacePool and CoderWorkspaceClaim controllers +// always write together, with one reason and message: ready, reconciling +// (neither ready nor stalled), or stalled. func readyReconcilingStalled(state, reason string) map[string]any { ready, reconciling, stalled := "False", "True", "False" switch state { @@ -236,6 +240,106 @@ func aiProviderCases() []healthCase { } } +// workspacePoolObject returns a CoderWorkspacePool at generation 2 with the +// given status. +func workspacePoolObject(status map[string]any) map[string]any { + obj := map[string]any{ + "apiVersion": "coder.com/v1alpha1", + "kind": "CoderWorkspacePool", + "metadata": map[string]any{"name": "ci-runners", "namespace": "coder", "generation": int64(2)}, + "spec": map[string]any{"template": "coder.ci-runner", "replicas": int64(3)}, + } + if status != nil { + obj["status"] = status + } + return obj +} + +func workspacePoolCases() []healthCase { + // A /scale write bumps the generation, so the pool is stale until the + // controller counts its members for the new replicas. + rescaled := readyReconcilingStalled("ready", "Ready") + rescaled["observedGeneration"] = int64(1) + noConditions := map[string]any{"observedGeneration": int64(2)} + staleConditions := readyReconcilingStalled("ready", "Ready") + for _, c := range staleConditions["conditions"].([]any) { + c.(map[string]any)["observedGeneration"] = int64(1) + } + deleting := workspacePoolObject(readyReconcilingStalled("ready", "Ready")) + deleting["metadata"].(map[string]any)["deletionTimestamp"] = "2026-11-01T10:00:00Z" + deletingWait := workspacePoolObject(readyReconcilingStalled("reconciling", "WaitingForMembers")) + deletingWait["metadata"].(map[string]any)["deletionTimestamp"] = "2026-11-01T10:00:00Z" + return []healthCase{ + {"no status", workspacePoolObject(nil), "Progressing", "Waiting for the controller", "Unknown", ""}, + {"rescaled", workspacePoolObject(rescaled), "Progressing", "Waiting for the controller", "InProgress", ""}, + {"no conditions", workspacePoolObject(noConditions), "Progressing", "Waiting for the controller", "InProgress", ""}, + {"stale conditions", workspacePoolObject(staleConditions), "Progressing", "Waiting for the controller", "InProgress", ""}, + {"ready", workspacePoolObject(readyReconcilingStalled("ready", "Ready")), "Healthy", "Ready: details", "Current", ""}, + {"waiting for members", workspacePoolObject(readyReconcilingStalled("reconciling", "WaitingForMembers")), "Progressing", "WaitingForMembers: details", "InProgress", ""}, + {"owner not allowed", workspacePoolObject(readyReconcilingStalled("reconciling", "OwnerNotAllowed")), "Progressing", "OwnerNotAllowed: details", "InProgress", ""}, + {"owner cap", workspacePoolObject(readyReconcilingStalled("reconciling", "OwnerWorkspaceLimit")), "Progressing", "OwnerWorkspaceLimit: details", "InProgress", ""}, + {"failure limit", workspacePoolObject(readyReconcilingStalled("stalled", "MemberFailureLimit")), "Degraded", "MemberFailureLimit: details", "Failed", ""}, + {"version archived", workspacePoolObject(readyReconcilingStalled("stalled", "TemplateVersionArchived")), "Degraded", "TemplateVersionArchived: details", "Failed", ""}, + // Flux has no deletion rule: it waits for pruned objects separately. + {"being deleted", deleting, "Progressing", "Deleting the workspace pool", "Current", "Deleting the workspace pool"}, + {"being deleted while members build", deletingWait, "Progressing", "Deleting the workspace pool", "InProgress", "Deleting the workspace pool"}, + } +} + +// workspaceClaimObject returns a CoderWorkspaceClaim at generation 2 with the +// given status. +func workspaceClaimObject(status map[string]any) map[string]any { + obj := map[string]any{ + "apiVersion": "coder.com/v1alpha1", + "kind": "CoderWorkspaceClaim", + "metadata": map[string]any{"name": "build-1234", "namespace": "coder", "generation": int64(2)}, + "spec": map[string]any{"poolName": "ci-runners"}, + } + if status != nil { + obj["status"] = status + } + return obj +} + +// claimStatus returns a claim status in phase with the conditions of state. +// A claim status has no observedGeneration: only its conditions carry one. +func claimStatus(phase, state, reason string) map[string]any { + st := readyReconcilingStalled(state, reason) + delete(st, "observedGeneration") + st["phase"] = phase + return st +} + +func workspaceClaimCases() []healthCase { + staleConditions := claimStatus("Bound", "ready", "WorkspaceReady") + for _, c := range staleConditions["conditions"].([]any) { + c.(map[string]any)["observedGeneration"] = int64(1) + } + deleting := workspaceClaimObject(claimStatus("Bound", "ready", "WorkspaceReady")) + deleting["metadata"].(map[string]any)["deletionTimestamp"] = "2026-11-01T10:00:00Z" + deletingWait := workspaceClaimObject(claimStatus("Pending", "reconciling", "WaitingForReadyWorkspace")) + deletingWait["metadata"].(map[string]any)["deletionTimestamp"] = "2026-11-01T10:00:00Z" + return []healthCase{ + {"no status", workspaceClaimObject(nil), "Progressing", "Waiting for the controller", "Unknown", ""}, + {"no conditions", workspaceClaimObject(map[string]any{"phase": "Pending"}), "Progressing", "Waiting for the controller", "InProgress", ""}, + {"stale conditions", workspaceClaimObject(staleConditions), "Progressing", "Waiting for the controller", "InProgress", ""}, + {"waiting for a workspace", workspaceClaimObject(claimStatus("Pending", "reconciling", "WaitingForReadyWorkspace")), "Progressing", "WaitingForReadyWorkspace: details", "InProgress", ""}, + {"pool not found", workspaceClaimObject(claimStatus("Pending", "reconciling", "PoolNotFound")), "Progressing", "PoolNotFound: details", "InProgress", ""}, + {"bound and ready", workspaceClaimObject(claimStatus("Bound", "ready", "WorkspaceReady")), "Healthy", "WorkspaceReady: details", "Current", ""}, + {"bound, agent not ready", workspaceClaimObject(claimStatus("Bound", "reconciling", "AgentNotReady")), "Progressing", "AgentNotReady: details", "InProgress", ""}, + {"member deleted", workspaceClaimObject(claimStatus("Lost", "stalled", "MemberDeleted")), "Degraded", "MemberDeleted: details", "Failed", ""}, + {"workspace deleted", workspaceClaimObject(claimStatus("Lost", "stalled", "WorkspaceDeleted")), "Degraded", "WorkspaceDeleted: details", "Failed", ""}, + {"idle timeout", workspaceClaimObject(claimStatus("Lost", "stalled", "IdleTimeout")), "Degraded", "IdleTimeout: details", "Failed", ""}, + // The controller writes Lost and Stalled=True together. Each one alone + // still counts. + {"lost without conditions", workspaceClaimObject(map[string]any{"phase": "Lost"}), "Degraded", "Waiting for the controller", "Failed", ""}, + {"stalled while bound", workspaceClaimObject(claimStatus("Bound", "stalled", "WorkspaceDeleted")), "Degraded", "WorkspaceDeleted: details", "Failed", ""}, + // Flux has no deletion rule: it waits for pruned objects separately. + {"being deleted", deleting, "Progressing", "Deleting the workspace claim", "Current", "Deleting the workspace claim"}, + {"being deleted while waiting", deletingWait, "Progressing", "Deleting the workspace claim", "InProgress", "Deleting the workspace claim"}, + } +} + func TestArgoCDHealth(t *testing.T) { t.Parallel() for _, rules := range ruleSets { diff --git a/docs/how-to/gitops.md b/docs/how-to/gitops.md index 5ec375a5..cdad0583 100644 --- a/docs/how-to/gitops.md +++ b/docs/how-to/gitops.md @@ -1,6 +1,6 @@ # Gate template promotion with GitOps -This guide shows how to make Argo CD or Flux wait for a [`CoderTemplateTest`](test-templates.md) result, and how to promote a template version only after its test passed. The health rules live in `config/gitops/` in this repository. This page copies them, and a unit test keeps the copies equal to the files. The page also holds the [health rules for `CoderAIProvider`](#health-rules-for-coderaiprovider). +This guide shows how to make Argo CD or Flux wait for a [`CoderTemplateTest`](test-templates.md) result, and how to promote a template version only after its test passed. The health rules live in `config/gitops/` in this repository. This page copies them, and a unit test keeps the copies equal to the files. The page also holds the [health rules for `CoderAIProvider`](#health-rules-for-coderaiprovider), and for [`CoderWorkspacePool` and `CoderWorkspaceClaim`](#health-rules-for-coderworkspacepool-and-coderworkspaceclaim). ## Why custom health rules are needed @@ -229,6 +229,141 @@ Add this entry (`config/gitops/flux-healthcheck-coderaiprovider.yaml`) to `spec. current: "status.conditions.exists(c, c.type == 'Ready' && c.status == 'True')" ``` +## Health rules for CoderWorkspacePool and CoderWorkspaceClaim + +Argo CD and Flux also need health rules to wait for a [workspace pool](workspace-pools.md) and its claims. Without them, Argo CD has no health check for the kinds, and Flux reports a new pool or claim as current before the controller writes its first status. The controller writes the conditions `Ready`, `Reconciling` and `Stalled` of a pool or a claim together, with one reason and message, and with the object's `metadata.generation` as the condition's `observedGeneration`. The rules ignore a condition whose `observedGeneration` is older. + +A pool also has `status.observedGeneration`, and both pool rules wait until it equals `metadata.generation`. Every `/scale` write raises the generation, so after `kubectl scale`, or a resize by an HPA or KEDA, the pool shows `Progressing` until the controller has counted its members for the new `spec.replicas`. + +| Pool conditions | Argo CD | Flux | +| --- | --- | --- | +| no status, or a stale `observedGeneration` on the status or the `Ready` condition | `Progressing` | `InProgress` (`Unknown` before the first status) | +| `Reconciling=True`, for example `WaitingForMembers` or `OwnerWorkspaceLimit` | `Progressing` | `InProgress` | +| `Ready=True`: at least `spec.replicas` current unclaimed workspaces are ready, and no outdated one is left | `Healthy` | `Current` | +| `Stalled=True`, for example `MemberFailureLimit` | `Degraded` | `Failed` | + +A claim has no `status.observedGeneration`, and its spec cannot change, so the claim rules read only the conditions and `status.phase`: + +| Claim | Argo CD | Flux | +| --- | --- | --- | +| no status, or no conditions of the current generation | `Progressing` | `InProgress` (`Unknown` before the first status) | +| `Pending`, or `Bound` to a workspace that is not ready (`Reconciling=True`) | `Progressing` | `InProgress` | +| `Bound` to a ready workspace (`Ready=True`) | `Healthy` | `Current` | +| `Lost` (`Stalled=True`) | `Degraded` | `Failed` | + +`Lost` is final: the claim never binds again. To get a workspace again, delete the claim, and let the GitOps tool create it again. The new object binds a new workspace. For every reason, see [Troubleshoot workspace pools](troubleshooting.md#troubleshoot-workspace-pools). The Argo CD message is `: ` of the `Ready` condition. While the object is being deleted, Argo CD shows `Deleting the workspace pool` or `Deleting the workspace claim`. + +Add the scripts as the `argocd-cm` keys `resource.customizations.health.coder.com_CoderWorkspacePool` (`config/gitops/argocd-health-coderworkspacepool.lua`): + +```lua +-- Argo CD health check for CoderWorkspacePool (coder.com/v1alpha1). +-- Install it as the argocd-cm key +-- resource.customizations.health.coder.com_CoderWorkspacePool. +-- docs/how-to/gitops.md copies this file. A test keeps the copy equal. +local hs = { status = "Progressing", message = "Waiting for the controller" } +if obj.metadata.deletionTimestamp ~= nil then + -- Argo CD shows deletionMessage, not message, for a resource being deleted. + hs.deletionMessage = "Deleting the workspace pool" + hs.message = hs.deletionMessage + return hs +end +local st = obj.status +if st == nil or st.observedGeneration == nil or st.observedGeneration ~= obj.metadata.generation then + return hs +end +-- A condition from an older generation does not describe the current spec. +local conditions = {} +for _, c in ipairs(st.conditions or {}) do + if c.observedGeneration == obj.metadata.generation then + table.insert(conditions, c) + end +end +for _, c in ipairs(conditions) do + if c.type == "Ready" and c.reason ~= nil and c.message ~= nil then + hs.message = c.reason .. ": " .. c.message + end +end +for _, c in ipairs(conditions) do + if c.type == "Ready" and c.status == "True" then + hs.status = "Healthy" + elseif c.type == "Stalled" and c.status == "True" then + hs.status = "Degraded" + end +end +return hs +``` + +and `resource.customizations.health.coder.com_CoderWorkspaceClaim` (`config/gitops/argocd-health-coderworkspaceclaim.lua`): + +```lua +-- Argo CD health check for CoderWorkspaceClaim (coder.com/v1alpha1). +-- Install it as the argocd-cm key +-- resource.customizations.health.coder.com_CoderWorkspaceClaim. +-- docs/how-to/gitops.md copies this file. A test keeps the copy equal. +local hs = { status = "Progressing", message = "Waiting for the controller" } +if obj.metadata.deletionTimestamp ~= nil then + -- Argo CD shows deletionMessage, not message, for a resource being deleted. + hs.deletionMessage = "Deleting the workspace claim" + hs.message = hs.deletionMessage + return hs +end +local st = obj.status +if st == nil then + return hs +end +-- A claim has no status.observedGeneration, so only the conditions tell +-- whether the controller saw the current spec. +local conditions = {} +for _, c in ipairs(st.conditions or {}) do + if c.observedGeneration == obj.metadata.generation then + table.insert(conditions, c) + end +end +for _, c in ipairs(conditions) do + if c.type == "Ready" and c.reason ~= nil and c.message ~= nil then + hs.message = c.reason .. ": " .. c.message + end +end +for _, c in ipairs(conditions) do + if c.type == "Ready" and c.status == "True" then + hs.status = "Healthy" + elseif c.type == "Stalled" and c.status == "True" then + hs.status = "Degraded" + end +end +-- Lost is final: the claim never binds again. +if st.phase == "Lost" then + hs.status = "Degraded" +end +return hs +``` + +Add these entries (`config/gitops/flux-healthcheck-coderworkspacepool.yaml` and `config/gitops/flux-healthcheck-coderworkspaceclaim.yaml`) to `spec.healthCheckExprs` of a Flux v2.5+ Kustomization with `spec.wait: true`: + +```yaml +# Flux health check expressions for CoderWorkspacePool (coder.com/v1alpha1). +# Add this entry to spec.healthCheckExprs of a Flux v2.5+ Kustomization +# with spec.wait: true. +# docs/how-to/gitops.md copies this file. A test keeps the copy equal. +- apiVersion: coder.com/v1alpha1 + kind: CoderWorkspacePool + inProgress: "!has(status.observedGeneration) || status.observedGeneration != metadata.generation || !has(status.conditions) || !status.conditions.exists(c, c.type == 'Ready' && has(c.observedGeneration) && c.observedGeneration == metadata.generation) || status.conditions.exists(c, c.type == 'Reconciling' && c.status == 'True')" + failed: "status.conditions.exists(c, c.type == 'Stalled' && c.status == 'True')" + current: "status.conditions.exists(c, c.type == 'Ready' && c.status == 'True')" +``` + +```yaml +# Flux health check expressions for CoderWorkspaceClaim (coder.com/v1alpha1). +# Add this entry to spec.healthCheckExprs of a Flux v2.5+ Kustomization +# with spec.wait: true. +# docs/how-to/gitops.md copies this file. A test keeps the copy equal. +- apiVersion: coder.com/v1alpha1 + kind: CoderWorkspaceClaim + inProgress: "(!has(status.phase) || status.phase != 'Lost') && (!has(status.conditions) || !status.conditions.exists(c, c.type == 'Ready' && has(c.observedGeneration) && c.observedGeneration == metadata.generation) || status.conditions.exists(c, c.type == 'Reconciling' && c.status == 'True'))" + failed: "(has(status.phase) && status.phase == 'Lost') || status.conditions.exists(c, c.type == 'Stalled' && c.status == 'True')" + current: "status.conditions.exists(c, c.type == 'Ready' && c.status == 'True')" +``` + ## What CI covers -The unit test in `config/gitops/` runs the Lua script with gopher-lua, the Lua engine of Argo CD, and the Flux expressions with cel-go, both over fixed test objects. The unit test in `examples/gitops-promotion/` renders the example with Kustomize and runs `promote.sh` against a fake `kubectl`. Its cases include a test that someone created first with other parameters, which the script must refuse, and a test that runs for the longest `timeoutSeconds`. CI does not run Argo CD or Flux. +The unit test in `config/gitops/` runs the Lua scripts with gopher-lua, the Lua engine of Argo CD, and the Flux expressions with cel-go, both over fixed test objects. The unit test in `examples/gitops-promotion/` renders the example with Kustomize and runs `promote.sh` against a fake `kubectl`. Its cases include a test that someone created first with other parameters, which the script must refuse, and a test that runs for the longest `timeoutSeconds`. CI does not run Argo CD or Flux. From 9ec119d99f6bb8c92ea7c2198d459336320c0c99 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sun, 1 Nov 2026 01:05:02 +0000 Subject: [PATCH 2/7] docs: list every workspace pool, member, and claim reason in troubleshooting (#153) Change-Id: I39a4bcca90489c23e5fcee27cc10bd611d5bdea1 Signed-off-by: Thomas Kosiewski --- docs/how-to/troubleshooting.md | 112 +++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/docs/how-to/troubleshooting.md b/docs/how-to/troubleshooting.md index 969b461c..33c6fa17 100644 --- a/docs/how-to/troubleshooting.md +++ b/docs/how-to/troubleshooting.md @@ -306,3 +306,115 @@ CAUTION: Delete the workspace in Coder before you remove the finalizer. Otherwis kubectl get namespace -o jsonpath='{range .status.conditions[*]}{.type}={.status} {.message}{"\n"}{end}' kubectl get codercontrolplanes -n -o jsonpath='{range .items[*]}{.metadata.name}: {.metadata.finalizers}{"\n"}{end}' ``` + +## Troubleshoot workspace pools + +`kubectl get` shows the reason of each [pool](workspace-pools.md), member, and claim in the `REASON` column. `status.message` says more: + +```bash +kubectl get coderworkspacepools,coderworkspacepoolmembers,coderworkspaceclaims -n +kubectl get coderworkspacepoolmember -n -o jsonpath='{.status.reason}: {.status.message}{"\n"}' +``` + +A wait ends by itself when its cause goes away. A failure is final for a member: its phase becomes `Failed`, and the pool deletes it and creates a new one. A failure of the pool sets `Stalled=True`. The pool reads Coder again every 60 s, also while it is stalled, so a fix in Coder ends a stall without a spec change. A claim does not show why its pool cannot create workspaces. If a claim waits for a long time, read the reason of its pool. + +### Reasons of a pool and of a member before its create request + +The pool checks its inputs in Coder, and each member checks them again right before it sends its create request. + +| Reason | Kind | Cause | Fix | +| --- | --- | --- | --- | +| `ControlPlaneNotReady`, `OperatorAccessNotReady` | wait | The control plane in `spec.controlPlaneRef` does not exist, is being deleted, or has no URL, operator access, or TLS trust yet. | See [`CoderControlPlane` stays `Pending`](#codercontrolplane-stays-pending). | +| `ControlPlaneDeleting` | wait | The control plane is being deleted. No member creates a workspace. | None. See [Cleanup](workspace-pools.md#cleanup). | +| `OwnerNotAllowed`, `TemplateNotAllowed` | wait | `spec.workspacePools.owners` of the control plane does not list the owner, or does not list the template for that owner. The pool deletes its unclaimed workspaces. | The control plane admin adds the owner or the template. | +| `OwnerNotEligible` | wait | The owner breaks an [owner rule](workspace-pools.md#1-create-the-pool-owner), and the message names the rule, or the owner is not a member of the template's organization. The pool deletes its unclaimed workspaces. | Fix the owner in Coder. For a dormant owner, see [The owner is dormant](#the-pool-owner-is-dormant). | +| `TemplateNotFound`, `TemplateVersionNotFound`, `PresetNotFound` | wait | `spec.template`, `spec.version`, or `spec.presetName` names nothing in Coder. | Create it in Coder, or fix `spec.version` or `spec.presetName`. `spec.template` cannot change: create a new pool. | +| `TemplateVersionImporting` | wait | The template version is still importing. | None. | +| `TemplateDeprecated`, `TemplateVersionArchived`, `TemplateVersionImportFailed`, `TemplateVersionMismatch` | failure | Coder accepts no new workspaces from the template or the version, or the version belongs to another template. | Remove the deprecation or the archive flag in Coder, or select another version. | +| `CoderUnavailable` | wait | A Coder request failed. The message holds the HTTP status and Coder's short message. After a `429`, a member waits 2 s, doubled per answer up to 2 min. | Check Coder and its logs. | +| `CoderAnswerMismatch` | wait | Coder, a proxy, or the SDK answered about another object than the one the operator asked for. The operator logs the error. | Check the proxies between the operator and Coder. | + +### Reasons of a pool + +| Reason | Kind | Cause | Fix | +| --- | --- | --- | --- | +| `Ready` | ready | At least `spec.replicas` current unclaimed workspaces are ready, and no outdated unclaimed workspace is left. | None. | +| `WaitingForMembers` | wait | Workspaces build, or outdated workspaces wait for their replacements. | Read the reasons of the members that are not `Ready`. | +| `OwnerWorkspaceLimit` | wait | The owner has `maxWorkspaces` members on this control plane, across all pools. Claimed and used members count. | Delete claims or used members, or raise `maxWorkspaces`. | +| `FailureBackoff` | wait | A member failed. The pool waits 30 s, doubled per failure in a row up to 10 min, before it creates the next member. `status.lastFailure` names the failed member and its reason. | Fix the cause of the failure. | +| `MemberFailureLimit` | failure | 5 members failed in a row, and the pool creates no more. `status.runFailures` lists them. | Fix the cause. Then change `spec.version`, `spec.presetName`, `spec.parameters`, or `spec.readyTimeoutSeconds`, or wait for a new active version. A resize does not reset the count. | + +### Reasons of a member + +| Reason | Kind | Cause | Fix | +| --- | --- | --- | --- | +| `NotOwned` | failure | No pool controls the member, or its pool was deleted and created again under the same name. Only the operator creates members. | Delete the member. | +| `PoolNotFound`, `PoolDeleting` | wait | The member's pool does not exist or is being deleted. | None. Garbage collection deletes the member. | +| `InputsMismatch` | failure | The member pins inputs that its pool no longer builds, for example after a spec change. | None. The pool replaces the member. | +| `WorkspaceNameConflict` | failure | Coder already has a workspace with the member's `kwp-` name, and nothing proves that the member created it. The operator never touches that workspace. | None. The pool replaces the member under a new name. If the member then stays `Terminating`, see [A member does not finish deleting](#a-pool-member-does-not-finish-deleting). | +| `CreatingWorkspace` | wait | The member stored its create marker and sends the create request. | None. | +| `CreateRetrying` | wait | Coder answered `401` or `429`, or the request was not sent, so nothing changed in Coder. The member sends the request again. | If it stays, check the operator token and Coder. | +| `CreateRejected` | failure | Coder refused the create request with another `4xx` answer. | Fix what the message names. | +| `ConfirmingCreate` | wait | The result of the create request is unknown, for example after a timeout or a `5xx`. The member reads Coder and never sends the request again. | None. | +| `CreateOutcomeUnknown` | failure | No workspace of the member appeared within 15 min of a create request with an unknown result. | None. The pool replaces the member. | +| `WaitingForBuild`, `WaitingForAgents` | wait | The start build runs, or an agent has not connected or is not ready yet. | None. | +| `BuildFailed`, `BuildCanceled` | failure | The start build failed, or someone else canceled it. The message holds only Coder's error code. | Read the build log in Coder. | +| `NoAgents` | failure | The workspace has no agent, so nothing proves that it works. | Add an agent to the template. | +| `AgentStartError`, `AgentStartTimeout`, `AgentStopped`, `AgentConnectionTimeout` | failure | The startup script of an agent failed or timed out, an agent stopped, or an agent did not connect in time. | Read the agent logs in Coder. | +| `DeadlineExceeded` | failure | The workspace was not ready within `spec.readyTimeoutSeconds` (default 900) of the member's creation. The message names the last wait. | Fix the cause of the last wait, or raise `readyTimeoutSeconds`. | +| `WorkspaceReady` | ready | Every agent is ready. A claim can take the workspace. | None. | +| `Used` | wait | The workspace's `last_used_at` moved after it became ready, so someone may have used it without a claim. The member gets `Used=True`, and the pool releases it. | See [Cleanup](workspace-pools.md#cleanup). | +| `AgentNotReady` | wait | An agent of a ready workspace disconnected or is no longer ready. No claim takes the workspace meanwhile. | None, unless it becomes `AgentLost`. | +| `AgentLost` | failure | An agent stayed not ready for 5 min. | Read the agent logs in Coder. | +| `ChangedExternally` | failure | Someone else started a build of the workspace, for example a stop, an update, or Coder's autostop. | None. The pool replaces the member. | +| `Dormant` | failure | Coder marked the workspace dormant, or scheduled its deletion. | Check the template's dormancy settings. | +| `MaxDeadline` | wait | The workspace's max deadline allows no extension of its autostop deadline. Coder stops the workspace at the max deadline, and the pool replaces it. | None. | +| `ExtendRefused` | wait | Coder refused to extend the autostop deadline. The member tries again at its next check, every 60 s. | Read Coder's message. | +| `OwnershipUnknown` | failure | Nothing proves that the workspace is the member's: someone renamed it, it has another owner or template, or the operator did not start its build. | See [A member does not finish deleting](#a-pool-member-does-not-finish-deleting). | +| `WorkspaceDeletedExternally` | failure | Someone deleted the workspace in Coder. A claim that holds the member becomes `Lost`. | None. | +| `Bound` | ready | A claim holds the member. The operator extends its autostop deadline at most once, at the bind. | None. | +| `Deleting`, `DeleteRetrying` | wait | The member is being deleted, and its delete build runs, or failed and runs again after 1 min, doubled per failure up to 30 min. The operator cancels an active start or stop build first. | Read the log of the delete build in Coder. | +| `Deleted` | done | The delete build succeeded, and the member's finalizer goes. | None. | + +The condition `WorkspaceDeleted` tells whether the member's workspace still exists. While it is `True`, with the reason `NotCreated`, `DeletedExternally`, or `Deleted`, a deleted member releases its finalizer without a Coder call. With `CleanupPending` (`False`), a delete of the member deletes its workspace. With `OwnershipUnknown` or `CreateOutcomeUnknown` (`Unknown`), the finalizer stays. `CreateOutcomeUnknown` during a delete means that the operator cannot prove whether a create request with an unknown result made a workspace. The member reads Coder again every 10 min. Check that the operator user is a site owner: only then does a `404` prove that no workspace exists. + +### Reasons of a claim + +| Reason | Phase | Cause | Fix | +| --- | --- | --- | --- | +| `WaitingForReadyWorkspace` | `Pending` | The pool has no ready unclaimed workspace for the claim yet. The pool creates one for each waiting claim, also at `replicas: 0`. | If it stays, read the reason of the pool. | +| `PoolNotFound` | `Pending` | `spec.poolName` names no pool, or the pool is being deleted. | Create the pool. | +| `WorkspaceReady` | `Bound` | The claimed workspace is ready. `status.ownerName` and `status.workspaceName` name it. | None. | +| a member reason, or `WorkspaceNotReady` | `Bound` | The claimed workspace is not ready. The claim shows the reason of its member. | See [Reasons of a member](#reasons-of-a-member). | +| `MemberDeleted` | `Lost` | The member that held the claim is gone. | Delete the claim, and create a new one. | +| `WorkspaceDeleted` | `Lost` | Someone deleted the claimed workspace in Coder. | Delete the claim, and create a new one. | +| `IdleTimeout` | `Lost` | Nobody used the workspace for `spec.claims.idleTimeoutSeconds` since the bind and since its last use, so the pool deletes it, unless its last check before the delete sees a use. | See [Idle reaping](workspace-pools.md#4-claim-a-workspace). | + +`Lost` is final: the claim never binds again. + +### The pool owner is dormant + +The pool waits with `OwnerNotEligible`, and the message names the status `dormant`. Coder creates users through its API as `dormant`, and every 15 minutes it marks each active user dormant whose last login or API use is older than 90 days. An activation does not count as a use. Activate the owner in Coder, then log in as it once, for example with `POST /api/v2/users/login`. After that, use the owner's credentials at least once every 90 days. The pool deletes its unclaimed workspaces while the owner is not eligible, and builds new ones once the owner is active. + +### A claim stays `Pending` and names a member + +Before the pool binds a claim, it stores the chosen member in the claim's `status.memberName`. That reservation holds until the member is gone, is being deleted, failed, was used, or holds another claim. A member that stays ready but whose workspace the pool cannot read keeps the reservation, and the claim waits. If the pool cannot read any workspace, fix Coder first. Otherwise, to end the wait, delete the member that `status.memberName` names. Only a cluster admin can delete members. The member's cleanup deletes its workspace, unless it sees a use. The pool then binds the claim to another member: + +```bash +kubectl delete coderworkspacepoolmember "$(kubectl get coderworkspaceclaim -n -o jsonpath='{.status.memberName}')" -n +``` + +### A pool member does not finish deleting + +A member keeps the finalizer `coder.com/workspace-pool-member-cleanup` until its cleanup ends: its workspace is deleted, the operator proves that the member made none, the cleanup keeps a workspace that someone used, or the control plane is gone. When the condition `WorkspaceDeleted` is `Unknown` with the reason `OwnershipUnknown`, nothing proves that the workspace is the member's, and the operator never deletes it. A renamed `kwp-` workspace is the usual cause: the operator proves that a workspace is the member's by its `kwp-` name, so a rename fails that proof. Do not rename these workspaces. A claimed member with `OwnershipUnknown` keeps its claim `Bound` and not ready. Delete the claim first. + +1. Find the workspace in Coder by the member's `status.workspaceID`, and delete it, or keep it on purpose. +2. Remove only the controller's finalizer. Find its position in the list, where the first entry has position 0: + + ```bash + kubectl get coderworkspacepoolmember -n -o jsonpath='{.metadata.finalizers}' + kubectl patch coderworkspacepoolmember -n --type json -p \ + '[{"op":"test","path":"/metadata/finalizers/","value":"coder.com/workspace-pool-member-cleanup"},{"op":"remove","path":"/metadata/finalizers/"}]' + ``` + +CAUTION: Delete the workspace in Coder before you remove the finalizer. Otherwise the workspace stays in Coder, and it keeps the pool owner's token from its start build. From 11f5401b3b6f64abb43e93857cfd04ab5758ad04 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sun, 1 Nov 2026 01:39:24 +0000 Subject: [PATCH 3/7] docs: cover Share mode reasons in troubleshooting and the claim health rules (#153) Change-Id: I770d1585d1de1c0ae371a3ab910ec1b4449e8ce7 Signed-off-by: Thomas Kosiewski --- config/gitops/health_test.go | 8 ++++++-- docs/how-to/gitops.md | 4 ++-- docs/how-to/troubleshooting.md | 28 +++++++++++++++++++++++----- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/config/gitops/health_test.go b/config/gitops/health_test.go index 605ad9b2..34eceb1b 100644 --- a/config/gitops/health_test.go +++ b/config/gitops/health_test.go @@ -330,10 +330,14 @@ func workspaceClaimCases() []healthCase { {"member deleted", workspaceClaimObject(claimStatus("Lost", "stalled", "MemberDeleted")), "Degraded", "MemberDeleted: details", "Failed", ""}, {"workspace deleted", workspaceClaimObject(claimStatus("Lost", "stalled", "WorkspaceDeleted")), "Degraded", "WorkspaceDeleted: details", "Failed", ""}, {"idle timeout", workspaceClaimObject(claimStatus("Lost", "stalled", "IdleTimeout")), "Degraded", "IdleTimeout: details", "Failed", ""}, - // The controller writes Lost and Stalled=True together. Each one alone + // The controller writes Lost and Stalled=True together. Lost alone // still counts. {"lost without conditions", workspaceClaimObject(map[string]any{"phase": "Lost"}), "Degraded", "Waiting for the controller", "Failed", ""}, - {"stalled while bound", workspaceClaimObject(claimStatus("Bound", "stalled", "WorkspaceDeleted")), "Degraded", "WorkspaceDeleted: details", "Failed", ""}, + {"claim mode mismatch", workspaceClaimObject(claimStatus("Pending", "stalled", "ClaimModeMismatch")), "Degraded", "ClaimModeMismatch: details", "Failed", ""}, + {"share user not eligible", workspaceClaimObject(claimStatus("Pending", "reconciling", "ShareUserNotEligible")), "Progressing", "ShareUserNotEligible: details", "InProgress", ""}, + {"sharing", workspaceClaimObject(claimStatus("Bound", "reconciling", "SharingWorkspace")), "Progressing", "SharingWorkspace: details", "InProgress", ""}, + {"share refused", workspaceClaimObject(claimStatus("Bound", "stalled", "ShareRefused")), "Degraded", "ShareRefused: details", "Failed", ""}, + {"share revoked", workspaceClaimObject(claimStatus("Bound", "stalled", "ShareRevoked")), "Degraded", "ShareRevoked: details", "Failed", ""}, // Flux has no deletion rule: it waits for pruned objects separately. {"being deleted", deleting, "Progressing", "Deleting the workspace claim", "Current", "Deleting the workspace claim"}, {"being deleted while waiting", deletingWait, "Progressing", "Deleting the workspace claim", "InProgress", "Deleting the workspace claim"}, diff --git a/docs/how-to/gitops.md b/docs/how-to/gitops.md index cdad0583..f4d7a4e9 100644 --- a/docs/how-to/gitops.md +++ b/docs/how-to/gitops.md @@ -249,9 +249,9 @@ A claim has no `status.observedGeneration`, and its spec cannot change, so the c | no status, or no conditions of the current generation | `Progressing` | `InProgress` (`Unknown` before the first status) | | `Pending`, or `Bound` to a workspace that is not ready (`Reconciling=True`) | `Progressing` | `InProgress` | | `Bound` to a ready workspace (`Ready=True`) | `Healthy` | `Current` | -| `Lost` (`Stalled=True`) | `Degraded` | `Failed` | +| `Lost`, or `Stalled=True` | `Degraded` | `Failed` | -`Lost` is final: the claim never binds again. To get a workspace again, delete the claim, and let the GitOps tool create it again. The new object binds a new workspace. For every reason, see [Troubleshoot workspace pools](troubleshooting.md#troubleshoot-workspace-pools). The Argo CD message is `: ` of the `Ready` condition. While the object is being deleted, Argo CD shows `Deleting the workspace pool` or `Deleting the workspace claim`. +`Lost` is final: the claim never binds again. To get a workspace again, delete the claim, and let the GitOps tool create it again. The new object binds a new workspace. In Share mode, a claim that is not `Lost` can also be `Stalled`: `ClaimModeMismatch` never binds, and `ShareRefused`, `ShareRevoked`, and `ShareRemoved` leave a bound workspace without its share. Only `ShareRefused` can become healthy again without a new claim. For every reason, see [Troubleshoot workspace pools](troubleshooting.md#troubleshoot-workspace-pools). The Argo CD message is `: ` of the `Ready` condition. While the object is being deleted, Argo CD shows `Deleting the workspace pool` or `Deleting the workspace claim`. Add the scripts as the `argocd-cm` keys `resource.customizations.health.coder.com_CoderWorkspacePool` (`config/gitops/argocd-health-coderworkspacepool.lua`): diff --git a/docs/how-to/troubleshooting.md b/docs/how-to/troubleshooting.md index 33c6fa17..436d153f 100644 --- a/docs/how-to/troubleshooting.md +++ b/docs/how-to/troubleshooting.md @@ -316,7 +316,7 @@ kubectl get coderworkspacepools,coderworkspacepoolmembers,coderworkspaceclaims - kubectl get coderworkspacepoolmember -n -o jsonpath='{.status.reason}: {.status.message}{"\n"}' ``` -A wait ends by itself when its cause goes away. A failure is final for a member: its phase becomes `Failed`, and the pool deletes it and creates a new one. A failure of the pool sets `Stalled=True`. The pool reads Coder again every 60 s, also while it is stalled, so a fix in Coder ends a stall without a spec change. A claim does not show why its pool cannot create workspaces. If a claim waits for a long time, read the reason of its pool. +A wait ends by itself when its cause goes away. A failure is final for a member: its phase becomes `Failed`, and the pool deletes it and creates a new one. A failure of the pool sets `Stalled=True`. The pool reads Coder again every 60 s, also while it is stalled, so a fix in Coder ends a stall without a spec change. A claim often does not show why its pool cannot create workspaces. If a claim waits for a long time, read the reason of its pool. ### Reasons of a pool and of a member before its create request @@ -325,7 +325,7 @@ The pool checks its inputs in Coder, and each member checks them again right bef | Reason | Kind | Cause | Fix | | --- | --- | --- | --- | | `ControlPlaneNotReady`, `OperatorAccessNotReady` | wait | The control plane in `spec.controlPlaneRef` does not exist, is being deleted, or has no URL, operator access, or TLS trust yet. | See [`CoderControlPlane` stays `Pending`](#codercontrolplane-stays-pending). | -| `ControlPlaneDeleting` | wait | The control plane is being deleted. No member creates a workspace. | None. See [Cleanup](workspace-pools.md#cleanup). | +| `ControlPlaneDeleting` | wait | The control plane is being deleted. No member creates a workspace, and members no longer check or remove shares ([#448](https://github.com/coder/coder-k8s/issues/448)). | None. See [Cleanup](workspace-pools.md#cleanup). | | `OwnerNotAllowed`, `TemplateNotAllowed` | wait | `spec.workspacePools.owners` of the control plane does not list the owner, or does not list the template for that owner. The pool deletes its unclaimed workspaces. | The control plane admin adds the owner or the template. | | `OwnerNotEligible` | wait | The owner breaks an [owner rule](workspace-pools.md#1-create-the-pool-owner), and the message names the rule, or the owner is not a member of the template's organization. The pool deletes its unclaimed workspaces. | Fix the owner in Coder. For a dormant owner, see [The owner is dormant](#the-pool-owner-is-dormant). | | `TemplateNotFound`, `TemplateVersionNotFound`, `PresetNotFound` | wait | `spec.template`, `spec.version`, or `spec.presetName` names nothing in Coder. | Create it in Coder, or fix `spec.version` or `spec.presetName`. `spec.template` cannot change: create a new pool. | @@ -342,6 +342,7 @@ The pool checks its inputs in Coder, and each member checks them again right bef | `WaitingForMembers` | wait | Workspaces build, or outdated workspaces wait for their replacements. | Read the reasons of the members that are not `Ready`. | | `OwnerWorkspaceLimit` | wait | The owner has `maxWorkspaces` members on this control plane, across all pools. Claimed and used members count. | Delete claims or used members, or raise `maxWorkspaces`. | | `FailureBackoff` | wait | A member failed. The pool waits 30 s, doubled per failure in a row up to 10 min, before it creates the next member. `status.lastFailure` names the failed member and its reason. | Fix the cause of the failure. | +| `ShareNotAllowed` | wait | The pool is in Share mode, and its owner may not share: the control plane lists no `shareGroupIDs` for the owner, the deployment disables sharing, or the organization's sharing setting does not let the owner share. The claims of the pool wait with the same reason and are no demand. | See [Share a workspace with a person](workspace-pools.md#6-share-a-workspace-with-a-person). | | `MemberFailureLimit` | failure | 5 members failed in a row, and the pool creates no more. `status.runFailures` lists them. | Fix the cause. Then change `spec.version`, `spec.presetName`, `spec.parameters`, or `spec.readyTimeoutSeconds`, or wait for a new active version. A resize does not reset the count. | ### Reasons of a member @@ -363,7 +364,7 @@ The pool checks its inputs in Coder, and each member checks them again right bef | `AgentStartError`, `AgentStartTimeout`, `AgentStopped`, `AgentConnectionTimeout` | failure | The startup script of an agent failed or timed out, an agent stopped, or an agent did not connect in time. | Read the agent logs in Coder. | | `DeadlineExceeded` | failure | The workspace was not ready within `spec.readyTimeoutSeconds` (default 900) of the member's creation. The message names the last wait. | Fix the cause of the last wait, or raise `readyTimeoutSeconds`. | | `WorkspaceReady` | ready | Every agent is ready. A claim can take the workspace. | None. | -| `Used` | wait | The workspace's `last_used_at` moved after it became ready, so someone may have used it without a claim. The member gets `Used=True`, and the pool releases it. | See [Cleanup](workspace-pools.md#cleanup). | +| `Used` | wait | The workspace's `last_used_at` moved after it became ready, so someone may have used it without a claim. Opening the workspace's page in the Coder dashboard counts ([#444](https://github.com/coder/coder-k8s/issues/444)). The member gets `Used=True`, and the pool releases it. | See [Cleanup](workspace-pools.md#cleanup). | | `AgentNotReady` | wait | An agent of a ready workspace disconnected or is no longer ready. No claim takes the workspace meanwhile. | None, unless it becomes `AgentLost`. | | `AgentLost` | failure | An agent stayed not ready for 5 min. | Read the agent logs in Coder. | | `ChangedExternally` | failure | Someone else started a build of the workspace, for example a stop, an update, or Coder's autostop. | None. The pool replaces the member. | @@ -373,6 +374,7 @@ The pool checks its inputs in Coder, and each member checks them again right bef | `OwnershipUnknown` | failure | Nothing proves that the workspace is the member's: someone renamed it, it has another owner or template, or the operator did not start its build. | See [A member does not finish deleting](#a-pool-member-does-not-finish-deleting). | | `WorkspaceDeletedExternally` | failure | Someone deleted the workspace in Coder. A claim that holds the member becomes `Lost`. | None. | | `Bound` | ready | A claim holds the member. The operator extends its autostop deadline at most once, at the bind. | None. | +| `Shared`, `ShareRefused`, `ShareRevoking`, `ShareRevoked`, `ShareRemoved` | ready | The state of the share for a Share claim, also in the member's condition `Shared`. `Shared` means that Coder lists the claim's user on the workspace. | See [Reasons of a claim](#reasons-of-a-claim). | | `Deleting`, `DeleteRetrying` | wait | The member is being deleted, and its delete build runs, or failed and runs again after 1 min, doubled per failure up to 30 min. The operator cancels an active start or stop build first. | Read the log of the delete build in Coder. | | `Deleted` | done | The delete build succeeded, and the member's finalizer goes. | None. | @@ -384,13 +386,29 @@ The condition `WorkspaceDeleted` tells whether the member's workspace still exis | --- | --- | --- | --- | | `WaitingForReadyWorkspace` | `Pending` | The pool has no ready unclaimed workspace for the claim yet. The pool creates one for each waiting claim, also at `replicas: 0`. | If it stays, read the reason of the pool. | | `PoolNotFound` | `Pending` | `spec.poolName` names no pool, or the pool is being deleted. | Create the pool. | -| `WorkspaceReady` | `Bound` | The claimed workspace is ready. `status.ownerName` and `status.workspaceName` name it. | None. | +| `ShareUserNotFound` | `Pending` | `spec.share.user` names no Coder user. The pool checks the claim again every minute. | Create the user in Coder, or create a claim with the right name. | +| `ShareUserNotEligible` | `Pending` | The user is not active, is not a member of the template's organization, or is in none of the groups in the owner's `shareGroupIDs`. The message names each group that the operator could not read. The pool checks the claim again every minute. | See [A Share claim waits with `ShareUserNotEligible`](#a-share-claim-waits-with-shareusernoteligible). | +| `ShareNotAllowed` | `Pending` | The pool's owner may not share. | See the pool reason `ShareNotAllowed`. | +| `ClaimModeMismatch` | `Pending` | The claim sets `spec.share`, but the pool is in Reserve mode, or the other way around. Both are immutable, so the claim is `Stalled` and never binds. | Delete the claim, and create one that matches the pool's `claims.mode`. | +| `WorkspaceReady` | `Bound` | The claimed workspace is ready, and in Share mode shared. `status.ownerName` and `status.workspaceName` name it. | None. | +| `SharingWorkspace` | `Bound` | The claim is bound, and its member shares the workspace with `spec.share.user`. | None. | +| `ShareRefused` | `Bound` | The share policy or Coder refused the share after the bind. The claim is `Stalled`, and the member checks again every minute. | Fix what the message names. | +| `ShareRevoking`, `ShareRevoked` | `Bound` | The admin withdrew what allowed the share, and the operator removes it (`ShareRevoking`) or removed it. The claim is `Stalled`. | See [A user keeps access after a share ends](#a-user-keeps-access-after-a-share-ends). | +| `ShareRemoved` | `Bound` | Coder no longer lists the user on the workspace: an admin tightened the organization's sharing setting, or a person removed the share. The claim is `Stalled`. | See [A user keeps access after a share ends](#a-user-keeps-access-after-a-share-ends). | | a member reason, or `WorkspaceNotReady` | `Bound` | The claimed workspace is not ready. The claim shows the reason of its member. | See [Reasons of a member](#reasons-of-a-member). | | `MemberDeleted` | `Lost` | The member that held the claim is gone. | Delete the claim, and create a new one. | | `WorkspaceDeleted` | `Lost` | Someone deleted the claimed workspace in Coder. | Delete the claim, and create a new one. | | `IdleTimeout` | `Lost` | Nobody used the workspace for `spec.claims.idleTimeoutSeconds` since the bind and since its last use, so the pool deletes it, unless its last check before the delete sees a use. | See [Idle reaping](workspace-pools.md#4-claim-a-workspace). | -`Lost` is final: the claim never binds again. +`Lost` is final: the claim never binds again. A claim of a Share pool can also wait with a reason of its pool, for example `OwnerNotAllowed` or `CoderUnavailable`. + +### A Share claim waits with `ShareUserNotEligible` + +The message says which check failed. When it says that a group could not be read, Coder answered `403` or `404` to the group read. Every group other than an organization's `Everyone` group needs a Premium license, and without one Coder answers `403`. Without a license, list the ID of the template's organization in `shareGroupIDs`: it stands for that organization's `Everyone` group, so every active member of the organization can get a share. The message tells the claim's author facts about the Coder user, such as its status ([#449](https://github.com/coder/coder-k8s/issues/449)). + +### A user keeps access after a share ends + +After `ShareRevoked` or `ShareRemoved`, Coder no longer lists the user on the workspace, but the user keeps access until the workspace restarts or stops. Coder applies the removal of a share only then, and the operator does not restart or stop the workspace ([#450](https://github.com/coder/coder-k8s/issues/450)). To end the access now, restart or stop the workspace in Coder, or delete the claim. The operator does not share the workspace again for this claim. ### The pool owner is dormant From 206bed3a8bc7440185713f289cf1545280911cd2 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sun, 1 Nov 2026 01:47:01 +0000 Subject: [PATCH 4/7] docs: correct pool troubleshooting details found in review (#153) Change-Id: I3d0054da6ecf6f6a8ed3336751173f0ac5fdc539 Signed-off-by: Thomas Kosiewski --- docs/how-to/gitops.md | 2 +- docs/how-to/troubleshooting.md | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/how-to/gitops.md b/docs/how-to/gitops.md index f4d7a4e9..3c95a8e8 100644 --- a/docs/how-to/gitops.md +++ b/docs/how-to/gitops.md @@ -251,7 +251,7 @@ A claim has no `status.observedGeneration`, and its spec cannot change, so the c | `Bound` to a ready workspace (`Ready=True`) | `Healthy` | `Current` | | `Lost`, or `Stalled=True` | `Degraded` | `Failed` | -`Lost` is final: the claim never binds again. To get a workspace again, delete the claim, and let the GitOps tool create it again. The new object binds a new workspace. In Share mode, a claim that is not `Lost` can also be `Stalled`: `ClaimModeMismatch` never binds, and `ShareRefused`, `ShareRevoked`, and `ShareRemoved` leave a bound workspace without its share. Only `ShareRefused` can become healthy again without a new claim. For every reason, see [Troubleshoot workspace pools](troubleshooting.md#troubleshoot-workspace-pools). The Argo CD message is `: ` of the `Ready` condition. While the object is being deleted, Argo CD shows `Deleting the workspace pool` or `Deleting the workspace claim`. +`Lost` is final: the claim never binds again. To get a workspace again, delete the claim, and let the GitOps tool create it again. The new object binds a new workspace. In Share mode, a claim that is not `Lost` can also be `Stalled`: `ClaimModeMismatch` never binds, and `ShareRefused`, `ShareRevoked`, and `ShareRemoved` leave a bound workspace without its share. `ShareRefused` clears once the policy and Coder allow the share. After `ShareRevoked` or `ShareRemoved`, the operator does not share the workspace again. For every reason, see [Troubleshoot workspace pools](troubleshooting.md#troubleshoot-workspace-pools). The Argo CD message is `: ` of the `Ready` condition. While the object is being deleted, Argo CD shows `Deleting the workspace pool` or `Deleting the workspace claim`. Add the scripts as the `argocd-cm` keys `resource.customizations.health.coder.com_CoderWorkspacePool` (`config/gitops/argocd-health-coderworkspacepool.lua`): diff --git a/docs/how-to/troubleshooting.md b/docs/how-to/troubleshooting.md index 436d153f..ca3ea6aa 100644 --- a/docs/how-to/troubleshooting.md +++ b/docs/how-to/troubleshooting.md @@ -316,7 +316,7 @@ kubectl get coderworkspacepools,coderworkspacepoolmembers,coderworkspaceclaims - kubectl get coderworkspacepoolmember -n -o jsonpath='{.status.reason}: {.status.message}{"\n"}' ``` -A wait ends by itself when its cause goes away. A failure is final for a member: its phase becomes `Failed`, and the pool deletes it and creates a new one. A failure of the pool sets `Stalled=True`. The pool reads Coder again every 60 s, also while it is stalled, so a fix in Coder ends a stall without a spec change. A claim often does not show why its pool cannot create workspaces. If a claim waits for a long time, read the reason of its pool. +A wait ends by itself when its cause goes away. A failure is final for a member: its phase becomes `Failed`. The pool deletes a failed unclaimed member and creates a new one. A claimed member that fails stays, and its claim stays `Bound` and not ready until you delete the claim. If its workspace is gone, the claim becomes `Lost` instead. A failure of the pool sets `Stalled=True`. The pool reads Coder again every 60 s, also while it is stalled, so a fix in Coder ends a stall without a spec change. A claim often does not show why its pool cannot create workspaces. If a claim waits for a long time, read the reason of its pool. ### Reasons of a pool and of a member before its create request @@ -325,11 +325,11 @@ The pool checks its inputs in Coder, and each member checks them again right bef | Reason | Kind | Cause | Fix | | --- | --- | --- | --- | | `ControlPlaneNotReady`, `OperatorAccessNotReady` | wait | The control plane in `spec.controlPlaneRef` does not exist, is being deleted, or has no URL, operator access, or TLS trust yet. | See [`CoderControlPlane` stays `Pending`](#codercontrolplane-stays-pending). | -| `ControlPlaneDeleting` | wait | The control plane is being deleted. No member creates a workspace, and members no longer check or remove shares ([#448](https://github.com/coder/coder-k8s/issues/448)). | None. See [Cleanup](workspace-pools.md#cleanup). | +| `ControlPlaneDeleting` | wait | A read without the cache shows that the control plane is being deleted, before the cache shows it. No member creates a workspace, and members no longer check or remove shares ([#448](https://github.com/coder/coder-k8s/issues/448)). | None. See [Cleanup](workspace-pools.md#cleanup). | | `OwnerNotAllowed`, `TemplateNotAllowed` | wait | `spec.workspacePools.owners` of the control plane does not list the owner, or does not list the template for that owner. The pool deletes its unclaimed workspaces. | The control plane admin adds the owner or the template. | | `OwnerNotEligible` | wait | The owner breaks an [owner rule](workspace-pools.md#1-create-the-pool-owner), and the message names the rule, or the owner is not a member of the template's organization. The pool deletes its unclaimed workspaces. | Fix the owner in Coder. For a dormant owner, see [The owner is dormant](#the-pool-owner-is-dormant). | | `TemplateNotFound`, `TemplateVersionNotFound`, `PresetNotFound` | wait | `spec.template`, `spec.version`, or `spec.presetName` names nothing in Coder. | Create it in Coder, or fix `spec.version` or `spec.presetName`. `spec.template` cannot change: create a new pool. | -| `TemplateVersionImporting` | wait | The template version is still importing. | None. | +| `TemplateVersionImporting` | wait | The template version is still importing, or Coder answered `406` to a member's create request. | None. | | `TemplateDeprecated`, `TemplateVersionArchived`, `TemplateVersionImportFailed`, `TemplateVersionMismatch` | failure | Coder accepts no new workspaces from the template or the version, or the version belongs to another template. | Remove the deprecation or the archive flag in Coder, or select another version. | | `CoderUnavailable` | wait | A Coder request failed. The message holds the HTTP status and Coder's short message. After a `429`, a member waits 2 s, doubled per answer up to 2 min. | Check Coder and its logs. | | `CoderAnswerMismatch` | wait | Coder, a proxy, or the SDK answered about another object than the one the operator asked for. The operator logs the error. | Check the proxies between the operator and Coder. | @@ -340,7 +340,7 @@ The pool checks its inputs in Coder, and each member checks them again right bef | --- | --- | --- | --- | | `Ready` | ready | At least `spec.replicas` current unclaimed workspaces are ready, and no outdated unclaimed workspace is left. | None. | | `WaitingForMembers` | wait | Workspaces build, or outdated workspaces wait for their replacements. | Read the reasons of the members that are not `Ready`. | -| `OwnerWorkspaceLimit` | wait | The owner has `maxWorkspaces` members on this control plane, across all pools. Claimed and used members count. | Delete claims or used members, or raise `maxWorkspaces`. | +| `OwnerWorkspaceLimit` | wait | The owner has `maxWorkspaces` members on this control plane, across all pools. Claimed and used members count. A member checks the cap again right before its create request, and waits with the same reason. | Delete claims or used members, or raise `maxWorkspaces`. | | `FailureBackoff` | wait | A member failed. The pool waits 30 s, doubled per failure in a row up to 10 min, before it creates the next member. `status.lastFailure` names the failed member and its reason. | Fix the cause of the failure. | | `ShareNotAllowed` | wait | The pool is in Share mode, and its owner may not share: the control plane lists no `shareGroupIDs` for the owner, the deployment disables sharing, or the organization's sharing setting does not let the owner share. The claims of the pool wait with the same reason and are no demand. | See [Share a workspace with a person](workspace-pools.md#6-share-a-workspace-with-a-person). | | `MemberFailureLimit` | failure | 5 members failed in a row, and the pool creates no more. `status.runFailures` lists them. | Fix the cause. Then change `spec.version`, `spec.presetName`, `spec.parameters`, or `spec.readyTimeoutSeconds`, or wait for a new active version. A resize does not reset the count. | @@ -352,12 +352,12 @@ The pool checks its inputs in Coder, and each member checks them again right bef | `NotOwned` | failure | No pool controls the member, or its pool was deleted and created again under the same name. Only the operator creates members. | Delete the member. | | `PoolNotFound`, `PoolDeleting` | wait | The member's pool does not exist or is being deleted. | None. Garbage collection deletes the member. | | `InputsMismatch` | failure | The member pins inputs that its pool no longer builds, for example after a spec change. | None. The pool replaces the member. | -| `WorkspaceNameConflict` | failure | Coder already has a workspace with the member's `kwp-` name, and nothing proves that the member created it. The operator never touches that workspace. | None. The pool replaces the member under a new name. If the member then stays `Terminating`, see [A member does not finish deleting](#a-pool-member-does-not-finish-deleting). | +| `WorkspaceNameConflict` | failure | Coder already has a workspace with the member's `kwp-` name, and nothing proves that the member created it. The operator never touches that workspace. | None. The pool replaces the member under a new name. If the member then does not finish deleting, see [A member does not finish deleting](#a-pool-member-does-not-finish-deleting). | | `CreatingWorkspace` | wait | The member stored its create marker and sends the create request. | None. | | `CreateRetrying` | wait | Coder answered `401` or `429`, or the request was not sent, so nothing changed in Coder. The member sends the request again. | If it stays, check the operator token and Coder. | | `CreateRejected` | failure | Coder refused the create request with another `4xx` answer. | Fix what the message names. | | `ConfirmingCreate` | wait | The result of the create request is unknown, for example after a timeout or a `5xx`. The member reads Coder and never sends the request again. | None. | -| `CreateOutcomeUnknown` | failure | No workspace of the member appeared within 15 min of a create request with an unknown result. | None. The pool replaces the member. | +| `CreateOutcomeUnknown` | failure | No workspace of the member appeared within 15 min of a create request with an unknown result. Only a site owner's reads prove that, so with another operator user the member waits with `ConfirmingCreate` until `DeadlineExceeded`. | None. The pool replaces the member. | | `WaitingForBuild`, `WaitingForAgents` | wait | The start build runs, or an agent has not connected or is not ready yet. | None. | | `BuildFailed`, `BuildCanceled` | failure | The start build failed, or someone else canceled it. The message holds only Coder's error code. | Read the build log in Coder. | | `NoAgents` | failure | The workspace has no agent, so nothing proves that it works. | Add an agent to the template. | @@ -367,27 +367,27 @@ The pool checks its inputs in Coder, and each member checks them again right bef | `Used` | wait | The workspace's `last_used_at` moved after it became ready, so someone may have used it without a claim. Opening the workspace's page in the Coder dashboard counts ([#444](https://github.com/coder/coder-k8s/issues/444)). The member gets `Used=True`, and the pool releases it. | See [Cleanup](workspace-pools.md#cleanup). | | `AgentNotReady` | wait | An agent of a ready workspace disconnected or is no longer ready. No claim takes the workspace meanwhile. | None, unless it becomes `AgentLost`. | | `AgentLost` | failure | An agent stayed not ready for 5 min. | Read the agent logs in Coder. | -| `ChangedExternally` | failure | Someone else started a build of the workspace, for example a stop, an update, or Coder's autostop. | None. The pool replaces the member. | +| `ChangedExternally` | failure | Someone else started a build of the workspace, for example a stop, an update, or Coder's autostop. | None. The pool replaces an unclaimed member. | | `Dormant` | failure | Coder marked the workspace dormant, or scheduled its deletion. | Check the template's dormancy settings. | | `MaxDeadline` | wait | The workspace's max deadline allows no extension of its autostop deadline. Coder stops the workspace at the max deadline, and the pool replaces it. | None. | | `ExtendRefused` | wait | Coder refused to extend the autostop deadline. The member tries again at its next check, every 60 s. | Read Coder's message. | -| `OwnershipUnknown` | failure | Nothing proves that the workspace is the member's: someone renamed it, it has another owner or template, or the operator did not start its build. | See [A member does not finish deleting](#a-pool-member-does-not-finish-deleting). | +| `OwnershipUnknown` | failure | Nothing proves that the workspace is the member's: someone renamed it, it has another owner or template, or the operator did not start its build. During a delete, the member waits with this reason and never deletes the workspace. | See [A member does not finish deleting](#a-pool-member-does-not-finish-deleting). | | `WorkspaceDeletedExternally` | failure | Someone deleted the workspace in Coder. A claim that holds the member becomes `Lost`. | None. | | `Bound` | ready | A claim holds the member. The operator extends its autostop deadline at most once, at the bind. | None. | | `Shared`, `ShareRefused`, `ShareRevoking`, `ShareRevoked`, `ShareRemoved` | ready | The state of the share for a Share claim, also in the member's condition `Shared`. `Shared` means that Coder lists the claim's user on the workspace. | See [Reasons of a claim](#reasons-of-a-claim). | | `Deleting`, `DeleteRetrying` | wait | The member is being deleted, and its delete build runs, or failed and runs again after 1 min, doubled per failure up to 30 min. The operator cancels an active start or stop build first. | Read the log of the delete build in Coder. | | `Deleted` | done | The delete build succeeded, and the member's finalizer goes. | None. | -The condition `WorkspaceDeleted` tells whether the member's workspace still exists. While it is `True`, with the reason `NotCreated`, `DeletedExternally`, or `Deleted`, a deleted member releases its finalizer without a Coder call. With `CleanupPending` (`False`), a delete of the member deletes its workspace. With `OwnershipUnknown` or `CreateOutcomeUnknown` (`Unknown`), the finalizer stays. `CreateOutcomeUnknown` during a delete means that the operator cannot prove whether a create request with an unknown result made a workspace. The member reads Coder again every 10 min. Check that the operator user is a site owner: only then does a `404` prove that no workspace exists. +The condition `WorkspaceDeleted` tells whether the member's workspace still exists. During a delete, it is `False` with the reason of the current delete step, for example `Deleting` or `CoderUnavailable`. While it is `True`, with the reason `NotCreated`, `DeletedExternally`, or `Deleted`, a deleted member releases its finalizer without a Coder call. `CleanupPending` (`False`) marks a failed member whose workspace can exist: a delete of the member deletes it. With `OwnershipUnknown` or `CreateOutcomeUnknown` (`Unknown`), the finalizer stays. `CreateOutcomeUnknown` during a delete means that the operator cannot prove whether a create request with an unknown result made a workspace. The member reads Coder again every 10 min. Check that the operator user is a site owner: only then does a `404` prove that no workspace exists. ### Reasons of a claim | Reason | Phase | Cause | Fix | | --- | --- | --- | --- | -| `WaitingForReadyWorkspace` | `Pending` | The pool has no ready unclaimed workspace for the claim yet. The pool creates one for each waiting claim, also at `replicas: 0`. | If it stays, read the reason of the pool. | +| `WaitingForReadyWorkspace` | `Pending` | The pool has no ready unclaimed workspace for the claim yet. The pool creates one for each waiting claim that can bind, also at `replicas: 0`. | If it stays, read the reason of the pool. | | `PoolNotFound` | `Pending` | `spec.poolName` names no pool, or the pool is being deleted. | Create the pool. | | `ShareUserNotFound` | `Pending` | `spec.share.user` names no Coder user. The pool checks the claim again every minute. | Create the user in Coder, or create a claim with the right name. | -| `ShareUserNotEligible` | `Pending` | The user is not active, is not a member of the template's organization, or is in none of the groups in the owner's `shareGroupIDs`. The message names each group that the operator could not read. The pool checks the claim again every minute. | See [A Share claim waits with `ShareUserNotEligible`](#a-share-claim-waits-with-shareusernoteligible). | +| `ShareUserNotEligible` | `Pending` | The user is not active, is not a member of the template's organization, or is in none of the groups in the owner's `shareGroupIDs`. The message names each group that the operator could not read. The pool checks the claim again every minute. A claim that names the prebuilds system user is `Stalled`. | See [A Share claim waits with `ShareUserNotEligible`](#a-share-claim-waits-with-shareusernoteligible). | | `ShareNotAllowed` | `Pending` | The pool's owner may not share. | See the pool reason `ShareNotAllowed`. | | `ClaimModeMismatch` | `Pending` | The claim sets `spec.share`, but the pool is in Reserve mode, or the other way around. Both are immutable, so the claim is `Stalled` and never binds. | Delete the claim, and create one that matches the pool's `claims.mode`. | | `WorkspaceReady` | `Bound` | The claimed workspace is ready, and in Share mode shared. `status.ownerName` and `status.workspaceName` name it. | None. | From c57c000f7570fa23be6faa38d1bd576576e57fb0 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sun, 1 Nov 2026 01:56:10 +0000 Subject: [PATCH 5/7] docs: plain wording in pool troubleshooting (#153) Change-Id: Icfe185a6bdc6a77437b376f9512b9c47ed9c3652 Signed-off-by: Thomas Kosiewski --- docs/how-to/troubleshooting.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/how-to/troubleshooting.md b/docs/how-to/troubleshooting.md index ca3ea6aa..3cfb0d52 100644 --- a/docs/how-to/troubleshooting.md +++ b/docs/how-to/troubleshooting.md @@ -342,7 +342,7 @@ The pool checks its inputs in Coder, and each member checks them again right bef | `WaitingForMembers` | wait | Workspaces build, or outdated workspaces wait for their replacements. | Read the reasons of the members that are not `Ready`. | | `OwnerWorkspaceLimit` | wait | The owner has `maxWorkspaces` members on this control plane, across all pools. Claimed and used members count. A member checks the cap again right before its create request, and waits with the same reason. | Delete claims or used members, or raise `maxWorkspaces`. | | `FailureBackoff` | wait | A member failed. The pool waits 30 s, doubled per failure in a row up to 10 min, before it creates the next member. `status.lastFailure` names the failed member and its reason. | Fix the cause of the failure. | -| `ShareNotAllowed` | wait | The pool is in Share mode, and its owner may not share: the control plane lists no `shareGroupIDs` for the owner, the deployment disables sharing, or the organization's sharing setting does not let the owner share. The claims of the pool wait with the same reason and are no demand. | See [Share a workspace with a person](workspace-pools.md#6-share-a-workspace-with-a-person). | +| `ShareNotAllowed` | wait | The pool is in Share mode, and its owner is not allowed to share: the control plane lists no `shareGroupIDs` for the owner, the deployment disables sharing, or the organization's sharing setting does not let the owner share. The claims of the pool wait with the same reason and are no demand. | See [Share a workspace with a person](workspace-pools.md#6-share-a-workspace-with-a-person). | | `MemberFailureLimit` | failure | 5 members failed in a row, and the pool creates no more. `status.runFailures` lists them. | Fix the cause. Then change `spec.version`, `spec.presetName`, `spec.parameters`, or `spec.readyTimeoutSeconds`, or wait for a new active version. A resize does not reset the count. | ### Reasons of a member @@ -364,7 +364,7 @@ The pool checks its inputs in Coder, and each member checks them again right bef | `AgentStartError`, `AgentStartTimeout`, `AgentStopped`, `AgentConnectionTimeout` | failure | The startup script of an agent failed or timed out, an agent stopped, or an agent did not connect in time. | Read the agent logs in Coder. | | `DeadlineExceeded` | failure | The workspace was not ready within `spec.readyTimeoutSeconds` (default 900) of the member's creation. The message names the last wait. | Fix the cause of the last wait, or raise `readyTimeoutSeconds`. | | `WorkspaceReady` | ready | Every agent is ready. A claim can take the workspace. | None. | -| `Used` | wait | The workspace's `last_used_at` moved after it became ready, so someone may have used it without a claim. Opening the workspace's page in the Coder dashboard counts ([#444](https://github.com/coder/coder-k8s/issues/444)). The member gets `Used=True`, and the pool releases it. | See [Cleanup](workspace-pools.md#cleanup). | +| `Used` | wait | The workspace's `last_used_at` moved after it became ready, so someone used it without a claim, or Coder counted something else as a use. Opening the workspace's page in the Coder dashboard counts ([#444](https://github.com/coder/coder-k8s/issues/444)). The member gets `Used=True`, and the pool releases it. | See [Cleanup](workspace-pools.md#cleanup). | | `AgentNotReady` | wait | An agent of a ready workspace disconnected or is no longer ready. No claim takes the workspace meanwhile. | None, unless it becomes `AgentLost`. | | `AgentLost` | failure | An agent stayed not ready for 5 min. | Read the agent logs in Coder. | | `ChangedExternally` | failure | Someone else started a build of the workspace, for example a stop, an update, or Coder's autostop. | None. The pool replaces an unclaimed member. | @@ -387,8 +387,8 @@ The condition `WorkspaceDeleted` tells whether the member's workspace still exis | `WaitingForReadyWorkspace` | `Pending` | The pool has no ready unclaimed workspace for the claim yet. The pool creates one for each waiting claim that can bind, also at `replicas: 0`. | If it stays, read the reason of the pool. | | `PoolNotFound` | `Pending` | `spec.poolName` names no pool, or the pool is being deleted. | Create the pool. | | `ShareUserNotFound` | `Pending` | `spec.share.user` names no Coder user. The pool checks the claim again every minute. | Create the user in Coder, or create a claim with the right name. | -| `ShareUserNotEligible` | `Pending` | The user is not active, is not a member of the template's organization, or is in none of the groups in the owner's `shareGroupIDs`. The message names each group that the operator could not read. The pool checks the claim again every minute. A claim that names the prebuilds system user is `Stalled`. | See [A Share claim waits with `ShareUserNotEligible`](#a-share-claim-waits-with-shareusernoteligible). | -| `ShareNotAllowed` | `Pending` | The pool's owner may not share. | See the pool reason `ShareNotAllowed`. | +| `ShareUserNotEligible` | `Pending` | The user is not active, is not a member of the template's organization, or is in none of the groups in the owner's `shareGroupIDs`. The message names each group that the operator failed to read. The pool checks the claim again every minute. A claim that names the prebuilds system user is `Stalled`. | See [A Share claim waits with `ShareUserNotEligible`](#a-share-claim-waits-with-shareusernoteligible). | +| `ShareNotAllowed` | `Pending` | The pool's owner is not allowed to share. | See the pool reason `ShareNotAllowed`. | | `ClaimModeMismatch` | `Pending` | The claim sets `spec.share`, but the pool is in Reserve mode, or the other way around. Both are immutable, so the claim is `Stalled` and never binds. | Delete the claim, and create one that matches the pool's `claims.mode`. | | `WorkspaceReady` | `Bound` | The claimed workspace is ready, and in Share mode shared. `status.ownerName` and `status.workspaceName` name it. | None. | | `SharingWorkspace` | `Bound` | The claim is bound, and its member shares the workspace with `spec.share.user`. | None. | @@ -404,7 +404,7 @@ The condition `WorkspaceDeleted` tells whether the member's workspace still exis ### A Share claim waits with `ShareUserNotEligible` -The message says which check failed. When it says that a group could not be read, Coder answered `403` or `404` to the group read. Every group other than an organization's `Everyone` group needs a Premium license, and without one Coder answers `403`. Without a license, list the ID of the template's organization in `shareGroupIDs`: it stands for that organization's `Everyone` group, so every active member of the organization can get a share. The message tells the claim's author facts about the Coder user, such as its status ([#449](https://github.com/coder/coder-k8s/issues/449)). +The message says which check failed. When it names a group that the operator failed to read, Coder answered `403` or `404` to the group read. Every group other than an organization's `Everyone` group needs a Premium license, and without one Coder answers `403`. Without a license, list the ID of the template's organization in `shareGroupIDs`: it stands for that organization's `Everyone` group, so every active member of the organization can get a share. The message tells the claim's author facts about the Coder user, such as its status ([#449](https://github.com/coder/coder-k8s/issues/449)). ### A user keeps access after a share ends From eada0e6b33f0f40f8007a3a338fa25b5b53ea5fe Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sun, 1 Nov 2026 02:08:43 +0000 Subject: [PATCH 6/7] docs: note ShareRevoking and claims that stay Progressing after a member failure (#153) Change-Id: Ic1ebf2845d1fca904fb5933fdc834b12cda6a172 Signed-off-by: Thomas Kosiewski --- docs/how-to/gitops.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/how-to/gitops.md b/docs/how-to/gitops.md index 3c95a8e8..e672749e 100644 --- a/docs/how-to/gitops.md +++ b/docs/how-to/gitops.md @@ -251,7 +251,9 @@ A claim has no `status.observedGeneration`, and its spec cannot change, so the c | `Bound` to a ready workspace (`Ready=True`) | `Healthy` | `Current` | | `Lost`, or `Stalled=True` | `Degraded` | `Failed` | -`Lost` is final: the claim never binds again. To get a workspace again, delete the claim, and let the GitOps tool create it again. The new object binds a new workspace. In Share mode, a claim that is not `Lost` can also be `Stalled`: `ClaimModeMismatch` never binds, and `ShareRefused`, `ShareRevoked`, and `ShareRemoved` leave a bound workspace without its share. `ShareRefused` clears once the policy and Coder allow the share. After `ShareRevoked` or `ShareRemoved`, the operator does not share the workspace again. For every reason, see [Troubleshoot workspace pools](troubleshooting.md#troubleshoot-workspace-pools). The Argo CD message is `: ` of the `Ready` condition. While the object is being deleted, Argo CD shows `Deleting the workspace pool` or `Deleting the workspace claim`. +`Lost` is final: the claim never binds again. To get a workspace again, delete the claim, and let the GitOps tool create it again. The new object binds a new workspace. In Share mode, a claim that is not `Lost` can also be `Stalled`: `ClaimModeMismatch` never binds, `ShareRevoking` means the operator removes the share, and `ShareRefused`, `ShareRevoked`, and `ShareRemoved` leave a bound workspace without its share. `ShareRefused` clears once the policy and Coder allow the share. After `ShareRevoked` or `ShareRemoved`, the operator does not share the workspace again. + +A bound claim whose member failed with `OwnershipUnknown` or `ChangedExternally` stays `Progressing` and never becomes `Degraded`, so a GitOps wait for it ends only at its timeout. The claim shows the member's reason. To recover, delete the claim, and let the GitOps tool create it again. See [A pool member does not finish deleting](troubleshooting.md#a-pool-member-does-not-finish-deleting). For every reason, see [Troubleshoot workspace pools](troubleshooting.md#troubleshoot-workspace-pools). The Argo CD message is `: ` of the `Ready` condition. While the object is being deleted, Argo CD shows `Deleting the workspace pool` or `Deleting the workspace claim`. Add the scripts as the `argocd-cm` keys `resource.customizations.health.coder.com_CoderWorkspacePool` (`config/gitops/argocd-health-coderworkspacepool.lua`): From c998543f7ae8fefe703522709be9bcc848c208e0 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sun, 1 Nov 2026 02:10:14 +0000 Subject: [PATCH 7/7] docs: link #454 from the claim health notes (#153) Change-Id: I5bee8847ee7906bdc82f8157c35249be7cc4b5dd Signed-off-by: Thomas Kosiewski --- docs/how-to/gitops.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-to/gitops.md b/docs/how-to/gitops.md index e672749e..feca7dc9 100644 --- a/docs/how-to/gitops.md +++ b/docs/how-to/gitops.md @@ -253,7 +253,7 @@ A claim has no `status.observedGeneration`, and its spec cannot change, so the c `Lost` is final: the claim never binds again. To get a workspace again, delete the claim, and let the GitOps tool create it again. The new object binds a new workspace. In Share mode, a claim that is not `Lost` can also be `Stalled`: `ClaimModeMismatch` never binds, `ShareRevoking` means the operator removes the share, and `ShareRefused`, `ShareRevoked`, and `ShareRemoved` leave a bound workspace without its share. `ShareRefused` clears once the policy and Coder allow the share. After `ShareRevoked` or `ShareRemoved`, the operator does not share the workspace again. -A bound claim whose member failed with `OwnershipUnknown` or `ChangedExternally` stays `Progressing` and never becomes `Degraded`, so a GitOps wait for it ends only at its timeout. The claim shows the member's reason. To recover, delete the claim, and let the GitOps tool create it again. See [A pool member does not finish deleting](troubleshooting.md#a-pool-member-does-not-finish-deleting). For every reason, see [Troubleshoot workspace pools](troubleshooting.md#troubleshoot-workspace-pools). The Argo CD message is `: ` of the `Ready` condition. While the object is being deleted, Argo CD shows `Deleting the workspace pool` or `Deleting the workspace claim`. +A bound claim whose member failed with `OwnershipUnknown` or `ChangedExternally` stays `Progressing` and never becomes `Degraded`, so a GitOps wait for it ends only at its timeout ([#454](https://github.com/coder/coder-k8s/issues/454)). The claim shows the member's reason. To recover, delete the claim, and let the GitOps tool create it again. See [A pool member does not finish deleting](troubleshooting.md#a-pool-member-does-not-finish-deleting). For every reason, see [Troubleshoot workspace pools](troubleshooting.md#troubleshoot-workspace-pools). The Argo CD message is `: ` of the `Ready` condition. While the object is being deleted, Argo CD shows `Deleting the workspace pool` or `Deleting the workspace claim`. Add the scripts as the `argocd-cm` keys `resource.customizations.health.coder.com_CoderWorkspacePool` (`config/gitops/argocd-health-coderworkspacepool.lua`):