From 133b44bc76f36545d7ce96315b211ee9fda6f275 Mon Sep 17 00:00:00 2001 From: Alex Firmani Date: Fri, 18 Sep 2026 11:36:27 -0700 Subject: [PATCH 1/2] Move someguy to needle --- k8s/{someguy => needle}/deployment-extra.yaml.j2 | 0 k8s/{someguy => needle}/deployment.yaml.j2 | 0 k8s/{someguy => needle}/kustomization.yaml | 0 k8s/{someguy => needle}/namespace.yaml | 0 k8s/{someguy => needle}/pvc.yaml.j2 | 0 roles/{someguy => needle}/defaults/main.yml | 0 roles/{someguy => needle}/tasks/main.yml | 0 roles/{someguy => needle}/tasks/roll_instance.yml | 0 roles/{someguy => needle}/tasks/wait_warm.yml | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename k8s/{someguy => needle}/deployment-extra.yaml.j2 (100%) rename k8s/{someguy => needle}/deployment.yaml.j2 (100%) rename k8s/{someguy => needle}/kustomization.yaml (100%) rename k8s/{someguy => needle}/namespace.yaml (100%) rename k8s/{someguy => needle}/pvc.yaml.j2 (100%) rename roles/{someguy => needle}/defaults/main.yml (100%) rename roles/{someguy => needle}/tasks/main.yml (100%) rename roles/{someguy => needle}/tasks/roll_instance.yml (100%) rename roles/{someguy => needle}/tasks/wait_warm.yml (100%) diff --git a/k8s/someguy/deployment-extra.yaml.j2 b/k8s/needle/deployment-extra.yaml.j2 similarity index 100% rename from k8s/someguy/deployment-extra.yaml.j2 rename to k8s/needle/deployment-extra.yaml.j2 diff --git a/k8s/someguy/deployment.yaml.j2 b/k8s/needle/deployment.yaml.j2 similarity index 100% rename from k8s/someguy/deployment.yaml.j2 rename to k8s/needle/deployment.yaml.j2 diff --git a/k8s/someguy/kustomization.yaml b/k8s/needle/kustomization.yaml similarity index 100% rename from k8s/someguy/kustomization.yaml rename to k8s/needle/kustomization.yaml diff --git a/k8s/someguy/namespace.yaml b/k8s/needle/namespace.yaml similarity index 100% rename from k8s/someguy/namespace.yaml rename to k8s/needle/namespace.yaml diff --git a/k8s/someguy/pvc.yaml.j2 b/k8s/needle/pvc.yaml.j2 similarity index 100% rename from k8s/someguy/pvc.yaml.j2 rename to k8s/needle/pvc.yaml.j2 diff --git a/roles/someguy/defaults/main.yml b/roles/needle/defaults/main.yml similarity index 100% rename from roles/someguy/defaults/main.yml rename to roles/needle/defaults/main.yml diff --git a/roles/someguy/tasks/main.yml b/roles/needle/tasks/main.yml similarity index 100% rename from roles/someguy/tasks/main.yml rename to roles/needle/tasks/main.yml diff --git a/roles/someguy/tasks/roll_instance.yml b/roles/needle/tasks/roll_instance.yml similarity index 100% rename from roles/someguy/tasks/roll_instance.yml rename to roles/needle/tasks/roll_instance.yml diff --git a/roles/someguy/tasks/wait_warm.yml b/roles/needle/tasks/wait_warm.yml similarity index 100% rename from roles/someguy/tasks/wait_warm.yml rename to roles/needle/tasks/wait_warm.yml From 7b8286595f03458a65e32b15cde7f1b1db15030c Mon Sep 17 00:00:00 2001 From: Alex Firmani Date: Fri, 18 Sep 2026 11:40:09 -0700 Subject: [PATCH 2/2] Rename someguy to needle, pin ghcr.io/ipni/needle:v1.0.0 The fork shipped as ipni/needle v1.0.0 (NEEDLE_ env prefix, needle_ metric prefix, no aliases). This moves the whole deployment onto it: - git mv roles/someguy -> roles/needle, k8s/someguy -> k8s/needle (previous commit) - every someguy_* Ansible variable to needle_*, every SOMEGUY_* env var to NEEDLE_*, every someguy_ metric reference to needle_ - K8s resources: namespace, ConfigMap needle-env-vars, Deployments needle/-b/-c/-d, PVCs -data, Envoy cluster and endpoints - image pin to the v1.0.0 index digest (amd64 manifest sha256:f31bd19df98216b6418cb6c007fe5ce2559c7ed430d60bf18c0929e74d5671f3) - README, AGENTS.md and prose in every role; ports unchanged (API 8190-8193, libp2p 4004-4007) Verified: yamllint, ansible-lint (production profile), syntax-check on all playbooks, render-manifests + kubectl kustomize for every build (4 Deployments / 4 PVCs at needle_instances=4, 1/1 at =1), Envoy cluster and endpoints render as needle. --- .github/workflows/ci.yml | 16 +-- AGENTS.md | 38 ++--- README.md | 135 +++++++++--------- group_vars/ipfs_nodes/main.yml | 12 +- k8s/bootstrap/deployment.yaml | 6 +- k8s/cert-manager/kustomization.yaml | 2 +- k8s/content/deployment.yaml | 4 +- k8s/needle/deployment-extra.yaml.j2 | 24 ++-- k8s/needle/deployment.yaml.j2 | 50 +++---- k8s/needle/kustomization.yaml | 80 ++++++----- k8s/needle/namespace.yaml | 2 +- k8s/needle/pvc.yaml.j2 | 10 +- k8s/route-origin/deployment.yaml | 4 +- k8s/route-origin/envoy.yaml.j2 | 40 +++--- roles/bootstrap/tasks/main.yml | 4 +- roles/content/tasks/main.yml | 2 +- roles/hardening/tasks/main.yml | 2 +- roles/kustomize_apply/tasks/main.yml | 4 +- .../tasks/prune_configmaps.yml | 2 +- roles/needle/defaults/main.yml | 46 +++--- roles/needle/tasks/main.yml | 96 ++++++------- roles/needle/tasks/roll_instance.yml | 38 ++--- roles/needle/tasks/wait_warm.yml | 80 +++++------ roles/route_origin/defaults/main.yml | 2 +- roles/route_origin/tasks/main.yml | 4 +- roles/route_origin/tasks/verify.yml | 12 +- routing.yml | 10 +- scripts/render-manifests.yml | 8 +- scripts/routing-compare.sh | 4 +- scripts/routing-load.sh | 8 +- scripts/routing-rate-test.sh | 50 +++---- 31 files changed, 399 insertions(+), 396 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d426464..497ba14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,22 +88,22 @@ jobs: kubectl kustomize "$dir" > /dev/null done - # The instance count is a knob (someguy_instances). It is only a real one + # The instance count is a knob (needle_instances). It is only a real one # if the Deployments, their snapshots and Envoy's endpoint list move # together, which is the whole point of rendering them from one table. - - name: someguy instance count drives all the manifests + - name: needle instance count drives all the manifests run: | for count in 1 2 4; do ansible-playbook scripts/render-manifests.yml \ - -e dest="rendered-$count" -e "someguy_instances=$count" > /dev/null - someguy_build=$(kubectl kustomize "rendered-$count/someguy") - deployments=$(printf '%s\n' "$someguy_build" | grep -c '^kind: Deployment') - pvcs=$(printf '%s\n' "$someguy_build" | grep -c '^kind: PersistentVolumeClaim') + -e dest="rendered-$count" -e "needle_instances=$count" > /dev/null + needle_build=$(kubectl kustomize "rendered-$count/needle") + deployments=$(printf '%s\n' "$needle_build" | grep -c '^kind: Deployment') + pvcs=$(printf '%s\n' "$needle_build" | grep -c '^kind: PersistentVolumeClaim') endpoints=$(kubectl kustomize "rendered-$count/route-origin" \ | grep -c 'port_value: 819') - echo "someguy_instances=$count -> $deployments Deployments, $pvcs PVCs, $endpoints endpoints" + echo "needle_instances=$count -> $deployments Deployments, $pvcs PVCs, $endpoints endpoints" if [ "$deployments" -ne "$count" ] || [ "$pvcs" -ne "$count" ] || [ "$endpoints" -ne "$count" ]; then - echo "::error::someguy_instances=$count produced $deployments Deployments, $pvcs PVCs and $endpoints Envoy endpoints" + echo "::error::needle_instances=$count produced $deployments Deployments, $pvcs PVCs and $endpoints Envoy endpoints" exit 1 fi done diff --git a/AGENTS.md b/AGENTS.md index 6b4b23f..7f2f959 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,8 +37,8 @@ it and there is no flag to open it. Two ways in: ```bash # 1. straight through ssh, no setup -ansible sing-1 -u ipni -m shell -a 'sudo k3s kubectl get pods -n someguy -o wide' -ssh ipni@5.199.165.77 'sudo k3s kubectl -n someguy logs deploy/someguy --tail=50' +ansible sing-1 -u ipni -m shell -a 'sudo k3s kubectl get pods -n needle -o wide' +ssh ipni@5.199.165.77 'sudo k3s kubectl -n needle logs deploy/needle --tail=50' # 2. an ssh tunnel plus the fetched kubeconfig, for real kubectl ./scripts/kubectl-tunnel.sh sing-1 # prints the KUBECONFIG line to use @@ -47,30 +47,30 @@ pkill -f "^ssh -f -N .*-L 6443:127.0.0.1:6443" # stop it `kubeconfigs/` holds cluster-admin credentials and is gitignored. -## someguy +## needle Four instances per box, `hostNetwork`, all on loopback. Defined in -`roles/someguy/defaults/main.yml` as `someguy_running_instances`: +`roles/needle/defaults/main.yml` as `needle_running_instances`: | Instance | HTTP API | libp2p | |-----------|------------------|--------| -| someguy | 127.0.0.1:8190 | 4004 | -| someguy-b | 127.0.0.1:8191 | 4005 | -| someguy-c | 127.0.0.1:8192 | 4006 | -| someguy-d | 127.0.0.1:8193 | 4007 | +| needle | 127.0.0.1:8190 | 4004 | +| needle-b | 127.0.0.1:8191 | 4005 | +| needle-c | 127.0.0.1:8192 | 4006 | +| needle-d | 127.0.0.1:8193 | 4007 | Loopback-only, so metrics have to be curled **on the box**: ```bash ansible sing-1 -u ipni -m shell -a \ - 'curl -s http://127.0.0.1:8190/debug/metrics/prometheus | grep ^someguy_dht' + 'curl -s http://127.0.0.1:8190/debug/metrics/prometheus | grep ^needle_dht' ``` -Each instance has an `-data` PVC mounted at `/data/someguy`, holding +Each instance has an `-data` PVC mounted at `/data/needle`, holding the autoconf cache and the `*.ndjson` snapshots. It is backed by k3s `local-path`, pointed at the NVMe by `default-local-storage-path` (`roles/k3s/tasks/main.yml`), so on the box the data is under -`/data/local-path-provisioner/_someguy_-data`. Find it by claim +`/data/local-path-provisioner/_needle_-data`. Find it by claim rather than guessing: ```bash @@ -92,10 +92,10 @@ ansible-playbook routing.yml -l sing-1 # one box ansible-playbook routing.yml -e production_rollout=true # one box at a time, warmed ``` -someguy stays degraded for tens of minutes after a restart, long after the pod +needle stays degraded for tens of minutes after a restart, long after the pod reports Ready — measured at 200 req/s: 11 minutes in, p95 2904 ms with 2937 rejected lookups; by 69 minutes, p95 588–691 ms. That is what -`production_rollout` and `roles/someguy/tasks/wait_warm.yml` exist for, and why +`production_rollout` and `roles/needle/tasks/wait_warm.yml` exist for, and why `-l ` is the habit rather than the exception. ## Locally built images @@ -109,12 +109,12 @@ into `ImagePullBackOff` — an outage for that box, on all four instances. # build once, on the controller, and ship the result - never build per box, # or the boxes end up running different bytes under the same tag docker buildx build --platform linux/amd64 --load --provenance=false --sbom=false \ - -t someguy: . -docker save someguy: -o someguy-snap.tar + -t needle: . +docker save needle: -o needle-snap.tar -scp someguy-snap.tar ipni@:/tmp/ -ssh ipni@ 'sudo k3s ctr images import /tmp/someguy-snap.tar' -ssh ipni@ 'sudo k3s ctr images ls | grep someguy' # digest must match everywhere +scp needle-snap.tar ipni@:/tmp/ +ssh ipni@ 'sudo k3s ctr images import /tmp/needle-snap.tar' +ssh ipni@ 'sudo k3s ctr images ls | grep needle' # digest must match everywhere ``` `--provenance=false --sbom=false` matters: buildx otherwise attaches an @@ -170,7 +170,7 @@ for d in rendered/*/; do [ -f "$d/kustomization.yaml" ] && kubectl kustomize "$d `rendered/` is gitignored and is the way to see what a change actually does to the manifests before it reaches a box. Diffing `kubectl kustomize -rendered/someguy` against the same render on `main` is how you check a manifest +rendered/needle` against the same render on `main` is how you check a manifest change does only what you meant. **Two traps when linting locally**, both of which make a clean branch look diff --git a/README.md b/README.md index b290900..99cc3da 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # IPFS/IPNI worker nodes Ansible base preparation for three geographically distributed hosts that will -run IPFS-related services (someguy, a bootstrapper node, and the "content +run IPFS-related services (needle, a bootstrapper node, and the "content cluster" node serving website content and documentation). `site.yml` prepares the boxes; `routing.yml` deploys the routing service -(someguy, the primary workload, behind an Envoy origin for Cloudflare); +(needle, the primary workload, behind an Envoy origin for Cloudflare); `bootstrap.yml` deploys the public bootstrap nodes; `content.yml` deploys the content cluster node (kubo + ipfs-cluster, one cluster peer per box) that hosts the website content. @@ -57,8 +57,8 @@ reason). Once hardening has run, every later connection is key-based and iterating. Set it to `true` - or pass `-e production_rollout=true` for one deploy - once these boxes serve real traffic: `routing.yml`, `bootstrap.yml`, `content.yml` and `k3s-upgrade.yml` then roll one box at a time, and -`routing.yml` additionally waits for each box to rebuild the state a someguy -restart throws away, about an hour per box (see "Capacity: four someguy +`routing.yml` additionally waits for each box to rebuild the state a needle +restart throws away, about an hour per box (see "Capacity: four needle instances per box"). Every box is a whole site with no failover between them, so a simultaneous rollout in production is a full outage. `site.yml` is the exception: it is not serialised, so re-run it against a live fleet one box at @@ -167,25 +167,26 @@ unless the two agree, so the target and the route to it stay in step. Then run - Each step waits for the node version, Ready, and every deployment. - Rolling back to an older minor needs the backup taken on that minor. -## someguy +## needle -[someguy](https://github.com/ipfs/someguy) is a Delegated Routing V1 HTTP +[needle](https://github.com/ipni/needle) is IPNI's fork of +[ipfs/someguy](https://github.com/ipfs/someguy), a Delegated Routing V1 HTTP server (`/routing/v1/providers`, `/peers`, `/ipns`, `/dht/closest/peers`). It answers lookups from the Amino DHT (accelerated client) and cid.contact. ```bash -ansible-playbook routing.yml # someguy + origin, all boxes +ansible-playbook routing.yml # needle + origin, all boxes ansible-playbook routing.yml -l chic-1 # one box -ansible-playbook routing.yml --tags someguy # someguy only +ansible-playbook routing.yml --tags needle # needle only ansible-playbook routing.yml --check --diff # dry run, validated by the API server ``` -Manifests live in `k8s/someguy/` (kustomize, same conventions as +Manifests live in `k8s/needle/` (kustomize, same conventions as `storetheindex/deploy`). The role opens the libp2p port, ships the manifests, applies them, waits for the rollout and checks `/version` on the box. -**Version:** `ghcr.io/ipni/someguy:v0.16.0-ipni.1`, pinned by index digest in -`k8s/someguy/kustomization.yaml`. To upgrade, change the digest. +**Version:** `ghcr.io/ipni/needle:v1.0.0`, pinned by index digest in +`k8s/needle/kustomization.yaml`. To upgrade, change the digest. That is IPNI's fork of someguy, not upstream `ghcr.io/ipfs/someguy`. It is upstream v0.16.0 plus eight changes measured on this fleet - pprof, per-router @@ -193,12 +194,12 @@ timing metrics, the cached address book snapshot, the negative TTL with singleflight, the DHT tail budget, the DHT crawl snapshot, the accelerated-ready gauge, and the FindPeer grace and dial timeout - and is built against `github.com/ipni/go-libp2p-kad-dht v0.42.2-ipni.2`. Each change is off -unless the matching `SOMEGUY_*` variable in the kustomization turns it on. See +unless the matching `NEEDLE_*` variable in the kustomization turns it on. See that repo's `FORK.md` for what the kad-dht fork carries and why. **Design points, and why** -- **`hostNetwork: true`.** someguy has no option to announce an address, and +- **`hostNetwork: true`.** needle has no option to announce an address, and libp2p advertises the addresses it sees on its interfaces. On the pod network those are unreachable `10.42.x` addresses. With the host network each box advertises its public IP on TCP, QUIC, WebTransport and WebRTC. @@ -207,8 +208,8 @@ that repo's `FORK.md` for what the kad-dht fork carries and why. loopback, where their single client (the Envoy origin, also `hostNetwork`) lives. They are unreachable from any interface even without the firewall. There is deliberately no Service for them. -- **Four instances per box, so a restart is no longer an outage.** `someguy`, - `someguy-b`, `someguy-c` and `someguy-d` own 8190-8193 (API, loopback) and +- **Four instances per box, so a restart is no longer an outage.** `needle`, + `needle-b`, `needle-c` and `needle-d` own 8190-8193 (API, loopback) and 4004-4007 (libp2p). Envoy round-robins across them, health-checks each on `/version`, and ejects one that stops answering (verified: with an instance scaled to zero, 40/40 requests still returned 200). Each Deployment is still @@ -216,12 +217,12 @@ that repo's `FORK.md` for what the kad-dht fork carries and why. only one instance is down at a time, so a rollout costs a quarter of the box's capacity rather than all of it. That matters because Cloudflare does **not** cover a gap: responses are not cached (`cf-cache-status: DYNAMIC`), - so someguy's `stale-if-error` header has no effect. There is no preStop + so needle's `stale-if-error` header has no effect. There is no preStop pause: with no Service in front, it would only lengthen rollouts. - **Upstreams.** The DHT plus the autoconf default endpoints, which means cid.contact for providers. **These boxes must never be configured as an upstream of cid.contact**, or provider lookups would loop. -- **No persistent state.** someguy keeps no identity or datastore on disk +- **No persistent state.** needle keeps no identity or datastore on disk (`--datadir` holds only the autoconf cache, and the address book is purely in-memory with a 48h TTL). Each restart gets a new PeerID - which is why four instances get four separate DHT identities - and re-crawls, about 1-2 minutes @@ -231,7 +232,7 @@ that repo's `FORK.md` for what the kad-dht fork carries and why. - **Memory.** The libp2p resource manager defaults to 85% of *host* RAM, ignoring the pod limit. It is capped explicitly at 16 GiB per instance, below the 21 GiB container limit, with `GOMEMLIMIT=20GiB`. -- **Health checks.** someguy has no health endpoint, so the probes use `/version`. +- **Health checks.** needle has no health endpoint, so the probes use `/version`. - **Runs as uid 10001,** a uid with no account on the host, rather than the image's default 1000. @@ -242,7 +243,7 @@ bootstrapper, the content cluster node and the OS. Memory is a reservation, not measured need - four instances serving 894 req/s held 1.2-1.5 GiB each. The CPU floor is measured: the same run used 4.4-4.9 cores each, 18.2 of 32. -**Rolling someguy.** Both the one-at-a-time order and the warm-up gate hang off +**Rolling needle.** Both the one-at-a-time order and the warm-up gate hang off `production_rollout`, which is **`false` today** in `group_vars/ipfs_nodes/main.yml`. So the default run - the one you get with no `-e` at all - rolls all four instances on all three boxes simultaneously. That @@ -252,11 +253,11 @@ in `group_vars` when they go live. | | boxes | instances per box | warm-up gate | cost | |---|---|---|---|---| | `production_rollout=false` (today) | all at once | all four at once | no | ~5 min, fleet-wide outage | -| `-e production_rollout=true -e someguy_rollout_wait_warm=false` | one at a time | one at a time | no | ~6 min/box, no outage | +| `-e production_rollout=true -e needle_rollout_wait_warm=false` | one at a time | one at a time | no | ~6 min/box, no outage | | `production_rollout=true` | one at a time | one at a time | yes | hours/box, no outage | With the gate on, each instance waits for its address book to reach -`someguy_warm_min_peers` before the next instance - or the next box - is +`needle_warm_min_peers` before the next instance - or the next box - is touched, so the box never drops below three quarters of *warm* capacity. The middle row is the break-glass path for an urgent fix. It still avoids an outage - Envoy ejects whichever instance is down and serves from the other @@ -266,25 +267,25 @@ minutes, so the box answers everything and answers it slowly for about an hour. ```bash # break glass: one instance at a time, no warm-up wait (~6 min/box) -ansible-playbook routing.yml -e production_rollout=true -e someguy_rollout_wait_warm=false +ansible-playbook routing.yml -e production_rollout=true -e needle_rollout_wait_warm=false ``` -Note that `routing.yml` runs the `someguy` role before `route_origin`, so the +Note that `routing.yml` runs the `needle` role before `route_origin`, so the **first** run that introduces a new instance is still a full outage for the box: Envoy only learns the new endpoints when the origin role applies, one role later. The no-outage property holds from the second run onward. -`someguy_instances` (default 4) lowers the count: the extra Deployments are +`needle_instances` (default 4) lowers the count: the extra Deployments are deleted and their firewall ports closed. It is not a full revert on its own - -see `roles/someguy/defaults/main.yml` for what else has to change. +see `roles/needle/defaults/main.yml` for what else has to change. ## Public endpoint: route-.ipni.io -The public names deliberately avoid "someguy", since the implementation may +The public names deliberately avoid "needle", since the implementation may change. ``` -client -> Cloudflare (proxied, Full strict) -> :443 Envoy on the box -> 127.0.0.1:8190 someguy +client -> Cloudflare (proxied, Full strict) -> :443 Envoy on the box -> 127.0.0.1:8190 needle ``` | Box | Hostname | @@ -297,12 +298,12 @@ Envoy (`k8s/route-origin`, `roles/route_origin`, v1.39.1 pinned by digest) terminates TLS with a **Cloudflare Origin CA certificate**. - **Paths are allowlisted, not blocklisted.** Only `/routing/v1/*` and - `/version` reach someguy. Everything else returns 404, including + `/version` reach needle. Everything else returns 404, including `/debug/metrics/prometheus`, which shares a port with the API. - **Paths are canonicalized before matching** (`normalize_path`, `merge_slashes`, escaped slashes rejected with 400). Without this, `/routing/v1/../debug/...` matched the allowlisted prefix and was forwarded - unchanged. `/debug` stayed hidden only because someguy happened to redirect + unchanged. `/debug` stayed hidden only because needle happened to redirect unclean paths. - **Host allowlist.** Only the three `route-.ipni.io` names are served; any other Host gets **421**. If the live Cloudflare router sends a different Host, @@ -315,7 +316,7 @@ terminates TLS with a **Cloudflare Origin CA certificate**. Pulls, is prepared but deferred** (see "Deferred: Authenticated Origin Pulls" below) until it is clear which Cloudflare zones will send traffic here. Until then, another Cloudflare account that rewrites Host can still reach - someguy through these origins. The data is public, but that bypasses any + needle through these origins. The data is public, but that bypasses any `ipni.io` zone controls such as rate limiting. - **Envoy runs with `hostNetwork` so the ufw rule applies at all.** Traffic to a LoadBalancer or hostPort is DNATed into the FORWARD chain, where kube-proxy @@ -325,7 +326,7 @@ terminates TLS with a **Cloudflare Origin CA certificate**. removed automatically, and `scripts/check-cloudflare-ranges.sh` compares the pinned list with Cloudflare's API (exit 1 on drift). - **IPNS publishing is open, deliberately.** `PUT /routing/v1/ipns/{name}` - reaches someguy, matching production `delegated-ipfs.dev`. Records are + reaches needle, matching production `delegated-ipfs.dev`. Records are self-certifying, but each PUT fans out to the DHT, so it has its own small rate limit. - **Rate limits.** A Cloudflare rate limiting rule limits per client. Envoy @@ -338,7 +339,7 @@ terminates TLS with a **Cloudflare Origin CA certificate**. traffic arrives. Circuit breakers are 20,000, matching the downstream connection cap and well under loopback's ~64k source ports. - **Streaming and timeouts.** NDJSON responses stream unbuffered (first byte in - about 0.1s). The route timeout is 60s: above someguy's 25s lookup cap, below + about 0.1s). The route timeout is 60s: above needle's 25s lookup cap, below Cloudflare's 100s proxy timeout. - **Binding 443 as non-root.** Kubernetes does not give added capabilities to non-root processes. The role instead sets the host's @@ -374,7 +375,7 @@ terminates TLS with a **Cloudflare Origin CA certificate**. **After deploy** it checks that Envoy serves exactly that certificate (by SHA-256 fingerprint), then runs `roles/route_origin/tasks/verify.yml`: -allowlisted and hidden paths, IPNS PUT reaching someguy, four path-traversal +allowlisted and hidden paths, IPNS PUT reaching needle, four path-traversal cases, and a wrong Host (421). With AOP enforced it also checks that a connection without the client certificate is refused. Any unexpected result fails the deploy. @@ -480,12 +481,12 @@ request at a time. - **Latency includes your path to Cloudflare**, the same for every endpoint. Box-to-box differences partly reflect where you ran it from. -**A restart no longer costs an hour.** It used to: someguy kept no state, so a +**A restart no longer costs an hour.** It used to: needle kept no state, so a restart began with an empty DHT routing table and an empty address book, both rebuilt only by serving traffic, and until they were our counts and latencies were understated. That is what this table recorded, on sing-1 at 200 req/s: -| Age | p50 | Address-book hit rate | someguy CPU per 30s | +| Age | p50 | Address-book hit rate | needle CPU per 30s | |-----|-----|-----------------------|---------------------| | 15 min | 5002ms | 51% | 300s | | 30 min | 2940ms | 58% | 187s | @@ -500,17 +501,17 @@ instance's own PVC - the address book every 15 minutes, the crawled routing table after every completed crawl - so a restarted instance restores both and serves properly in seconds. Re-measured on sing-1 at 200 req/s, same conditions: -| Age | p50 | p95 | `someguy_cached_addr_book_peer_state_size` | +| Age | p50 | p95 | `needle_cached_addr_book_peer_state_size` | |-----|-----|-----|--------------------------------------------| | 2 min | _measured 2026-09-18_ | | | | 15 min | _measured 2026-09-18_ | | | -Check the pod's age first (`kubectl -n someguy get pods`) and use `--out` to +Check the pod's age first (`kubectl -n needle get pods`) and use `--out` to track how results change, but the hour-long embargo above no longer applies to -an instance that restored its snapshots - `someguy_dht_accelerated_ready` at `1` +an instance that restored its snapshots - `needle_dht_accelerated_ready` at `1` with both `*_snapshot_restored_peers` gauges non-zero is the thing to confirm. -### Capacity: four someguy instances per box +### Capacity: four needle instances per box `routing-compare.sh` shows the boxes answer correctly; it says nothing about load. Two scripts measure that: @@ -524,7 +525,7 @@ load. Two scripts measure that: `routing-load.sh` holds N requests in flight, so its throughput is `concurrency / latency` and a slow link caps the answer. `routing-rate-test.sh` drives a fixed **arrival rate** instead, so a box that cannot keep up shows -growing latency rather than quietly lower throughput, and it samples someguy's +growing latency rather than quietly lower throughput, and it samples needle's own counters (CPU, address-book hit rate, rejected lookups, open FDs) around every stage. **Run it on the box** (`ansible -m script -a ...`): from a workstation, the round trip and the client's own thread pool become the limit @@ -534,8 +535,8 @@ takes seconds, so the client silently becomes the ceiling. The 2026-09-16 numbers below marked "client-bound" were measured that way and are too low; re-measured with `--workers 12288` the same boxes went 50% higher. -**Each box runs FOUR someguy instances** (`someguy`, `someguy-b`, `someguy-c`, -`someguy-d`) on 8190-8193 (API, loopback) and 4004-4007 (libp2p), with Envoy +**Each box runs FOUR needle instances** (`needle`, `needle-b`, `needle-c`, +`needle-d`) on 8190-8193 (API, loopback) and 4004-4007 (libp2p), with Envoy round-robining across them. Each is a separate process with its own PeerID, concurrency budget and connection pool. Measured on sing-1, 2026-09-16, warm, fixture mix through Cloudflare, zero HTTP errors at every stage: @@ -637,7 +638,7 @@ penalty: it costs tail latency at high rates, not capacity. **Watch chic-1's tail.** Its p95 degrades at the top of the ladder (1427ms at 850/s, 2739ms at 1000/s, p99 at the 5s wall) while lith-1 holds p95 at 283ms, -and its someguy CPU was the highest of the three (~190s per 30s stage). Nothing +and its needle CPU was the highest of the three (~190s per 30s stage). Nothing failed, but it is the first box that would. For scale, 1.1B requests/month is ~424 req/s across the fleet, or ~141 req/s @@ -651,7 +652,7 @@ instances not the FindPeer cap either. It is **DHT round-trip distance**. A look records arrive without addresses needs a FindPeer walk, several sequential hops each costing a round trip, and every operation inside the accelerated client is capped at 5s (`timeoutPerOp` in go-libp2p-kad-dht `fullrt/dht.go`, not exposed -by someguy - it is why p95 pins at almost exactly 5000ms under load). Walks +by needle - it is why p95 pins at almost exactly 5000ms under load). Walks that hit the cap return records without addresses, so fewer addresses are cached, so more walks are needed: the further a box sits from the DHT's centre of mass, the worse the loop. sing-1 pays it hardest - cold lookups take 594ms @@ -659,10 +660,10 @@ against chic-1's 303ms, and its address-book hit rate settles at 57-69% against chic-1's 76-82%, on identical hardware and identical configuration. **The 5s cap costs results, on every box.** Running sing-1 on -`SOMEGUY_DHT=standard` for one comparison (2026-09-16) showed what the +`NEEDLE_DHT=standard` for one comparison (2026-09-16) showed what the accelerated client drops: on sparse DHT content a full iterative walk found 45, 18, 16 and 17 providers where the accelerated boxes returned 38, 14, 14 and 12. -Well-provided content is unaffected - every box hits `SOMEGUY_RECORDS_LIMIT` +Well-provided content is unaffected - every box hits `NEEDLE_RECORDS_LIMIT` either way - so the loss is invisible except on exactly the content that has few providers to begin with. This is the strongest argument for exposing `fullrt`'s `timeoutPerOp`: a value between 5s and 25s would likely recover most @@ -713,34 +714,34 @@ below still stand on their own merits: **What has already been tried:** - **Kept: four instances per box** (2026-09-16). See the table above. Rejected - along the way: raising `SOMEGUY_CACHED_ADDR_BOOK_MAX_CONCURRENT_FIND_PEERS` + along the way: raising `NEEDLE_CACHED_ADDR_BOOK_MAX_CONCURRENT_FIND_PEERS` again. It bounds *background* address-book fills, not the foreground lookup path; the earlier 512 -> 2048 raise cut CPU but never moved the ceiling, and there was no reason to expect a third raise to differ. -- **Kept:** `SOMEGUY_RECORDS_LIMIT` 100 -> 50 (all boxes). Measured on sing-1 +- **Kept:** `NEEDLE_RECORDS_LIMIT` 100 -> 50 (all boxes). Measured on sing-1 at 25: p50 halved at 200 req/s (2208ms -> 1180ms) and response bytes fell - ~70%; warm-up was quicker at every checkpoint. But someguy's CPU (100s per + ~70%; warm-up was quicker at every checkpoint. But needle's CPU (100s per 30s stage), its p95 (5018ms) and its ceiling (~355 req/s) did not move, so the gain is in building and shipping the response, **not** in fewer provider lookups as expected - the tail is DHT walks, and they happen whatever we return. 50 trades half that measured gain for half the loss in completeness; the spec recommends 100. Bandwidth was never the constraint here, so the ~70% saving is incidental. -- **Kept:** `SOMEGUY_CACHED_ADDR_BOOK_MAX_CONCURRENT_FIND_PEERS` 512 -> 2048. +- **Kept:** `NEEDLE_CACHED_ADDR_BOOK_MAX_CONCURRENT_FIND_PEERS` 512 -> 2048. At 200 req/s sing-1 was rejecting 7,809 background FindPeer lookups per 30s while using 5.5 of 32 cores - the cap, not the box, was the limit. Raising it removed the rejections and cut CPU about 2.3x. Inert on chic-1, which never reached the old cap. -- **Rejected:** `SOMEGUY_DHT=standard` on sing-1. It answered *more* +- **Rejected:** `NEEDLE_DHT=standard` on sing-1. It answered *more* completely - more providers on 7 of 28 fixtures - but many requests then ran - someguy's full 25s `routing-timeout` where the accelerated boxes answer in + needle's full 25s `routing-timeout` where the accelerated boxes answer in 0.1-0.5s. At that latency a public endpoint holds connections and goroutines open until they time out, so throughput would fall below the ~355 req/s - sing-1 already manages. `SOMEGUY_DHT=disabled` (cid.contact only) is ruled + sing-1 already manages. `NEEDLE_DHT=disabled` (cid.contact only) is ruled out by requirement: these endpoints must serve DHT-only content. - **Reverted:** connection manager 1000/8000 with a 96h address TTL. It doubled - someguy's CPU (2.7 -> 5.5 cores at 200 req/s) and made p50 about 1.5x worse + needle's CPU (2.7 -> 5.5 cores at 200 req/s) and made p50 about 1.5x worse for a marginal hit-rate change. More retained connections cost more to maintain than the dials they saved. @@ -810,7 +811,7 @@ clients; if it changed, every bootstrap address would break. (`API.Authorizations`, per-box `vault_bootstrap_api_token`). With `hostNetwork`, loopback is shared with every host process and the other host-network pods, so loopback alone did not protect the admin RPC. Verified: - without the token, `/api/v0` returns 403, including from inside someguy's + without the token, `/api/v0` returns 403, including from inside needle's container. kubo does not guard `/debug/metrics` and `/debug/pprof` with it; - connection manager 4000/8000 (a bootstrapper's job is to accept peers), resource manager 8 GiB; @@ -943,7 +944,7 @@ browser --wss--> :4443 Envoy (TLS, Let's Encrypt) --ws--> 127.0.0.1:4002 kubo - **Kubo (uid 10002) may not open new loopback connections** except to its RPC API. Removing the loopback filter also let kubo *dial* loopback, and a WSS client (a loopback peer to kubo) could hand it 127.0.0.1 addresses to probe - the k3s API, kubelet, someguy and Envoy admin ports. Replies to the proxy's + the k3s API, kubelet, needle and Envoy admin ports. Replies to the proxy's connections are unaffected. **Testing like a browser.** `scripts/wss-check` dials with js-libp2p, WebSockets @@ -1091,7 +1092,7 @@ k3s kubectl -n content exec deploy/content -c cluster -- \ starts from a root (for example DNSLink to the site root, then path resolution) finds these nodes. A client asking the DHT for any other CID (a direct `ipfs://` link to a file or subdirectory) finds no provider unless it - is already connected to one of them. The reason was contention with someguy, + is already connected to one of them. The reason was contention with needle, which runs about 416 DHT lookups/sec on the same boxes. The two do not share a libp2p host or resource manager, only the host's CPU and sockets, and kubo 0.43's sweeping provider batches reprovides by keyspace region, so the cost of @@ -1115,9 +1116,9 @@ k3s kubectl -n content exec deploy/content -c cluster -- \ Not at `/data`: both images declare `VOLUME /data/...`, and containerd mounts an anonymous volume on the OS disk over those paths unless a pod volume is mounted exactly there. `hostNetwork`, `Recreate`, uid/gid 10003 (distinct - from someguy's 10001 and the bootstrapper's 10002), read-only root + from needle's 10001 and the bootstrapper's 10002), read-only root filesystems. -- From the ~30% of each box not reserved for someguy, shared with the +- From the ~30% of each box not reserved for needle, shared with the bootstrapper: kubo requests 500m / 2 GiB (limit 4 CPU / 8 GiB), cluster 250m / 512 MiB (limit 2 CPU / 4 GiB). **Initial guesses**, to tune once content is pinned. @@ -1126,7 +1127,7 @@ k3s kubectl -n content exec deploy/content -c cluster -- \ `ipfs-cluster-ctl id`. - **The APIs are unauthenticated on loopback, by decision.** Loopback is shared with the host and the other host-network pods, all of them this project's own - workloads (someguy, both Envoys and the bootstrapper, several internet-facing). + workloads (needle, both Envoys and the bootstrapper, several internet-facing). A request from any of them to 9094 or 9095 changes the shared pinset, so **every box** would fetch, store and announce that content from this project's IPs. Changes to kubo's config through 5021 last only until the @@ -1195,7 +1196,7 @@ means three copies on three boxes; the pins do not override it with a fixed **Only roots are reprovided.** `Provide.Strategy` is `roots` (the kubo 0.43 name for `Reprovider.Strategy`): each pin's root CID is announced to the DHT, -not every block. someguy runs about 416 DHT lookups/sec on the same boxes, and +not every block. needle runs about 416 DHT lookups/sec on the same boxes, and reproviding every block of every site would compete with it. Clients reach a site through its root (DNSLink, then path resolution) and fetch the rest over Bitswap. See **kubo** above. @@ -1307,7 +1308,7 @@ same path a deploy takes), so what is checked is what would be applied: | `ansible-playbook --syntax-check` on all five playbooks | a role edit that breaks a playbook nobody ran | | `ansible-lint`, `yamllint` | the rest, minus the style rules in `.ansible-lint` | | `kubectl kustomize` on every rendered `k8s/*/` | a patch or digest pin that no longer applies | -| rendering at `someguy_instances` 1, 2 and 4 | the Deployments and Envoy's endpoint list falling out of step | +| rendering at `needle_instances` 1, 2 and 4 | the Deployments and Envoy's endpoint list falling out of step | | `sha256sum -c` on `cert-manager.yaml` | the vendored release manifest changing under its pin | | `bootstrap-dns.py \| diff - dns.txt` | `dns.txt` drifting from its generator | | every `pinset.yml` section is in the role's `known_sections` | a section the content role silently ignores | @@ -1351,7 +1352,7 @@ turn an unrelated pull request red. - [ ] **Enable Authenticated Origin Pulls** once the zones are known (see "Deferred: Authenticated Origin Pulls"). - [ ] Add the Cloudflare rate limiting rule (see Cloudflare configuration). -- [ ] Tune someguy resources and Envoy's rate limits against real traffic metrics. +- [ ] Tune needle resources and Envoy's rate limits against real traffic metrics. - [ ] **Three sites have no CID anywhere:** `docs.libp2p.io`, `ipld.io` and `dnslink.io` are under `recover_from_upstream`, and the 2026-09-16 search found them nowhere in the upstream pinset listings committed beside @@ -1405,12 +1406,12 @@ inventory/hosts.yml the three hosts group_vars/ipfs_nodes/ tunables (admin user, firewall, k3s, sysctl) host_vars//vault.yml encrypted per-box credentials (gitignored, see Secrets) roles/{common,storage,hardening,k3s}/ base preparation -roles/someguy/ someguy firewall + deploy +roles/needle/ needle firewall + deploy roles/route_origin/ Envoy origin: cert preflight, Cloudflare allowlist, TLS secret tasks/verify.yml post-deploy request checks (paths, Host, traversal; AOP when enforced) roles/kustomize_apply/ shared: ship, dry-run/apply, wait, prune stale ConfigMaps -k8s/someguy/ someguy kustomize manifests; the Deployments are .j2, rendered from someguy_all_instances -k8s/route-origin/ Envoy kustomize manifests and envoy.yaml.j2 (someguy endpoints rendered) +k8s/needle/ needle kustomize manifests; the Deployments are .j2, rendered from needle_all_instances +k8s/route-origin/ Envoy kustomize manifests and envoy.yaml.j2 (needle endpoints rendered) k8s/bootstrap/ kubo bootstrapper manifests (deployment, repo PVC) k8s/bootstrap-wss/ Envoy TLS proxy for the bootstrappers' WSS listener k8s/content/ content cluster node manifests (kubo + ipfs-cluster, repo PVC) @@ -1445,7 +1446,7 @@ scripts/recover-upstream-pinset.sh capture the upstream collab cluster's pinset dns.txt Cloudflare-importable bootstrap DNS records (generated) site.yml base preparation playbook k3s-upgrade.yml k3s upgrade, one minor version at a time, backup per step -routing.yml routing service playbook (someguy + origin) +routing.yml routing service playbook (needle + origin) bootstrap.yml bootstrap node playbook (with cert-manager) content.yml content cluster node playbook (deploy, load the pinset, wait for PINNED, summary) .github/workflows/ci.yml drift checks: playbook syntax, lint, kustomize build, generated files, shellcheck diff --git a/group_vars/ipfs_nodes/main.yml b/group_vars/ipfs_nodes/main.yml index be629d3..381cc52 100644 --- a/group_vars/ipfs_nodes/main.yml +++ b/group_vars/ipfs_nodes/main.yml @@ -43,7 +43,7 @@ firewall_allowed_udp: [] # Rollout mode # --------------------------------------------------------------------------- # false (dev): playbooks roll every box at once, which is fastest to iterate on. -# true (production): one box at a time, and someguy's rollout waits for each box +# true (production): one box at a time, and needle's rollout waits for each box # to rebuild the state a restart throws away before the next one is touched. # # This governs routing.yml, bootstrap.yml, content.yml and k3s-upgrade.yml, via @@ -54,13 +54,13 @@ firewall_allowed_udp: [] # # Every box is a whole site behind its own hostname with no failover between # them, so once these are the only boxes serving traffic, a simultaneous -# rollout is a full outage - and someguy stays degraded for tens of minutes +# rollout is a full outage - and needle stays degraded for tens of minutes # after a restart, long after its pod reports Ready. Measured at 200 req/s: # 11 minutes in, p95 2904ms with 2937 rejected lookups and 3x the CPU; by 69 # minutes, p95 588-691ms, no rejections, warm CPU. # # Flip this to true when the boxes go live. A production rollout then takes -# roughly an hour per box, which is the real cost of restarting someguy. +# roughly an hour per box, which is the real cost of restarting needle. production_rollout: false # --------------------------------------------------------------------------- @@ -109,17 +109,17 @@ sysctl_tuning: net.ipv4.tcp_max_syn_backlog: 8192 net.ipv4.ip_local_port_range: "1024 65535" # Host-network listeners that sit INSIDE the ephemeral range above. Without - # this, any outbound socket (someguy's DHT crawl alone opens ~800) can be + # this, any outbound socket (needle's DHT crawl alone opens ~800) can be # handed one of these numbers as its source port; a pod restarting under # `Recreate` then fails to re-bind with "address already in use". # Reserving only stops the kernel choosing them for outbound connections; # explicit binds still work. Update when a host-network port is added: # 443 Envoy (outside the range today, kept for safety) # 4001 bootstrapper libp2p 4002 bootstrapper /ws (loopback) - # 4004-4007 someguy libp2p (four instances) 4101 content kubo libp2p + # 4004-4007 needle libp2p (four instances) 4101 content kubo libp2p # 4443 bootstrapper WSS (Envoy) 5011 bootstrapper API # 5021 content kubo RPC API (loopback) - # 6443-6444 k3s API 8190-8193 someguy APIs (loopback, four instances) + # 6443-6444 k3s API 8190-8193 needle APIs (loopback, four instances) # 8472/udp flannel VXLAN # 9094 content ipfs-cluster REST API (loopback) # 9095 content ipfs-cluster IPFS proxy (loopback) diff --git a/k8s/bootstrap/deployment.yaml b/k8s/bootstrap/deployment.yaml index 8836adb..891bb05 100644 --- a/k8s/bootstrap/deployment.yaml +++ b/k8s/bootstrap/deployment.yaml @@ -132,7 +132,7 @@ spec: # RPC bearer token for probes and CLI calls (kubo's --api-auth takes # no env var or config fallback). Expanded inside `sh -c` below, so # it only appears in argv of processes in THIS container's PID - # namespace - not visible to someguy's or Envoy's containers. + # namespace - not visible to needle's or Envoy's containers. - name: KUBO_API_AUTH valueFrom: secretKeyRef: @@ -148,9 +148,9 @@ spec: containerPort: 4001 protocol: UDP - # Shares the ~30% of each box not reserved for someguy with the + # Shares the ~30% of each box not reserved for needle with the # future content cluster node. A CPU limit is set here (unlike - # someguy): Go sizes GOMAXPROCS from it, and a bootstrapper is not + # needle): Go sizes GOMAXPROCS from it, and a bootstrapper is not # latency-critical. # # Requests follow observed use (about 20 minutes after deploy: 15-43m diff --git a/k8s/cert-manager/kustomization.yaml b/k8s/cert-manager/kustomization.yaml index 611f18d..0730808 100644 --- a/k8s/cert-manager/kustomization.yaml +++ b/k8s/cert-manager/kustomization.yaml @@ -30,7 +30,7 @@ images: - name: quay.io/jetstack/cert-manager-webhook digest: sha256:a60e2dac46dbb8a7f3df95c54ce941012f54c2fe022f0ee55aaa1ab40ed957ae -# The release sets no resources. Small requests keep these off someguy's +# The release sets no resources. Small requests keep these off needle's # share; memory limits bound the Secret caches of the controller and cainjector. patches: - target: { kind: Deployment, namespace: cert-manager } diff --git a/k8s/content/deployment.yaml b/k8s/content/deployment.yaml index 0baebd3..946a32b 100644 --- a/k8s/content/deployment.yaml +++ b/k8s/content/deployment.yaml @@ -29,7 +29,7 @@ spec: terminationGracePeriodSeconds: 45 securityContext: - # A uid with no host account, distinct from someguy (10001) and the + # A uid with no host account, distinct from needle (10001) and the # bootstrapper (10002), so the repos on the shared data volume cannot # be read or written across workloads. runAsNonRoot: true @@ -132,7 +132,7 @@ spec: containerPort: 4101 protocol: UDP - # From the ~30% of each box not reserved for someguy, shared with the + # From the ~30% of each box not reserved for needle, shared with the # bootstrapper (requests 500m/2Gi). Initial guesses, to be tuned from # observed use once content is pinned (libp2p_rcmgr_* and Bitswap # metrics), as the bootstrapper's were. diff --git a/k8s/needle/deployment-extra.yaml.j2 b/k8s/needle/deployment-extra.yaml.j2 index 477d1a5..9906833 100644 --- a/k8s/needle/deployment-extra.yaml.j2 +++ b/k8s/needle/deployment-extra.yaml.j2 @@ -1,8 +1,8 @@ {# - Rendered by roles/kustomize_apply from someguy_all_instances and - someguy_instances (roles/someguy/defaults/main.yml). + Rendered by roles/kustomize_apply from needle_all_instances and + needle_instances (roles/needle/defaults/main.yml). - Every instance after the first, identical to k8s/someguy/deployment.yaml + Every instance after the first, identical to k8s/needle/deployment.yaml (instance a) except for the name and the two ports it owns. Every comment explaining why the pod looks like this lives in that file; this one is deliberately mechanical, so the instances cannot drift apart. @@ -11,16 +11,16 @@ and in k8s/route-origin/envoy.yaml. Now the defaults hold it and the other two are rendered, so a port is written down in exactly one place. #} -# GENERATED from someguy_all_instances in roles/someguy/defaults/main.yml. +# GENERATED from needle_all_instances in roles/needle/defaults/main.yml. # Edit that table, not this file. Rendered by roles/kustomize_apply. # # Instances running on this box: # # instance API (loopback) libp2p (public) -{% for instance in someguy_running_instances %} +{% for instance in needle_running_instances %} # {{ '%-10s %-16s %s' | format(instance.name, instance.api, instance.libp2p) }} {% endfor %} -{% for instance in someguy_running_instances[1:] %} +{% for instance in needle_running_instances[1:] %} --- apiVersion: apps/v1 kind: Deployment @@ -49,15 +49,15 @@ spec: seccompProfile: type: RuntimeDefault containers: - - name: someguy - image: someguy + - name: needle + image: needle envFrom: - configMapRef: - name: someguy-env-vars + name: needle-env-vars env: - - name: SOMEGUY_LISTEN_ADDRESS + - name: NEEDLE_LISTEN_ADDRESS value: 127.0.0.1:{{ instance.api }} - - name: SOMEGUY_LIBP2P_LISTEN_ADDRS + - name: NEEDLE_LIBP2P_LISTEN_ADDRS value: /ip4/0.0.0.0/tcp/{{ instance.libp2p }},/ip4/0.0.0.0/udp/{{ instance.libp2p }}/quic-v1,/ip4/0.0.0.0/udp/{{ instance.libp2p }}/webrtc-direct,/ip4/0.0.0.0/udp/{{ instance.libp2p }}/quic-v1/webtransport,/ip6/::/tcp/{{ instance.libp2p }},/ip6/::/udp/{{ instance.libp2p }}/quic-v1,/ip6/::/udp/{{ instance.libp2p }}/webrtc-direct,/ip6/::/udp/{{ instance.libp2p }}/quic-v1/webtransport ports: - name: http @@ -100,7 +100,7 @@ spec: drop: ["ALL"] volumeMounts: - name: data - mountPath: /data/someguy + mountPath: /data/needle volumes: - name: data persistentVolumeClaim: diff --git a/k8s/needle/deployment.yaml.j2 b/k8s/needle/deployment.yaml.j2 index c70eade..c595bf7 100644 --- a/k8s/needle/deployment.yaml.j2 +++ b/k8s/needle/deployment.yaml.j2 @@ -1,7 +1,7 @@ {# - Rendered by roles/kustomize_apply from someguy_all_instances - (roles/someguy/defaults/main.yml): instance a, the first entry in that table. - Instances b onward are k8s/someguy/deployment-extra.yaml.j2, which is + Rendered by roles/kustomize_apply from needle_all_instances + (roles/needle/defaults/main.yml): instance a, the first entry in that table. + Instances b onward are k8s/needle/deployment-extra.yaml.j2, which is deliberately mechanical and points back here for the reasoning. Only the name and the two ports come from the table. Everything else is @@ -11,7 +11,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ someguy_first_instance.name }} + name: {{ needle_first_instance.name }} spec: replicas: 1 # hostNetwork binds ports 8190 and 4004 on the node itself, so a second pod @@ -21,32 +21,32 @@ spec: # A restart of THIS instance is no longer an outage for the box: Envoy fans # out across all four, health-checks them, and ejects one that stops # answering (verified: with an instance scaled to zero, 40/40 requests still - # returned 200). roles/someguy rolls them one at a time, so a rollout costs + # returned 200). roles/needle rolls them one at a time, so a rollout costs # a quarter of the box's capacity rather than all of it. That matters - # because responses are not cached (cf-cache-status: DYNAMIC), so someguy's + # because responses are not cached (cf-cache-status: DYNAMIC), so needle's # stale-if-error header has no effect. strategy: type: Recreate selector: matchLabels: - app: {{ someguy_first_instance.name }} + app: {{ needle_first_instance.name }} template: metadata: labels: - app: {{ someguy_first_instance.name }} + app: {{ needle_first_instance.name }} spec: # libp2p has no announce-address option and advertises the addresses it # sees on its interfaces. On the pod network those would be unreachable # 10.42.x addresses; on the host network they include the public IP. hostNetwork: true - # The host's resolvers, deliberately not cluster DNS. someguy only needs + # The host's resolvers, deliberately not cluster DNS. needle only needs # public names (autoconf, cid.contact, /dnsaddr). With hostNetwork, # ClusterFirst would also fall back to the host resolvers; Default states # that intent explicitly (ClusterFirstWithHostNet would opt in to # cluster DNS). dnsPolicy: Default - # someguy shuts its HTTP server down within 5s of SIGTERM, then writes the + # needle shuts its HTTP server down within 5s of SIGTERM, then writes the # address book snapshot (sub-second at 25k peers) and closes the DHT. 30s # covers all three; SIGKILL mid-write leaves the previous snapshot intact # because the write is temp file plus rename. @@ -67,32 +67,32 @@ spec: type: RuntimeDefault containers: - - name: someguy - image: someguy + - name: needle + image: needle envFrom: - configMapRef: - name: someguy-env-vars + name: needle-env-vars # The only per-instance settings: which ports this one owns. Set here # rather than in a per-instance ConfigMap so the generated-ConfigMap # prune in roles/kustomize_apply still sees every ConfigMap in use. - # someguy keeps no identity on disk, so each instance also gets its + # needle keeps no identity on disk, so each instance also gets its # own PeerID at start - the point of running more than one. env: - - name: SOMEGUY_LISTEN_ADDRESS - value: 127.0.0.1:{{ someguy_first_instance.api }} - - name: SOMEGUY_LIBP2P_LISTEN_ADDRS - value: /ip4/0.0.0.0/tcp/{{ someguy_first_instance.libp2p }},/ip4/0.0.0.0/udp/{{ someguy_first_instance.libp2p }}/quic-v1,/ip4/0.0.0.0/udp/{{ someguy_first_instance.libp2p }}/webrtc-direct,/ip4/0.0.0.0/udp/{{ someguy_first_instance.libp2p }}/quic-v1/webtransport,/ip6/::/tcp/{{ someguy_first_instance.libp2p }},/ip6/::/udp/{{ someguy_first_instance.libp2p }}/quic-v1,/ip6/::/udp/{{ someguy_first_instance.libp2p }}/webrtc-direct,/ip6/::/udp/{{ someguy_first_instance.libp2p }}/quic-v1/webtransport + - name: NEEDLE_LISTEN_ADDRESS + value: 127.0.0.1:{{ needle_first_instance.api }} + - name: NEEDLE_LIBP2P_LISTEN_ADDRS + value: /ip4/0.0.0.0/tcp/{{ needle_first_instance.libp2p }},/ip4/0.0.0.0/udp/{{ needle_first_instance.libp2p }}/quic-v1,/ip4/0.0.0.0/udp/{{ needle_first_instance.libp2p }}/webrtc-direct,/ip4/0.0.0.0/udp/{{ needle_first_instance.libp2p }}/quic-v1/webtransport,/ip6/::/tcp/{{ needle_first_instance.libp2p }},/ip6/::/udp/{{ needle_first_instance.libp2p }}/quic-v1,/ip6/::/udp/{{ needle_first_instance.libp2p }}/webrtc-direct,/ip6/::/udp/{{ needle_first_instance.libp2p }}/quic-v1/webtransport ports: - name: http - containerPort: {{ someguy_first_instance.api }} + containerPort: {{ needle_first_instance.api }} protocol: TCP - # 4004 is someguy's built-in libp2p default; roles/someguy opens the + # 4004 is needle's built-in libp2p default; roles/needle opens the # same port in ufw. Change both together. - name: libp2p-tcp - containerPort: {{ someguy_first_instance.libp2p }} + containerPort: {{ needle_first_instance.libp2p }} protocol: TCP - name: libp2p-udp - containerPort: {{ someguy_first_instance.libp2p }} + containerPort: {{ needle_first_instance.libp2p }} protocol: UDP # One of FOUR instances per box (see kustomization.yaml). Together @@ -115,7 +115,7 @@ spec: limits: memory: 21Gi - # someguy has no health endpoint. /version answers as soon as the + # needle has no health endpoint. /version answers as soon as the # HTTP server is up, and the server keeps answering while the # accelerated DHT crawls (lookups fall back to the standard client). # host: 127.0.0.1 - the API only listens on loopback, and with @@ -156,7 +156,7 @@ spec: volumeMounts: - name: data - mountPath: /data/someguy + mountPath: /data/needle volumes: # The autoconf cache and the address book snapshot survive on this @@ -165,4 +165,4 @@ spec: # pvc.yaml.j2, one claim per running instance. - name: data persistentVolumeClaim: - claimName: {{ someguy_first_instance.name }}-data + claimName: {{ needle_first_instance.name }}-data diff --git a/k8s/needle/kustomization.yaml b/k8s/needle/kustomization.yaml index 62b38f9..d2b6f38 100644 --- a/k8s/needle/kustomization.yaml +++ b/k8s/needle/kustomization.yaml @@ -2,43 +2,45 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -namespace: someguy +namespace: needle resources: - namespace.yaml - # Both are RENDERED from someguy_all_instances (roles/someguy/defaults) by + # Both are RENDERED from needle_all_instances (roles/needle/defaults) by # roles/kustomize_apply; the repo holds deployment.yaml.j2 and # deployment-extra.yaml.j2. The names below are what lands on the node. - deployment.yaml - # Instances b onward. Four someguy processes per box, each with its own + # Instances b onward. Four needle processes per box, each with its own # PeerID, concurrency budget and connection pool; Envoy fans out across them # (k8s/route-origin/envoy.yaml.j2). Measured on sing-1 2026-09-16: one # instance topped out at 355 req/s with p50 5004ms at 400 req/s, four reached # 894 req/s with p50 217ms at 400 - see README, "Capacity". At - # someguy_instances: 1 this renders to a comment, which kustomize accepts as + # needle_instances: 1 this renders to a comment, which kustomize accepts as # an empty resource. - deployment-extra.yaml - # RENDERED from someguy_running_instances: one per instance, the volume + # RENDERED from needle_running_instances: one per instance, the volume # behind the data mount (see deployment.yaml.j2). - pvc.yaml labels: - pairs: - app.kubernetes.io/name: someguy + app.kubernetes.io/name: needle app.kubernetes.io/part-of: ipfs-workers # Pinned by digest so every box runs byte-identical images and a re-pushed tag # cannot change what is deployed. This is the index digest of -# ghcr.io/ipni/someguy:v0.16.0-ipni.1 (2026-09-18), a multi-arch index covering +# ghcr.io/ipni/needle:v1.0.0 (2026-09-18), a multi-arch index covering # amd64, arm/v7 and arm64; the amd64 manifest these boxes run is -# sha256:32e6f3f66c8468a80776d22dff8b313059c62cef72a499aae88f778389356138. +# sha256:f31bd19df98216b6418cb6c007fe5ce2559c7ed430d60bf18c0929e74d5671f3. # -# ghcr.io/ipni is IPNI's fork of ipfs/someguy, not a re-tag of upstream. The -# build carries eight things upstream v0.16.0 does not: pprof endpoints, the -# per-router timing metrics, the cached address book snapshot, the negative TTL -# with singleflight, the DHT tail budget, the DHT crawl snapshot, the -# accelerated-ready gauge, and the FindPeer grace and dial timeout. Every one is -# off unless the matching variable below turns it on. +# ghcr.io/ipni is IPNI's fork of ipfs/someguy, not a re-tag of upstream. At +# v1.0.0 the fork was renamed to needle: module github.com/ipni/needle, image +# ghcr.io/ipni/needle, NEEDLE_ env prefix and needle_ metric prefix (no +# someguy_ aliases). The build carries everything v0.16.0-ipni.2 did: pprof +# endpoints, the per-router timing metrics, the cached address book snapshot, +# the negative TTL with singleflight, the DHT tail budget, the DHT crawl +# snapshot, the accelerated-ready gauge, and the FindPeer grace and dial +# timeout. Every one is off unless the matching variable below turns it on. # # It is built against github.com/ipni/go-libp2p-kad-dht v0.42.2-ipni.2, which is # what makes the crawl snapshot work at all: fullrt's default route table filter @@ -46,12 +48,12 @@ labels: # peer, deliberately never dialled, would otherwise be dropped and the table # would come up empty. See that fork's FORK.md. images: - - name: someguy - newName: ghcr.io/ipni/someguy - digest: sha256:723094f7dedc9899b6cd15893a363f36b66a888870005a1c6c23839ef33e3526 + - name: needle + newName: ghcr.io/ipni/needle + digest: sha256:65cfdabeb5eb1a616960ac9aab1e2c4663802c8fabb3ae84fb87151006b6ada6 configMapGenerator: - - name: someguy-env-vars + - name: needle-env-vars behavior: create literals: # Loopback only. Under hostNetwork this is the HOST's loopback, which is @@ -61,49 +63,49 @@ configMapGenerator: # firewall. The public path is Cloudflare -> Envoy :443 -> here. # # The PORT here is a fallback that nothing uses: every Deployment sets - # SOMEGUY_LISTEN_ADDRESS in its own `env:` (container `env` wins over + # NEEDLE_LISTEN_ADDRESS in its own `env:` (container `env` wins over # `envFrom`), because the four instances need four different ports. The # address is kept here so the loopback-only decision stays documented in # one place, and so a new instance that forgot its override would still # not be able to listen on a public interface. - - SOMEGUY_LISTEN_ADDRESS=127.0.0.1:8190 + - NEEDLE_LISTEN_ADDRESS=127.0.0.1:8190 # Accelerated DHT plus the autoconf `auto` endpoints (cid.contact for # providers). These boxes must never become an upstream of cid.contact, # or provider lookups would loop. - - SOMEGUY_DHT=accelerated - - SOMEGUY_PROVIDER_ENDPOINTS=auto - - SOMEGUY_PEER_ENDPOINTS=auto - - SOMEGUY_IPNS_ENDPOINTS=auto + - NEEDLE_DHT=accelerated + - NEEDLE_PROVIDER_ENDPOINTS=auto + - NEEDLE_PEER_ENDPOINTS=auto + - NEEDLE_IPNS_ENDPOINTS=auto # The autoconf cache and the address book snapshot live here; there is # still no identity or datastore. - - SOMEGUY_DATADIR=/data/someguy + - NEEDLE_DATADIR=/data/needle # Providers or peers per `Accept: application/json` response. The HTTP - # Routing v1 spec recommends 100 (someguy's default); 50 still gives a + # Routing v1 spec recommends 100 (needle's default); 50 still gives a # client plenty of providers to try. # # Measured on sing-1 at 200 req/s, warm, 2026-09-16: dropping this to 25 - # halved p50 (2208ms -> 1180ms) and cut response bytes ~70%. someguy's + # halved p50 (2208ms -> 1180ms) and cut response bytes ~70%. needle's # CPU and p95 did not move, so the gain is in building and shipping the # response, NOT in fewer provider-address lookups - the tail is still - # bounded by DHT walks (see README, "Capacity: four someguy instances + # bounded by DHT walks (see README, "Capacity: four needle instances # per box"). 50 is the compromise: most of a measured gain, half the # loss in completeness. 25 is what was measured; 50 is interpolation. - - SOMEGUY_RECORDS_LIMIT=50 + - NEEDLE_RECORDS_LIMIT=50 # Background FindPeer lookups for provider records that arrive without # addresses. The default (512) is reached under load: a rate test at - # 200 req/s on sing-1 rejected 7809 of them while someguy used only 5.5 + # 200 req/s on sing-1 rejected 7809 of them while needle used only 5.5 # of 32 cores, so the cap - not CPU - was the limit. Rejected lookups # mean provider records returned without addresses. - - SOMEGUY_CACHED_ADDR_BOOK_MAX_CONCURRENT_FIND_PEERS=2048 + - NEEDLE_CACHED_ADDR_BOOK_MAX_CONCURRENT_FIND_PEERS=2048 # Snapshot the address book to the volume every 15 minutes, plus once on # a clean shutdown. The periodic write is the guarantee: an OOM kill # skips shutdown, so the interval bounds what a crash can lose. - - SOMEGUY_CACHED_ADDR_BOOK_SNAPSHOT_INTERVAL=15m + - NEEDLE_CACHED_ADDR_BOOK_SNAPSHOT_INTERVAL=15m # Answer /routing/v1/peers as not-found from a recorded earlier failure # instead of repeating a full-timeout DHT walk. Measured on sing-1: four @@ -112,7 +114,7 @@ configMapGenerator: # fell from 28,558 to 15,081. The trade-off is staleness in one direction # only: a peer that comes online is invisible on that instance for up to # 1m, which is one client retry cycle. - - SOMEGUY_CACHED_ADDR_BOOK_NEGATIVE_TTL=1m + - NEEDLE_CACHED_ADDR_BOOK_NEGATIVE_TTL=1m # Once every non-DHT router in a request has finished, give the DHT this # much longer and then cancel it, so one slow DHT walk stops holding the @@ -120,7 +122,7 @@ configMapGenerator: # The trade-off is recall, and it is small and measured: 2% of the # providers that only the DHT would have found, 0.4% of all exclusive # records. - - SOMEGUY_DHT_TAIL_BUDGET=500ms + - NEEDLE_DHT_TAIL_BUDGET=500ms # Persist the accelerated client's crawled routing table to the volume # after every completed crawl, and replay it at startup when the file is @@ -129,20 +131,20 @@ configMapGenerator: # trade-off is staleness: a table older than two crawl intervals has # drifted enough that crawling from cold is the safer start, so past 2h # the file is ignored and the instance crawls as it always did. - - SOMEGUY_DHT_CRAWL_SNAPSHOT_MAX_AGE=2h + - NEEDLE_DHT_CRAWL_SNAPSHOT_MAX_AGE=2h # Go pprof endpoints on the API address. That address is loopback-only - # (see SOMEGUY_LISTEN_ADDRESS above), so this is exactly the exposure the + # (see NEEDLE_LISTEN_ADDRESS above), so this is exactly the exposure the # metrics endpoint already has, and the profiles are how every number in # this file was measured. It also turns on mutex and block sampling, # which is the only reason the per-peer lock contention was ever visible. - - SOMEGUY_PPROF=true + - NEEDLE_PPROF=true # The libp2p resource manager defaults to 85% of HOST RAM and ignores the # cgroup limit, which would let it plan past the pod's 21Gi and get # OOM-killed. 16 GiB, in bytes: a quarter of the 64 GiB the single # instance was given, as there are now four of them. - - SOMEGUY_LIBP2P_MAX_MEMORY=17179869184 + - NEEDLE_LIBP2P_MAX_MEMORY=17179869184 # Soft Go heap target below the 21Gi container limit, so the GC works # harder before the kernel OOM-kills the pod. Each instance actually @@ -160,5 +162,5 @@ configMapGenerator: # makes the trade safe: the heap cannot run away to the container limit. - GOGC=400 - - GOLOG_LOG_LEVEL=error,someguy=info + - GOLOG_LOG_LEVEL=error,needle=info - GOLOG_LOG_FMT=json diff --git a/k8s/needle/namespace.yaml b/k8s/needle/namespace.yaml index 9c07c7e..aaffcd2 100644 --- a/k8s/needle/namespace.yaml +++ b/k8s/needle/namespace.yaml @@ -2,4 +2,4 @@ apiVersion: v1 kind: Namespace metadata: - name: someguy + name: needle diff --git a/k8s/needle/pvc.yaml.j2 b/k8s/needle/pvc.yaml.j2 index 0d01681..6fe3bf0 100644 --- a/k8s/needle/pvc.yaml.j2 +++ b/k8s/needle/pvc.yaml.j2 @@ -1,7 +1,7 @@ {# - Rendered by roles/kustomize_apply from someguy_running_instances - (roles/someguy/defaults/main.yml): one PersistentVolumeClaim per running - instance, mounted at /data/someguy by its Deployment. + Rendered by roles/kustomize_apply from needle_running_instances + (roles/needle/defaults/main.yml): one PersistentVolumeClaim per running + instance, mounted at /data/needle by its Deployment. One per instance. It holds the autoconf cache and the cached address book snapshot (cached-addr-book.ndjson); losing one costs only that instance's @@ -9,11 +9,11 @@ Deployment is Recreate: the old pod is stopped before the new one mounts the same claim, so two pods never share it. 1Gi documents intent, not a bound: local-path is hostPath-backed and does - not enforce the request. The real bounds are someguy's PeerCacheSize (a + not enforce the request. The real bounds are needle's PeerCacheSize (a 26k-peer snapshot is about 6 MiB, ~250 MiB at the 1,000,000-peer cap) and the NVMe behind default-local-storage-path. #} -{% for instance in someguy_running_instances %} +{% for instance in needle_running_instances %} --- apiVersion: v1 kind: PersistentVolumeClaim diff --git a/k8s/route-origin/deployment.yaml b/k8s/route-origin/deployment.yaml index ab95afc..137e955 100644 --- a/k8s/route-origin/deployment.yaml +++ b/k8s/route-origin/deployment.yaml @@ -22,7 +22,7 @@ spec: # would DNAT the traffic into FORWARD, where kube-proxy's rules accept it # before ufw's and the allowlist would silently not apply. # - # It also makes the upstream plain loopback: someguy is hostNetwork too. + # It also makes the upstream plain loopback: needle is hostNetwork too. hostNetwork: true dnsPolicy: Default terminationGracePeriodSeconds: 45 @@ -42,7 +42,7 @@ spec: - -c - /etc/envoy/config/envoy.yaml # Envoy defaults to one worker per core (32 here). A small pool is - # ample for proxying to loopback and leaves CPU for someguy. + # ample for proxying to loopback and leaves CPU for needle. - --concurrency - "4" # Hot restart uses shared memory under /dev/shm and is not needed: diff --git a/k8s/route-origin/envoy.yaml.j2 b/k8s/route-origin/envoy.yaml.j2 index c34fb7c..f275deb 100644 --- a/k8s/route-origin/envoy.yaml.j2 +++ b/k8s/route-origin/envoy.yaml.j2 @@ -1,7 +1,7 @@ # Envoy as the Cloudflare origin for route-.ipni.io. # -# Cloudflare (Full strict) -> :443 TLS here -> 127.0.0.1:8190 (someguy). -# Both Envoy and someguy use hostNetwork, so the upstream is plain loopback. +# Cloudflare (Full strict) -> :443 TLS here -> 127.0.0.1:8190 (needle). +# Both Envoy and needle use hostNetwork, so the upstream is plain loopback. admin: # Host loopback only (hostNetwork). ufw would block it anyway; this keeps the @@ -65,14 +65,14 @@ static_resources: # Canonicalize the path BEFORE route matching. Without this the # "/routing/v1/" prefix also matched /routing/v1/../debug/... and # forwarded it unchanged; /debug stayed hidden only because - # someguy's router happens to redirect unclean paths. The + # needle's router happens to redirect unclean paths. The # allowlist must not depend on an upstream implementation detail. normalize_path: true merge_slashes: true path_with_escaped_slashes_action: REJECT_REQUEST request_headers_timeout: 10s - # someguy keeps a response open for up to 25s - # (SOMEGUY_ROUTING_TIMEOUT) while lookups run, and NDJSON + # needle keeps a response open for up to 25s + # (NEEDLE_ROUTING_TIMEOUT) while lookups run, and NDJSON # streams can be silent between records for most of that. stream_idle_timeout: 60s common_http_protocol_options: @@ -94,7 +94,7 @@ static_resources: bytes: "%BYTES_SENT%" duration_ms: "%DURATION%" upstream_ms: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%" - # Which someguy instance served it. With four behind + # Which needle instance served it. With four behind # the round robin, a slow tail is only actionable if # it can be attributed to one of them - a cold # instance after a rollout looks exactly like a slow @@ -131,10 +131,10 @@ static_resources: - route-sing-1.ipni.io - route-lith-1.ipni.io - route-chic-1.ipni.io - # ALLOWLIST, not a blocklist. someguy serves Prometheus + # ALLOWLIST, not a blocklist. needle serves Prometheus # metrics on the same port as the API (/debug/...); # anything not listed here - including endpoints a future - # someguy release adds - is never exposed publicly. + # needle release adds - is never exposed publicly. routes: # IPNS publishing is deliberately open, matching the # production delegated-ipfs.dev: records are @@ -146,7 +146,7 @@ static_resources: - name: ":method" string_match: { exact: "PUT" } route: - cluster: someguy + cluster: needle timeout: 60s typed_per_filter_config: envoy.filters.http.local_ratelimit: @@ -163,7 +163,7 @@ static_resources: - append_action: OVERWRITE_IF_EXISTS_OR_ADD header: { key: x-local-rate-limit, value: "ipns-put" } # Lookups. A miss can hold a request for up to 25s while - # someguy queries the DHT and cid.contact. The bucket is a + # needle queries the DHT and cid.contact. The bucket is a # per-box ceiling, not a per-client limit (that is # Cloudflare's job). # @@ -184,7 +184,7 @@ static_resources: # box (see README, "Capacity"). - match: { prefix: "/routing/v1/" } route: - cluster: someguy + cluster: needle # Above the 25s routing timeout, below Cloudflare's # 100s proxy read timeout. timeout: 60s @@ -204,7 +204,7 @@ static_resources: header: { key: x-local-rate-limit, value: "lookups" } - match: { path: "/version" } route: - cluster: someguy + cluster: needle timeout: 5s - match: { prefix: "/" } direct_response: @@ -222,7 +222,7 @@ static_resources: body: { inline_string: "misdirected request\n" } clusters: - - name: someguy + - name: needle type: STATIC connect_timeout: 2s # Envoy's default of 1024 concurrent requests is too low when each lookup @@ -235,14 +235,14 @@ static_resources: max_connections: 20000 max_pending_requests: 20000 max_requests: 20000 - # Four someguy instances per box (k8s/someguy). Round robin across them: + # Four needle instances per box (k8s/needle). Round robin across them: # each is a separate process with its own PeerID and concurrency budget, # and requests queue behind slow DHT rounds per process, so spreading # them is what raised sing-1 from 355 to 894 req/s (README, "Capacity"). lb_policy: ROUND_ROBIN # An instance that stops answering must be taken out of rotation, not - # sent traffic that then fails: roles/someguy restarts them one at a time. + # sent traffic that then fails: roles/needle restarts them one at a time. # An ACTIVE health check is what does that here - passive outlier # detection alone cannot, because with four endpoints the default # max_ejection_percent of 10% never reaches one whole endpoint. @@ -253,7 +253,7 @@ static_resources: # # The 2s timeout is safe because /version is a trivial handler that # touches neither the DHT nor the address book: lookup p50 runs to - # seconds under load, but /version does not. WATCH THIS if someguy ever + # seconds under load, but /version does not. WATCH THIS if needle ever # grows a real health endpoint that does work, or if health-check # failures start appearing in the Envoy logs at peak - an instance that # flaps out for 5-10s sheds its share onto the other three and makes the @@ -279,16 +279,16 @@ static_resources: interval: 5s base_ejection_time: 10s max_ejection_percent: 50 - # Rendered from someguy_running_instances (roles/someguy/defaults) by - # roles/kustomize_apply, so lowering someguy_instances trims this list + # Rendered from needle_running_instances (roles/needle/defaults) by + # roles/kustomize_apply, so lowering needle_instances trims this list # too. It used to be written out by hand, which meant a lowered count # left Envoy health-checking ports that no longer existed and logging # the failures forever. load_assignment: - cluster_name: someguy + cluster_name: needle endpoints: - lb_endpoints: -{% for instance in someguy_running_instances %} +{% for instance in needle_running_instances %} # {{ instance.name }} - endpoint: address: diff --git a/roles/bootstrap/tasks/main.yml b/roles/bootstrap/tasks/main.yml index 26b0539..98e0c3e 100644 --- a/roles/bootstrap/tasks/main.yml +++ b/roles/bootstrap/tasks/main.yml @@ -87,7 +87,7 @@ # kubo to accept connections from 127.0.0.1, so the loopback entry is gone # from Swarm.AddrFilters, which also stopped kubo DIALING loopback. A # WSS client could otherwise hand kubo 127.0.0.1 addresses (identify accepts -# them from loopback peers) and make it probe the k3s API, kubelet, someguy +# them from loopback peers) and make it probe the k3s API, kubelet, needle # and Envoy admin ports. Replies to accepted connections are not NEW, so # serving the proxy is unaffected. This one stays IPv4-only on purpose: # ::1 is still in Swarm.AddrFilters, so kubo never dials v6 loopback in the @@ -154,7 +154,7 @@ {{ entries }} ansible.builtin.set_fact: # API.Authorizations: with hostNetwork, 127.0.0.1 is shared with every host - # process and the other hostNetwork pods (someguy, Envoy), so loopback alone + # process and the other hostNetwork pods (needle, Envoy), so loopback alone # does not protect the admin RPC. Every /api/v0 call now needs the token. # (kubo does not guard /debug/metrics or /debug/pprof with it.) # diff --git a/roles/content/tasks/main.yml b/roles/content/tasks/main.yml index 4a851b0..0ae9ca9 100644 --- a/roles/content/tasks/main.yml +++ b/roles/content/tasks/main.yml @@ -90,7 +90,7 @@ # content-cluster the cluster identity, swarm secret and peer lists # --------------------------------------------------------------------------- # Choices in files/kubo-config.json (JSON cannot carry comments): -# Provide.Strategy "roots" - announce only each pin's root CID. someguy runs +# Provide.Strategy "roots" - announce only each pin's root CID. needle runs # ~416 DHT lookups/sec on these boxes, and reproviding every block of the # website content would compete with it for the host's CPU and sockets. # Clients find a root's providers and fetch the rest over Bitswap. (kubo 0.43 diff --git a/roles/hardening/tasks/main.yml b/roles/hardening/tasks/main.yml index 3f6c265..2ef6eb4 100644 --- a/roles/hardening/tasks/main.yml +++ b/roles/hardening/tasks/main.yml @@ -63,7 +63,7 @@ # adm). Nothing uses it: its password is locked, it has no SSH keys, and SSH # only admits the admin user. Console recovery uses root. It is removed so uid # 1000 is not a real, privileged login - several images default to uid 1000 -# (someguy's, and k3s's own metrics-server, which still runs as 1000), and a +# (needle's, and k3s's own metrics-server, which still runs as 1000), and a # container escape as that uid would otherwise land in this account. # # Removal is safe while containers run as uid 1000: userdel only refuses for diff --git a/roles/kustomize_apply/tasks/main.yml b/roles/kustomize_apply/tasks/main.yml index cf5acf0..971e16c 100644 --- a/roles/kustomize_apply/tasks/main.yml +++ b/roles/kustomize_apply/tasks/main.yml @@ -10,7 +10,7 @@ # A file in k8s// ending in .j2 is rendered with the play's variables and # shipped under its name minus the suffix; the .j2 itself never reaches the # node. This is how manifests that repeat a list already held in a role's -# defaults (the someguy instance table) stay in step with it instead of being +# defaults (the needle instance table) stay in step with it instead of being # maintained twice. - name: "Create the manifest directory for {{ kustomize_app }}" @@ -189,7 +189,7 @@ # a time: between the apply and the last resume, the instances still waiting # run pods from the OLD ReplicaSet, which names the OLD ConfigMap. Pruning it # then leaves those pods unable to restart (CreateContainerConfigError) until -# their turn comes - hours, with a warm-up gate in between. roles/someguy sets +# their turn comes - hours, with a warm-up gate in between. roles/needle sets # kustomize_prune_configmaps: false and includes prune_configmaps.yml itself # once every instance is back. # --------------------------------------------------------------------------- diff --git a/roles/kustomize_apply/tasks/prune_configmaps.yml b/roles/kustomize_apply/tasks/prune_configmaps.yml index 9d6e94f..031d576 100644 --- a/roles/kustomize_apply/tasks/prune_configmaps.yml +++ b/roles/kustomize_apply/tasks/prune_configmaps.yml @@ -6,7 +6,7 @@ # (kube-root-ca.crt has no labels, so it is never a candidate). # # EVERY Deployment in the namespace, not just kustomize_deployment: an app can -# have several (someguy runs four), and each names the ConfigMap it uses. The +# have several (needle runs four), and each names the ConfigMap it uses. The # union can only ever protect more ConfigMaps than it should, never fewer. # # The reference scan is deliberately narrow: Deployment volumes[].configMap diff --git a/roles/needle/defaults/main.yml b/roles/needle/defaults/main.yml index b605c85..f834c0e 100644 --- a/roles/needle/defaults/main.yml +++ b/roles/needle/defaults/main.yml @@ -2,7 +2,7 @@ # --------------------------------------------------------------------------- # Instances per box # --------------------------------------------------------------------------- -# Four someguy processes per box, each with its own PeerID, concurrency budget +# Four needle processes per box, each with its own PeerID, concurrency budget # and connection pool, behind Envoy's round-robin fan-out. # # Measured on sing-1 (2026-09-16, warm, through Cloudflare), achieved rate and @@ -28,33 +28,33 @@ # until someone hand-edited k8s/route-origin/envoy.yaml to match. Both of those # came from the same cause - this table was written out by hand in two other # files - which is why it is now rendered instead. -someguy_instances: 4 +needle_instances: 4 # Name, API port (loopback) and libp2p port (public) of each instance, in # rollout order. # # THE source of truth for the instance layout. roles/kustomize_apply renders # all three consumers from it: -# k8s/someguy/deployment.yaml.j2 instance a -# k8s/someguy/deployment-extra.yaml.j2 instances b onward +# k8s/needle/deployment.yaml.j2 instance a +# k8s/needle/deployment-extra.yaml.j2 instances b onward # k8s/route-origin/envoy.yaml.j2 the load-balancer endpoint list # so a port is written down exactly once. Adding an instance is a line here. -someguy_all_instances: - - { name: someguy, api: 8190, libp2p: 4004 } - - { name: someguy-b, api: 8191, libp2p: 4005 } - - { name: someguy-c, api: 8192, libp2p: 4006 } - - { name: someguy-d, api: 8193, libp2p: 4007 } +needle_all_instances: + - { name: needle, api: 8190, libp2p: 4004 } + - { name: needle-b, api: 8191, libp2p: 4005 } + - { name: needle-c, api: 8192, libp2p: 4006 } + - { name: needle-d, api: 8193, libp2p: 4007 } -someguy_running_instances: "{{ someguy_all_instances[: someguy_instances | int] }}" -someguy_retired_instances: "{{ someguy_all_instances[someguy_instances | int :] }}" +needle_running_instances: "{{ needle_all_instances[: needle_instances | int] }}" +needle_retired_instances: "{{ needle_all_instances[needle_instances | int :] }}" -# Instance a, the one k8s/someguy/deployment.yaml.j2 renders. Named rather than +# Instance a, the one k8s/needle/deployment.yaml.j2 renders. Named rather than # indexed inline so the template reads as "the first instance" instead of # "[0]". -someguy_first_instance: "{{ someguy_all_instances[0] }}" +needle_first_instance: "{{ needle_all_instances[0] }}" # First start pulls the image; later rollouts wait for the old pod to stop. -someguy_rollout_timeout: 600s +needle_rollout_timeout: 600s # --------------------------------------------------------------------------- # Rollout safety @@ -62,15 +62,15 @@ someguy_rollout_timeout: 600s # BOTH halves of the state a restart used to throw away now survive it, on the # instance's own PVC: # -# * the cached address book (SOMEGUY_CACHED_ADDR_BOOK_SNAPSHOT_INTERVAL), +# * the cached address book (NEEDLE_CACHED_ADDR_BOOK_SNAPSHOT_INTERVAL), # written every 15 minutes and on a clean shutdown, restored at start; # * the accelerated DHT client's crawled routing table -# (SOMEGUY_DHT_CRAWL_SNAPSHOT_MAX_AGE), written after every completed crawl +# (NEEDLE_DHT_CRAWL_SNAPSHOT_MAX_AGE), written after every completed crawl # and replayed at start when it is younger than 2h, with a real crawl # triggered the moment the replayed table is in place. # # With both restored an instance is serving properly in seconds rather than -# after an hour, so roles/someguy/tasks/wait_warm.yml skips its poll when both +# after an hour, so roles/needle/tasks/wait_warm.yml skips its poll when both # restored gauges are non-zero and polls as before when they are not. The poll # is still the fallback that matters: a first rollout onto a box that has never # written a snapshot, a snapshot older than the max age, or a lost PVC all @@ -93,7 +93,7 @@ someguy_rollout_timeout: 600s # is off while these boxes are still a dev fleet and rollouts stay fast. Set # production_rollout: true - or pass -e production_rollout=true - once they are # the only boxes serving traffic. -someguy_rollout_wait_warm: "{{ production_rollout | default(false) }}" +needle_rollout_wait_warm: "{{ production_rollout | default(false) }}" # Roll the instances ONE AT A TIME rather than all at once. With four instances # behind Envoy's health checks, a serial rollout keeps the box serving from the @@ -101,18 +101,18 @@ someguy_rollout_wait_warm: "{{ production_rollout | default(false) }}" # takes hours instead of minutes. That is the right trade only once these boxes # carry traffic, so it rides the same production switch: in dev all four roll # together and a rollout is over in a couple of minutes. -someguy_rollout_serial: "{{ production_rollout | default(false) }}" +needle_rollout_serial: "{{ production_rollout | default(false) }}" # Peers in the cached address book that count as warm. A settled box on this # fleet holds 23k-26k; 15k is comfortably past the painful phase without # waiting for the last slow arrivals. -someguy_warm_min_peers: 15000 +needle_warm_min_peers: 15000 # How long to wait for that, and how often to look. The whole gate is skipped # when this run did not actually restart the pod. -someguy_warm_timeout_seconds: 3600 -someguy_warm_poll_seconds: 60 +needle_warm_timeout_seconds: 3600 +needle_warm_poll_seconds: 60 # Fail the play when a box does not warm up in time, rather than rolling the # next box on the assumption that this one recovered. -someguy_warm_fail_on_timeout: true +needle_warm_fail_on_timeout: true diff --git a/roles/needle/tasks/main.yml b/roles/needle/tasks/main.yml index 25cfe4d..5efbb61 100644 --- a/roles/needle/tasks/main.yml +++ b/roles/needle/tasks/main.yml @@ -1,40 +1,40 @@ --- -# someguy: Delegated Routing V1 HTTP server, the primary workload on each box. -# Manifests live in k8s/someguy (kustomize). This role owns someguy's firewall +# needle: Delegated Routing V1 HTTP server, the primary workload on each box. +# Manifests live in k8s/needle (kustomize). This role owns needle's firewall # ports and the rollout order; shipping and applying are shared via # roles/kustomize_apply. # -# FOUR instances per box (someguy_instances), each a separate process with its +# FOUR instances per box (needle_instances), each a separate process with its # own PeerID, concurrency budget and connection pool, behind Envoy's round-robin -# fan-out. See roles/someguy/defaults/main.yml for what that is worth. +# fan-out. See roles/needle/defaults/main.yml for what that is worth. # --------------------------------------------------------------------------- # Firewall. Only libp2p is opened; every instance's HTTP API is loopback only. # -# The ports are fixed in someguy_all_instances and in -# k8s/someguy/deployment*.yaml, and must be changed in both together. +# The ports are fixed in needle_all_instances and in +# k8s/needle/deployment*.yaml, and must be changed in both together. # --------------------------------------------------------------------------- -- name: Allow someguy libp2p traffic (TCP) +- name: Allow needle libp2p traffic (TCP) community.general.ufw: rule: allow port: "{{ item.libp2p }}" proto: tcp comment: "{{ item.name }} libp2p" - loop: "{{ someguy_running_instances }}" + loop: "{{ needle_running_instances }}" loop_control: label: "{{ item.name }} {{ item.libp2p }}/tcp" -- name: Allow someguy libp2p traffic (UDP - QUIC, WebTransport, WebRTC) +- name: Allow needle libp2p traffic (UDP - QUIC, WebTransport, WebRTC) community.general.ufw: rule: allow port: "{{ item.libp2p }}" proto: udp comment: "{{ item.name }} libp2p" - loop: "{{ someguy_running_instances }}" + loop: "{{ needle_running_instances }}" loop_control: label: "{{ item.name }} {{ item.libp2p }}/udp" -# Lowering someguy_instances has to close the ports it opened, or a box keeps +# Lowering needle_instances has to close the ports it opened, or a box keeps # holes for instances that no longer exist. - name: Close the libp2p ports of instances that are not running community.general.ufw: @@ -42,28 +42,28 @@ port: "{{ item.0.libp2p }}" proto: "{{ item.1 }}" delete: true - loop: "{{ someguy_retired_instances | product(['tcp', 'udp']) | list }}" + loop: "{{ needle_retired_instances | product(['tcp', 'udp']) | list }}" loop_control: label: "{{ item.0.name }} {{ item.0.libp2p }}/{{ item.1 }}" -# someguy used to have a ClusterIP Service. It was removed when the API moved +# needle used to have a ClusterIP Service. It was removed when the API moved # to loopback-only: its endpoint (node IP :8190) no longer answers. Removing a # file from a kustomization does not delete the live object - `kubectl apply` # does not prune - so delete it explicitly. Safe to keep: absent is a no-op. -- name: Remove the obsolete someguy Service +- name: Remove the obsolete needle Service kubernetes.core.k8s: kubeconfig: /etc/rancher/k3s/k3s.yaml api_version: v1 kind: Service - name: someguy - namespace: someguy + name: needle + namespace: needle state: absent # --------------------------------------------------------------------------- # Rollout. Every instance's pod template changes together (they share the # generated env ConfigMap), and each Deployment is `Recreate`, so applying # without care stops all four at once - the outage this design exists to avoid. -# With someguy_rollout_serial (production only) each instance is paused before +# With needle_rollout_serial (production only) each instance is paused before # the apply and resumed in turn, so only one is ever down; in dev they all roll # together, which is a short outage but keeps the edit-deploy loop fast. # @@ -71,49 +71,49 @@ # the box keeps serving from the others throughout (verified: with an instance # scaled to zero, 40/40 requests still returned 200). # --------------------------------------------------------------------------- -- name: Check which someguy deployments exist +- name: Check which needle deployments exist ansible.builtin.command: - argv: [k3s, kubectl, --namespace=someguy, get, deployments, -o, "jsonpath={.items[*].metadata.name}"] - register: someguy_deployments + argv: [k3s, kubectl, --namespace=needle, get, deployments, -o, "jsonpath={.items[*].metadata.name}"] + register: needle_deployments changed_when: false check_mode: false - name: Pause every instance but the first, so they do not all roll at once ansible.builtin.command: - argv: [k3s, kubectl, --namespace=someguy, rollout, pause, "deployment/{{ item.name }}"] - loop: "{{ someguy_running_instances[1:] }}" + argv: [k3s, kubectl, --namespace=needle, rollout, pause, "deployment/{{ item.name }}"] + loop: "{{ needle_running_instances[1:] }}" loop_control: label: "{{ item.name }}" when: - - someguy_rollout_serial | bool + - needle_rollout_serial | bool - not ansible_check_mode - - item.name in someguy_deployments.stdout.split() + - item.name in needle_deployments.stdout.split() changed_when: false # a rollout control, not a change to the box -- name: Deploy the someguy manifests +- name: Deploy the needle manifests ansible.builtin.include_role: name: kustomize_apply vars: - kustomize_app: someguy - kustomize_namespace: someguy + kustomize_app: needle + kustomize_namespace: needle # The rollout this waits for is the first instance's; the rest are paused # and are rolled one at a time below. - kustomize_deployment: "{{ someguy_running_instances[0].name }}" - kustomize_rollout_timeout: "{{ someguy_rollout_timeout }}" + kustomize_deployment: "{{ needle_running_instances[0].name }}" + kustomize_rollout_timeout: "{{ needle_rollout_timeout }}" # Deferred to the end of this file. The paused instances still run pods # from the old ReplicaSet, which names the old ConfigMap; deleting it now # would strand them if one restarted before its turn. kustomize_prune_configmaps: false -- name: Delete instances above someguy_instances +- name: Delete instances above needle_instances kubernetes.core.k8s: kubeconfig: /etc/rancher/k3s/k3s.yaml api_version: apps/v1 kind: Deployment name: "{{ item.name }}" - namespace: someguy + namespace: needle state: absent - loop: "{{ someguy_retired_instances }}" + loop: "{{ needle_retired_instances }}" loop_control: label: "{{ item.name }}" @@ -129,9 +129,9 @@ api_version: v1 kind: PersistentVolumeClaim name: "{{ item.name }}-data" - namespace: someguy + namespace: needle state: absent - loop: "{{ someguy_retired_instances }}" + loop: "{{ needle_retired_instances }}" loop_control: label: "{{ item.name }}" @@ -139,20 +139,20 @@ # roll_instance.yml checks the others. - name: Confirm the first instance's API answers locally ansible.builtin.uri: - url: "http://127.0.0.1:{{ someguy_running_instances[0].api }}/version" + url: "http://127.0.0.1:{{ needle_running_instances[0].api }}/version" return_content: true - register: someguy_version + register: needle_version retries: 6 delay: 5 - until: someguy_version.status == 200 + until: needle_version.status == 200 when: not ansible_check_mode - name: Report the running version ansible.builtin.debug: msg: >- - someguy on {{ inventory_hostname }}: {{ someguy_version.content | trim }} - ({{ someguy_running_instances | length }} instance(s): - {{ someguy_running_instances | map(attribute='api') | join(', ') }}) + needle on {{ inventory_hostname }}: {{ needle_version.content | trim }} + ({{ needle_running_instances | length }} instance(s): + {{ needle_running_instances | map(attribute='api') | join(', ') }}) when: not ansible_check_mode # --------------------------------------------------------------------------- @@ -168,12 +168,12 @@ # --------------------------------------------------------------------------- - name: Wait for the first instance's address book to warm up when: - - someguy_rollout_wait_warm | bool + - needle_rollout_wait_warm | bool - not ansible_check_mode - kustomize_deployment_rolled | default(false) | bool ansible.builtin.include_tasks: wait_warm.yml vars: - someguy_warm_instance: "{{ someguy_running_instances[0] }}" + needle_warm_instance: "{{ needle_running_instances[0] }}" # --------------------------------------------------------------------------- # One instance at a time: resume it, wait for its pod, prove its API answers, @@ -181,18 +181,18 @@ # --------------------------------------------------------------------------- - name: Roll the remaining instances one at a time ansible.builtin.include_tasks: roll_instance.yml - loop: "{{ someguy_running_instances[1:] }}" + loop: "{{ needle_running_instances[1:] }}" loop_control: - loop_var: someguy_instance - label: "{{ someguy_instance.name }}" + loop_var: needle_instance + label: "{{ needle_instance.name }}" when: not ansible_check_mode # Deferred from the apply: safe now that every instance runs a pod from the new # ReplicaSet, so no live pod names the superseded ConfigMap any more. -- name: Prune superseded someguy ConfigMaps +- name: Prune superseded needle ConfigMaps ansible.builtin.include_role: name: kustomize_apply tasks_from: prune_configmaps vars: - kustomize_app: someguy - kustomize_namespace: someguy + kustomize_app: needle + kustomize_namespace: needle diff --git a/roles/needle/tasks/roll_instance.yml b/roles/needle/tasks/roll_instance.yml index 11c0167..c21f1cb 100644 --- a/roles/needle/tasks/roll_instance.yml +++ b/roles/needle/tasks/roll_instance.yml @@ -1,46 +1,46 @@ --- -# Roll ONE someguy instance: resume the rollout the apply left paused, wait for +# Roll ONE needle instance: resume the rollout the apply left paused, wait for # its pod, prove its API answers, and (in production) wait for its address book # before the caller moves to the next one. Included per instance by main.yml -# with someguy_instance set. +# with needle_instance set. # # The other instances keep serving throughout; Envoy ejects this one while it # is down and takes it back when /version answers again. -- name: "Resume {{ someguy_instance.name }}" +- name: "Resume {{ needle_instance.name }}" ansible.builtin.command: - argv: [k3s, kubectl, --namespace=someguy, rollout, resume, "deployment/{{ someguy_instance.name }}"] - register: someguy_resume + argv: [k3s, kubectl, --namespace=needle, rollout, resume, "deployment/{{ needle_instance.name }}"] + register: needle_resume # Resuming a deployment that was never paused is an error, not a failure: # it happens whenever this instance was created by the apply just now. failed_when: - - someguy_resume.rc != 0 - - "'is not paused' not in someguy_resume.stderr" - changed_when: someguy_resume.rc == 0 + - needle_resume.rc != 0 + - "'is not paused' not in needle_resume.stderr" + changed_when: needle_resume.rc == 0 -- name: "Wait for {{ someguy_instance.name }} to roll out" +- name: "Wait for {{ needle_instance.name }} to roll out" ansible.builtin.command: argv: - k3s - kubectl - - --namespace=someguy + - --namespace=needle - rollout - status - - "deployment/{{ someguy_instance.name }}" - - "--timeout={{ someguy_rollout_timeout }}" + - "deployment/{{ needle_instance.name }}" + - "--timeout={{ needle_rollout_timeout }}" changed_when: false -- name: "Confirm {{ someguy_instance.name }} answers on 127.0.0.1:{{ someguy_instance.api }}" +- name: "Confirm {{ needle_instance.name }} answers on 127.0.0.1:{{ needle_instance.api }}" ansible.builtin.uri: - url: "http://127.0.0.1:{{ someguy_instance.api }}/version" + url: "http://127.0.0.1:{{ needle_instance.api }}/version" return_content: true - register: someguy_instance_version + register: needle_instance_version retries: 6 delay: 5 - until: someguy_instance_version.status == 200 + until: needle_instance_version.status == 200 -- name: "Wait for {{ someguy_instance.name }}'s address book to warm up" +- name: "Wait for {{ needle_instance.name }}'s address book to warm up" ansible.builtin.include_tasks: wait_warm.yml vars: - someguy_warm_instance: "{{ someguy_instance }}" - when: someguy_rollout_wait_warm | bool + needle_warm_instance: "{{ needle_instance }}" + when: needle_rollout_wait_warm | bool diff --git a/roles/needle/tasks/wait_warm.yml b/roles/needle/tasks/wait_warm.yml index 81adbc2..de79f66 100644 --- a/roles/needle/tasks/wait_warm.yml +++ b/roles/needle/tasks/wait_warm.yml @@ -1,6 +1,6 @@ --- # Wait until one instance is warm enough to roll the next. Included with -# someguy_warm_instance set. See roles/someguy/defaults/main.yml for the +# needle_warm_instance set. See roles/needle/defaults/main.yml for the # measurements behind the threshold. # # A restarted instance answers within a minute but, with nothing to restore from, @@ -19,85 +19,85 @@ # two gauges already state is dead time, so the poll is skipped. # # * Anything else - no snapshot yet, a snapshot older than -# SOMEGUY_DHT_CRAWL_SNAPSHOT_MAX_AGE, a lost PVC, or a build that does not +# NEEDLE_DHT_CRAWL_SNAPSHOT_MAX_AGE, a lost PVC, or a build that does not # write them - falls through to the original poll, unchanged. That is the # genuinely cold start, and it is exactly the case the measurements above # describe, so it still gets the full wait. # # The restored gauges are read once, together, and only decide WHICH path runs. -# Neither is a readiness signal on its own: someguy_dht_crawl_snapshot_restored_peers +# Neither is a readiness signal on its own: needle_dht_crawl_snapshot_restored_peers # counts what the replay reported before the routing table filter decides what to # keep, so on a build without the kad-dht fork it reads thousands while the table -# is empty. someguy_dht_accelerated_ready is the signal for that, and the image +# is empty. needle_dht_accelerated_ready is the signal for that, and the image # this fleet pins carries the fork, so a replay that reports peers does keep them. -- name: "Read {{ someguy_warm_instance.name }}'s restored-snapshot gauges" +- name: "Read {{ needle_warm_instance.name }}'s restored-snapshot gauges" ansible.builtin.shell: cmd: >- set -o pipefail; - curl -sf --max-time 10 http://127.0.0.1:{{ someguy_warm_instance.api }}/debug/metrics/prometheus - | awk '/^someguy_cached_addr_book_snapshot_restored_peers[ {]/ {book = int($2)} - /^someguy_dht_crawl_snapshot_restored_peers[ {]/ {crawl = int($2)} + curl -sf --max-time 10 http://127.0.0.1:{{ needle_warm_instance.api }}/debug/metrics/prometheus + | awk '/^needle_cached_addr_book_snapshot_restored_peers[ {]/ {book = int($2)} + /^needle_dht_crawl_snapshot_restored_peers[ {]/ {crawl = int($2)} END {printf "%d %d\n", book, crawl}' executable: /bin/bash - register: someguy_restored + register: needle_restored changed_when: false failed_when: false -- name: "Decide whether {{ someguy_warm_instance.name }} came up on its snapshots" +- name: "Decide whether {{ needle_warm_instance.name }} came up on its snapshots" ansible.builtin.set_fact: - someguy_warm_restored: >- - {{ (someguy_restored_fields[0] | int) > 0 and (someguy_restored_fields[1] | int) > 0 }} - someguy_warm_restored_book: "{{ someguy_restored_fields[0] | int }}" - someguy_warm_restored_crawl: "{{ someguy_restored_fields[1] | int }}" + needle_warm_restored: >- + {{ (needle_restored_fields[0] | int) > 0 and (needle_restored_fields[1] | int) > 0 }} + needle_warm_restored_book: "{{ needle_restored_fields[0] | int }}" + needle_warm_restored_crawl: "{{ needle_restored_fields[1] | int }}" vars: # A failed curl leaves stdout empty; awk prints "0 0" for a reachable # instance that exports neither gauge. Both mean "not restored", which is # the safe answer because it takes the poll. - someguy_restored_fields: >- - {{ ((someguy_restored | default({})).stdout | default('', true) + needle_restored_fields: >- + {{ ((needle_restored | default({})).stdout | default('', true) | trim | default('0 0', true)).split() }} -- name: "Report that {{ someguy_warm_instance.name }} restored and needs no warm-up" +- name: "Report that {{ needle_warm_instance.name }} restored and needs no warm-up" ansible.builtin.debug: msg: >- - {{ inventory_hostname }} {{ someguy_warm_instance.name }} restored - {{ someguy_warm_restored_book }} address book peers and - {{ someguy_warm_restored_crawl }} crawled routing table peers from its + {{ inventory_hostname }} {{ needle_warm_instance.name }} restored + {{ needle_warm_restored_book }} address book peers and + {{ needle_warm_restored_crawl }} crawled routing table peers from its snapshots; skipping the warm-up poll and rolling the next. - when: someguy_warm_restored | bool + when: needle_warm_restored | bool -- name: "Poll {{ someguy_warm_instance.name }} until it holds {{ someguy_warm_min_peers }} peers" +- name: "Poll {{ needle_warm_instance.name }} until it holds {{ needle_warm_min_peers }} peers" ansible.builtin.shell: cmd: >- set -o pipefail; - curl -sf --max-time 10 http://127.0.0.1:{{ someguy_warm_instance.api }}/debug/metrics/prometheus - | awk '/^someguy_cached_addr_book_peer_state_size/ {print int($2); found = 1} + curl -sf --max-time 10 http://127.0.0.1:{{ needle_warm_instance.api }}/debug/metrics/prometheus + | awk '/^needle_cached_addr_book_peer_state_size/ {print int($2); found = 1} END {if (!found) print 0}' executable: /bin/bash - register: someguy_warm + register: needle_warm changed_when: false failed_when: false - retries: "{{ (someguy_warm_timeout_seconds | int // (someguy_warm_poll_seconds | int)) | int }}" - delay: "{{ someguy_warm_poll_seconds | int }}" - until: (someguy_warm.stdout | trim | default('0', true) | int) >= (someguy_warm_min_peers | int) - when: not (someguy_warm_restored | bool) + retries: "{{ (needle_warm_timeout_seconds | int // (needle_warm_poll_seconds | int)) | int }}" + delay: "{{ needle_warm_poll_seconds | int }}" + until: (needle_warm.stdout | trim | default('0', true) | int) >= (needle_warm_min_peers | int) + when: not (needle_warm_restored | bool) -- name: "Report whether {{ someguy_warm_instance.name }} warmed up in time" +- name: "Report whether {{ needle_warm_instance.name }} warmed up in time" ansible.builtin.assert: - that: (someguy_warm.stdout | trim | default('0', true) | int) >= (someguy_warm_min_peers | int) + that: (needle_warm.stdout | trim | default('0', true) | int) >= (needle_warm_min_peers | int) fail_msg: >- - {{ inventory_hostname }} {{ someguy_warm_instance.name }} still holds only - {{ someguy_warm.stdout | trim | default('0', true) }} cached peers after - {{ someguy_warm_timeout_seconds }}s (wanted {{ someguy_warm_min_peers }}). + {{ inventory_hostname }} {{ needle_warm_instance.name }} still holds only + {{ needle_warm.stdout | trim | default('0', true) }} cached peers after + {{ needle_warm_timeout_seconds }}s (wanted {{ needle_warm_min_peers }}). Not rolling the next instance or box on the assumption that this one - recovered. Re-run with -e someguy_rollout_wait_warm=false to skip this gate. + recovered. Re-run with -e needle_rollout_wait_warm=false to skip this gate. success_msg: >- - {{ inventory_hostname }} {{ someguy_warm_instance.name }}: address book holds - {{ someguy_warm.stdout | trim }} peers; safe to roll the next. + {{ inventory_hostname }} {{ needle_warm_instance.name }}: address book holds + {{ needle_warm.stdout | trim }} peers; safe to roll the next. quiet: true - # Only when the poll actually ran: the restored path leaves someguy_warm unset, + # Only when the poll actually ran: the restored path leaves needle_warm unset, # or stale from a previous instance in the loop. when: - - someguy_warm_fail_on_timeout | bool - - not (someguy_warm_restored | bool) + - needle_warm_fail_on_timeout | bool + - not (needle_warm_restored | bool) diff --git a/roles/route_origin/defaults/main.yml b/roles/route_origin/defaults/main.yml index e83b13e..43e2bf4 100644 --- a/roles/route_origin/defaults/main.yml +++ b/roles/route_origin/defaults/main.yml @@ -1,6 +1,6 @@ --- # Public hostname Cloudflare proxies to this box, e.g. route-chic-1.ipni.io. -# Deliberately not named after someguy: the implementation may change. +# Deliberately not named after what it fronts: the implementation may change. route_hostname: "route-{{ inventory_hostname }}.ipni.io" # Cloudflare Origin CA certificate covering every route_hostname. Public, so it diff --git a/roles/route_origin/tasks/main.yml b/roles/route_origin/tasks/main.yml index 2be584c..6ca9a42 100644 --- a/roles/route_origin/tasks/main.yml +++ b/roles/route_origin/tasks/main.yml @@ -1,6 +1,6 @@ --- # route-origin: Envoy on :443 as the Cloudflare origin for route-.ipni.io, -# proxying an allowlist of paths to someguy on loopback. +# proxying an allowlist of paths to needle on loopback. # # Cloudflare edge --(Full strict, Origin CA cert)--> :443 Envoy --> 127.0.0.1:8190 # @@ -272,7 +272,7 @@ # --------------------------------------------------------------------------- # Verification, on the box: the right certificate is served, allowlisted -# paths reach someguy, and everything else (notably /debug/) does not. +# paths reach needle, and everything else (notably /debug/) does not. # --------------------------------------------------------------------------- - name: Read the certificate Envoy is actually serving # noqa risky-shell-pipe ansible.builtin.shell: diff --git a/roles/route_origin/tasks/verify.yml b/roles/route_origin/tasks/verify.yml index 9721863..eeac0e9 100644 --- a/roles/route_origin/tasks/verify.yml +++ b/roles/route_origin/tasks/verify.yml @@ -17,17 +17,17 @@ # assert below reports failures. # # Every negative case is a regression test: if any returns something else -# (a redirect from someguy, a 200), the deploy fails. +# (a redirect from needle, a 200), the deploy fails. - name: Run the origin request checks vars: origin_checks: - # Allowlisted paths reach someguy. + # Allowlisted paths reach needle. - { path: /version, status: 200 } - # Proves the Envoy -> someguy wiring for lookups. It does NOT prove the - # DHT works: someguy answers 200 for unknown peers too. + # Proves the Envoy -> needle wiring for lookups. It does NOT prove the + # DHT works: needle answers 200 for unknown peers too. - { path: /routing/v1/peers/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN, status: 200 } # IPNS publishing is deliberately open: a malformed record must reach - # someguy and be rejected BY SOMEGUY (400), not by Envoy. + # needle and be rejected BY NEEDLE (400), not by Envoy. - { path: /routing/v1/ipns/k2k4r8jl0yz8qjgqbmc2cdu5hkqek5rj6flgnlkyywynci20j0iuyfuj, method: PUT, status: 400 } # Everything else is hidden. - { path: /debug/metrics/prometheus, status: 404 } @@ -39,7 +39,7 @@ - { path: /routing/v1/%2e%2e/debug/metrics/prometheus, status: 404 } # Here ".." only removes the EMPTY segment from the double slash (RFC # 3986), giving /routing/v1/debug/metrics/prometheus: still inside the - # allowlisted prefix, so Envoy forwards it and someguy answers 404. Not an + # allowlisted prefix, so Envoy forwards it and needle answers 404. Not an # escape - it can never become /debug/metrics/prometheus. - { path: //routing/v1//..//debug/metrics/prometheus, status: 404 } # Escaped slashes are rejected by Envoy outright. diff --git a/routing.yml b/routing.yml index dc05432..16b1728 100644 --- a/routing.yml +++ b/routing.yml @@ -1,6 +1,6 @@ --- # Deploy the routing service to the prepared boxes: -# someguy Delegated Routing V1 server (hostNetwork, four instances, +# needle Delegated Routing V1 server (hostNetwork, four instances, # libp2p on 4004-4007, HTTP API on 127.0.0.1:8190-8193) # route_origin Envoy on :443 as the Cloudflare origin for route-.ipni.io # @@ -17,9 +17,9 @@ # # ROLLOUT MODE. Each box is a whole site behind its own hostname with no # failover between them, so in production a simultaneous rollout takes the -# entire routing service down - and someguy stays degraded for tens of minutes +# entire routing service down - and needle stays degraded for tens of minutes # after a restart, long after the pod reports Ready (see -# roles/someguy/defaults/main.yml for the measurements). +# roles/needle/defaults/main.yml for the measurements). # # production_rollout: false (default, dev) all boxes at once, no warm-up wait # production_rollout: true one box at a time, each warmed up @@ -37,7 +37,7 @@ become: true gather_facts: false roles: - - role: someguy - tags: [someguy] + - role: needle + tags: [needle] - role: route_origin tags: [origin] diff --git a/scripts/render-manifests.yml b/scripts/render-manifests.yml index f0830a5..a268ee2 100644 --- a/scripts/render-manifests.yml +++ b/scripts/render-manifests.yml @@ -9,11 +9,11 @@ # # ansible-playbook scripts/render-manifests.yml # -> ./rendered/ # ansible-playbook scripts/render-manifests.yml -e dest=/tmp/out -# ansible-playbook scripts/render-manifests.yml -e someguy_instances=1 +# ansible-playbook scripts/render-manifests.yml -e needle_instances=1 # # A relative dest is taken from the repo root, not your shell's cwd. # -# The someguy instance table is read straight from the role's defaults, so this +# The needle instance table is read straight from the role's defaults, so this # cannot disagree with what a deploy would render. - name: Render the k8s manifest templates @@ -21,7 +21,7 @@ gather_facts: false connection: local vars_files: - - "{{ playbook_dir }}/../roles/someguy/defaults/main.yml" + - "{{ playbook_dir }}/../roles/needle/defaults/main.yml" vars: repo_root: "{{ (playbook_dir ~ '/..') | realpath }}" dest: rendered @@ -91,4 +91,4 @@ {{ manifest_dirs.files | length }} manifest directories into {{ render_dest }}, {{ manifest_templates.files | length }} templates rendered ({{ manifest_templates.files | map(attribute='path') | map('basename') | sort | join(', ') }}); - someguy_instances={{ someguy_instances }} + needle_instances={{ needle_instances }} diff --git a/scripts/routing-compare.sh b/scripts/routing-compare.sh index 217669c..0a50a9c 100755 --- a/scripts/routing-compare.sh +++ b/scripts/routing-compare.sh @@ -102,7 +102,7 @@ candidates = endpoints[1:] # Why these Accept headers: providers and peers are asked for as one JSON # document (the default is NDJSON streaming). IPNS has no JSON form; both -# delegated-ipfs.dev and someguy answer application/json with 406, so records +# delegated-ipfs.dev and needle answer application/json with 406, so records # are requested as application/vnd.ipfs.ipns-record. ACCEPT = { "providers": "application/json", @@ -110,7 +110,7 @@ ACCEPT = { "ipns": "application/vnd.ipfs.ipns-record", } # A missing record comes back as `200 text/plain` with this body (seen on both -# someguy and delegated-ipfs.dev), or as 404. Both mean "no result", not error. +# needle and delegated-ipfs.dev), or as 404. Both mean "no result", not error. NOT_FOUND = b"routing: not found" ERROR_RATE_MARGIN = 0.05 USER_AGENT = "ipni-workers-routing-compare/1 (+https://github.com/ipni/workers)" diff --git a/scripts/routing-load.sh b/scripts/routing-load.sh index 5f9cf63..860afc9 100755 --- a/scripts/routing-load.sh +++ b/scripts/routing-load.sh @@ -7,10 +7,10 @@ # # Endpoints are tested ONE AT A TIME so a stage's numbers belong to one box. # -# Load reaches real third parties: every lookup someguy cannot serve from its +# Load reaches real third parties: every lookup needle cannot serve from its # cache becomes queries to the Amino DHT and cid.contact. By default the same -# fixtures repeat, so someguy's cache absorbs most of them and this mainly -# measures our own path (Cloudflare -> Envoy -> someguy). --cache-bust forces +# fixtures repeat, so needle's cache absorbs most of them and this mainly +# measures our own path (Cloudflare -> Envoy -> needle). --cache-bust forces # every request to miss that cache; keep concurrency and duration low with it. # # Exit 0 when every endpoint completed all stages within thresholds, 1 when any @@ -40,7 +40,7 @@ Options: (default: scripts/routing-compare-cids.txt) --timeout SECONDS per-request timeout (default: 30) --max-error-rate F stop climbing an endpoint above this rate (default: 0.05) - --cache-bust unique query string per request, so someguy answers + --cache-bust unique query string per request, so needle answers every request from the DHT/cid.contact instead of cache --out FILE also write the full results as JSON -h, --help show this help diff --git a/scripts/routing-rate-test.sh b/scripts/routing-rate-test.sh index b7a304e..261d721 100755 --- a/scripts/routing-rate-test.sh +++ b/scripts/routing-rate-test.sh @@ -8,13 +8,13 @@ # ones have finished, so queueing shows up as growing latency instead of # silently lowering throughput. # -# It also samples what the box itself is doing around every stage: someguy's +# It also samples what the box itself is doing around every stage: needle's # CPU, its address-book cache hit/miss counters, in-flight and rejected -# lookups, open FDs, and NIC bytes. That is what separates "someguy is at its +# lookups, open FDs, and NIC bytes. That is what separates "needle is at its # limit" from "the client or the network is". # # WORKLOAD. --workload cold generates random CIDs that nobody provides, so -# every request is a real, uncached lookup (the expensive case: someguy has no +# every request is a real, uncached lookup (the expensive case: needle has no # provider-response cache, only a cached address book). --workload fixtures # replays scripts/routing-compare-cids.txt, which is mostly popular CIDs that # cid.contact answers quickly. --cold-fraction mixes the two. @@ -36,7 +36,7 @@ usage() { Usage: ./scripts/routing-rate-test.sh --url-base URL [options] Options: - --url-base URL required, e.g. http://127.0.0.1:8190 (someguy direct) + --url-base URL required, e.g. http://127.0.0.1:8190 (needle direct) or https://route-chic-1.ipni.io (through Cloudflare) --rates LIST comma-separated target rates in requests/sec (default: 25,50,100,200) @@ -46,7 +46,7 @@ Options: 1.0 for --workload cold, 0.0 for fixtures --fixtures FILE fixture list for the non-cold share (default: scripts/routing-compare-cids.txt) - --metrics-url URL someguy Prometheus endpoint to sample around each stage + --metrics-url URL needle Prometheus endpoint to sample around each stage (default: http://127.0.0.1:8190/debug/metrics/prometheus) --timeout S per-request timeout (default: 30) --max-error-rate F stop climbing above this error rate (default: 0.05) @@ -176,7 +176,7 @@ def b32(raw): def cold_cid(): """A CIDv1 raw/sha2-256 over random bytes: valid, and nobody provides it, - so someguy must do the full lookup.""" + so needle must do the full lookup.""" return "b" + b32(bytes([0x01, 0x55, 0x12, 0x20]) + hashlib.sha256(os.urandom(32)).digest()) @@ -210,9 +210,9 @@ def ms(v): # --- what the box is doing -------------------------------------------------- METRICS = ( - "someguy_cached_router_peer_addr_lookups", - "someguy_cached_router_find_peer_lookups_rejected", - "someguy_cached_router_find_peer_lookups_in_flight", + "needle_cached_router_peer_addr_lookups", + "needle_cached_router_find_peer_lookups_rejected", + "needle_cached_router_find_peer_lookups_in_flight", "process_cpu_seconds_total", "process_open_fds", "go_goroutines", @@ -220,7 +220,7 @@ METRICS = ( def scrape(): - """someguy's own counters. Returns {} if the endpoint is not reachable.""" + """needle's own counters. Returns {} if the endpoint is not reachable.""" out = {} try: with urllib.request.urlopen(metrics_url, timeout=10) as resp: @@ -402,10 +402,10 @@ def run_stage(rate): else: statuses[r["status"]] = statuses.get(r["status"], 0) + 1 - hits = sum_by(m_after, "someguy_cached_router_peer_addr_lookups", cache="hit") - \ - sum_by(m_before, "someguy_cached_router_peer_addr_lookups", cache="hit") - misses = sum_by(m_after, "someguy_cached_router_peer_addr_lookups", cache="miss") - \ - sum_by(m_before, "someguy_cached_router_peer_addr_lookups", cache="miss") + hits = sum_by(m_after, "needle_cached_router_peer_addr_lookups", cache="hit") - \ + sum_by(m_before, "needle_cached_router_peer_addr_lookups", cache="hit") + misses = sum_by(m_after, "needle_cached_router_peer_addr_lookups", cache="miss") - \ + sum_by(m_before, "needle_cached_router_peer_addr_lookups", cache="miss") return { "target_rate": rate, "workers": pool_size(rate), "seconds": round(wall, 1), "requests": len(results), "errors": len(failed), @@ -422,13 +422,13 @@ def run_stage(rate): "queue_p50_ms": percentile(queued, 50), "queue_p95_ms": percentile(queued, 95), "bytes_total": sum(r.get("bytes", 0) for r in good), "statuses": statuses, "error_kinds": errors, - "someguy_cpu_s": delta(m_before, m_after, "process_cpu_seconds_total"), - "someguy_open_fds": m_after.get("process_open_fds"), - "someguy_goroutines": m_after.get("go_goroutines"), - "someguy_lookups_rejected": delta( - m_before, m_after, "someguy_cached_router_find_peer_lookups_rejected"), - "someguy_lookups_in_flight": m_after.get( - "someguy_cached_router_find_peer_lookups_in_flight"), + "needle_cpu_s": delta(m_before, m_after, "process_cpu_seconds_total"), + "needle_open_fds": m_after.get("process_open_fds"), + "needle_goroutines": m_after.get("go_goroutines"), + "needle_lookups_rejected": delta( + m_before, m_after, "needle_cached_router_find_peer_lookups_rejected"), + "needle_lookups_in_flight": m_after.get( + "needle_cached_router_find_peer_lookups_in_flight"), "addr_cache_hits": hits, "addr_cache_misses": misses, "addr_cache_hit_rate": hits / (hits + misses) if (hits + misses) else None, "client_cpu_s": round(cpu1 - cpu0, 1), @@ -457,7 +457,7 @@ for rate in rates: doc["stages"].append(stage) print(f" {rate:>5}/s -> {stage['achieved_rps']:>7.1f}/s p50 {ms(stage['p50_ms']):>8}" f" p95 {ms(stage['p95_ms']):>8} drain {stage['drain_s']}s" - f" err {stage['error_rate']:.1%} someguy_cpu {stage['someguy_cpu_s']}s", + f" err {stage['error_rate']:.1%} needle_cpu {stage['needle_cpu_s']}s", file=sys.stderr) throttled = stage["statuses"].get(429, 0) if throttled: @@ -477,15 +477,15 @@ print(head) print("-" * len(head)) for s in doc["stages"]: hit = "-" if s["addr_cache_hit_rate"] is None else f"{s['addr_cache_hit_rate']:.0%}" - cpu = "-" if s["someguy_cpu_s"] is None else f"{s['someguy_cpu_s']:.1f}s" + cpu = "-" if s["needle_cpu_s"] is None else f"{s['needle_cpu_s']:.1f}s" mbs = s["bytes_total"] / 1e6 / s["seconds"] if s["seconds"] else 0 print(f"{s['target_rate']:>7} {s['achieved_rps']:>8.1f} {ms(s['p50_ms']):>8} " f"{ms(s['p95_ms']):>8} {ms(s['p99_ms']):>8} {ms(s['queue_p95_ms']):>8} " f"{s['error_rate']:>6.1%} {cpu:>7} {hit:>6} {mbs:>6.1f}") last = doc["stages"][-1] print(f"client cpu {last['client_cpu_s']}s/stage, nic rx {last['net_rx_mbps']}Mb/s " - f"tx {last['net_tx_mbps']}Mb/s, someguy fds {last['someguy_open_fds']}, " - f"goroutines {last['someguy_goroutines']}, lookups rejected {last['someguy_lookups_rejected']}") + f"tx {last['net_tx_mbps']}Mb/s, needle fds {last['needle_open_fds']}, " + f"goroutines {last['needle_goroutines']}, lookups rejected {last['needle_lookups_rejected']}") print("STOPPED: " + stopped if stopped else "OK: every stage met its target rate") doc.update(finished_at=now(), exit_code=1 if stopped else 0, stopped=stopped)