Gateway origin for dist.ipfs.tech, POST /add on the pinning API

ansible.cfg

@@ -3,7 +3,8 @@ inventory            = inventory/hosts.yml
 roles_path           = roles
 host_key_checking    = True
 interpreter_python   = auto_silent
-stdout_callback      = yaml
+stdout_callback      = default
+callback_result_format = yaml
 callbacks_enabled    = profile_tasks
 retry_files_enabled  = False
 forks                = 3

group_vars/managed_boxes/main.yml

@@ -146,10 +146,12 @@ sysctl_tuning:
   #   9096 content ipfs-cluster swarm
   #   9097 content ipfs-cluster Pinning Service API (loopback)
   #   9443 content pinning API over TLS (Envoy)  9904 its Envoy admin (loopback)
+  #   2053 content gateway origin over TLS (Envoy)  8181 kubo gateway (loopback)
+  #   9905 the gateway Envoy's admin (loopback)
   #   9901 route-origin Envoy admin  9902 bootstrap-wss Envoy admin
   #   9903 bootstrap-b's bootstrap-wss Envoy admin
   #   10010 containerd  10248-10259 kubelet/kube-proxy/k3s
-  net.ipv4.ip_local_reserved_ports: "443,4001-4002,4004-4007,4011-4012,4101,4317-4318,4443,4453,5011-5012,5021,6443-6444,8000,8190-8193,8443,8472,8888,9094-9097,9100,9443,9800-9801,9810-9812,9901-9904,10010,10248-10259"
+  net.ipv4.ip_local_reserved_ports: "443,2053,4001-4002,4004-4007,4011-4012,4101,4317-4318,4443,4453,5011-5012,5021,6443-6444,8000,8181,8190-8193,8443,8472,8888,9094-9097,9100,9443,9800-9801,9810-9812,9901-9905,10010,10248-10259"
   net.ipv4.tcp_fin_timeout: 15
   # Must equal what k3s's kube-proxy sets on every start: max(32768 * CPUs,
   # 131072), i.e. 1048576 on these 32-core boxes. A fixed lower value both

k8s/content-gateway/deployment.yaml.j2

@@ -0,0 +1,97 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: content-gateway
+spec:
+  replicas: 1
+  # hostNetwork binds the gateway port on the node; Recreate avoids a bind deadlock.
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: content-gateway
+  template:
+    metadata:
+      labels:
+        app: content-gateway
+    spec:
+      # hostNetwork: kubo's gateway is on the host's loopback.
+      hostNetwork: true
+      dnsPolicy: Default
+      terminationGracePeriodSeconds: 30
+
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65532     # the distroless image's nonroot user
+        runAsGroup: 65532
+        fsGroup: 65532       # lets Envoy read the mounted certificate
+        seccompProfile:
+          type: RuntimeDefault
+
+      containers:
+        - name: envoy
+          image: envoy
+          args:
+            - -c
+            - /etc/envoy/config/envoy.yaml
+            - --concurrency
+            - "2"
+            - --disable-hot-restart
+          ports:
+            - name: gw-https
+              containerPort: {{ content_gateway_port }}
+              protocol: TCP
+
+          # From the content node's share of the box (see k8s/content).
+          resources:
+            requests:
+              cpu: 50m
+              memory: 64Mi
+            limits:
+              memory: 512Mi
+
+          readinessProbe:
+            httpGet:
+              host: 127.0.0.1
+              path: /ready
+              port: {{ content_gateway_admin_port }}
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 3
+          livenessProbe:
+            httpGet:
+              host: 127.0.0.1
+              path: /ready
+              port: {{ content_gateway_admin_port }}
+            initialDelaySeconds: 15
+            periodSeconds: 20
+            timeoutSeconds: 3
+            failureThreshold: 6
+
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+
+          volumeMounts:
+            - name: config
+              mountPath: /etc/envoy/config
+              readOnly: true
+            # A whole-volume mount, never subPath: subPath mounts do not see
+            # Secret updates, so renewals would never reach Envoy.
+            - name: tls
+              mountPath: /etc/envoy/tls
+              readOnly: true
+
+      volumes:
+        - name: config
+          configMap:
+            name: content-gateway-envoy-config
+        # Not optional: roles/content waits for the Certificate before
+        # deploying this, so Envoy never starts without it.
+        - name: tls
+          secret:
+            secretName: content-gateway-tls
+            defaultMode: 0440

k8s/content-gateway/envoy.yaml.j2

@@ -0,0 +1,112 @@
+---
+# Envoy terminating TLS for the content node's read-only kubo gateway, the
+# origin behind the dist.ipfs.tech Worker: https://{{ content_gateway_hostname }}:{{ content_gateway_port }}.
+#
+# client -> :{{ content_gateway_port }} TLS (Let's Encrypt, cert-manager) -> 127.0.0.1:{{ content_gateway_kubo_port }} plain HTTP (kubo gateway)
+#
+# Only GET and HEAD under /ipfs/ get through; everything else (/ipns/, /api,
+# POST) is refused here, and kubo's RPC API is not on this port at all. kubo runs the
+# gateway with NoFetch, so it serves blocks this node already holds and never
+# fetches for a stranger. Range requests pass through untouched.
+
+admin:
+  # Host loopback only (hostNetwork).
+  address:
+    socket_address: { address: 127.0.0.1, port_value: {{ content_gateway_admin_port }} }
+
+# SDS needs a node id and cluster, even for file-based sources.
+node: { id: content-gateway, cluster: content-gateway }
+
+overload_manager:
+  resource_monitors:
+    - name: envoy.resource_monitors.global_downstream_max_connections
+      typed_config:
+        "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig
+        max_active_downstream_connections: 1000
+
+static_resources:
+  listeners:
+    - name: gw_https
+      address:
+        socket_address: { address: 0.0.0.0, port_value: {{ content_gateway_port }} }
+      additional_addresses:
+        - address:
+            socket_address: { address: "::", port_value: {{ content_gateway_port }} }
+      filter_chains:
+        - transport_socket_connect_timeout: 10s
+          transport_socket:
+            name: envoy.transport_sockets.tls
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
+              common_tls_context:
+                alpn_protocols: ["http/1.1"]
+                tls_params:
+                  tls_minimum_protocol_version: TLSv1_2
+                # File-based SDS: renewals are picked up without a restart.
+                tls_certificate_sds_secret_configs:
+                  - name: gw_cert
+                    sds_config:
+                      resource_api_version: V3
+                      path_config_source:
+                        path: /etc/envoy/config/sds.yaml
+                        watched_directory: { path: /etc/envoy/config }
+          filters:
+            - name: envoy.filters.network.http_connection_manager
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                stat_prefix: gw_https
+                codec_type: HTTP1
+                request_headers_timeout: 10s
+                # A tarball download can sit idle between reads on a slow
+                # client; the route timeout is off so large files finish.
+                stream_idle_timeout: 300s
+                common_http_protocol_options: { idle_timeout: 300s, max_headers_count: 50 }
+                max_request_headers_kb: 16
+                route_config:
+                  name: gateway
+                  virtual_hosts:
+                    - name: gateway
+                      domains: ["*"]
+                      routes:
+                        - match:
+                            path: /__gateway/health
+                          direct_response: { status: 200, body: { inline_string: "ok\n" } }
+                        - match:
+                            prefix: /ipfs/
+                            headers: [{ name: ":method", string_match: { safe_regex: { regex: "GET|HEAD" } } }]
+                          route: { cluster: kubo_gateway, timeout: 0s }
+                        - match: { prefix: / }
+                          direct_response:
+                            status: 403
+                            body:
+                              inline_string: "this origin serves GET and HEAD under /ipfs/ only.\n"
+                http_filters:
+                  - name: envoy.filters.http.router
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+                access_log:
+                  - name: envoy.access_loggers.stdout
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
+                      log_format:
+                        json_format:
+                          time: "%START_TIME%"
+                          client: "%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT%"
+                          method: "%REQ(:METHOD)%"
+                          path: "%REQ(:PATH)%"
+                          status: "%RESPONSE_CODE%"
+                          bytes: "%BYTES_SENT%"
+                          duration_ms: "%DURATION%"
+                          flags: "%RESPONSE_FLAGS%"
+
+  clusters:
+    - name: kubo_gateway
+      type: STATIC
+      connect_timeout: 2s
+      load_assignment:
+        cluster_name: kubo_gateway
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address: { address: 127.0.0.1, port_value: {{ content_gateway_kubo_port }} }

k8s/content-gateway/kustomization.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# TLS in front of the content node's read-only kubo gateway (k8s/content), in the
+# content namespace next to the Certificate whose Secret it mounts. A separate
+# Deployment from the content node, so proxy changes do not restart it.
+namespace: content
+
+resources:
+  - deployment.yaml
+
+labels:
+  - pairs:
+      app.kubernetes.io/name: content-gateway
+      app.kubernetes.io/part-of: ipfs-workers
+
+# Same Envoy as k8s/route-origin and k8s/bootstrap-wss: envoyproxy/envoy
+# distroless-v1.39.1 (2026-08-27), pinned by index digest.
+images:
+  - name: envoy
+    newName: envoyproxy/envoy
+    digest: sha256:eb2c01c13125d1629637cb4e4cce7207009fb7cc2c8027f9742758549d15b6f4
+
+configMapGenerator:
+  - name: content-gateway-envoy-config
+    behavior: create
+    files:
+      - envoy.yaml
+      - sds.yaml
+
+# The certificate Secret (content-gateway-tls) is NOT here: cert-manager writes it
+# from the Certificate that roles/content creates per box.

k8s/content-gateway/sds.yaml

@@ -0,0 +1,12 @@
+---
+# The gateway origin's certificate for envoy.yaml, as a file-based SDS resource.
+# The files come from the content-gateway-tls Secret that cert-manager writes. Envoy
+# watches the directory: kubelet swaps a renewed Secret in with an atomic
+# rename there.
+resources:
+  - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret
+    name: gw_cert
+    tls_certificate:
+      certificate_chain: { filename: /etc/envoy/tls/tls.crt }
+      private_key: { filename: /etc/envoy/tls/tls.key }
+      watched_directory: { path: /etc/envoy/tls }

k8s/content-pin/envoy.yaml.j2

@@ -11,6 +11,12 @@
 # and refuses the rest (DELETE /pins/<id>, and POST /pins/<id>, which replaces
 # a pin and so unpins the old CID) with 403. Pins are removed by the operators
 # with ipfs-cluster-ctl. The client address is in the access log.
+#
+# One more route, for `ipfs-cluster-ctl add` (what ipfs-deploy-action runs):
+# POST /add goes to ipfs-cluster's REST API on loopback {{ content_cluster_api_port }}, which has no auth of
+# its own. Envoy checks HTTP basic auth against the same users instead
+# ({{ content_pin_basic_users | length }} here, as SHA1 hashes computed on the controller). Nothing else of
+# the REST API is reachable: it can delete and replace pins and manage peers.
 
 admin:
   # Host loopback only (hostNetwork).
@@ -59,10 +65,10 @@ static_resources:
                 "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
                 stat_prefix: pin_https
                 codec_type: HTTP1
-                # Requests are small and answered quickly: POST /pins queues the
-                # pin and returns; clients then poll with GET.
+                # POST /pins queues the pin and returns; clients then poll with GET.
+                # POST /add streams a CAR, so the idle limit allows a slow upload.
                 request_headers_timeout: 10s
-                stream_idle_timeout: 60s
+                stream_idle_timeout: 300s
                 common_http_protocol_options: { idle_timeout: 300s, max_headers_count: 50 }
                 max_request_headers_kb: 16
                 route_config:
@@ -72,24 +78,54 @@ static_resources:
                       domains: ["*"]
                       routes:
                         # path matches without the query string.
+                        # CAR uploads: large bodies, so no route timeout.
+                        - match:
+                            path: /add
+                            headers: [{ name: ":method", string_match: { exact: POST } }]
+                          route: { cluster: clusterrest, timeout: 0s, idle_timeout: 300s }
                         - match:
                             path: /pins
                             headers: [{ name: ":method", string_match: { exact: POST } }]
                           route: { cluster: pinsvcapi, timeout: 60s }
+                          typed_per_filter_config:
+                            envoy.filters.http.basic_auth:
+                              "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig
+                              disabled: true
                         - match:
                             path: /token
                             headers: [{ name: ":method", string_match: { exact: POST } }]
                           route: { cluster: pinsvcapi, timeout: 60s }
+                          typed_per_filter_config:
+                            envoy.filters.http.basic_auth:
+                              "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig
+                              disabled: true
                         - match:
                             prefix: /
                             headers: [{ name: ":method", string_match: { exact: GET } }]
                           route: { cluster: pinsvcapi, timeout: 60s }
+                          typed_per_filter_config:
+                            envoy.filters.http.basic_auth:
+                              "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig
+                              disabled: true
                         - match: { prefix: / }
+                          typed_per_filter_config:
+                            envoy.filters.http.basic_auth:
+                              "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig
+                              disabled: true
                           direct_response:
                             status: 403
                             body:
                               inline_string: "pin.ipni.io only adds pins; ask the operators to remove or replace one.\n"
                 http_filters:
+                  # Applies to POST /add only: every other route disables it
+                  # (ipfs-cluster authenticates those itself, with tokens too).
+                  - name: envoy.filters.http.basic_auth
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.basic_auth.v3.BasicAuth
+                      users:
+                        inline_string: |
+                          {{ content_pin_basic_users | join('\n') | indent(26) }}
+                      forward_username_header: x-pin-user
                   - name: envoy.filters.http.router
                     typed_config:
                       "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
@@ -118,3 +154,13 @@ static_resources:
               - endpoint:
                   address:
                     socket_address: { address: 127.0.0.1, port_value: {{ content_pinsvc_port }} }
+    - name: clusterrest
+      type: STATIC
+      connect_timeout: 2s
+      load_assignment:
+        cluster_name: clusterrest
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address: { address: 127.0.0.1, port_value: {{ content_cluster_api_port }} }

k8s/content/pvc.yaml

@@ -5,6 +5,12 @@
 # local-path ignores it, and kubo's Datastore.StorageMax is only the automatic
 # GC threshold (GC is not enabled and never removes pinned blocks). What is
 # pinned - on every box - is the only bound; check sizes before pinning.
+#
+# The 100Gi is a label, not a limit, and it is not raised for larger pins: the
+# storage class (k3s's local-storage addon) does not allow volume expansion, so
+# Kubernetes rejects a size change on the bound claim and the apply would fail.
+# The real limit is the NVMe under /data (about 820 GB free per box on
+# 2026-10-21). dist.ipfs.tech (~128 GB) is pinned past the label on purpose.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata: