diff --git a/ansible.cfg b/ansible.cfg index c8ce70e..ebcf53e 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -3,7 +3,8 @@ inventory = inventory/hosts.yml roles_path = roles host_key_checking = True interpreter_python = auto_silent -stdout_callback = yaml +stdout_callback = default +callback_result_format = yaml callbacks_enabled = profile_tasks retry_files_enabled = False forks = 3 diff --git a/group_vars/managed_boxes/main.yml b/group_vars/managed_boxes/main.yml index a0d61f5..9ef0a64 100644 --- a/group_vars/managed_boxes/main.yml +++ b/group_vars/managed_boxes/main.yml @@ -146,10 +146,12 @@ sysctl_tuning: # 9096 content ipfs-cluster swarm # 9097 content ipfs-cluster Pinning Service API (loopback) # 9443 content pinning API over TLS (Envoy) 9904 its Envoy admin (loopback) + # 2053 content gateway origin over TLS (Envoy) 8181 kubo gateway (loopback) + # 9905 the gateway Envoy's admin (loopback) # 9901 route-origin Envoy admin 9902 bootstrap-wss Envoy admin # 9903 bootstrap-b's bootstrap-wss Envoy admin # 10010 containerd 10248-10259 kubelet/kube-proxy/k3s - net.ipv4.ip_local_reserved_ports: "443,4001-4002,4004-4007,4011-4012,4101,4317-4318,4443,4453,5011-5012,5021,6443-6444,8000,8190-8193,8443,8472,8888,9094-9097,9100,9443,9800-9801,9810-9812,9901-9904,10010,10248-10259" + net.ipv4.ip_local_reserved_ports: "443,2053,4001-4002,4004-4007,4011-4012,4101,4317-4318,4443,4453,5011-5012,5021,6443-6444,8000,8181,8190-8193,8443,8472,8888,9094-9097,9100,9443,9800-9801,9810-9812,9901-9905,10010,10248-10259" net.ipv4.tcp_fin_timeout: 15 # Must equal what k3s's kube-proxy sets on every start: max(32768 * CPUs, # 131072), i.e. 1048576 on these 32-core boxes. A fixed lower value both diff --git a/k8s/content-gateway/deployment.yaml.j2 b/k8s/content-gateway/deployment.yaml.j2 new file mode 100644 index 0000000..5357cca --- /dev/null +++ b/k8s/content-gateway/deployment.yaml.j2 @@ -0,0 +1,97 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: content-gateway +spec: + replicas: 1 + # hostNetwork binds the gateway port on the node; Recreate avoids a bind deadlock. + strategy: + type: Recreate + selector: + matchLabels: + app: content-gateway + template: + metadata: + labels: + app: content-gateway + spec: + # hostNetwork: kubo's gateway is on the host's loopback. + hostNetwork: true + dnsPolicy: Default + terminationGracePeriodSeconds: 30 + + securityContext: + runAsNonRoot: true + runAsUser: 65532 # the distroless image's nonroot user + runAsGroup: 65532 + fsGroup: 65532 # lets Envoy read the mounted certificate + seccompProfile: + type: RuntimeDefault + + containers: + - name: envoy + image: envoy + args: + - -c + - /etc/envoy/config/envoy.yaml + - --concurrency + - "2" + - --disable-hot-restart + ports: + - name: gw-https + containerPort: {{ content_gateway_port }} + protocol: TCP + + # From the content node's share of the box (see k8s/content). + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 512Mi + + readinessProbe: + httpGet: + host: 127.0.0.1 + path: /ready + port: {{ content_gateway_admin_port }} + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 3 + livenessProbe: + httpGet: + host: 127.0.0.1 + path: /ready + port: {{ content_gateway_admin_port }} + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 6 + + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + + volumeMounts: + - name: config + mountPath: /etc/envoy/config + readOnly: true + # A whole-volume mount, never subPath: subPath mounts do not see + # Secret updates, so renewals would never reach Envoy. + - name: tls + mountPath: /etc/envoy/tls + readOnly: true + + volumes: + - name: config + configMap: + name: content-gateway-envoy-config + # Not optional: roles/content waits for the Certificate before + # deploying this, so Envoy never starts without it. + - name: tls + secret: + secretName: content-gateway-tls + defaultMode: 0440 diff --git a/k8s/content-gateway/envoy.yaml.j2 b/k8s/content-gateway/envoy.yaml.j2 new file mode 100644 index 0000000..7472ee4 --- /dev/null +++ b/k8s/content-gateway/envoy.yaml.j2 @@ -0,0 +1,112 @@ +--- +# Envoy terminating TLS for the content node's read-only kubo gateway, the +# origin behind the dist.ipfs.tech Worker: https://{{ content_gateway_hostname }}:{{ content_gateway_port }}. +# +# client -> :{{ content_gateway_port }} TLS (Let's Encrypt, cert-manager) -> 127.0.0.1:{{ content_gateway_kubo_port }} plain HTTP (kubo gateway) +# +# Only GET and HEAD under /ipfs/ get through; everything else (/ipns/, /api, +# POST) is refused here, and kubo's RPC API is not on this port at all. kubo runs the +# gateway with NoFetch, so it serves blocks this node already holds and never +# fetches for a stranger. Range requests pass through untouched. + +admin: + # Host loopback only (hostNetwork). + address: + socket_address: { address: 127.0.0.1, port_value: {{ content_gateway_admin_port }} } + +# SDS needs a node id and cluster, even for file-based sources. +node: { id: content-gateway, cluster: content-gateway } + +overload_manager: + resource_monitors: + - name: envoy.resource_monitors.global_downstream_max_connections + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 1000 + +static_resources: + listeners: + - name: gw_https + address: + socket_address: { address: 0.0.0.0, port_value: {{ content_gateway_port }} } + additional_addresses: + - address: + socket_address: { address: "::", port_value: {{ content_gateway_port }} } + filter_chains: + - transport_socket_connect_timeout: 10s + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext + common_tls_context: + alpn_protocols: ["http/1.1"] + tls_params: + tls_minimum_protocol_version: TLSv1_2 + # File-based SDS: renewals are picked up without a restart. + tls_certificate_sds_secret_configs: + - name: gw_cert + sds_config: + resource_api_version: V3 + path_config_source: + path: /etc/envoy/config/sds.yaml + watched_directory: { path: /etc/envoy/config } + filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: gw_https + codec_type: HTTP1 + request_headers_timeout: 10s + # A tarball download can sit idle between reads on a slow + # client; the route timeout is off so large files finish. + stream_idle_timeout: 300s + common_http_protocol_options: { idle_timeout: 300s, max_headers_count: 50 } + max_request_headers_kb: 16 + route_config: + name: gateway + virtual_hosts: + - name: gateway + domains: ["*"] + routes: + - match: + path: /__gateway/health + direct_response: { status: 200, body: { inline_string: "ok\n" } } + - match: + prefix: /ipfs/ + headers: [{ name: ":method", string_match: { safe_regex: { regex: "GET|HEAD" } } }] + route: { cluster: kubo_gateway, timeout: 0s } + - match: { prefix: / } + direct_response: + status: 403 + body: + inline_string: "this origin serves GET and HEAD under /ipfs/ only.\n" + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + access_log: + - name: envoy.access_loggers.stdout + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog + log_format: + json_format: + time: "%START_TIME%" + client: "%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT%" + method: "%REQ(:METHOD)%" + path: "%REQ(:PATH)%" + status: "%RESPONSE_CODE%" + bytes: "%BYTES_SENT%" + duration_ms: "%DURATION%" + flags: "%RESPONSE_FLAGS%" + + clusters: + - name: kubo_gateway + type: STATIC + connect_timeout: 2s + load_assignment: + cluster_name: kubo_gateway + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: { address: 127.0.0.1, port_value: {{ content_gateway_kubo_port }} } diff --git a/k8s/content-gateway/kustomization.yaml b/k8s/content-gateway/kustomization.yaml new file mode 100644 index 0000000..7b6d50a --- /dev/null +++ b/k8s/content-gateway/kustomization.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# TLS in front of the content node's read-only kubo gateway (k8s/content), in the +# content namespace next to the Certificate whose Secret it mounts. A separate +# Deployment from the content node, so proxy changes do not restart it. +namespace: content + +resources: + - deployment.yaml + +labels: + - pairs: + app.kubernetes.io/name: content-gateway + app.kubernetes.io/part-of: ipfs-workers + +# Same Envoy as k8s/route-origin and k8s/bootstrap-wss: envoyproxy/envoy +# distroless-v1.39.1 (2026-08-27), pinned by index digest. +images: + - name: envoy + newName: envoyproxy/envoy + digest: sha256:eb2c01c13125d1629637cb4e4cce7207009fb7cc2c8027f9742758549d15b6f4 + +configMapGenerator: + - name: content-gateway-envoy-config + behavior: create + files: + - envoy.yaml + - sds.yaml + +# The certificate Secret (content-gateway-tls) is NOT here: cert-manager writes it +# from the Certificate that roles/content creates per box. diff --git a/k8s/content-gateway/sds.yaml b/k8s/content-gateway/sds.yaml new file mode 100644 index 0000000..ecd8a6a --- /dev/null +++ b/k8s/content-gateway/sds.yaml @@ -0,0 +1,12 @@ +--- +# The gateway origin's certificate for envoy.yaml, as a file-based SDS resource. +# The files come from the content-gateway-tls Secret that cert-manager writes. Envoy +# watches the directory: kubelet swaps a renewed Secret in with an atomic +# rename there. +resources: + - "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret + name: gw_cert + tls_certificate: + certificate_chain: { filename: /etc/envoy/tls/tls.crt } + private_key: { filename: /etc/envoy/tls/tls.key } + watched_directory: { path: /etc/envoy/tls } diff --git a/k8s/content-pin/envoy.yaml.j2 b/k8s/content-pin/envoy.yaml.j2 index 6e51bbc..1f8dd39 100644 --- a/k8s/content-pin/envoy.yaml.j2 +++ b/k8s/content-pin/envoy.yaml.j2 @@ -11,6 +11,12 @@ # and refuses the rest (DELETE /pins/, and POST /pins/, which replaces # a pin and so unpins the old CID) with 403. Pins are removed by the operators # with ipfs-cluster-ctl. The client address is in the access log. +# +# One more route, for `ipfs-cluster-ctl add` (what ipfs-deploy-action runs): +# POST /add goes to ipfs-cluster's REST API on loopback {{ content_cluster_api_port }}, which has no auth of +# its own. Envoy checks HTTP basic auth against the same users instead +# ({{ content_pin_basic_users | length }} here, as SHA1 hashes computed on the controller). Nothing else of +# the REST API is reachable: it can delete and replace pins and manage peers. admin: # Host loopback only (hostNetwork). @@ -59,10 +65,10 @@ static_resources: "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager stat_prefix: pin_https codec_type: HTTP1 - # Requests are small and answered quickly: POST /pins queues the - # pin and returns; clients then poll with GET. + # POST /pins queues the pin and returns; clients then poll with GET. + # POST /add streams a CAR, so the idle limit allows a slow upload. request_headers_timeout: 10s - stream_idle_timeout: 60s + stream_idle_timeout: 300s common_http_protocol_options: { idle_timeout: 300s, max_headers_count: 50 } max_request_headers_kb: 16 route_config: @@ -72,24 +78,54 @@ static_resources: domains: ["*"] routes: # path matches without the query string. + # CAR uploads: large bodies, so no route timeout. + - match: + path: /add + headers: [{ name: ":method", string_match: { exact: POST } }] + route: { cluster: clusterrest, timeout: 0s, idle_timeout: 300s } - match: path: /pins headers: [{ name: ":method", string_match: { exact: POST } }] route: { cluster: pinsvcapi, timeout: 60s } + typed_per_filter_config: + envoy.filters.http.basic_auth: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true - match: path: /token headers: [{ name: ":method", string_match: { exact: POST } }] route: { cluster: pinsvcapi, timeout: 60s } + typed_per_filter_config: + envoy.filters.http.basic_auth: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true - match: prefix: / headers: [{ name: ":method", string_match: { exact: GET } }] route: { cluster: pinsvcapi, timeout: 60s } + typed_per_filter_config: + envoy.filters.http.basic_auth: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true - match: { prefix: / } + typed_per_filter_config: + envoy.filters.http.basic_auth: + "@type": type.googleapis.com/envoy.config.route.v3.FilterConfig + disabled: true direct_response: status: 403 body: inline_string: "pin.ipni.io only adds pins; ask the operators to remove or replace one.\n" http_filters: + # Applies to POST /add only: every other route disables it + # (ipfs-cluster authenticates those itself, with tokens too). + - name: envoy.filters.http.basic_auth + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.basic_auth.v3.BasicAuth + users: + inline_string: | + {{ content_pin_basic_users | join('\n') | indent(26) }} + forward_username_header: x-pin-user - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router @@ -118,3 +154,13 @@ static_resources: - endpoint: address: socket_address: { address: 127.0.0.1, port_value: {{ content_pinsvc_port }} } + - name: clusterrest + type: STATIC + connect_timeout: 2s + load_assignment: + cluster_name: clusterrest + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: { address: 127.0.0.1, port_value: {{ content_cluster_api_port }} } diff --git a/k8s/content/pvc.yaml b/k8s/content/pvc.yaml index 3f59580..206ea24 100644 --- a/k8s/content/pvc.yaml +++ b/k8s/content/pvc.yaml @@ -5,6 +5,12 @@ # local-path ignores it, and kubo's Datastore.StorageMax is only the automatic # GC threshold (GC is not enabled and never removes pinned blocks). What is # pinned - on every box - is the only bound; check sizes before pinning. +# +# The 100Gi is a label, not a limit, and it is not raised for larger pins: the +# storage class (k3s's local-storage addon) does not allow volume expansion, so +# Kubernetes rejects a size change on the bound claim and the apply would fail. +# The real limit is the NVMe under /data (about 820 GB free per box on +# 2026-10-21). dist.ipfs.tech (~128 GB) is pinned past the label on purpose. apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/roles/content/defaults/main.yml b/roles/content/defaults/main.yml index 7ffd613..41ad900 100644 --- a/roles/content/defaults/main.yml +++ b/roles/content/defaults/main.yml @@ -27,6 +27,22 @@ content_pin_issuer: letsencrypt-prod # README), so this only catches a hand-typed one. content_pin_min_password_length: 32 +# --------------------------------------------------------------------------- +# Read-only HTTP gateway for dist.ipfs.tech: kubo's gateway on loopback +# (NoFetch: it serves only blocks this node holds), published by Envoy +# (k8s/content-gateway) over TLS on every box under one name, as the origin +# behind the dist.ipfs.tech Cloudflare Worker. The port is one Cloudflare +# can reach (8443 is taken by another workload on lith-1). The certificate +# comes from DNS-01 in the ipfs.io zone, with its own issuers and token +# (vault_cloudflare_dns_token_ipfs_io: the ipni.io token cannot see ipfs.io). +# Must match k8s/content-gateway and files/kubo-config.json. +# --------------------------------------------------------------------------- +content_gateway_hostname: dist-origin.ipfs.io +content_gateway_port: 2053 # public, TLS +content_gateway_kubo_port: 8181 # loopback (kubo gateway, behind Envoy) +content_gateway_admin_port: 9905 # loopback, the gateway Envoy's admin +content_gateway_issuer: letsencrypt-ipfs-io-prod + # Public bootstrap peers the content kubo dials, after this project's own # bootstrappers (/dnsaddr/bootstrap.ipni.io, one entry per bootstrapper # instance, added by the role). The same public list the bootstrappers use @@ -67,8 +83,11 @@ content_size_gate_gb: 5 # Explicit per-entry override of the size gate, e.g. # -e '{"content_size_gate_allow": ["dist.ipfs.tech"]}' # dist.ipfs.tech (binary distributions, not a website) estimated ~128 GB on -# 2026-09-15, more than the 100 GiB content-repo claim: resize first. -content_size_gate_allow: [] +# 2026-09-15, more than the 100 GiB content-repo claim, which is only a label +# (see k8s/content/pvc.yaml): the NVMe is the bound. Allowed by name, and only +# this entry; it is pinned directly with ipfs-cluster-ctl, not through the +# playbook's pin timeout. +content_size_gate_allow: [dist.ipfs.tech] # Entries in the pinset that are not to be pinned, by name. Never pinned, and # reported by name on every run. Empty: everything in pinset.yml is in scope. diff --git a/roles/content/files/kubo-config.json b/roles/content/files/kubo-config.json index 8693f3b..914b5ee 100644 --- a/roles/content/files/kubo-config.json +++ b/roles/content/files/kubo-config.json @@ -6,7 +6,7 @@ "API": "/ip4/127.0.0.1/tcp/5021", "Announce": [], "AppendAnnounce": [], - "Gateway": null, + "Gateway": "/ip4/127.0.0.1/tcp/8181", "NoAnnounce": [ "/ip4/10.0.0.0/ipcidr/8", "/ip4/100.64.0.0/ipcidr/10", @@ -102,8 +102,8 @@ "DisableHTMLErrors": null, "ExposeRoutingAPI": null, "HTTPHeaders": {}, - "NoDNSLink": false, - "NoFetch": false, + "NoDNSLink": true, + "NoFetch": true, "PublicGateways": null, "RootRedirect": "" }, diff --git a/roles/content/tasks/gateway.yml b/roles/content/tasks/gateway.yml new file mode 100644 index 0000000..356458a --- /dev/null +++ b/roles/content/tasks/gateway.yml @@ -0,0 +1,149 @@ +--- +# Read-only HTTP gateway: kubo's gateway (loopback {{ content_gateway_kubo_port }}, NoFetch) published by +# Envoy (k8s/content-gateway) over TLS as +# https://{{ content_gateway_hostname }}:{{ content_gateway_port }} on every box. It is the origin +# behind the dist.ipfs.tech Worker. The certificate comes from cert-manager by +# DNS-01 in the ipfs.io zone, with its own token: the ipni.io token used by +# pin_api.yml cannot see that zone. cert-manager is installed by bootstrap.yml. + +- name: Fail unless cert-manager and the ipfs.io token are available + ansible.builtin.assert: + that: + - content_cm_crd.stdout | length > 0 + - vault_cloudflare_dns_token_ipfs_io is defined and vault_cloudflare_dns_token_ipfs_io | length > 0 + fail_msg: >- + The gateway certificate needs cert-manager (checked by pin_api.yml) and + vault_cloudflare_dns_token_ipfs_io (group_vars/ipfs_nodes/vault.yml, + Zone:DNS:Edit on ipfs.io). + quiet: true + +- name: Create the gateway certificate + when: not ansible_check_mode or (content_ns.stdout | length > 0) + block: + - name: Install the ipfs.io Cloudflare API token for DNS-01 challenges + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: cloudflare-api-token-ipfs-io + namespace: content + data: + api-token: "{{ vault_cloudflare_dns_token_ipfs_io | b64encode }}" + no_log: true + + - name: Create the Let's Encrypt issuers for ipfs.io + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: cert-manager.io/v1 + kind: Issuer + metadata: + name: "{{ item.name }}" + namespace: content + spec: + acme: + server: "{{ item.server }}" + privateKeySecretRef: { name: "{{ item.name }}-account-key" } + solvers: + - selector: { dnsZones: ["{{ content_gateway_hostname | regex_replace('^[^.]+[.]', '') }}"] } + dns01: + cloudflare: + apiTokenSecretRef: { name: cloudflare-api-token-ipfs-io, key: api-token } + loop: + - { name: letsencrypt-ipfs-io-prod, server: "https://acme-v02.api.letsencrypt.org/directory" } + - { name: letsencrypt-ipfs-io-staging, server: "https://acme-staging-v02.api.letsencrypt.org/directory" } + loop_control: + label: "{{ item.name }}" + + - name: Request the gateway certificate + kubernetes.core.k8s: + kubeconfig: /etc/rancher/k3s/k3s.yaml + state: present + definition: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: content-gateway + namespace: content + spec: + secretName: content-gateway-tls + dnsNames: ["{{ content_gateway_hostname }}"] + privateKey: { algorithm: ECDSA, size: 256, rotationPolicy: Always } + issuerRef: { group: cert-manager.io, kind: Issuer, name: "{{ content_gateway_issuer }}" } + + - name: Wait for the gateway certificate to be issued + ansible.builtin.command: + argv: + - k3s + - kubectl + - --namespace=content + - get + - certificate/content-gateway + - secret/content-gateway-tls + - --ignore-not-found + - -o + - >- + jsonpath={range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{.metadata.annotations.cert-manager\.io/issuer-name}{" "}{end} + register: content_gateway_cert_state + changed_when: false + retries: 60 + delay: 10 + until: content_gateway_cert_state.stdout.split() == ['True', content_gateway_issuer] + failed_when: false + when: not ansible_check_mode + + - name: Show why the gateway certificate is not issued + ansible.builtin.command: + argv: [k3s, kubectl, --namespace=content, describe, certificate/content-gateway, + certificaterequests, orders.acme.cert-manager.io, challenges.acme.cert-manager.io] + changed_when: false + failed_when: true + when: + - not ansible_check_mode + - content_gateway_cert_state.stdout.split() != ['True', content_gateway_issuer] + + - name: Deploy the gateway proxy + ansible.builtin.include_role: + name: kustomize_apply + vars: + kustomize_app: content-gateway + kustomize_namespace: content + kustomize_deployment: content-gateway + +# From the controller, as a client sees it: the certificate verifies for the +# published name, and the proxy answers health and refuses what it should. +# --resolve pins the request to this box, whatever DNS says. +- name: Check the gateway origin answers health over TLS + ansible.builtin.command: + argv: [curl, -sS, -o, /dev/null, -w, "%{http_code}", --max-time, "20", + --resolve, "{{ content_gateway_hostname }}:{{ content_gateway_port }}:{{ ansible_host }}", + "https://{{ content_gateway_hostname }}:{{ content_gateway_port }}/__gateway/health"] + delegate_to: localhost + become: false + register: content_gateway_health + changed_when: false + failed_when: content_gateway_health.stdout != '200' + when: not ansible_check_mode + +- name: Check the gateway origin refuses everything but GET and HEAD under /ipfs/ + ansible.builtin.command: + argv: [curl, -sS, -o, /dev/null, -w, "%{http_code}", --max-time, "20", -X, "{{ item.method }}", + --resolve, "{{ content_gateway_hostname }}:{{ content_gateway_port }}:{{ ansible_host }}", + "https://{{ content_gateway_hostname }}:{{ content_gateway_port }}{{ item.path }}"] + delegate_to: localhost + become: false + register: content_gateway_refused + changed_when: false + failed_when: content_gateway_refused.stdout != '403' + loop: + - { method: POST, path: /ipfs/bafkqaaa } + - { method: GET, path: /ipns/dist.ipfs.tech } + - { method: GET, path: /api/v0/version } + loop_control: + label: "{{ item.method }} {{ item.path }}" + when: not ansible_check_mode diff --git a/roles/content/tasks/main.yml b/roles/content/tasks/main.yml index 715b5b7..4fe3142 100644 --- a/roles/content/tasks/main.yml +++ b/roles/content/tasks/main.yml @@ -121,6 +121,13 @@ proto: tcp comment: content pinning API +- name: Allow gateway origin clients (read-only kubo gateway over TLS, via Envoy) + community.general.ufw: + rule: allow + port: "{{ content_gateway_port }}" + proto: tcp + comment: content gateway origin + # --------------------------------------------------------------------------- # Per-box settings, as two Secrets: # kubo-config the full kubo config: files/kubo-config.json (committed, @@ -137,7 +144,9 @@ # bootstrapper). # AutoConf and AutoTLS off, no delegated routers/publishers, relay service # off - no dependency on Shipyard-operated services. -# Addresses.Gateway null - no HTTP gateway. API on 127.0.0.1:5021, swarm 4101. +# Addresses.Gateway 127.0.0.1:8181, read-only: NoFetch (serves only blocks this +# node holds) and NoDNSLink, published only through k8s/content-gateway. +# API on 127.0.0.1:5021, swarm 4101. # Datastore.StorageMax 90GB - only the GC threshold (GC is not enabled). # Swarm.ConnMgr 600/200 - what this node really uses (about 300 conns). # Resource manager: System.ConnsInbound 65536, set in @@ -389,6 +398,9 @@ - name: Publish the pinning API over TLS ansible.builtin.include_tasks: pin_api.yml +- name: Publish the read-only gateway over TLS + ansible.builtin.include_tasks: gateway.yml + # --------------------------------------------------------------------------- # Pinset. roles/content/vars/pinset.yml is the source of truth. It is read # fresh on every run and may be mid-edit (recovered entries gaining CIDs), so diff --git a/roles/content/tasks/pin_api.yml b/roles/content/tasks/pin_api.yml index e85467f..c511c70 100644 --- a/roles/content/tasks/pin_api.yml +++ b/roles/content/tasks/pin_api.yml @@ -117,6 +117,36 @@ - not ansible_check_mode - content_pin_cert_state.stdout.split() != ['True', content_pin_issuer] + # Envoy's basic auth takes htpasswd SHA1 entries ("name:{SHA}"). + # They are computed here, on the controller, from the same vault users + # ipfs-cluster authenticates its own routes with, so the two stay in step. + # Sorted, so an unchanged user list renders an unchanged config and does + # not restart the proxy. The password goes in the environment, never on a + # command line. + - name: Hash the pinning users for Envoy's basic auth on POST /add + ansible.builtin.shell: set -o pipefail; printf %s "$PW" | openssl dgst -sha1 -binary | base64 + args: + executable: /bin/bash + environment: + PW: "{{ item.value }}" + delegate_to: localhost + become: false + loop: "{{ vault_content_pin_users | dict2items | sort(attribute='key') }}" + loop_control: + label: "{{ item.key }}" + register: content_pin_hashes + changed_when: false + check_mode: false + no_log: true + + - name: Collect the htpasswd entries + ansible.builtin.set_fact: + content_pin_basic_users: >- + {{ content_pin_hashes.results | map(attribute='item.key') + | zip(content_pin_hashes.results | map(attribute='stdout')) + | map('join', ':{SHA}') | list }} + no_log: true + - name: Deploy the pinning API proxy ansible.builtin.include_role: name: kustomize_apply @@ -157,6 +187,43 @@ failed_when: content_pin_delete.stdout != '403' when: not ansible_check_mode +# POST /add is the one route of ipfs-cluster's REST API that is published +# (`ipfs-cluster-ctl add`, used by ipfs-deploy-action), authenticated by Envoy. +# Without credentials, or with wrong ones, it must be refused before ipfs-cluster +# sees it; no body is sent, so nothing is ever added. +- name: Check POST /add needs credentials + ansible.builtin.command: + argv: [curl, -sS, -o, /dev/null, -w, "%{http_code}", --max-time, "20", -X, POST, + "--resolve", "{{ content_pin_hostname }}:{{ content_pin_port }}:{{ ansible_host }}", + "https://{{ content_pin_hostname }}:{{ content_pin_port }}/add"] + delegate_to: localhost + become: false + register: content_pin_add_anon + changed_when: false + failed_when: content_pin_add_anon.stdout != '401' + when: not ansible_check_mode + +# Everything but POST /add stays refused. (GETs go to the Pinning Service API +# on 9097, which has no REST API endpoints, so they cannot reach it either.) +- name: Check the rest of the REST API is not published + ansible.builtin.command: + argv: [curl, -sS, -o, /dev/null, -w, "%{http_code}", --max-time, "20", -X, "{{ item.method }}", + "--resolve", "{{ content_pin_hostname }}:{{ content_pin_port }}:{{ ansible_host }}", + "https://{{ content_pin_hostname }}:{{ content_pin_port }}{{ item.path }}"] + delegate_to: localhost + become: false + register: content_pin_rest_refused + changed_when: false + failed_when: content_pin_rest_refused.stdout != '403' + loop: + - { method: PUT, path: /add } + - { method: DELETE, path: /add } + - { method: POST, path: /peers } + - { method: POST, path: /pins/bafkqaaa } + loop_control: + label: "{{ item.method }} {{ item.path }}" + when: not ansible_check_mode + # The credentials go to curl on stdin (-K -), never on its command line. - name: Check the first pinning user can list pins vars: diff --git a/scripts/render-manifests.yml b/scripts/render-manifests.yml index 91d98cb..781336a 100644 --- a/scripts/render-manifests.yml +++ b/scripts/render-manifests.yml @@ -34,6 +34,10 @@ # so `-e dest=rendered` would report success and write somewhere else # entirely. Anchor it to the repo root and say so. render_dest: "{{ dest if dest.startswith('/') else repo_root ~ '/' ~ dest }}" + # Envoy's basic-auth users for POST /add on the pinning API. A deploy hashes + # them from the vault (roles/content/tasks/pin_api.yml); one placeholder + # entry here, in the same htpasswd SHA1 form. + content_pin_basic_users: ["render:{SHA}AAAAAAAAAAAAAAAAAAAAAAAAAAA="] tasks: - name: Find the manifest directories ansible.builtin.find: