From e9713c0d8e19e147c42199ed602a71be12632a3f Mon Sep 17 00:00:00 2001 From: Kazuhiro NISHIYAMA Date: Thu, 6 Aug 2026 18:49:09 +0900 Subject: [PATCH 1/9] Move docs-dev to S3 backends as the docs.ruby-lang.org migration canary Add a public-read docs.r-l.o bucket (us-east-1, the cheapest tier; every request reaches it through Fastly so the client distance does not matter) and route docs-dev to it, with the doxygen pages still proxied from the rubyci bucket and docs-origin left as the fallback for unflagged paths. The custom VCL takes over everything the nginx origin does today: the five redirect rules, per-version Surrogate-Key and Cache-Control (the purge key scheme is unchanged), directory index.html, text/markdown for the .md twins, and the 50x page. On top of that it resolves /ja/latest and /ja/master through an edge dictionary instead of filesystem symlinks, temporarily redirects the unreleased version (en/4.1 and ja/4.1 to the master alias, 302, keyed off the same dictionary so a release only updates dictionary values), and serves Markdown content negotiation on /ja/ pages (Accept: text/markdown fetches the .md twin, falling back to .html where no twin exists). Co-Authored-By: Claude Fable 5 --- cdn/docs_dev.tf | 112 ++++++++++++++++++++++- cdn/s3.tf | 84 +++++++++++++++++ cdn/vcl/docs_dev.vcl | 212 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 403 insertions(+), 5 deletions(-) diff --git a/cdn/docs_dev.tf b/cdn/docs_dev.tf index f506c53..166d9c6 100644 --- a/cdn/docs_dev.tf +++ b/cdn/docs_dev.tf @@ -1,6 +1,16 @@ +# The S3-backend canary for docs.ruby-lang.org. The production service keeps +# pointing at docs-origin until this proves out, then docs.tf adopts the same +# shape. Backend selection goes through request_conditions on a header flag the +# custom VCL sets before #FASTLY recv (assigning req.backend in VCL would +# bypass shielding, see cache.tf); the docs-origin backend stays as the +# fallback for unflagged requests so paths can be moved over one at a time. resource "fastly_service_vcl" "docs_dev" { activate = true stage = false + # A shielded fetch needs a Host that is a domain of this service, so the + # bucket endpoint doubles as default_host and as a domain below, the same + # arrangement as cache.tf. The other backends override_host instead. + default_host = "docs.r-l.o.s3.amazonaws.com" default_ttl = 60 http3 = true name = "docs-dev.ruby-lang.org" @@ -19,8 +29,10 @@ resource "fastly_service_vcl" "docs_dev" { max_lifetime = 0 max_use = 0 name = "docs origin server" + override_host = "docs.ruby-lang.org" port = 443 prefer_ipv6 = false + request_condition = "backend-is-origin" shield = "tyo-tokyo-jp" ssl_cert_hostname = "docs-origin.ruby-lang.org" ssl_check_cert = true @@ -28,6 +40,77 @@ resource "fastly_service_vcl" "docs_dev" { weight = 100 } + # The docs bucket (aws_s3_bucket.docs in s3.tf): public/ root files, en and + # ja. us-east-1, so shielded near it like the cache service's S3 backend. + backend { + address = "s3.amazonaws.com" + auto_loadbalance = false + between_bytes_timeout = 10000 + connect_timeout = 1000 + error_threshold = 0 + first_byte_timeout = 15000 + keepalive_time = 0 + max_conn = 200 + max_lifetime = 0 + max_use = 0 + name = "s3-docs" + port = 443 + prefer_ipv6 = false + request_condition = "backend-is-docs-s3" + shield = "iad-va-us" + ssl_cert_hostname = "s3.amazonaws.com" + ssl_check_cert = true + use_ssl = true + weight = 100 + } + + # Doxygen stays in the rubyci bucket for now (step 1 of the migration); + # dropping this backend and pointing doxygen.yml at the docs bucket's + # capi/en/master/ prefix is step 2. The bucket name has no dots, so the + # virtual-hosted endpoint works as the TLS hostname directly. Unshielded: + # it refreshes every three hours and carries little traffic. + backend { + address = "rubyci.s3.amazonaws.com" + auto_loadbalance = false + between_bytes_timeout = 10000 + connect_timeout = 1000 + error_threshold = 0 + first_byte_timeout = 15000 + keepalive_time = 0 + max_conn = 200 + max_lifetime = 0 + max_use = 0 + name = "s3-rubyci-doxygen" + port = 443 + prefer_ipv6 = false + request_condition = "backend-is-doxygen-s3" + ssl_cert_hostname = "rubyci.s3.amazonaws.com" + ssl_check_cert = true + use_ssl = true + weight = 100 + } + + condition { + name = "backend-is-origin" + priority = 10 + statement = "!req.http.X-Docs-Backend" + type = "REQUEST" + } + + condition { + name = "backend-is-docs-s3" + priority = 10 + statement = "req.http.X-Docs-Backend == \"s3\"" + type = "REQUEST" + } + + condition { + name = "backend-is-doxygen-s3" + priority = 10 + statement = "req.http.X-Docs-Backend == \"doxygen\"" + type = "REQUEST" + } + # A shielded miss runs the logging endpoint at both POPs, so one request # becomes two events carrying the same byte count. The edge sets Fastly-FF when # it forwards to the shield, so this keeps the edge line, which is the one with @@ -39,6 +122,18 @@ resource "fastly_service_vcl" "docs_dev" { type = "RESPONSE" } + # What /ja/latest and /ja/master resolve to (the bucket holds no symlink + # objects), and what the unreleased-version redirects key off. A release + # updates these values and everything else follows. + dictionary { + name = "docs_versions" + } + + domain { + comment = "For shielding" + name = "docs.r-l.o.s3.amazonaws.com" + } + # Reached only through the Fastly-provided domain, same as cache-dev. That # needs neither a ruby-lang.org zone change nor a TLS subscription, since the # shared certificate already covers it. @@ -47,8 +142,8 @@ resource "fastly_service_vcl" "docs_dev" { } gzip { - content_types = ["text/html", "application/x-javascript", "text/css", "application/javascript", "text/javascript", "application/json", "application/vnd.ms-fontobject", "application/x-font-opentype", "application/x-font-truetype", "application/x-font-ttf", "application/xml", "font/eot", "font/opentype", "font/otf", "image/svg+xml", "image/vnd.microsoft.icon", "text/plain", "text/xml"] - extensions = ["css", "js", "html", "eot", "ico", "otf", "ttf", "json", "svg"] + content_types = ["text/html", "application/x-javascript", "text/css", "application/javascript", "text/javascript", "application/json", "application/vnd.ms-fontobject", "application/x-font-opentype", "application/x-font-truetype", "application/x-font-ttf", "application/xml", "font/eot", "font/opentype", "font/otf", "image/svg+xml", "image/vnd.microsoft.icon", "text/plain", "text/xml", "text/markdown"] + extensions = ["css", "js", "html", "eot", "ico", "otf", "ttf", "json", "svg", "md", "xml", "txt"] name = "Default Gzip Policy" } @@ -72,12 +167,19 @@ resource "fastly_service_vcl" "docs_dev" { xff = "append" } - # Production docs intentionally has no custom VCL. This file is the - # behavior-neutral boilerplate equivalent, uploaded so VCL-level changes can - # be canaried on docs-dev before deciding how to apply them to production. vcl { content = file("${path.module}/vcl/docs_dev.vcl") main = true name = "default" } } + +resource "fastly_service_dictionary_items" "docs_dev_versions" { + service_id = fastly_service_vcl.docs_dev.id + dictionary_id = one([for d in fastly_service_vcl.docs_dev.dictionary : d.dictionary_id if d.name == "docs_versions"]) + + items = { + latest = "4.0" + master = "4.1" + } +} diff --git a/cdn/s3.tf b/cdn/s3.tf index da1179a..57ab39b 100644 --- a/cdn/s3.tf +++ b/cdn/s3.tf @@ -86,6 +86,90 @@ resource "aws_s3_bucket_lifecycle_configuration" "ftp" { } } +# Origin of the docs service (docs.ruby-lang.org), canaried on docs-dev first. +# Prefixes and their writers: +# (root) -- ruby/docs.ruby-lang.org public/ (robots.txt, llms.txt, sitemap.xml, index pages, assets) +# en// -- ruby/actions docs.yml (extracted RDoc HTML; frozen 3.0/3.1 synced once by hand) +# ja// -- rurema/generated-documents html/ja/* (latest/master are resolved at the +# edge from the docs_versions dictionary; no symlink objects in the bucket) +# capi/en/master/ -- ruby/actions doxygen.yml, once it moves off the rubyci bucket +# us-east-1 like ftp.r-l.o: every request comes through Fastly, so client +# latency does not depend on the bucket region and the cheapest tier wins. +resource "aws_s3_bucket" "docs" { + bucket = "docs.r-l.o" + region = "us-east-1" + + tags = { + Name = "docs.r-l.o" + } + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_s3_bucket_policy" "docs" { + bucket = aws_s3_bucket.docs.bucket + region = "us-east-1" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "Allow Public Access to All Objects" + Effect = "Allow" + Principal = "*" + Action = ["s3:GetObject", "s3:GetObjectTagging"] + Resource = ["arn:aws:s3:::docs.r-l.o", "arn:aws:s3:::docs.r-l.o/*"] + }, + ] + }) +} + +resource "aws_s3_bucket_versioning" "docs" { + bucket = aws_s3_bucket.docs.bucket + region = "us-east-1" + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "docs" { + bucket = aws_s3_bucket.docs.bucket + region = "us-east-1" + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_lifecycle_configuration" "docs" { + bucket = aws_s3_bucket.docs.bucket + region = "us-east-1" + + transition_default_minimum_object_size = "varies_by_storage_class" + + rule { + id = "lifecycle" + status = "Enabled" + + expiration { + expired_object_delete_marker = true + } + + noncurrent_version_expiration { + noncurrent_days = 7 + } + + abort_incomplete_multipart_upload { + days_after_initiation = 1 + } + } +} + # Origin of the logs.rubyci.org service (chkbuild logs). resource "aws_s3_bucket" "rubyci" { bucket = "rubyci" diff --git a/cdn/vcl/docs_dev.vcl b/cdn/vcl/docs_dev.vcl index 7f4aa3f..b6d317e 100644 --- a/cdn/vcl/docs_dev.vcl +++ b/cdn/vcl/docs_dev.vcl @@ -1,4 +1,135 @@ sub vcl_recv { + + # ---- S3 backend routing and rewrites (docs-dev canary) ---- + # Runs before #FASTLY recv so the generated backend-selection code + # (request_condition) can see the X-Docs-Backend flag, same as cache.vcl. + # Client-visible URLs never change here: everything except the synthetic + # redirects (error 601/602) is an internal rewrite, and the cache key is + # the rewritten URL. + + declare local var.lang STRING; + declare local var.ver STRING; + declare local var.rest STRING; + + if (req.request == "HEAD" || req.request == "GET") { + + # Surrogate-Key and redirects are computed from what the client asked + # for, so keep the pre-rewrite URL. It survives restarts (markdown + # fallback below), so only set it on the first pass. + if (!req.http.X-Orig-Url) { + set req.http.X-Orig-Url = req.url; + } + + # Vary: Accept is served on the negotiable /ja/ pages below. Normalize + # Accept to two values first so the variants cannot explode per client. + if (req.url ~ "^/ja/") { + if (req.http.Accept ~ "text/markdown") { + set req.http.Accept = "text/markdown"; + } else { + unset req.http.Accept; + } + } + + # ---- redirects, same rules and order as the nginx origin ---- + if (req.url ~ "^/en/trunk(.*)$") { + set req.http.X-Redirect-Location = "https://docs.ruby-lang.org/en/master" re.group.1; + error 601; + } + # https://github.com/ruby/docs.ruby-lang.org/issues/130 + if (req.url ~ "^/en/([^/]+)/doc/(.*)$") { + set req.http.X-Redirect-Location = "/en/" re.group.1 "/" re.group.2; + error 601; + } + # 2.8.0 was renamed to 3.0.0, and the directory is 3.0 + if (req.url ~ "^/(en|ja)/(2\.8\.0|3\.0\.0)(.*)$") { + set req.http.X-Redirect-Location = "/" re.group.1 "/3.0" re.group.3; + error 601; + } + # Old rurema-search /ja/search/query:WORD/ URLs; ?q= keeps the + # percent-encoding because req.url is still encoded here. + if (req.url ~ "^/ja/search/" && req.url ~ "query:") { + if (req.url ~ "^/ja/search/(?:[^?]*?/)??query:([^/?]+)") { + set req.http.X-Redirect-Location = "/ja/search/?q=" re.group.1; + } else { + set req.http.X-Redirect-Location = "/ja/search/"; + } + error 601; + } + + # The unreleased version is not public until the release: ja/master + # pages point at en/ through their [rdoc] links (which only + # exists as en/master), and ja/ itself is not linked from the + # version index. Both go to the master alias with a temporary redirect + # that disappears once the docs_versions dictionary moves at release. + if (req.url ~ "^/(en|ja)/([^/?]+)(/[^?]*)?$") { + set var.lang = re.group.1; + set var.ver = re.group.2; + set var.rest = re.group.3; + if (var.ver == table.lookup(docs_versions, "master")) { + if (var.rest == "") { + set var.rest = "/"; + } + set req.http.X-Redirect-Location = "/" var.lang "/master" var.rest; + error 602; + } + } + + # S3 interprets query strings as API parameters and the static site + # never varies on them (/ja/search/?q= is read by client-side JS), so + # drop them from the cache key and the backend request. + set req.url = req.url.path; + + # Directory-looking URL without the trailing slash: redirect to the + # slash form, like nginx did for directories. Every real page has an + # extension, so no dot in the last segment is a safe heuristic. + if (req.url ~ "^/(en|ja|capi)(/|$)" && req.url !~ "\.[^/]+$" && req.url !~ "/$") { + set req.http.X-Redirect-Location = req.url "/"; + error 601; + } + + # /ja/latest and /ja/master have no symlink objects in the bucket; + # rewrite them to the real version from the docs_versions dictionary. + # The client URL stays on the alias, and a release only needs a + # dictionary update. + if (req.url ~ "^/ja/(latest|master)/") { + set req.http.X-Docs-Symlink = re.group.1; + set req.http.X-Docs-Version = table.lookup(docs_versions, req.http.X-Docs-Symlink); + if (req.http.X-Docs-Version) { + set req.url = regsub(req.url, "^/ja/(latest|master)/", "/ja/" req.http.X-Docs-Version "/"); + } + } + + # Directory index: a trailing-slash URL fetches index.html from S3 + # (the nginx index directive, as an internal rewrite). + if (req.url ~ "/$") { + set req.url = req.url "index.html"; + } + + # ---- Markdown content negotiation (ja only, where .md twins exist) ---- + # Accept: text/markdown on an .html URL fetches the .md twin instead. + # Pages without a twin (generated indexes, frozen RD versions) fall + # back: vcl_fetch restarts with X-Md-Negotiate=fallback on the 404 and + # this block then restores the .html URL. + if (req.http.X-Md-Negotiate == "fallback") { + if (req.url ~ "\.md$") { + set req.url = regsub(req.url, "\.md$", ".html"); + } + } else if (req.url ~ "^/ja/" && req.url ~ "\.html$" && req.http.Accept == "text/markdown") { + set req.http.X-Md-Negotiate = "md"; + set req.url = regsub(req.url, "\.html$", ".md"); + } + + # ---- backend selection flags, read by the request_conditions ---- + if (req.url ~ "^/capi/en/master/") { + set req.http.X-Docs-Backend = "doxygen"; + set req.url = regsub(req.url, "^/capi/en/master/", "/doxygen-latest-html/"); + } else { + # Narrow this per prefix to move paths over one at a time; anything + # unflagged still goes to docs-origin. + set req.http.X-Docs-Backend = "s3"; + } + } + #FASTLY recv if (req.request != "HEAD" && req.request != "GET" && req.request != "FASTLYPURGE") { @@ -15,6 +146,13 @@ sub vcl_fetch { restart; } + # A negotiated .md that does not exist falls back to the .html twin + # (403 is what public-read S3 answers for a missing key). + if ((beresp.status == 403 || beresp.status == 404) && req.http.X-Md-Negotiate == "md" && req.restarts < 3) { + set req.http.X-Md-Negotiate = "fallback"; + restart; + } + if(req.restarts > 0 ) { set beresp.http.Fastly-Restarts = req.restarts; } @@ -36,6 +174,55 @@ sub vcl_fetch { return (deliver); } + if (req.http.X-Docs-Backend) { + # The public-read policy only grants GetObject, so a missing key is + # AccessDenied. Serve it as 404 with a short negative-cache TTL. + if (beresp.status == 403) { + set beresp.status = 404; + set beresp.response = "Not Found"; + } + if (beresp.status == 404) { + set beresp.ttl = 60s; + } + + # Cache-Control and Surrogate-Key move here from the nginx origin; the + # fastly-purge-key key scheme is unchanged. An alias URL (/ja/latest) + # and its real version share one object after the rewrite, so the key + # carries both names: bc-static-all purges ja/ as well as + # ja/latest and ja/master. + if (req.http.X-Orig-Url ~ "^/capi/en/master/") { + set beresp.http.Surrogate-Key = "doxygen-latest-html"; + } else if (req.url ~ "^/(en|ja)/([^/?]+)/") { + set beresp.http.Surrogate-Key = "docs " re.group.1 " " re.group.2 " " re.group.1 "/" re.group.2; + if (req.http.X-Docs-Symlink) { + set beresp.http.Surrogate-Key = beresp.http.Surrogate-Key " ja/" req.http.X-Docs-Symlink; + } + } else { + set beresp.http.Surrogate-Key = "index"; + } + set beresp.http.Cache-Control = "public, max-age=43200, s-maxage=172800, stale-while-revalidate=86400, stale-if-error=604800"; + + # The generated Markdown twins (given text/markdown by nginx since + # ruby/docs.ruby-lang.org#200; aws s3 sync cannot guess a type for .md). + if (req.url ~ "\.md$") { + set beresp.http.Content-Type = "text/markdown; charset=utf-8"; + } + + # The negotiable pages answer differently by Accept, so downstream + # caches need Vary (the edge already keys on the rewritten URL plus + # the normalized Accept). + if (req.http.X-Orig-Url ~ "^/ja/" && req.url ~ "\.(html|md)$") { + set beresp.http.Vary = "Accept"; + } + + unset beresp.http.x-amz-id-2; + unset beresp.http.x-amz-request-id; + unset beresp.http.x-amz-version-id; + unset beresp.http.x-amz-delete-marker; + unset beresp.http.x-amz-server-side-encryption; + unset beresp.http.Server; + } + if (beresp.http.Expires || beresp.http.Surrogate-Control ~ "max-age" || beresp.http.Cache-Control ~"(s-maxage|max-age)") { # keep the ttl here } else { @@ -67,6 +254,31 @@ sub vcl_deliver { sub vcl_error { #FASTLY error + + # Synthetic redirects from vcl_recv: 601 is the permanent kind (the + # nginx return 301 rules), 602 the temporary kind (unreleased-version + # aliases, gone once the docs_versions dictionary moves at a release). + if (obj.status == 601 || obj.status == 602) { + if (obj.status == 601) { + set obj.status = 301; + set obj.response = "Moved Permanently"; + set obj.http.Cache-Control = "public, max-age=3600"; + } else { + set obj.status = 302; + set obj.response = "Found"; + set obj.http.Cache-Control = "public, max-age=60"; + } + set obj.http.Location = req.http.X-Redirect-Location; + synthetic ""; + return(deliver); + } + + # The nginx error_page 50x equivalent. + if (obj.status >= 500 && obj.status < 600) { + set obj.http.Content-Type = "text/html; charset=utf-8"; + synthetic {"Error

An error occurred.

"}; + return(deliver); + } } sub vcl_pass { From ba9c86d40507219aa8e8e142699583f1153645c9 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 19 Aug 2026 13:07:23 +0900 Subject: [PATCH 2/9] Send the rubyci bucket its own Host on the doxygen backend default_host rewrites Host to the docs bucket endpoint for every backend without an override, and S3 routes virtual-hosted requests by Host, so /capi/en/master/ fetches were answered by the docs bucket and came back 404. Use the regional endpoint like logs_rubyci.tf while here. Co-Authored-By: Claude Fable 5 --- cdn/docs_dev.tf | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cdn/docs_dev.tf b/cdn/docs_dev.tf index 166d9c6..a52220d 100644 --- a/cdn/docs_dev.tf +++ b/cdn/docs_dev.tf @@ -67,10 +67,13 @@ resource "fastly_service_vcl" "docs_dev" { # Doxygen stays in the rubyci bucket for now (step 1 of the migration); # dropping this backend and pointing doxygen.yml at the docs bucket's # capi/en/master/ prefix is step 2. The bucket name has no dots, so the - # virtual-hosted endpoint works as the TLS hostname directly. Unshielded: + # virtual-hosted endpoint works as the TLS hostname directly, the same + # arrangement as logs_rubyci.tf. override_host is required: default_host + # rewrites Host to the docs bucket endpoint, and S3 routes by Host, so + # without it these requests would hit the docs bucket. Unshielded: # it refreshes every three hours and carries little traffic. backend { - address = "rubyci.s3.amazonaws.com" + address = "rubyci.s3.ap-northeast-1.amazonaws.com" auto_loadbalance = false between_bytes_timeout = 10000 connect_timeout = 1000 @@ -81,10 +84,11 @@ resource "fastly_service_vcl" "docs_dev" { max_lifetime = 0 max_use = 0 name = "s3-rubyci-doxygen" + override_host = "rubyci.s3.ap-northeast-1.amazonaws.com" port = 443 prefer_ipv6 = false request_condition = "backend-is-doxygen-s3" - ssl_cert_hostname = "rubyci.s3.amazonaws.com" + ssl_cert_hostname = "rubyci.s3.ap-northeast-1.amazonaws.com" ssl_check_cert = true use_ssl = true weight = 100 From 975b3aa9de66708267fee05d6ba0ce6dbcb00f33 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 19 Aug 2026 13:08:10 +0900 Subject: [PATCH 3/9] Run the redirect rules only on the first pass at the edge The shield POP runs this VCL again on the edge's rewritten URL, and a restart re-enters recv the same way. /ja/master resolved to /ja/ then matched the unreleased-version rule there, producing a 302 to itself that the edge cached. The markdown fallback restart is likewise kept at the edge so the fallback .html is never cached under a .md key. Co-Authored-By: Claude Fable 5 --- cdn/vcl/docs_dev.vcl | 96 ++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 40 deletions(-) diff --git a/cdn/vcl/docs_dev.vcl b/cdn/vcl/docs_dev.vcl index b6d317e..8534370 100644 --- a/cdn/vcl/docs_dev.vcl +++ b/cdn/vcl/docs_dev.vcl @@ -10,6 +10,18 @@ sub vcl_recv { declare local var.lang STRING; declare local var.ver STRING; declare local var.rest STRING; + declare local var.edge_first_pass BOOL; + + # The redirect rules match client-shaped URLs, so they may only run on + # the first pass at the edge. Rewritten URLs come through recv again in + # two ways: a restart re-enters it, and a shielded fetch runs this whole + # VCL once more at the shield POP. /ja/master rewritten to /ja/ + # would match the unreleased-version rule there and 302 back to itself. + # ff_visits_this_service, unlike the Fastly-FF header, cannot be spoofed + # by the client. + if (req.restarts == 0 && fastly.ff_visits_this_service == 0) { + set var.edge_first_pass = true; + } if (req.request == "HEAD" || req.request == "GET") { @@ -31,46 +43,48 @@ sub vcl_recv { } # ---- redirects, same rules and order as the nginx origin ---- - if (req.url ~ "^/en/trunk(.*)$") { - set req.http.X-Redirect-Location = "https://docs.ruby-lang.org/en/master" re.group.1; - error 601; - } - # https://github.com/ruby/docs.ruby-lang.org/issues/130 - if (req.url ~ "^/en/([^/]+)/doc/(.*)$") { - set req.http.X-Redirect-Location = "/en/" re.group.1 "/" re.group.2; - error 601; - } - # 2.8.0 was renamed to 3.0.0, and the directory is 3.0 - if (req.url ~ "^/(en|ja)/(2\.8\.0|3\.0\.0)(.*)$") { - set req.http.X-Redirect-Location = "/" re.group.1 "/3.0" re.group.3; - error 601; - } - # Old rurema-search /ja/search/query:WORD/ URLs; ?q= keeps the - # percent-encoding because req.url is still encoded here. - if (req.url ~ "^/ja/search/" && req.url ~ "query:") { - if (req.url ~ "^/ja/search/(?:[^?]*?/)??query:([^/?]+)") { - set req.http.X-Redirect-Location = "/ja/search/?q=" re.group.1; - } else { - set req.http.X-Redirect-Location = "/ja/search/"; + if (var.edge_first_pass) { + if (req.url ~ "^/en/trunk(.*)$") { + set req.http.X-Redirect-Location = "https://docs.ruby-lang.org/en/master" re.group.1; + error 601; + } + # https://github.com/ruby/docs.ruby-lang.org/issues/130 + if (req.url ~ "^/en/([^/]+)/doc/(.*)$") { + set req.http.X-Redirect-Location = "/en/" re.group.1 "/" re.group.2; + error 601; + } + # 2.8.0 was renamed to 3.0.0, and the directory is 3.0 + if (req.url ~ "^/(en|ja)/(2\.8\.0|3\.0\.0)(.*)$") { + set req.http.X-Redirect-Location = "/" re.group.1 "/3.0" re.group.3; + error 601; + } + # Old rurema-search /ja/search/query:WORD/ URLs; ?q= keeps the + # percent-encoding because req.url is still encoded here. + if (req.url ~ "^/ja/search/" && req.url ~ "query:") { + if (req.url ~ "^/ja/search/(?:[^?]*?/)??query:([^/?]+)") { + set req.http.X-Redirect-Location = "/ja/search/?q=" re.group.1; + } else { + set req.http.X-Redirect-Location = "/ja/search/"; + } + error 601; } - error 601; - } - # The unreleased version is not public until the release: ja/master - # pages point at en/ through their [rdoc] links (which only - # exists as en/master), and ja/ itself is not linked from the - # version index. Both go to the master alias with a temporary redirect - # that disappears once the docs_versions dictionary moves at release. - if (req.url ~ "^/(en|ja)/([^/?]+)(/[^?]*)?$") { - set var.lang = re.group.1; - set var.ver = re.group.2; - set var.rest = re.group.3; - if (var.ver == table.lookup(docs_versions, "master")) { - if (var.rest == "") { - set var.rest = "/"; + # The unreleased version is not public until the release: ja/master + # pages point at en/ through their [rdoc] links (which only + # exists as en/master), and ja/ itself is not linked from the + # version index. Both go to the master alias with a temporary redirect + # that disappears once the docs_versions dictionary moves at release. + if (req.url ~ "^/(en|ja)/([^/?]+)(/[^?]*)?$") { + set var.lang = re.group.1; + set var.ver = re.group.2; + set var.rest = re.group.3; + if (var.ver == table.lookup(docs_versions, "master")) { + if (var.rest == "") { + set var.rest = "/"; + } + set req.http.X-Redirect-Location = "/" var.lang "/master" var.rest; + error 602; } - set req.http.X-Redirect-Location = "/" var.lang "/master" var.rest; - error 602; } } @@ -82,7 +96,7 @@ sub vcl_recv { # Directory-looking URL without the trailing slash: redirect to the # slash form, like nginx did for directories. Every real page has an # extension, so no dot in the last segment is a safe heuristic. - if (req.url ~ "^/(en|ja|capi)(/|$)" && req.url !~ "\.[^/]+$" && req.url !~ "/$") { + if (var.edge_first_pass && req.url ~ "^/(en|ja|capi)(/|$)" && req.url !~ "\.[^/]+$" && req.url !~ "/$") { set req.http.X-Redirect-Location = req.url "/"; error 601; } @@ -147,8 +161,10 @@ sub vcl_fetch { } # A negotiated .md that does not exist falls back to the .html twin - # (403 is what public-read S3 answers for a missing key). - if ((beresp.status == 403 || beresp.status == 404) && req.http.X-Md-Negotiate == "md" && req.restarts < 3) { + # (403 is what public-read S3 answers for a missing key). Edge only: if + # the shield restarted instead, the edge would cache the fallback .html + # response under the .md cache key it asked the shield for. + if ((beresp.status == 403 || beresp.status == 404) && req.http.X-Md-Negotiate == "md" && req.restarts < 3 && fastly.ff_visits_this_service == 0) { set req.http.X-Md-Negotiate = "fallback"; restart; } From 615f648a4eb1e4b359bae4c843bf2841c3de2efa Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 19 Aug 2026 13:08:23 +0900 Subject: [PATCH 4/9] Redirect bare version directories to the slash form The no-dot-in-last-segment heuristic treated /ja/4.0 and /en/3.4 as pages with an extension, so they fetched a nonexistent key and came back 404 instead of redirecting like /ja/latest does. Co-Authored-By: Claude Fable 5 --- cdn/vcl/docs_dev.vcl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cdn/vcl/docs_dev.vcl b/cdn/vcl/docs_dev.vcl index 8534370..c1f3885 100644 --- a/cdn/vcl/docs_dev.vcl +++ b/cdn/vcl/docs_dev.vcl @@ -95,8 +95,10 @@ sub vcl_recv { # Directory-looking URL without the trailing slash: redirect to the # slash form, like nginx did for directories. Every real page has an - # extension, so no dot in the last segment is a safe heuristic. - if (var.edge_first_pass && req.url ~ "^/(en|ja|capi)(/|$)" && req.url !~ "\.[^/]+$" && req.url !~ "/$") { + # extension, so no dot in the last segment is the heuristic; a bare + # version directory (/ja/4.0, /en/3.4) has dots of its own, so a + # two-segment lang/version URL is a directory regardless. + if (var.edge_first_pass && req.url ~ "^/(en|ja|capi)(/|$)" && req.url !~ "/$" && (req.url !~ "\.[^/]+$" || req.url ~ "^/(en|ja)/[0-9][^/]*$")) { set req.http.X-Redirect-Location = req.url "/"; error 601; } From e9ddc28f984d335102ec5fdab90b6f538bfd6928 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 19 Aug 2026 13:08:37 +0900 Subject: [PATCH 5/9] Strip the internal routing headers off incoming requests X-Orig-Url and friends are only trusted because later passes set them themselves; on the first pass at the edge they can only be a client spoof. A forged X-Orig-Url poisons the Surrogate-Key of the cached object, making it unpurgeable by the real keys. Co-Authored-By: Claude Fable 5 --- cdn/vcl/docs_dev.vcl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cdn/vcl/docs_dev.vcl b/cdn/vcl/docs_dev.vcl index c1f3885..e9c82f8 100644 --- a/cdn/vcl/docs_dev.vcl +++ b/cdn/vcl/docs_dev.vcl @@ -23,6 +23,19 @@ sub vcl_recv { set var.edge_first_pass = true; } + # These headers carry state across restarts and to the shield, so on + # the first pass anything already present is a client spoof. A forged + # X-Orig-Url would change the Surrogate-Key of the cached object, + # leaving it stale and unpurgeable by the real keys. + if (var.edge_first_pass) { + unset req.http.X-Orig-Url; + unset req.http.X-Md-Negotiate; + unset req.http.X-Docs-Symlink; + unset req.http.X-Docs-Version; + unset req.http.X-Redirect-Location; + unset req.http.X-Docs-Backend; + } + if (req.request == "HEAD" || req.request == "GET") { # Surrogate-Key and redirects are computed from what the client asked From 491c895982c24de12bd991c9d80d9abef5e0df0f Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 19 Aug 2026 13:35:50 +0900 Subject: [PATCH 6/9] Lift Block Public Access policy blocks on the docs bucket A new bucket starts with all four blocks enabled, so PutBucketPolicy on the public-read policy came back 403. The legacy buckets predate the feature and stay unmanaged; ACLs remain blocked here. Co-Authored-By: Claude Fable 5 --- cdn/s3.tf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cdn/s3.tf b/cdn/s3.tf index 57ab39b..d97e99a 100644 --- a/cdn/s3.tf +++ b/cdn/s3.tf @@ -108,10 +108,26 @@ resource "aws_s3_bucket" "docs" { } } +# The legacy buckets predate Block Public Access and their settings are left +# unmanaged (see README), but a new bucket starts with all four blocks on and +# rejects PutBucketPolicy until the policy blocks are lifted. ACLs stay +# blocked; only the bucket policy grants public reads. +resource "aws_s3_bucket_public_access_block" "docs" { + bucket = aws_s3_bucket.docs.bucket + region = "us-east-1" + + block_public_acls = true + ignore_public_acls = true + block_public_policy = false + restrict_public_buckets = false +} + resource "aws_s3_bucket_policy" "docs" { bucket = aws_s3_bucket.docs.bucket region = "us-east-1" + depends_on = [aws_s3_bucket_public_access_block.docs] + policy = jsonencode({ Version = "2012-10-17" Statement = [ From 4b88f8d698c971c9b04ce2468f82e650110a1f82 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 19 Aug 2026 13:35:56 +0900 Subject: [PATCH 7/9] Detect the shield hop with Fastly-FF instead of a nonexistent variable fastly.ff_visits_this_service is not a Fastly VCL variable and the service rejected the upload. Fastly-FF is what the logging conditions already use; since a client can forge it, vcl_fetch now derives the Surrogate-Key and Vary decisions from the rewritten req.url instead of the X-Orig-Url request header, which is dropped entirely. Co-Authored-By: Claude Fable 5 --- cdn/vcl/docs_dev.vcl | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/cdn/vcl/docs_dev.vcl b/cdn/vcl/docs_dev.vcl index e9c82f8..17efb7a 100644 --- a/cdn/vcl/docs_dev.vcl +++ b/cdn/vcl/docs_dev.vcl @@ -17,18 +17,16 @@ sub vcl_recv { # two ways: a restart re-enters it, and a shielded fetch runs this whole # VCL once more at the shield POP. /ja/master rewritten to /ja/ # would match the unreleased-version rule there and 302 back to itself. - # ff_visits_this_service, unlike the Fastly-FF header, cannot be spoofed - # by the client. - if (req.restarts == 0 && fastly.ff_visits_this_service == 0) { + # Fastly-FF marks the shield hop, same as the logging condition. A client + # forging it only opts itself out of the redirects; vcl_fetch computes + # everything cacheable from the rewritten req.url, not from these headers. + if (req.restarts == 0 && !req.http.Fastly-FF) { set var.edge_first_pass = true; } # These headers carry state across restarts and to the shield, so on - # the first pass anything already present is a client spoof. A forged - # X-Orig-Url would change the Surrogate-Key of the cached object, - # leaving it stale and unpurgeable by the real keys. + # the first pass anything already present is a client spoof. if (var.edge_first_pass) { - unset req.http.X-Orig-Url; unset req.http.X-Md-Negotiate; unset req.http.X-Docs-Symlink; unset req.http.X-Docs-Version; @@ -38,13 +36,6 @@ sub vcl_recv { if (req.request == "HEAD" || req.request == "GET") { - # Surrogate-Key and redirects are computed from what the client asked - # for, so keep the pre-rewrite URL. It survives restarts (markdown - # fallback below), so only set it on the first pass. - if (!req.http.X-Orig-Url) { - set req.http.X-Orig-Url = req.url; - } - # Vary: Accept is served on the negotiable /ja/ pages below. Normalize # Accept to two values first so the variants cannot explode per client. if (req.url ~ "^/ja/") { @@ -179,7 +170,7 @@ sub vcl_fetch { # (403 is what public-read S3 answers for a missing key). Edge only: if # the shield restarted instead, the edge would cache the fallback .html # response under the .md cache key it asked the shield for. - if ((beresp.status == 403 || beresp.status == 404) && req.http.X-Md-Negotiate == "md" && req.restarts < 3 && fastly.ff_visits_this_service == 0) { + if ((beresp.status == 403 || beresp.status == 404) && req.http.X-Md-Negotiate == "md" && req.restarts < 3 && !req.http.Fastly-FF) { set req.http.X-Md-Negotiate = "fallback"; restart; } @@ -220,8 +211,10 @@ sub vcl_fetch { # fastly-purge-key key scheme is unchanged. An alias URL (/ja/latest) # and its real version share one object after the rewrite, so the key # carries both names: bc-static-all purges ja/ as well as - # ja/latest and ja/master. - if (req.http.X-Orig-Url ~ "^/capi/en/master/") { + # ja/latest and ja/master. Everything derives from the rewritten + # req.url; the only header input is X-Docs-Symlink, which recv resets + # on the first pass. + if (req.url ~ "^/doxygen-latest-html/") { set beresp.http.Surrogate-Key = "doxygen-latest-html"; } else if (req.url ~ "^/(en|ja)/([^/?]+)/") { set beresp.http.Surrogate-Key = "docs " re.group.1 " " re.group.2 " " re.group.1 "/" re.group.2; @@ -242,7 +235,7 @@ sub vcl_fetch { # The negotiable pages answer differently by Accept, so downstream # caches need Vary (the edge already keys on the rewritten URL plus # the normalized Accept). - if (req.http.X-Orig-Url ~ "^/ja/" && req.url ~ "\.(html|md)$") { + if (req.url ~ "^/ja/" && req.url ~ "\.(html|md)$") { set beresp.http.Vary = "Accept"; } From 0681d55eba11c4c089278d16e52a236c87524d49 Mon Sep 17 00:00:00 2001 From: Kazuhiro NISHIYAMA Date: Wed, 19 Aug 2026 14:43:42 +0900 Subject: [PATCH 8/9] Fix the md fallback: Fastly-FF is set even in the edge's vcl_fetch The fallback restart for a missing .md twin never fired: the intra-POP clustering hop (delivery node to fetch node) also sets Fastly-FF, so the "edge only" guard in vcl_fetch was false everywhere and a negotiated request for a page without a twin returned the S3 error as a 404 instead of falling back to the .html. Observed on docs-rlo-dev: GET /ja/4.0/class/index.html with Accept: text/markdown answered 404 text/markdown with no Fastly-Restarts, while /ja/4.0/class/index.html itself is 200. Decide edge/shield in vcl_recv instead, where Fastly-FF still means "another Fastly POP sent this": the edge plants an X-Docs-Md-Fallback marker (overwriting any client spoof) and the shield strips it, and vcl_fetch restarts only when the marker is present. Co-Authored-By: Claude Fable 5 --- cdn/vcl/docs_dev.vcl | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cdn/vcl/docs_dev.vcl b/cdn/vcl/docs_dev.vcl index 17efb7a..69e0f89 100644 --- a/cdn/vcl/docs_dev.vcl +++ b/cdn/vcl/docs_dev.vcl @@ -34,6 +34,20 @@ sub vcl_recv { unset req.http.X-Docs-Backend; } + # The md fallback restart must run at the edge only, but vcl_fetch cannot + # use Fastly-FF to tell the edge from the shield: the clustering hop + # between the delivery and fetch node inside a POP also sets it, so it is + # present in fetch even at the edge and the fallback never fired. Decide + # here instead, where Fastly-FF still means "another Fastly POP sent + # this": the edge plants a marker and the shield strips it. On the edge + # the marker overwrites any client spoof; a client sending Fastly-FF only + # opts itself out of the fallback. + if (req.http.Fastly-FF) { + unset req.http.X-Docs-Md-Fallback; + } elsif (req.restarts == 0) { + set req.http.X-Docs-Md-Fallback = "edge"; + } + if (req.request == "HEAD" || req.request == "GET") { # Vary: Accept is served on the negotiable /ja/ pages below. Normalize @@ -169,8 +183,10 @@ sub vcl_fetch { # A negotiated .md that does not exist falls back to the .html twin # (403 is what public-read S3 answers for a missing key). Edge only: if # the shield restarted instead, the edge would cache the fallback .html - # response under the .md cache key it asked the shield for. - if ((beresp.status == 403 || beresp.status == 404) && req.http.X-Md-Negotiate == "md" && req.restarts < 3 && !req.http.Fastly-FF) { + # response under the .md cache key it asked the shield for. The marker + # comes from vcl_recv; Fastly-FF itself is unusable here because the + # intra-POP clustering hop sets it on the edge's fetch node too. + if ((beresp.status == 403 || beresp.status == 404) && req.http.X-Md-Negotiate == "md" && req.restarts < 3 && req.http.X-Docs-Md-Fallback == "edge") { set req.http.X-Md-Negotiate = "fallback"; restart; } From f37bbb0103fc963abbd917b2696d38b0f5f01093 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 19 Aug 2026 16:10:50 +0900 Subject: [PATCH 9/9] Keep the 404 negative cache short downstream too The 60s beresp.ttl only shortened Fastly's own copy; the emitted Cache-Control still told browsers to keep the 404 for 12 hours, well past the content sync that fixes most of them. Also zero the stale windows so an expired 404 is never served stale. Co-Authored-By: Claude Fable 5 --- cdn/vcl/docs_dev.vcl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cdn/vcl/docs_dev.vcl b/cdn/vcl/docs_dev.vcl index 69e0f89..78ef4fe 100644 --- a/cdn/vcl/docs_dev.vcl +++ b/cdn/vcl/docs_dev.vcl @@ -221,6 +221,8 @@ sub vcl_fetch { } if (beresp.status == 404) { set beresp.ttl = 60s; + set beresp.stale_while_revalidate = 0s; + set beresp.stale_if_error = 0s; } # Cache-Control and Surrogate-Key move here from the nginx origin; the @@ -240,7 +242,14 @@ sub vcl_fetch { } else { set beresp.http.Surrogate-Key = "index"; } - set beresp.http.Cache-Control = "public, max-age=43200, s-maxage=172800, stale-while-revalidate=86400, stale-if-error=604800"; + # The 404s are mostly content that has not been synced yet, so the + # negative cache must stay short downstream too; the long policy would + # sit in a browser for 12 hours. + if (beresp.status == 404) { + set beresp.http.Cache-Control = "public, max-age=60"; + } else { + set beresp.http.Cache-Control = "public, max-age=43200, s-maxage=172800, stale-while-revalidate=86400, stale-if-error=604800"; + } # The generated Markdown twins (given text/markdown by nginx since # ruby/docs.ruby-lang.org#200; aws s3 sync cannot guess a type for .md).