Move docs-dev to S3 backends as the docs.ruby-lang.org migration canary

cdn/docs_dev.tf

@@ -1,6 +1,16 @@
+# The S3-backend canary for docs.ruby-lang.org. The production service keeps
+# pointing at docs-origin until this proves out, then docs.tf adopts the same
+# shape. Backend selection goes through request_conditions on a header flag the
+# custom VCL sets before #FASTLY recv (assigning req.backend in VCL would
+# bypass shielding, see cache.tf); the docs-origin backend stays as the
+# fallback for unflagged requests so paths can be moved over one at a time.
 resource "fastly_service_vcl" "docs_dev" {
   activate           = true
   stage              = false
+  # A shielded fetch needs a Host that is a domain of this service, so the
+  # bucket endpoint doubles as default_host and as a domain below, the same
+  # arrangement as cache.tf. The other backends override_host instead.
+  default_host       = "docs.r-l.o.s3.amazonaws.com"
   default_ttl        = 60
   http3              = true
   name               = "docs-dev.ruby-lang.org"
@@ -19,15 +29,92 @@ resource "fastly_service_vcl" "docs_dev" {
     max_lifetime          = 0
     max_use               = 0
     name                  = "docs origin server"
+    override_host         = "docs.ruby-lang.org"
     port                  = 443
     prefer_ipv6           = false
+    request_condition     = "backend-is-origin"
     shield                = "tyo-tokyo-jp"
     ssl_cert_hostname     = "docs-origin.ruby-lang.org"
     ssl_check_cert        = true
     use_ssl               = true
     weight                = 100
   }
 
+  # The docs bucket (aws_s3_bucket.docs in s3.tf): public/ root files, en and
+  # ja. us-east-1, so shielded near it like the cache service's S3 backend.
+  backend {
+    address               = "s3.amazonaws.com"
+    auto_loadbalance      = false
+    between_bytes_timeout = 10000
+    connect_timeout       = 1000
+    error_threshold       = 0
+    first_byte_timeout    = 15000
+    keepalive_time        = 0
+    max_conn              = 200
+    max_lifetime          = 0
+    max_use               = 0
+    name                  = "s3-docs"
+    port                  = 443
+    prefer_ipv6           = false
+    request_condition     = "backend-is-docs-s3"
+    shield                = "iad-va-us"
+    ssl_cert_hostname     = "s3.amazonaws.com"
+    ssl_check_cert        = true
+    use_ssl               = true
+    weight                = 100
+  }
+
+  # Doxygen stays in the rubyci bucket for now (step 1 of the migration);
+  # dropping this backend and pointing doxygen.yml at the docs bucket's
+  # capi/en/master/ prefix is step 2. The bucket name has no dots, so the
+  # virtual-hosted endpoint works as the TLS hostname directly, the same
+  # arrangement as logs_rubyci.tf. override_host is required: default_host
+  # rewrites Host to the docs bucket endpoint, and S3 routes by Host, so
+  # without it these requests would hit the docs bucket. Unshielded:
+  # it refreshes every three hours and carries little traffic.
+  backend {
+    address               = "rubyci.s3.ap-northeast-1.amazonaws.com"
+    auto_loadbalance      = false
+    between_bytes_timeout = 10000
+    connect_timeout       = 1000
+    error_threshold       = 0
+    first_byte_timeout    = 15000
+    keepalive_time        = 0
+    max_conn              = 200
+    max_lifetime          = 0
+    max_use               = 0
+    name                  = "s3-rubyci-doxygen"
+    override_host         = "rubyci.s3.ap-northeast-1.amazonaws.com"
+    port                  = 443
+    prefer_ipv6           = false
+    request_condition     = "backend-is-doxygen-s3"
+    ssl_cert_hostname     = "rubyci.s3.ap-northeast-1.amazonaws.com"
+    ssl_check_cert        = true
+    use_ssl               = true
+    weight                = 100
+  }
+
+  condition {
+    name      = "backend-is-origin"
+    priority  = 10
+    statement = "!req.http.X-Docs-Backend"
+    type      = "REQUEST"
+  }
+
+  condition {
+    name      = "backend-is-docs-s3"
+    priority  = 10
+    statement = "req.http.X-Docs-Backend == \"s3\""
+    type      = "REQUEST"
+  }
+
+  condition {
+    name      = "backend-is-doxygen-s3"
+    priority  = 10
+    statement = "req.http.X-Docs-Backend == \"doxygen\""
+    type      = "REQUEST"
+  }
+
   # A shielded miss runs the logging endpoint at both POPs, so one request
   # becomes two events carrying the same byte count. The edge sets Fastly-FF when
   # it forwards to the shield, so this keeps the edge line, which is the one with
@@ -39,6 +126,18 @@ resource "fastly_service_vcl" "docs_dev" {
     type      = "RESPONSE"
   }
 
+  # What /ja/latest and /ja/master resolve to (the bucket holds no symlink
+  # objects), and what the unreleased-version redirects key off. A release
+  # updates these values and everything else follows.
+  dictionary {
+    name = "docs_versions"
+  }
+
+  domain {
+    comment = "For shielding"
+    name    = "docs.r-l.o.s3.amazonaws.com"
+  }
+
   # Reached only through the Fastly-provided domain, same as cache-dev. That
   # needs neither a ruby-lang.org zone change nor a TLS subscription, since the
   # shared certificate already covers it.
@@ -47,8 +146,8 @@ resource "fastly_service_vcl" "docs_dev" {
   }
 
   gzip {
-    content_types = ["text/html", "application/x-javascript", "text/css", "application/javascript", "text/javascript", "application/json", "application/vnd.ms-fontobject", "application/x-font-opentype", "application/x-font-truetype", "application/x-font-ttf", "application/xml", "font/eot", "font/opentype", "font/otf", "image/svg+xml", "image/vnd.microsoft.icon", "text/plain", "text/xml"]
-    extensions    = ["css", "js", "html", "eot", "ico", "otf", "ttf", "json", "svg"]
+    content_types = ["text/html", "application/x-javascript", "text/css", "application/javascript", "text/javascript", "application/json", "application/vnd.ms-fontobject", "application/x-font-opentype", "application/x-font-truetype", "application/x-font-ttf", "application/xml", "font/eot", "font/opentype", "font/otf", "image/svg+xml", "image/vnd.microsoft.icon", "text/plain", "text/xml", "text/markdown"]
+    extensions    = ["css", "js", "html", "eot", "ico", "otf", "ttf", "json", "svg", "md", "xml", "txt"]
     name          = "Default Gzip Policy"
   }
 
@@ -72,12 +171,19 @@ resource "fastly_service_vcl" "docs_dev" {
     xff              = "append"
   }
 
-  # Production docs intentionally has no custom VCL. This file is the
-  # behavior-neutral boilerplate equivalent, uploaded so VCL-level changes can
-  # be canaried on docs-dev before deciding how to apply them to production.
   vcl {
     content = file("${path.module}/vcl/docs_dev.vcl")
     main    = true
     name    = "default"
   }
 }
+
+resource "fastly_service_dictionary_items" "docs_dev_versions" {
+  service_id    = fastly_service_vcl.docs_dev.id
+  dictionary_id = one([for d in fastly_service_vcl.docs_dev.dictionary : d.dictionary_id if d.name == "docs_versions"])
+
+  items = {
+    latest = "4.0"
+    master = "4.1"
+  }
+}

cdn/s3.tf

@@ -86,6 +86,106 @@ resource "aws_s3_bucket_lifecycle_configuration" "ftp" {
   }
 }
 
+# Origin of the docs service (docs.ruby-lang.org), canaried on docs-dev first.
+# Prefixes and their writers:
+#   (root)          -- ruby/docs.ruby-lang.org public/ (robots.txt, llms.txt, sitemap.xml, index pages, assets)
+#   en/<version>/   -- ruby/actions docs.yml (extracted RDoc HTML; frozen 3.0/3.1 synced once by hand)
+#   ja/<version>/   -- rurema/generated-documents html/ja/* (latest/master are resolved at the
+#                      edge from the docs_versions dictionary; no symlink objects in the bucket)
+#   capi/en/master/ -- ruby/actions doxygen.yml, once it moves off the rubyci bucket
+# us-east-1 like ftp.r-l.o: every request comes through Fastly, so client
+# latency does not depend on the bucket region and the cheapest tier wins.
+resource "aws_s3_bucket" "docs" {
+  bucket = "docs.r-l.o"
+  region = "us-east-1"
+
+  tags = {
+    Name = "docs.r-l.o"
+  }
+
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+# The legacy buckets predate Block Public Access and their settings are left
+# unmanaged (see README), but a new bucket starts with all four blocks on and
+# rejects PutBucketPolicy until the policy blocks are lifted. ACLs stay
+# blocked; only the bucket policy grants public reads.
+resource "aws_s3_bucket_public_access_block" "docs" {
+  bucket = aws_s3_bucket.docs.bucket
+  region = "us-east-1"
+
+  block_public_acls       = true
+  ignore_public_acls      = true
+  block_public_policy     = false
+  restrict_public_buckets = false
+}
+
+resource "aws_s3_bucket_policy" "docs" {
+  bucket = aws_s3_bucket.docs.bucket
+  region = "us-east-1"
+
+  depends_on = [aws_s3_bucket_public_access_block.docs]
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Sid       = "Allow Public Access to All Objects"
+        Effect    = "Allow"
+        Principal = "*"
+        Action    = ["s3:GetObject", "s3:GetObjectTagging"]
+        Resource  = ["arn:aws:s3:::docs.r-l.o", "arn:aws:s3:::docs.r-l.o/*"]
+      },
+    ]
+  })
+}
+
+resource "aws_s3_bucket_versioning" "docs" {
+  bucket = aws_s3_bucket.docs.bucket
+  region = "us-east-1"
+
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "docs" {
+  bucket = aws_s3_bucket.docs.bucket
+  region = "us-east-1"
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "docs" {
+  bucket = aws_s3_bucket.docs.bucket
+  region = "us-east-1"
+
+  transition_default_minimum_object_size = "varies_by_storage_class"
+
+  rule {
+    id     = "lifecycle"
+    status = "Enabled"
+
+    expiration {
+      expired_object_delete_marker = true
+    }
+
+    noncurrent_version_expiration {
+      noncurrent_days = 7
+    }
+
+    abort_incomplete_multipart_upload {
+      days_after_initiation = 1
+    }
+  }
+}
+
 # Origin of the logs.rubyci.org service (chkbuild logs).
 resource "aws_s3_bucket" "rubyci" {
   bucket = "rubyci"