src, scripts: use kv to cache directories

Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com>

Author: flakey5

.env.example

@@ -0,0 +1,30 @@
+# Your Cloudflare account tag.
+# 
+# Needed for:
+#  - Directory cache scripts
+CLOUDFLARE_ACCOUNT_ID=
+
+# Cloudflare V4 API token.
+# 
+# Needed for:
+#  - Directory cache scripts
+# 
+# Required permissions:
+#  - `Workers KV Storage`: Edit
+#  - `Workers R2 Storage`: Read
+# 
+# See https://developers.cloudflare.com/fundamentals/api/get-started/create-token/
+CLOUDFLARE_API_TOKEN=
+
+# S3 credentials for your R2 bucket.
+# 
+# Needed for:
+#  - Directory listings in the worker.
+#  - Directory cache scripts
+# 
+# Required permissions:
+#  - `Object Read Only`
+# 
+# See https://dash.cloudflare.com/?account=/r2/api-tokens
+S3_ACCESS_KEY_ID=
+S3_ACCESS_KEY_SECRET=

.github/workflows/update-links.yml

@@ -6,6 +6,11 @@ permissions:
 on:
   # Triggered by https://github.com/nodejs/node/blob/main/.github/workflows/update-release-links.yml
   workflow_dispatch:
+    inputs:
+      version:
+        description: 'Node.js version (ex/ `v20.0.0`)'
+        required: true
+        type: string
 
 concurrency:
   group: update-redirect-links
@@ -49,11 +54,13 @@ jobs:
       - name: Install dependencies
         run: npm ci && npm update nodejs-latest-linker --save
 
-      - name: Update Redirect Links
-        run: node scripts/build-r2-symlinks.mjs && node --run format
+      - name: Update Directory Cache
+        run: node scripts/update-directory-cache.mjs "$VERSION_INPUT" && node --run format
         env:
-          CF_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }}
-          CF_SECRET_ACCESS_KEY: ${{ secrets.CF_SECRET_ACCESS_KEY }}
+          VERSION_INPUT: '${{ inputs.version }}'
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
+          S3_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }}
+          S3_ACCESS_KEY_SECRET: ${{ secrets.CF_SECRET_ACCESS_KEY }}
 
       - name: Commit Changes
         id: git_auto_commit

.gitignore

@@ -3,3 +3,4 @@ node_modules/
 dist/
 .dev.vars
 .sentryclirc
+.env

e2e-tests/directory.kv.test.ts

@@ -0,0 +1,89 @@
+import { env, createExecutionContext } from 'cloudflare:test';
+import { test, beforeAll, expect, vi } from 'vitest';
+import {
+  populateDirectoryCacheWithDevBucket,
+  populateR2WithDevBucket,
+} from './util';
+import worker from '../src/worker';
+import type { Env } from '../src/env';
+import { CACHE_HEADERS } from '../src/constants/cache';
+
+const mockedEnv: Env = {
+  ...env,
+  ENVIRONMENT: 'e2e-tests',
+  CACHING: false,
+  LOG_ERRORS: true,
+  USE_KV: true,
+};
+
+beforeAll(async () => {
+  await populateR2WithDevBucket();
+  await populateDirectoryCacheWithDevBucket();
+
+  vi.mock(
+    import('../src/constants/latestVersions.json'),
+    async importOriginal => {
+      const original = await importOriginal();
+
+      // Point all `latest-` directories to one that exists in the dev bucket
+      Object.keys(original.default).forEach(branch => {
+        let updatedValue: string;
+        if (branch === 'node-latest.tar.gz') {
+          updatedValue = 'latest/node-v20.0.0.tar.gz';
+        } else {
+          updatedValue = 'v20.0.0';
+        }
+
+        // @ts-expect-error
+        original.default[branch] = updatedValue;
+      });
+
+      return original;
+    }
+  );
+});
+
+// Ensure essential endpoints are routable
+for (const path of ['/dist/', '/docs/', '/api/', '/download/', '/metrics/']) {
+  test(`GET \`${path}\` returns 200`, async () => {
+    const ctx = createExecutionContext();
+
+    const res = await worker.fetch(
+      new Request(`https://localhost${path}`),
+      mockedEnv,
+      ctx
+    );
+
+    // Consume body promise
+    await res.text();
+
+    expect(res.status).toBe(200);
+  });
+}
+
+test('GET `/dist/unknown-directory/` returns 404', async () => {
+  const ctx = createExecutionContext();
+
+  const res = await worker.fetch(
+    new Request('https://localhost/dist/unknown-directory/'),
+    mockedEnv,
+    ctx
+  );
+
+  expect(res.status).toBe(404);
+  expect(res.headers.get('cache-control')).toStrictEqual(CACHE_HEADERS.failure);
+  expect(await res.text()).toStrictEqual('Directory not found');
+});
+
+test('GET `/dist` redirects to `/dist/`', async () => {
+  const ctx = createExecutionContext();
+
+  const res = await worker.fetch(
+    new Request('https://localhost/dist'),
+    mockedEnv,
+    ctx
+  );
+
+  expect(res.status).toBe(301);
+  expect(res.headers.get('location')).toStrictEqual('https://localhost/dist/');
+});

e2e-tests/directory.test.ts e2e-tests/directory.s3.test.tse2e-tests/directory.test.ts renamed to e2e-tests/directory.s3.test.ts

@@ -1,7 +1,9 @@
 import { env } from 'cloudflare:test';
+import { join } from 'node:path';
 import { inject } from 'vitest';
-import type { Env } from '../env';
-import type { Directory } from '../../vitest-setup';
+import type { Env } from '../src/env';
+import type { Directory } from '../vitest-setup';
+import type { ReadDirectoryResult, File } from '../src/providers/provider';
 
 async function populateR2BucketDirectory(directory: Directory): Promise<void> {
   const promises: Array<Promise<unknown>> = [];
@@ -10,7 +12,7 @@ async function populateR2BucketDirectory(directory: Directory): Promise<void> {
     const file = directory.files[path];
 
     promises.push(
-      env.R2_BUCKET.put(path, file.contents, {
+      env.R2_BUCKET.put(join(directory.name, path), file.contents, {
         customMetadata: {
           // This is added by rclone when copying the release assets to the
           //  bucket.
@@ -27,6 +29,41 @@ async function populateR2BucketDirectory(directory: Directory): Promise<void> {
   await Promise.all(promises);
 }
 
+async function populateDirectoryCache(directory: Directory): Promise<void> {
+  let hasIndexHtmlFile = false;
+
+  const files: File[] = Object.keys(directory.files).map(name => {
+    const file = directory.files[name];
+
+    if (!hasIndexHtmlFile && name.match(/index.htm(?:l)$/)) {
+      hasIndexHtmlFile = true;
+    }
+
+    return {
+      name,
+      lastModified: new Date(file.lastModified),
+      size: file.size,
+    };
+  });
+
+  const cachedDirectory: ReadDirectoryResult = {
+    subdirectories: Object.keys(directory.subdirectories),
+    files,
+    hasIndexHtmlFile,
+    lastModified: new Date(),
+  };
+
+  const promises: Array<Promise<void>> = [
+    env.DIRECTORY_CACHE.put(
+      `${directory.name}/`,
+      JSON.stringify(cachedDirectory)
+    ),
+    ...Object.values(directory.subdirectories).map(populateDirectoryCache),
+  ];
+
+  await Promise.all(promises);
+}
+
 /**
  * Writes the contents of the dev bucket into the R2 bucket given in {@link env}
  */
@@ -38,6 +75,14 @@ export async function populateR2WithDevBucket(): Promise<void> {
   await populateR2BucketDirectory(devBucket);
 }
 
+export async function populateDirectoryCacheWithDevBucket(): Promise<void> {
+  // Grab the contents of the dev bucket
+  const devBucket = inject('devBucket');
+
+  // Write it to KV
+  await populateDirectoryCache(devBucket);
+}
+
 declare module 'cloudflare:test' {
   // eslint-disable-next-line @typescript-eslint/no-empty-object-type
   interface ProvidedEnv extends Env {}

e2e-tests/util.ts

@@ -0,0 +1,3 @@
+# `lib/`
+
+Utilities used in local scripts and in the deployed worker.

lib/README.md

@@ -0,0 +1,19 @@
+/**
+ * Max amount of retries for requests to R2
+ */
+export const R2_RETRY_LIMIT = 5;
+
+/**
+ * Max amount of retries for requests to KV
+ */
+export const KV_RETRY_LIMIT = 5;
+
+/**
+ * Max amount of keys to be returned in a S3 request
+ */
+export const S3_MAX_KEYS = 1000;
+
+/**
+ * Max amount of keys we can have in a KV request
+ */
+export const KV_MAX_KEYS = 10_000;

lib/limits.mjs

@@ -0,0 +1,111 @@
+import { ListObjectsV2Command } from '@aws-sdk/client-s3';
+import { R2_RETRY_LIMIT, S3_MAX_KEYS } from './limits.mjs';
+
+/**
+ * List the contents of a directory in R2.
+ *
+ * @param {import('@aws-sdk/client-s3').S3Client} client
+ * @param {string} bucket
+ * @param {string | undefined} [directory=undefined]
+ * @param {number} retryCount
+ * @returns {Promise<import('../src/providers/provider.js').ReadDirectoryResult | undefined>}
+ */
+export async function listR2Directory(
+  client,
+  bucket,
+  directory = undefined,
+  retryCount = R2_RETRY_LIMIT
+) {
+  /**
+   * @type {Set<string>}
+   */
+  const subdirectories = new Set();
+
+  /**
+   * @type {Set<import('../src/providers/provider.js').File>}
+   */
+  const files = new Set();
+
+  let hasIndexHtmlFile = false;
+  let directoryLastModified = new Date(0);
+
+  let isTruncated;
+  let continuationToken;
+  do {
+    /**
+     * @type {import('@aws-sdk/client-s3').ListObjectsV2Output | undefined}
+     */
+    let data = undefined;
+
+    let retriesLeft = retryCount;
+    while (retriesLeft) {
+      try {
+        data = await client.send(
+          new ListObjectsV2Command({
+            Bucket: bucket,
+            Delimiter: '/',
+            Prefix: directory,
+            ContinuationToken: continuationToken,
+            MaxKeys: S3_MAX_KEYS,
+          })
+        );
+
+        break;
+      } catch (err) {
+        retriesLeft--;
+
+        if (retriesLeft === 0) {
+          throw new Error('exhausted R2 retries', { cause: err });
+        }
+      }
+    }
+
+    if (!data) {
+      return undefined;
+    }
+
+    isTruncated = data.IsTruncated;
+    continuationToken = data.NextContinuationToken;
+
+    data.CommonPrefixes?.forEach(subdirectory => {
+      if (subdirectory.Prefix) {
+        subdirectories.add(
+          subdirectory.Prefix.substring(directory?.length ?? 0)
+        );
+      }
+    });
+
+    data.Contents?.forEach(file => {
+      if (!file.Key) {
+        return;
+      }
+
+      if (!hasIndexHtmlFile && file.Key.match(/index.htm(?:l)$/)) {
+        hasIndexHtmlFile = true;
+      }
+
+      files.add({
+        name: file.Key.substring(directory?.length ?? 0),
+        lastModified: file.LastModified,
+        size: file.Size,
+      });
+
+      // Set the directory's last modified date to be the same as the most
+      // recently updated file
+      if (file.LastModified > directoryLastModified) {
+        directoryLastModified = file.LastModified;
+      }
+    });
+  } while (isTruncated);
+
+  if (subdirectories.size === 0 && files.size === 0) {
+    return undefined;
+  }
+
+  return {
+    subdirectories: Array.from(subdirectories),
+    hasIndexHtmlFile,
+    files: Array.from(files),
+    lastModified: directoryLastModified,
+  };
+}