Skip to content

Commit cd0ec08

Browse files
flakey5flakey5
committed
src, scripts: use kv to cache directories
Closes #159 Closes #314 Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com>
1 parent 20bcac4 commit cd0ec08

12 files changed

Lines changed: 1132 additions & 46 deletions

‎package-lock.json‎

Lines changed: 373 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
},
3939
"dependencies": {
4040
"@aws-sdk/client-s3": "^3.859.0",
41+
"cloudflare": "^5.2.0",
4142
"itty-router": "^5.0.22",
4243
"mustache": "^4.2.0",
4344
"toucan-js": "^4.1.1"

‎scripts/build-directory-cache.mjs‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
#!/usr/bin/env node
2+
// @ts-check
3+
4+
/**
5+
* This is probably not the script you want to run.
6+
*
7+
* This script builds the directory cache from scratch, which uses a _lot_ of
8+
* memory and a noteable amount of time when ran against the dist-prod bucket.
9+
*
10+
* This script should only be ran when there isn't already a directory cache,
11+
* for other scenarios try `./update-directory-cache.mjs`.
12+
*
13+
* Usage: build-directory-cache.mjs [--purge]
14+
* --purge: Use with caution. It will delete all elements in the cache and
15+
* then rebuild it.
16+
*/
17+
18+
import {
19+
DIRECTORY_CACHE_NAMESPACE_ID,
20+
PROD_BUCKET,
21+
RELEASE_DIR,
22+
} from './constants.mjs';
23+
import {
24+
addSymlinksToDirectoryCache,
25+
createCloudflareClient,
26+
createS3Client,
27+
deleteKvNamespaceKeys,
28+
getLatestVersionMapping,
29+
handleReleaseDirectory,
30+
listKvNamespace,
31+
listR2DirectoryRecursive,
32+
writeKeysToKv,
33+
} from './utils.mjs';
34+
35+
// Ensure all necessary environment variables are set
36+
for (const envVar of [
37+
// 'CLOUDFLARE_API_TOKEN',
38+
'CF_ACCESS_KEY_ID',
39+
'CF_SECRET_ACCESS_KEY',
40+
]) {
41+
if (!process.env[envVar]) {
42+
throw new TypeError(`${envVar} missing from process.env`);
43+
}
44+
}
45+
46+
const s3Client = createS3Client(
47+
process.env.CF_ACCESS_KEY_ID,
48+
process.env.CF_SECRET_ACCESS_KEY
49+
);
50+
51+
const cfClient = createCloudflareClient();
52+
53+
// List the entire R2 bucket
54+
const directories = await listR2DirectoryRecursive(s3Client, PROD_BUCKET);
55+
console.log(`Listed ${directories.size} directories from ${PROD_BUCKET}`);
56+
57+
const latestVersions = await getLatestVersionMapping(
58+
s3Client,
59+
PROD_BUCKET,
60+
directories.get(RELEASE_DIR),
61+
directories
62+
);
63+
64+
await handleReleaseDirectory(s3Client, directories, latestVersions);
65+
66+
await addSymlinksToDirectoryCache(
67+
s3Client,
68+
directories,
69+
latestVersions['latest']
70+
);
71+
72+
// Purge directory cache if wanted
73+
if (process.argv[2] === '--purge') {
74+
const cachedDirectoryKeys = await listKvNamespace(
75+
cfClient,
76+
DIRECTORY_CACHE_NAMESPACE_ID
77+
);
78+
console.log(
79+
`Listed ${cachedDirectoryKeys.size} keys from directory cache to purge`
80+
);
81+
82+
await deleteKvNamespaceKeys(
83+
cfClient,
84+
DIRECTORY_CACHE_NAMESPACE_ID,
85+
Array.from(cachedDirectoryKeys)
86+
);
87+
}
88+
89+
// Update keys in the directory cache
90+
console.log(`Writing ${directories.size} keys to directory cache...`);
91+
await writeKeysToKv(cfClient, DIRECTORY_CACHE_NAMESPACE_ID, directories);

‎scripts/build-r2-symlinks.mjs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,12 @@ import {
99
S3Client,
1010
} from '@aws-sdk/client-s3';
1111
import { Linker } from 'nodejs-latest-linker/common.js';
12-
import { DOCS_DIR, ENDPOINT, PROD_BUCKET, RELEASE_DIR } from './constants.mjs';
12+
import {
13+
DOCS_DIR,
14+
R2_ENDPOINT,
15+
PROD_BUCKET,
16+
RELEASE_DIR,
17+
} from './constants.mjs';
1318

1419
const DOCS_DIRECTORY_OUT = join(
1520
import.meta.dirname,
@@ -52,7 +57,7 @@ if (!process.env.CF_SECRET_ACCESS_KEY) {
5257
}
5358

5459
const client = new S3Client({
55-
endpoint: ENDPOINT,
60+
endpoint: R2_ENDPOINT,
5661
region: 'auto',
5762
credentials: {
5863
accessKeyId: process.env.CF_ACCESS_KEY_ID,

‎scripts/constants.mjs‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,26 @@
1-
'use strict';
2-
31
import { dirname, join } from 'node:path';
42
import { readdir, readFile, stat } from 'node:fs/promises';
53

6-
export const ENDPOINT =
7-
process.env.ENDPOINT ??
8-
'https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com';
4+
export const CF_ACCOUNT_TAG =
5+
process.env.CF_ACCOUNT_TAG ?? '07be8d2fbc940503ca1be344714cb0d1';
96

10-
export const PROD_BUCKET = process.env.PROD_BUCKET ?? 'dist-prod';
7+
export const R2_ENDPOINT =
8+
process.env.R2_ENDPOINT ??
9+
`https://${CF_ACCOUNT_TAG}.r2.cloudflarestorage.com`;
1110

12-
export const STAGING_BUCKET = process.env.STAGING_BUCKET ?? 'dist-staging';
11+
/**
12+
* The id of the KV namespace used for caching directories
13+
*/
14+
export const DIRECTORY_CACHE_NAMESPACE_ID =
15+
process.env.DIRECTORY_CACHE_NAMESPACE_ID ?? '';
1316

14-
export const R2_RETRY_COUNT = 3;
17+
export const R2_RETRY_COUNT = 5;
18+
export const KV_RETRY_COUNT = 5;
1519

16-
export const RELEASE_DIR = 'nodejs/release/';
20+
export const PROD_BUCKET = process.env.PROD_BUCKET ?? 'dist-prod';
21+
export const STAGING_BUCKET = process.env.STAGING_BUCKET ?? 'dist-staging';
1722

23+
export const RELEASE_DIR = 'nodejs/release/';
1824
export const DOCS_DIR = 'nodejs/docs/';
1925

2026
export const DEV_BUCKET_PATH = join(import.meta.dirname, '..', 'dev-bucket');

‎scripts/update-directory-cache.mjs‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
#!/usr/bin/env node
2+
// @ts-check
3+
4+
/**
5+
* Update the most commonly updated paths in the directory cache for a Node.js
6+
* release.
7+
*
8+
* Usage: update-directory-cache.mjs <version>
9+
*/
10+
11+
import {
12+
DIRECTORY_CACHE_NAMESPACE_ID,
13+
DOCS_DIR,
14+
PROD_BUCKET,
15+
RELEASE_DIR,
16+
} from './constants.mjs';
17+
import {
18+
createCloudflareClient,
19+
createS3Client,
20+
listR2DirectoryRecursive,
21+
listR2Directory,
22+
writeKeysToKv,
23+
getLatestVersionMapping,
24+
handleReleaseDirectory,
25+
addSymlinksToDirectoryCache,
26+
} from './utils.mjs';
27+
28+
const VERSION = process.argv[2];
29+
30+
if (!VERSION) {
31+
throw new TypeError('version missing from args');
32+
}
33+
34+
// Ensure all necessary environment variables are set
35+
for (const envVar of [
36+
// 'CLOUDFLARE_API_TOKEN',
37+
'CF_ACCESS_KEY_ID',
38+
'CF_SECRET_ACCESS_KEY',
39+
]) {
40+
if (!process.env[envVar]) {
41+
throw new TypeError(`${envVar} missing from process.env`);
42+
}
43+
}
44+
45+
const s3Client = createS3Client(
46+
process.env.CF_ACCESS_KEY_ID,
47+
process.env.CF_SECRET_ACCESS_KEY
48+
);
49+
50+
const cfClient = createCloudflareClient();
51+
52+
const ALWAYS_UPDATED_PATHS = [RELEASE_DIR, DOCS_DIR];
53+
54+
// List the directory of the new release (and subdirectories)
55+
const directories = await listR2DirectoryRecursive(
56+
s3Client,
57+
`nodejs/release/${VERSION}/`
58+
);
59+
60+
// Non-recursively list the paths that we always want to update
61+
await Promise.all(
62+
ALWAYS_UPDATED_PATHS.map(path =>
63+
listR2Directory(s3Client, PROD_BUCKET, path).then(value => {
64+
directories.set(path, value);
65+
})
66+
)
67+
);
68+
69+
const latestVersions = await getLatestVersionMapping(
70+
s3Client,
71+
PROD_BUCKET,
72+
directories.get(RELEASE_DIR),
73+
directories
74+
);
75+
76+
await handleReleaseDirectory(s3Client, directories, latestVersions);
77+
78+
await addSymlinksToDirectoryCache(
79+
s3Client,
80+
directories,
81+
latestVersions['latest']
82+
);
83+
84+
// Update keys in the directory cache
85+
console.log(`Writing ${directories.size} keys to directory cache...`);
86+
await writeKeysToKv(cfClient, DIRECTORY_CACHE_NAMESPACE_ID, directories);

0 commit comments

Comments
 (0)