From ecc6ea78551036639f987c4dd1a40f0fd2ea0e88 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 16 Sep 2026 16:43:15 +0200 Subject: [PATCH] feat(fs-storage): adopt out-of-band key-value files as regular records (#4128) --- docs/guides/result_storage.mdx | 7 +- docs/upgrading/upgrading_v4.md | 18 +- packages/fs-storage/package.json | 2 +- .../fs-storage/src/file-system-storage.ts | 71 +++- .../src/resource-clients/key-value-store.ts | 135 +------- .../test/configured-input-key.test.ts | 100 ------ packages/fs-storage/test/fs-fallback.test.ts | 314 ------------------ .../test/key-value-store/adoption.test.ts | 264 +++++++++++++++ pnpm-lock.yaml | 74 ++--- test/core/storages/storage_purge.test.ts | 3 +- 10 files changed, 401 insertions(+), 587 deletions(-) delete mode 100644 packages/fs-storage/test/configured-input-key.test.ts delete mode 100644 packages/fs-storage/test/fs-fallback.test.ts create mode 100644 packages/fs-storage/test/key-value-store/adoption.test.ts diff --git a/docs/guides/result_storage.mdx b/docs/guides/result_storage.mdx index c6f9b5379a18..e0890fa7a61c 100644 --- a/docs/guides/result_storage.mdx +++ b/docs/guides/result_storage.mdx @@ -21,12 +21,15 @@ In Crawlee, the key-value store is represented by the .__metadata__.json` sidecar that marks a record — the Apify CLI's input, a project template, a v3 store directory, a file you dropped in with an editor. Opening the store writes the missing sidecar (the value bytes are never touched), and from then on the file is an ordinary record: read by `getValue`, enumerated by `listKeys`, removed by `deleteValue`. Two rules decide the key: -- **Only `INPUT` and `INPUT.json` are probed.** v3 also fell back to `INPUT.txt` and `INPUT.bin`. In v4 those files are not read (`getValue('INPUT')` returns `undefined`), are not listed, and are no longer exempt from the purge of the default store on start, so they get deleted like any other untracked file. Rename a `.txt`/`.bin` input to `INPUT.json` (or drop the extension) before upgrading. -- **Extensionless bare files report `application/octet-stream`.** In v3 a bare value file with no extension was read as `text/plain`. In v4 the client is a plain byte transport and only infers a content type from a real extension, so an extensionless file now comes back as `application/octet-stream`. Give the file a `.json` extension if you need a more specific type. -- **Malformed bare files are no longer silently swallowed.** In v3 a bare `INPUT.json` containing invalid JSON was treated as a missing record (`getValue` returned `undefined`). In v4 the raw bytes are returned verbatim and parsing happens in the `KeyValueStore` frontend, so a malformed value now surfaces a parse error at read time instead of looking absent. -- **Bare files are enumerated by `listKeys` under their actual on-disk name.** A bare `INPUT.json` shows up in `listKeys` as `INPUT.json` and reads back cleanly under that key via `getValue` / `recordExists` / `getPublicUrl`; the logical `INPUT` lookup keeps resolving the same file as well. An extensionless bare file is listed as `INPUT`. If both a tracked `INPUT` record and a bare `INPUT.json` exist, the tracked record wins and the bare variant is not listed. Everything `listKeys` needs is read from the filesystem index, so this no longer triggers the per-read O(n) directory scans the v3 fallback performed. +- In the **default** store, the run-input keys (`INPUT` and the configured `inputKey`) claim a bare `INPUT` or `INPUT.json`. The key is `INPUT` while the file keeps its name, so `listKeys` reports `INPUT`, `getValue('INPUT.json')` is `undefined`, and `getPublicUrl('INPUT')` points at `INPUT.json`. If both files are present, opening the store fails instead of guessing which one is the input. +- Every other sidecar-less file becomes a record **keyed by its filename**, in every store: a hand-placed `some-key.json` is the key `some-key.json`, and so is an `INPUT.json` in a store other than the default one. Dotfiles are skipped. + +A `.json` file is adopted as `application/json; charset=utf-8` and anything else as `application/octet-stream`; there is no content sniffing. Adopted records are subject to the purge of the default store on start like any other record — only the run-input keys are spared. + +Beyond the literal keys, three v3 behaviors are gone: + +- **`INPUT.txt` and `INPUT.bin` are no longer the input.** v3 probed those extensions too. In v4 they are adopted under their own names, so `getValue('INPUT')` returns `undefined` and the purge on start deletes them. Rename such an input to `INPUT.json` (or drop the extension) before upgrading. +- **Extensionless input files report `application/octet-stream`.** In v3 a bare value file with no extension was read as `text/plain`. Give the file a `.json` extension if you need a more specific type. +- **Malformed input files are no longer silently swallowed.** In v3 an `INPUT.json` containing invalid JSON was treated as a missing record (`getValue` returned `undefined`). In v4 the raw bytes are returned verbatim and parsing happens in the `KeyValueStore` frontend, so a malformed value surfaces a parse error at read time instead of looking absent. ## Only if you tuned autoscaling diff --git a/packages/fs-storage/package.json b/packages/fs-storage/package.json index 35414342ff91..f15178fba773 100644 --- a/packages/fs-storage/package.json +++ b/packages/fs-storage/package.json @@ -42,7 +42,7 @@ "access": "public" }, "dependencies": { - "@crawlee/fs-storage-native": ">=0.2.1-beta.0 <0.3", + "@crawlee/fs-storage-native": ">=0.2.1 <0.3", "@crawlee/types": "workspace:*", "@crawlee/utils": "workspace:*", "zod": "catalog:" diff --git a/packages/fs-storage/src/file-system-storage.ts b/packages/fs-storage/src/file-system-storage.ts index 274c16b992bd..8fffc2269d2a 100644 --- a/packages/fs-storage/src/file-system-storage.ts +++ b/packages/fs-storage/src/file-system-storage.ts @@ -6,6 +6,7 @@ import type { CrawleeLogger } from '@crawlee/types'; import { parseArgument, schemas } from '@crawlee/utils/internal'; import { z } from 'zod'; +import type { AdoptionCandidate } from '@crawlee/fs-storage-native'; import { DatasetBackend } from './resource-clients/dataset.js'; import { KeyValueStoreBackend } from './resource-clients/key-value-store.js'; import { RequestQueueBackend } from './resource-clients/request-queue.js'; @@ -33,6 +34,17 @@ const DEFAULT_STORAGE_ALIAS = '__default__'; /** The directory the default storage lives in, one level below `datasets` / `key_value_stores` / etc. */ const DEFAULT_STORAGE_DIRECTORY = 'default'; +/** The conventional run-input key, always treated as one alongside the configured `inputKey`. */ +const DEFAULT_INPUT_KEY = 'INPUT'; + +/** + * Content types declared for adopted files. The native client infers nothing from extensions, and + * neither do we beyond this: a `.json` file is JSON, anything else is bytes for the + * {@apilink KeyValueStore} frontend to make sense of. + */ +const ADOPTED_JSON_CONTENT_TYPE = 'application/json; charset=utf-8'; +const ADOPTED_BINARY_CONTENT_TYPE = 'application/octet-stream'; + export interface FileSystemStorageOptions { /** * Path to directory where the data will be saved. @@ -64,11 +76,11 @@ export interface FileSystemStorageOptions { /** * The key the run input is read from — Crawlee's `inputKey` (`CRAWLEE_INPUT_KEY`). * - * Like the conventional `INPUT`, this key may live in the default key-value store as a bare value + * Like the conventional `INPUT`, this key may arrive in the default key-value store as a bare value * file with no metadata sidecar (e.g. the Apify CLI writes the effective input to `__CLI_INPUT.json` - * and points the run at that key). It is therefore readable out-of-band (``, `.json`) and - * preserved when the default store is purged, exactly like `INPUT`, - * which is always kept regardless of this setting. + * and points the run at that key). Such a file is adopted into a record under this key when the + * store is opened, and the key is preserved when the default store is purged — exactly like + * `INPUT`, which is always treated as an input key regardless of this setting. * * @default 'INPUT' */ @@ -90,7 +102,8 @@ export class FileSystemStorageBackend implements storage.StorageBackend { readonly requestQueuesDirectory: string; readonly logger?: CrawleeLogger; readonly requestQueueAccess: 'single' | 'shared'; - readonly #inputKey: string; + /** `INPUT` plus the configured `inputKey`, deduplicated. */ + readonly #inputKeys: string[]; readonly #keyValueStoreBackendCache: KeyValueStoreBackend[] = []; readonly #datasetBackendCache: DatasetBackend[] = []; @@ -104,7 +117,7 @@ export class FileSystemStorageBackend implements storage.StorageBackend { this.logger = logger; this.requestQueueAccess = requestQueueAccess; - this.#inputKey = inputKey; + this.#inputKeys = [...new Set([DEFAULT_INPUT_KEY, inputKey])]; this.localDataDirectory = localDataDirectory; this.datasetsDirectory = resolve(this.localDataDirectory, 'datasets'); @@ -182,13 +195,21 @@ export class FileSystemStorageBackend implements storage.StorageBackend { const nativeBackend = await ( await importNativeModule() - ).FileSystemKeyValueStoreClient.open(id, name, alias, this.localDataDirectory); + ).FileSystemKeyValueStoreClient.open( + id, + name, + alias, + this.localDataDirectory, + // useTestClock — always real wall-clock outside of native tests. + undefined, + this.#adoptionCandidates(cacheKey === DEFAULT_STORAGE_DIRECTORY), + ); const newStore = await KeyValueStoreBackend.create({ name: alias ? undefined : (name ?? cacheKey), cacheKey, nativeBackend, logger: this.logger, - inputKey: this.#inputKey, + inputKeys: this.#inputKeys, }); this.#keyValueStoreBackendCache.push(newStore); @@ -230,6 +251,40 @@ export class FileSystemStorageBackend implements storage.StorageBackend { return newStore; } + /** + * What the native `open` may turn into records: value files sitting in the store directory with no + * metadata sidecar, written out-of-band by a CLI, a project template, a v3 Crawlee or a text editor. + * + * Each run-input key claims a bare `` or `.json` first — that is the layout the Apify CLI + * and the templates produce, and the key must end up being `INPUT` rather than `INPUT.json`. Only + * the default store holds the run input, so elsewhere an `INPUT.json` is just a file named + * `INPUT.json`, adopted by the trailing sweep like every other one. + * + * Once adopted, the file is an ordinary record: readable, listed, deletable, and — in a run-scoped + * store — purged on start unless its key is a run-input key. + */ + #adoptionCandidates(isDefaultStore: boolean): AdoptionCandidate[] { + const inputCandidates: AdoptionCandidate[] = isDefaultStore + ? this.#inputKeys.map((key) => ({ + key, + files: [ + { filename: key, contentType: ADOPTED_BINARY_CONTENT_TYPE }, + { filename: `${key}.json`, contentType: ADOPTED_JSON_CONTENT_TYPE }, + ], + })) + : []; + + return [ + ...inputCandidates, + { + files: [ + { filename: '*.json', contentType: ADOPTED_JSON_CONTENT_TYPE }, + { filename: '*', contentType: ADOPTED_BINARY_CONTENT_TYPE }, + ], + }, + ]; + } + async storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise { let backends: (KeyValueStoreBackend | DatasetBackend | RequestQueueBackend)[]; let baseDir: string; diff --git a/packages/fs-storage/src/resource-clients/key-value-store.ts b/packages/fs-storage/src/resource-clients/key-value-store.ts index 148ddbd232e0..adbe2b87caee 100644 --- a/packages/fs-storage/src/resource-clients/key-value-store.ts +++ b/packages/fs-storage/src/resource-clients/key-value-store.ts @@ -5,30 +5,10 @@ import type { CrawleeLogger } from '@crawlee/types'; import { parseArgument, schemas } from '@crawlee/utils/internal'; import { z } from 'zod'; -import type { - FileSystemKeyValueStoreClient as NativeFileSystemKeyValueStoreBackend, - ListBareFallback, -} from '@crawlee/fs-storage-native'; +import type { FileSystemKeyValueStoreClient as NativeFileSystemKeyValueStoreBackend } from '@crawlee/fs-storage-native'; import { isStream } from '../utils.js'; import { CachedIdClient } from './cached-id-client.js'; -/** - * Out-of-band ("bare") value-file fallbacks tried when a run-input lookup misses the tracked record, so a - * lookup for `INPUT` also matches a hand-placed `INPUT.json`. Passed to the native - * `resolveValue`/`resolveExistingKey`, which do the probing and re-keying. - * - * Each entry declares the content type to report on a match — the native client does no MIME - * inference. An empty `contentType` is its sentinel for "keep the synthesized - * `application/octet-stream`", used for the extensionless key. - */ -const BARE_FILE_FALLBACKS: { extension: string; contentType: string }[] = [ - { extension: '', contentType: '' }, - { extension: '.json', contentType: 'application/json; charset=utf-8' }, -]; - -/** The conventional run-input key, always treated as one alongside the configured `inputKey`. */ -const DEFAULT_INPUT_KEY = 'INPUT'; - const keySchema = z.string(); const inputRecordShape = z.object({ @@ -55,8 +35,11 @@ export interface KeyValueStoreBackendOptions { cacheKey: string; nativeBackend: NativeFileSystemKeyValueStoreBackend; logger?: CrawleeLogger; - /** The configured run-input key, see `FileSystemStorageOptions.inputKey`. Treated like `INPUT`. */ - inputKey?: string; + /** + * The run-input keys of this store, deduplicated — `INPUT` plus `FileSystemStorageOptions.inputKey`. + * All that is left of the run-input special case: the keys {@link purgeExceptInput} spares. + */ + inputKeys: string[]; } /** @@ -73,29 +56,15 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyV readonly #nativeBackend: NativeFileSystemKeyValueStoreBackend; - /** `INPUT` plus the configured `inputKey`, deduplicated. */ + /** See {@link KeyValueStoreBackendOptions.inputKeys}. */ readonly #inputKeys: string[]; - /** Bare files the native `listKeys` should surface, under their on-disk name (e.g. `INPUT.json`). */ - readonly #listBareFallbacks: ListBareFallback[]; - - /** Bare-file on-disk name (`INPUT.json`) to logical key (`INPUT`). */ - readonly #bareFileLogicalKeys: Map; - constructor(options: KeyValueStoreBackendOptions) { super(); this.name = options.name; this.cacheKey = options.cacheKey; this.#nativeBackend = options.nativeBackend; - this.#inputKeys = [...new Set([DEFAULT_INPUT_KEY, options.inputKey ?? DEFAULT_INPUT_KEY])]; - this.#listBareFallbacks = this.#inputKeys.flatMap((key) => - BARE_FILE_FALLBACKS.map(({ extension, contentType }) => ({ name: `${key}${extension}`, contentType })), - ); - this.#bareFileLogicalKeys = new Map( - this.#inputKeys.flatMap((key) => - BARE_FILE_FALLBACKS.map(({ extension }) => [`${key}${extension}`, key] as const), - ), - ); + this.#inputKeys = options.inputKeys; } get keyValueStoreDirectory(): string { @@ -123,61 +92,30 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyV /** * Remove every record from the store except the run input. Used by * {@link FileSystemStorageBackend.purge} to clean the default key-value store at the start of a run - * while preserving the run's input. The native keep-list matches exact filenames, so every extension - * variant of every input key is listed. + * while preserving the run's input. */ async purgeExceptInput(): Promise { - const keep = this.#inputKeys.flatMap((key) => BARE_FILE_FALLBACKS.map(({ extension }) => `${key}${extension}`)); - await this.#nativeBackend.purge(keep); + await this.#nativeBackend.purge(this.#inputKeys); } async listKeys(options: storage.KeyValueStoreListKeysOptions = {}): Promise { const { prefix, exclusiveStartKey, limit } = parseArgument(options, schemas.keyValueStoreListKeysOptions); - // Pass the bare-file fallbacks so out-of-band value files (e.g. a hand-placed `INPUT.json`) - // are enumerated alongside tracked records, under their actual on-disk name. The native reads - // everything it needs off the filesystem index — no per-file reads — so this stays cheap. // The native `listKeys` already returns a self-describing page (items + pagination cursors) - // matching the `KeyValueStoreListKeysResult` contract, so we only post-process the items. - const page = await this.#nativeBackend.listKeys(exclusiveStartKey, limit, prefix, this.#listBareFallbacks); - - const presentKeys = new Set(page.items.map((record) => record.key)); - - // A bare value file is listed under its actual name (`INPUT.json`), which already round-trips - // through `getValue`/`recordExists`. The only collision is a tracked record occupying the - // logical key itself (`INPUT`): it shadows the extension-bearing bare variants (`INPUT.json` - // etc.) for the same logical key, so drop those. The extensionless bare file *is* the logical - // key, so it is never a separate duplicate. - const items = page.items.filter((record) => { - const logicalKey = this.#bareFileLogicalKeys.get(record.key); - const isExtensionBearingBareFile = logicalKey !== undefined && logicalKey !== record.key; - return !(isExtensionBearingBareFile && presentKeys.has(logicalKey)); - }); - - return { - items, - count: items.length, - limit: page.limit, - exclusiveStartKey: page.exclusiveStartKey, - isTruncated: page.isTruncated, - nextExclusiveStartKey: page.nextExclusiveStartKey, - }; + // matching the `KeyValueStoreListKeysResult` contract. + return this.#nativeBackend.listKeys(exclusiveStartKey, limit, prefix); } /** * Generates a public `file://` URL for accessing a specific record in the key-value store. * - * The native `getPublicUrl` derives the URL from the key without probing bare-file extensions, so - * an `INPUT` that lives on disk as a hand-placed `INPUT.json` is resolved first. Nothing on disk - * means nothing to resolve — the requested key is used as-is, and the URL is the one the record will - * have once written. + * Existence-agnostic per the {@apilink KeyValueStoreBackend} contract: the URL points at the file + * the key is bound to if there is a record, and at the file it would be written to otherwise. * @param key The key of the record to generate the public URL for. */ async getPublicUrl(key: string): Promise { parseArgument(key, keySchema); - - const resolvedKey = (await this.resolveExistingKey(key)) ?? key; - return this.#nativeBackend.getPublicUrl(resolvedKey); + return this.#nativeBackend.getPublicUrl(key); } /** @@ -188,17 +126,13 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyV */ async recordExists(key: string): Promise { parseArgument(key, keySchema); - return (await this.resolveExistingKey(key)) !== undefined; + return this.#nativeBackend.recordExists(key); } async getValue(key: string): Promise { parseArgument(key, keySchema); - const fallbacks = this.#bareFallbacksFor(key); - const record = fallbacks - ? await this.#nativeBackend.resolveValue(key, fallbacks) - : await this.#nativeBackend.getValue(key); - + const record = await this.#nativeBackend.getValue(key); if (record) { return { key: record.key, @@ -246,39 +180,4 @@ export class KeyValueStoreBackend extends CachedIdClient implements storage.KeyV parseArgument(key, keySchema); await this.#nativeBackend.deleteValue(key); } - - /** - * Resolve `key` to the on-disk key that actually exists, or `undefined` if nothing does. Run-input - * keys fall back to bare files, in which case the matched on-disk key is returned so callers like - * `getPublicUrl` point at the file that exists. - */ - private async resolveExistingKey(key: string): Promise { - const fallbacks = this.#bareFallbacksFor(key); - if (fallbacks) { - return ( - (await this.#nativeBackend.resolveExistingKey( - key, - fallbacks.map(({ extension }) => extension), - )) ?? undefined - ); - } - return (await this.#nativeBackend.recordExists(key)) ? key : undefined; - } - - /** - * Bare-file fallbacks for `key`, or `undefined` for a plain tracked-record lookup. A logical input - * key (`INPUT`) probes the whole extension ladder; a literal bare filename (`INPUT.json`, as listed - * by `listKeys`) probes only itself, so a listed key reads back under its own name. - */ - #bareFallbacksFor(key: string): { extension: string; contentType: string }[] | undefined { - if (this.#inputKeys.includes(key)) { - return BARE_FILE_FALLBACKS; - } - const logicalKey = this.#bareFileLogicalKeys.get(key); - if (logicalKey === undefined) { - return undefined; - } - const { contentType } = BARE_FILE_FALLBACKS.find(({ extension }) => `${logicalKey}${extension}` === key)!; - return [{ extension: '', contentType }]; - } } diff --git a/packages/fs-storage/test/configured-input-key.test.ts b/packages/fs-storage/test/configured-input-key.test.ts deleted file mode 100644 index 706c7126e99d..000000000000 --- a/packages/fs-storage/test/configured-input-key.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { mkdir, readdir, rm, writeFile } from 'node:fs/promises'; -import { resolve } from 'node:path'; - -import { FileSystemStorageBackend } from '@crawlee/fs-storage'; -import type { KeyValueStoreRecord } from '@crawlee/types'; - -// A run may be pointed at a different input key than `INPUT` (Crawlee's `inputKey` / `CRAWLEE_INPUT_KEY`). -// The Apify CLI does exactly that: it writes the effective input to a bare `__CLI_INPUT.json` (no metadata -// sidecar) in the default store and sets the input key to `__CLI_INPUT`. That key must get the same -// out-of-band read fallback and purge exemption as `INPUT`, otherwise the input is unreadable — or -// deleted by the purge on start before the run ever reads it. -describe('the configured input key', () => { - const tmpLocation = resolve(import.meta.dirname, './tmp/configured-input-key'); - const inputKey = '__CLI_INPUT'; - const payload = JSON.stringify({ hello: 'from the cli' }); - - const seedDefaultStore = async (storage: FileSystemStorageBackend, files: Record) => { - const dir = resolve(storage.keyValueStoresDirectory, 'default'); - await mkdir(dir, { recursive: true }); - for (const [file, content] of Object.entries(files)) { - await writeFile(resolve(dir, file), content); - } - }; - - afterEach(async () => { - await rm(tmpLocation, { force: true, recursive: true }); - }); - - test('a bare .json is readable under the configured key', async () => { - const storage = new FileSystemStorageBackend({ localDataDirectory: tmpLocation, inputKey }); - await seedDefaultStore(storage, { [`${inputKey}.json`]: payload }); - - const store = await storage.createKeyValueStoreBackend(); - - expect(await store.getValue(inputKey)).toStrictEqual({ - key: inputKey, - value: Buffer.from(payload), - contentType: 'application/json; charset=utf-8', - }); - expect(await store.recordExists(inputKey)).toBe(true); - expect(await store.getPublicUrl(inputKey)).toMatch(/\/__CLI_INPUT\.json$/); - }); - - test('a listed bare .json round-trips under its literal name', async () => { - const storage = new FileSystemStorageBackend({ localDataDirectory: tmpLocation, inputKey }); - await seedDefaultStore(storage, { [`${inputKey}.json`]: payload }); - - const store = await storage.createKeyValueStoreBackend(); - const { items } = await store.listKeys(); - - expect(items.map((item) => item.key)).toEqual([`${inputKey}.json`]); - expect((await store.getValue(`${inputKey}.json`))?.value.toString()).toBe(payload); - expect(await store.recordExists(`${inputKey}.json`)).toBe(true); - }); - - test('the same bare file is not readable when a different input key is configured', async () => { - const storage = new FileSystemStorageBackend({ localDataDirectory: tmpLocation }); - await seedDefaultStore(storage, { [`${inputKey}.json`]: payload }); - - const store = await storage.createKeyValueStoreBackend(); - - expect(await store.getValue(inputKey)).toBeUndefined(); - expect(await store.recordExists(inputKey)).toBe(false); - expect((await store.listKeys()).items).toEqual([]); - }); - - test('purge keeps both the configured input key and INPUT in the default store', async () => { - const storage = new FileSystemStorageBackend({ localDataDirectory: tmpLocation, inputKey }); - await seedDefaultStore(storage, { - [`${inputKey}.json`]: payload, - 'INPUT.json': JSON.stringify({ hello: 'from the user' }), - 'OUTPUT.json': JSON.stringify({ leftover: true }), - }); - - await storage.purge(); - - const remaining = await readdir(resolve(storage.keyValueStoresDirectory, 'default')); - expect(remaining.filter((file) => !file.startsWith('__metadata__')).sort()).toEqual([ - 'INPUT.json', - `${inputKey}.json`, - ]); - - const store = await storage.createKeyValueStoreBackend(); - expect((await store.getValue(inputKey))?.value.toString()).toBe(payload); - expect((await store.getValue('INPUT'))?.value.toString()).toBe(JSON.stringify({ hello: 'from the user' })); - }); - - test('purge does not keep the configured input key in a non-default alias store', async () => { - const storage = new FileSystemStorageBackend({ localDataDirectory: tmpLocation, inputKey }); - const dir = resolve(storage.keyValueStoresDirectory, 'other'); - await mkdir(dir, { recursive: true }); - await writeFile(resolve(dir, `${inputKey}.json`), payload); - await storage.createKeyValueStoreBackend({ alias: 'other' }); - - await storage.purge(); - - // Only the default store holds the run input; every other run-scoped store is swept clean. - expect(await readdir(dir)).not.toContain(`${inputKey}.json`); - }); -}); diff --git a/packages/fs-storage/test/fs-fallback.test.ts b/packages/fs-storage/test/fs-fallback.test.ts deleted file mode 100644 index e00a39f1bed1..000000000000 --- a/packages/fs-storage/test/fs-fallback.test.ts +++ /dev/null @@ -1,314 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { mkdir, rm, writeFile } from 'node:fs/promises'; -import { resolve } from 'node:path'; - -import { FileSystemStorageBackend } from '@crawlee/fs-storage'; -import type { KeyValueStoreRecord } from '@crawlee/types'; - -// The storage is backed by the native `@crawlee/fs-storage-native` extension, which only serves -// key-value records it has written itself (tracked via per-record metadata sidecars). The -// `KeyValueStoreBackend` adapter layers a fallback on top so that value files placed into the store -// directory out-of-band — e.g. a hand-written or platform-provided `INPUT.json` — are still readable. -// These tests pin both the store-identity metadata fallback and that bare-file fallback. -// -// The client is a plain byte transport: bare-file reads return the raw bytes plus a content type -// inferred from the file extension (falling back to the native `application/octet-stream` when there -// is none). Parsing those bytes — and surfacing any error from a malformed value — is the -// `KeyValueStore` frontend's job, so the client does not validate them. Bare files are readable by -// known key and are also enumerated by `listKeys` under their actual on-disk name (e.g. a bare -// `INPUT.json` is listed as `INPUT.json` and reads back under that key), while the logical `INPUT` -// lookup keeps resolving the same bare file. -describe('fallback to fs for reading', () => { - const tmpLocation = resolve(import.meta.dirname, './tmp/fs-fallback'); - const storage = new FileSystemStorageBackend({ - localDataDirectory: tmpLocation, - }); - - const expectedFsDate = new Date(2022, 0, 1); - - beforeAll(async () => { - // "default" store: metadata file + a bare INPUT.json (no per-record metadata sidecar). - await mkdir(resolve(storage.keyValueStoresDirectory, 'default'), { recursive: true }); - await writeFile( - resolve(storage.keyValueStoresDirectory, 'default/__metadata__.json'), - JSON.stringify({ - id: randomUUID(), - name: 'default', - createdAt: expectedFsDate, - accessedAt: expectedFsDate, - modifiedAt: expectedFsDate, - }), - ); - await writeFile( - resolve(storage.keyValueStoresDirectory, 'default/INPUT.json'), - JSON.stringify({ foo: 'bar but from fs' }), - ); - - // "other" store: a bare INPUT.json with no store metadata file at all. - await mkdir(resolve(storage.keyValueStoresDirectory, 'other'), { recursive: true }); - await writeFile( - resolve(storage.keyValueStoresDirectory, 'other/INPUT.json'), - JSON.stringify({ foo: 'bar but from fs' }), - ); - - // "no-ext" store: a value file with no extension — loaded as raw text. - await mkdir(resolve(storage.keyValueStoresDirectory, 'no-ext'), { recursive: true }); - await writeFile( - resolve(storage.keyValueStoresDirectory, 'no-ext/INPUT'), - JSON.stringify({ foo: 'bar but from fs' }), - ); - - // "invalid-json" store: a malformed INPUT.json — ignored. - await mkdir(resolve(storage.keyValueStoresDirectory, 'invalid-json'), { recursive: true }); - await writeFile(resolve(storage.keyValueStoresDirectory, 'invalid-json/INPUT.json'), '{'); - - // "non-input" store: a bare value file under a non-INPUT key. The bare-file fallback is scoped - // to the run input, so this file is NOT readable out-of-band — only `INPUT`-keyed bare files are. - await mkdir(resolve(storage.keyValueStoresDirectory, 'non-input'), { recursive: true }); - await writeFile( - resolve(storage.keyValueStoresDirectory, 'non-input/some-key.json'), - JSON.stringify({ foo: 'bar but from fs' }), - ); - }); - - afterAll(async () => { - await rm(tmpLocation, { force: true, recursive: true }); - }); - - test('reads store identity from the on-disk metadata, and a bare INPUT.json value', async () => { - const defaultStore = await storage.createKeyValueStoreBackend({ name: 'default' }); - const defaultStoreInfo = await defaultStore.getMetadata(); - - expect(defaultStoreInfo.name).toEqual('default'); - expect(defaultStoreInfo.createdAt).toEqual(expectedFsDate); - - // The client is a byte transport: it returns the raw on-disk bytes verbatim and leaves - // parsing to the KeyValueStore frontend codec. So we expect a Buffer, not a parsed object. - const input = await defaultStore.getValue('INPUT'); - expect(input).toStrictEqual({ - key: 'INPUT', - value: Buffer.from(JSON.stringify({ foo: 'bar but from fs' })), - contentType: 'application/json; charset=utf-8', - }); - }); - - test('reads a bare INPUT.json even with no store metadata present', async () => { - const otherStore = await storage.createKeyValueStoreBackend({ name: 'other' }); - - // Byte transport: raw bytes out, parsing is the frontend's job. - const input = await otherStore.getValue('INPUT'); - expect(input).toStrictEqual({ - key: 'INPUT', - value: Buffer.from(JSON.stringify({ foo: 'bar but from fs' })), - contentType: 'application/json; charset=utf-8', - }); - }); - - test('a store with no data on disk is still accessible after creation', async () => { - const default2Store = await storage.createKeyValueStoreBackend({ name: 'default_2' }); - const info = await default2Store.getMetadata(); - expect(info.name).toEqual('default_2'); - }); - - test('loads a value file with no extension as raw bytes with a generic content type', async () => { - const noExtStore = await storage.createKeyValueStoreBackend({ name: 'no-ext' }); - - // Byte transport: the no-extension fallback returns raw bytes. With no extension to infer a - // content type from, the native client reports the generic `application/octet-stream`. - const input = await noExtStore.getValue('INPUT'); - expect(input).toStrictEqual({ - key: 'INPUT', - value: Buffer.from(JSON.stringify({ foo: 'bar but from fs' })), - contentType: 'application/octet-stream', - }); - }); - - test('returns an invalid-JSON bare value file verbatim', async () => { - const invalidJsonStore = await storage.createKeyValueStoreBackend({ name: 'invalid-json' }); - - // Byte transport: the client no longer validates parseability. Malformed JSON is returned - // verbatim as raw bytes; parsing (and any resulting error) is the KeyValueStore frontend's job. - const input = await invalidJsonStore.getValue('INPUT'); - expect(input).toStrictEqual({ - key: 'INPUT', - value: Buffer.from('{'), - contentType: 'application/json; charset=utf-8', - }); - }); - - test('bare files are visible to recordExists, getPublicUrl, and listKeys', async () => { - const otherStore = await storage.createKeyValueStoreBackend({ name: 'other' }); - - expect(await otherStore.recordExists('INPUT')).toBe(true); - expect(await otherStore.recordExists('does-not-exist')).toBe(false); - - const url = await otherStore.getPublicUrl('INPUT'); - expect(url).toMatch(/^file:\/\/.*INPUT\.json$/); - - // A bare `INPUT.json` is enumerated under its actual on-disk name, not the logical `INPUT`. - const { items } = await otherStore.listKeys(); - expect(items.map((item) => item.key)).toContain('INPUT.json'); - }); - - test('a listed bare key round-trips through getValue / recordExists / getPublicUrl', async () => { - const otherStore = await storage.createKeyValueStoreBackend({ name: 'other' }); - - // The key `listKeys` reports for a bare file must be readable back verbatim, while the logical - // `INPUT` lookup keeps resolving the same bare file (matching how Crawlee reads run input). - const [listed] = (await otherStore.listKeys()).items.map((item) => item.key); - expect(listed).toBe('INPUT.json'); - - const expected: KeyValueStoreRecord = { - key: 'INPUT.json', - value: Buffer.from(JSON.stringify({ foo: 'bar but from fs' })), - contentType: 'application/json; charset=utf-8', - }; - expect(await otherStore.getValue('INPUT.json')).toStrictEqual(expected); - expect(await otherStore.recordExists('INPUT.json')).toBe(true); - expect(await otherStore.getPublicUrl('INPUT.json')).toMatch(/^file:\/\/.*INPUT\.json$/); - - // The logical key still resolves the same underlying file (reported under the requested key). - expect(await otherStore.getValue('INPUT')).toStrictEqual({ ...expected, key: 'INPUT' }); - }); - - test('the bare-file fallback is scoped to INPUT: a non-INPUT bare file is ignored', async () => { - const nonInputStore = await storage.createKeyValueStoreBackend({ name: 'non-input' }); - - // `some-key.json` sits on disk with no metadata sidecar. Only `INPUT` keys probe bare files, - // so this is invisible to every read path: it has no tracked record, and the `.json` extension - // probing that would resolve a bare `INPUT` is never attempted for other keys. `getPublicUrl` - // is existence-agnostic, so it still answers — with the extensionless key, not the bare file. - expect(await nonInputStore.getValue('some-key')).toBeUndefined(); - expect(await nonInputStore.recordExists('some-key')).toBe(false); - expect(await nonInputStore.getPublicUrl('some-key')).toMatch(/^file:\/\/.*\/some-key$/); - - // `listKeys` only surfaces bare files for the run-input keys, so `some-key` is not enumerated. - const { items } = await nonInputStore.listKeys(); - expect(items.map((item) => item.key)).not.toContain('some-key'); - }); - - test('a tracked INPUT record shadows the bare INPUT.json variant in listKeys', async () => { - const collisionStore = await storage.createKeyValueStoreBackend({ name: 'input-collision' }); - - // Write a tracked `INPUT` record (value file + metadata sidecar), then drop a sidecar-less bare - // `INPUT.json` next to it. Both belong to the logical key `INPUT`; the tracked record wins, so - // only `INPUT` is listed and the bare `INPUT.json` variant is suppressed. - await collisionStore.setValue({ key: 'INPUT', value: 'tracked', contentType: 'text/plain; charset=utf-8' }); - await writeFile( - resolve(storage.keyValueStoresDirectory, 'input-collision/INPUT.json'), - JSON.stringify({ foo: 'bare' }), - ); - - const keys = (await collisionStore.listKeys()).items.map((item) => item.key); - expect(keys).toContain('INPUT'); - expect(keys).not.toContain('INPUT.json'); - }); -}); - -// For each run-input bare file: the on-disk filename, the literal key that reads it directly, the -// content type the client reports (`.json` infers from the extension; the extensionless `INPUT` -// reports the synthesized `application/octet-stream`), and a unique payload so a read can be proven to -// have returned *this* file and not a sibling. -const BARE_VARIANTS = [ - { file: 'INPUT', literalKey: 'INPUT', contentType: 'application/octet-stream' }, - { file: 'INPUT.json', literalKey: 'INPUT.json', contentType: 'application/json; charset=utf-8' }, -].map((variant) => ({ ...variant, payload: `payload of ${variant.file}` })); - -// Each run-input bare file must be reachable by exactly two keys — the logical `INPUT` (which probes -// the `['', '.json']` ladder, first match wins) and its own literal on-disk name — and NOT via a -// *different* extension's literal name (a bare `INPUT` is not `INPUT.json`). Here each -// variant lives in its own store so the logical-`INPUT` lookup resolves it unambiguously. -describe('run-input bare-file reachability (one variant per store)', () => { - const tmpLocation = resolve(import.meta.dirname, './tmp/fs-reachability-isolated'); - const storage = new FileSystemStorageBackend({ localDataDirectory: tmpLocation }); - - const storeNameFor = (file: string) => `reach-${file.toLowerCase().replace('.', '-')}`; - - beforeAll(async () => { - for (const { file, payload } of BARE_VARIANTS) { - const dir = resolve(storage.keyValueStoresDirectory, storeNameFor(file)); - await mkdir(dir, { recursive: true }); - await writeFile(resolve(dir, file), payload); - } - }); - - afterAll(async () => { - await rm(tmpLocation, { force: true, recursive: true }); - }); - - describe.each(BARE_VARIANTS)('a bare $file', ({ file, literalKey, contentType, payload }) => { - // Keys that should resolve this variant: the logical `INPUT` and the file's own literal name - // (deduplicated — the extensionless variant's literal key *is* `INPUT`). - const reachableKeys = [...new Set(['INPUT', literalKey])]; - // The literal names of the *other* extensions, which must never resolve this variant. - const unreachableKeys = BARE_VARIANTS.map((variant) => variant.literalKey).filter( - (key) => !reachableKeys.includes(key), - ); - - test.each(reachableKeys)('is reachable via %s', async (key) => { - const store = await storage.createKeyValueStoreBackend({ name: storeNameFor(file) }); - - expect(await store.getValue(key)).toStrictEqual({ - key, - value: Buffer.from(payload), - contentType, - }); - expect(await store.recordExists(key)).toBe(true); - expect(await store.getPublicUrl(key)).toMatch(new RegExp(`^file://.*${file.replace('.', '\\.')}$`)); - }); - - test.each(unreachableKeys)('is not reachable via %s', async (key) => { - const store = await storage.createKeyValueStoreBackend({ name: storeNameFor(file) }); - - expect(await store.getValue(key)).toBeUndefined(); - expect(await store.recordExists(key)).toBe(false); - // Existence-agnostic: the URL falls back to the requested key rather than resolving to a - // sibling variant's file. - expect(await store.getPublicUrl(key)).toMatch(new RegExp(`^file://.*/${key.replace('.', '\\.')}$`)); - }); - }); -}); - -// The sharper cross-talk check: with *both* variants in one store, each literal key must read back -// its own bytes (never a sibling's), and the logical `INPUT` must resolve the first ladder match — the -// extensionless `INPUT`. This is what fails if literal-name probing ever widens to other extensions. -describe('run-input bare-file reachability (all variants in one store)', () => { - const tmpLocation = resolve(import.meta.dirname, './tmp/fs-reachability-shared'); - const storage = new FileSystemStorageBackend({ localDataDirectory: tmpLocation }); - - beforeAll(async () => { - const dir = resolve(storage.keyValueStoresDirectory, 'all-variants'); - await mkdir(dir, { recursive: true }); - for (const { file, payload } of BARE_VARIANTS) { - await writeFile(resolve(dir, file), payload); - } - }); - - afterAll(async () => { - await rm(tmpLocation, { force: true, recursive: true }); - }); - - test.each(BARE_VARIANTS)( - 'the literal key $literalKey reads its own file, not a sibling', - async ({ literalKey, contentType, payload }) => { - const store = await storage.createKeyValueStoreBackend({ name: 'all-variants' }); - - expect(await store.getValue(literalKey)).toStrictEqual({ - key: literalKey, - value: Buffer.from(payload), - contentType, - }); - }, - ); - - test('the logical INPUT key resolves the extensionless file (first ladder match)', async () => { - const store = await storage.createKeyValueStoreBackend({ name: 'all-variants' }); - - const extensionless = BARE_VARIANTS.find((variant) => variant.file === 'INPUT')!; - expect(await store.getValue('INPUT')).toStrictEqual({ - key: 'INPUT', - value: Buffer.from(extensionless.payload), - contentType: extensionless.contentType, - }); - }); -}); diff --git a/packages/fs-storage/test/key-value-store/adoption.test.ts b/packages/fs-storage/test/key-value-store/adoption.test.ts new file mode 100644 index 000000000000..a7996774fb33 --- /dev/null +++ b/packages/fs-storage/test/key-value-store/adoption.test.ts @@ -0,0 +1,264 @@ +import { randomUUID } from 'node:crypto'; +import { mkdir, readdir, rm, writeFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; + +import { FileSystemStorageBackend } from '@crawlee/fs-storage'; +import type { KeyValueStoreRecord } from '@crawlee/types'; + +// Value files land in a key-value store directory without the metadata sidecar that makes them a +// record all the time: written by the Apify CLI, shipped in a project template, left behind by +// Crawlee v3, or edited by hand. `FileSystemStorageBackend` declares such files as adoption +// candidates when it opens a store, so the native client writes the missing sidecar once and +// everything afterwards — reads, listings, deletes, purge — deals in ordinary records. +// +// The run-input keys claim a bare `` or `.json` in the default store; every other file is +// adopted under its own filename as the key. Content types come from the extension alone: `.json` is +// JSON, anything else is bytes for the `KeyValueStore` frontend to make sense of. + +const payload = JSON.stringify({ hello: 'from disk' }); + +/** A fresh backend over `directory`, seeded with sidecar-less files in one key-value store. */ +async function seedStore( + directory: string, + store: string, + files: Record, + options: { inputKey?: string } = {}, +): Promise { + const storage = new FileSystemStorageBackend({ localDataDirectory: directory, ...options }); + const storeDirectory = resolve(storage.keyValueStoresDirectory, store); + await mkdir(storeDirectory, { recursive: true }); + for (const [file, content] of Object.entries(files)) { + await writeFile(resolve(storeDirectory, file), content); + } + return storage; +} + +describe('a sidecar-less run-input file in the default store', () => { + const tmpLocation = resolve(import.meta.dirname, './tmp/adoption-input'); + + afterEach(async () => { + await rm(tmpLocation, { force: true, recursive: true }); + }); + + test('is a record under the input key, not under its filename', async () => { + const storage = await seedStore(tmpLocation, 'default', { 'INPUT.json': payload }); + const store = await storage.createKeyValueStoreBackend(); + + expect(await store.getValue('INPUT')).toStrictEqual({ + key: 'INPUT', + value: Buffer.from(payload), + contentType: 'application/json; charset=utf-8', + }); + expect((await store.listKeys()).items.map((item) => item.key)).toEqual(['INPUT']); + expect(await store.recordExists('INPUT')).toBe(true); + expect(await store.getPublicUrl('INPUT')).toMatch(/\/INPUT\.json$/); + + // The extension is the file's, not the key's. + expect(await store.getValue('INPUT.json')).toBeUndefined(); + expect(await store.recordExists('INPUT.json')).toBe(false); + }); + + test('is deleted by deleteValue instead of resurrecting the key', async () => { + const storage = await seedStore(tmpLocation, 'default', { 'INPUT.json': payload }); + const store = await storage.createKeyValueStoreBackend(); + + await store.deleteValue('INPUT'); + + expect(await store.getValue('INPUT')).toBeUndefined(); + expect(await readdir(resolve(storage.keyValueStoresDirectory, 'default'))).not.toContain('INPUT.json'); + }); + + test('reads as bytes when it has no extension', async () => { + const storage = await seedStore(tmpLocation, 'default', { INPUT: payload }); + const store = await storage.createKeyValueStoreBackend(); + + // No sniffing: the extension is the only thing that makes a file JSON, so an extensionless + // one is bytes. Turning those into a parsed input is the caller's job. + expect(await store.getValue('INPUT')).toStrictEqual({ + key: 'INPUT', + value: Buffer.from(payload), + contentType: 'application/octet-stream', + }); + }); + + test('is adopted verbatim even when it is malformed JSON', async () => { + const storage = await seedStore(tmpLocation, 'default', { 'INPUT.json': '{' }); + const store = await storage.createKeyValueStoreBackend(); + + // The backend is a byte transport — parsing, and any error from it, belongs to the frontend. + expect(await store.getValue('INPUT')).toStrictEqual({ + key: 'INPUT', + value: Buffer.from('{'), + contentType: 'application/json; charset=utf-8', + }); + }); + + test('fails the open when both candidate files are present', async () => { + const storage = await seedStore(tmpLocation, 'default', { INPUT: 'bytes', 'INPUT.json': payload }); + + // Picking one would silently ignore the other, and there is no way to guess which one the + // user means. + await expect(storage.createKeyValueStoreBackend()).rejects.toThrow(/Multiple candidate files for key 'INPUT'/); + }); + + test('is left alone when the input key already has a record', async () => { + const storage = await seedStore(tmpLocation, 'default', {}); + const store = await storage.createKeyValueStoreBackend(); + await store.setValue({ key: 'INPUT', value: 'tracked', contentType: 'text/plain; charset=utf-8' }); + await writeFile(resolve(storage.keyValueStoresDirectory, 'default', 'INPUT.json'), payload); + + // Reopening must not rebind the key: a stray file is not allowed to take over a record the + // run wrote itself, and adopting it under its own filename would make `INPUT.json` a second + // key for what the user thinks is the input. + const reopened = await new FileSystemStorageBackend({ + localDataDirectory: tmpLocation, + }).createKeyValueStoreBackend(); + + expect((await reopened.getValue('INPUT'))?.value.toString()).toBe('tracked'); + expect((await reopened.listKeys()).items.map((item) => item.key)).toEqual(['INPUT']); + }); + + test('is adopted under the configured input key', async () => { + const inputKey = '__CLI_INPUT'; + const storage = await seedStore(tmpLocation, 'default', { [`${inputKey}.json`]: payload }, { inputKey }); + const store = await storage.createKeyValueStoreBackend(); + + expect(await store.getValue(inputKey)).toStrictEqual({ + key: inputKey, + value: Buffer.from(payload), + contentType: 'application/json; charset=utf-8', + }); + expect((await store.listKeys()).items.map((item) => item.key)).toEqual([inputKey]); + }); + + test('is adopted under its filename when a different input key is configured', async () => { + const storage = await seedStore(tmpLocation, 'default', { '__CLI_INPUT.json': payload }); + const store = await storage.createKeyValueStoreBackend(); + + // Not this run's input, so it is an ordinary file: a record named after itself. + expect(await store.getValue('__CLI_INPUT')).toBeUndefined(); + expect((await store.getValue('__CLI_INPUT.json'))?.value.toString()).toBe(payload); + }); +}); + +describe('sidecar-less files outside the run input', () => { + const tmpLocation = resolve(import.meta.dirname, './tmp/adoption-sweep'); + + afterEach(async () => { + await rm(tmpLocation, { force: true, recursive: true }); + }); + + test('become records keyed by their filename', async () => { + const storage = await seedStore(tmpLocation, 'named-store', { + 'some-key.json': payload, + 'photo.png': 'not really a png', + '.hidden': 'tooling leftover', + }); + const store = await storage.createKeyValueStoreBackend({ name: 'named-store' }); + + expect((await store.listKeys()).items).toEqual([ + { key: 'photo.png', contentType: 'application/octet-stream', size: 16 }, + { key: 'some-key.json', contentType: 'application/json; charset=utf-8', size: payload.length }, + ]); + expect((await store.getValue('some-key.json'))?.value.toString()).toBe(payload); + + // Dotfiles are tooling droppings (including the native client's own `.tmp.*` write + // leftovers), never someone's record. + expect(await store.getValue('.hidden')).toBeUndefined(); + }); + + test('include an INPUT.json in a store that is not the default one', async () => { + const storage = await seedStore(tmpLocation, 'named-store', { 'INPUT.json': payload }); + const store = await storage.createKeyValueStoreBackend({ name: 'named-store' }); + + // Only the default store holds the run input, so here `INPUT.json` is just a file that + // happens to be called that. + expect(await store.getValue('INPUT')).toBeUndefined(); + expect((await store.getValue('INPUT.json'))?.value.toString()).toBe(payload); + }); + + test('are adopted in the default store too', async () => { + const storage = await seedStore(tmpLocation, 'default', { 'leftover.json': payload }); + const store = await storage.createKeyValueStoreBackend(); + + expect((await store.getValue('leftover.json'))?.value.toString()).toBe(payload); + }); +}); + +describe('a hand-written store directory', () => { + const tmpLocation = resolve(import.meta.dirname, './tmp/adoption-identity'); + const expectedDate = new Date(2022, 0, 1); + + afterEach(async () => { + await rm(tmpLocation, { force: true, recursive: true }); + }); + + test('keeps the identity its own metadata declares', async () => { + const storage = await seedStore(tmpLocation, 'hand-written', { + '__metadata__.json': JSON.stringify({ + id: randomUUID(), + name: 'hand-written', + createdAt: expectedDate, + accessedAt: expectedDate, + modifiedAt: expectedDate, + }), + 'INPUT.json': payload, + }); + const store = await storage.createKeyValueStoreBackend({ name: 'hand-written' }); + + const metadata = await store.getMetadata(); + expect(metadata.name).toBe('hand-written'); + expect(metadata.createdAt).toEqual(expectedDate); + + // The store's own metadata file is not a value file, so adoption leaves it be. + expect((await store.listKeys()).items.map((item) => item.key)).toEqual(['INPUT.json']); + }); + + test('works with no metadata file at all', async () => { + const storage = await seedStore(tmpLocation, 'no-metadata', { 'INPUT.json': payload }); + const store = await storage.createKeyValueStoreBackend({ name: 'no-metadata' }); + + expect((await store.getMetadata()).name).toBe('no-metadata'); + expect((await store.getValue('INPUT.json'))?.value.toString()).toBe(payload); + }); +}); + +describe('purging a store with adopted records', () => { + const tmpLocation = resolve(import.meta.dirname, './tmp/adoption-purge'); + + afterEach(async () => { + await rm(tmpLocation, { force: true, recursive: true }); + }); + + test('keeps the run input and drops everything else', async () => { + const inputKey = '__CLI_INPUT'; + const storage = await seedStore( + tmpLocation, + 'default', + { + 'INPUT.json': payload, + [`${inputKey}.json`]: JSON.stringify({ hello: 'from the cli' }), + 'leftover.json': JSON.stringify({ leftover: true }), + }, + { inputKey }, + ); + + // Purge-on-start opens the store, so adoption runs first and the input survives as a record — + // the very thing the keep-list is expressed in. + await storage.purge(); + + const store = await storage.createKeyValueStoreBackend(); + expect((await store.listKeys()).items.map((item) => item.key)).toEqual(['INPUT', inputKey]); + expect((await store.getValue('INPUT'))?.value.toString()).toBe(payload); + expect(await readdir(resolve(storage.keyValueStoresDirectory, 'default'))).not.toContain('leftover.json'); + }); + + test('drops the input of a non-default store', async () => { + const storage = await seedStore(tmpLocation, 'other', { 'INPUT.json': payload }); + await storage.createKeyValueStoreBackend({ alias: 'other' }); + + await storage.purge(); + + expect(await readdir(resolve(storage.keyValueStoresDirectory, 'other'))).not.toContain('INPUT.json'); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d02051eaacda..1bf31d3b68e2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -835,8 +835,8 @@ importers: packages/fs-storage: dependencies: '@crawlee/fs-storage-native': - specifier: '>=0.2.1-beta.0 <0.3' - version: 0.2.1-beta.0 + specifier: '>=0.2.1 <0.3' + version: 0.2.1 '@crawlee/types': specifier: workspace:* version: link:../types @@ -2233,60 +2233,60 @@ packages: resolution: {integrity: sha512-TzlTVpKPjaqW6qOYjQcYUDuGsLCNsvFHVBXkYGTAnf5V37jCWrE5haKNXzz0WZUtVHjrpV76L1buANjwXMfT8w==} engines: {node: '>=22'} - '@crawlee/fs-storage-native-darwin-arm64@0.2.1-beta.0': - resolution: {integrity: sha512-Ra0bQfxAGbfhDOwZ32GJuHTGc1WrIyE23mE1sLKNnZ4ukPnDSyy//ZO23LIkZ9i6YxyLsA6X3SwmZ/TFpewVVg==} + '@crawlee/fs-storage-native-darwin-arm64@0.2.1': + resolution: {integrity: sha512-vpjIyzhv2XB8oz5UbcXVMm6RwyfJgAE7xEiXG3pgxYjjtWOP2xGZX17dJiCw9fcOjH3bawfNZTZiKSDvxF0Q1A==} engines: {node: '>= 20'} cpu: [arm64] os: [darwin] - '@crawlee/fs-storage-native-darwin-x64@0.2.1-beta.0': - resolution: {integrity: sha512-Uf+hMZBrKMyKrJLUAer6V4W1xkyuABzgtsh2iByJE0CgPsyOS3mY4RdQBnKgS++XMQs5w94/DbfSWA90xV54Nw==} + '@crawlee/fs-storage-native-darwin-x64@0.2.1': + resolution: {integrity: sha512-os5MWPkUoNKM+JSLN5MxeVoRxlwqVlef2gMqXc+a359kSS3iqGjP+tUZQ8MTIU5s370igxIx6dnJl+xr6FplJA==} engines: {node: '>= 20'} cpu: [x64] os: [darwin] - '@crawlee/fs-storage-native-linux-arm64-gnu@0.2.1-beta.0': - resolution: {integrity: sha512-P+Be5g8YxnA7f93GOnnm+5jpFb5uhI0iFXcWzm1aLpQU0fhE6hZOuiYSxKYRqn2AU6yPLjqwHm0eKuRm/UO83A==} + '@crawlee/fs-storage-native-linux-arm64-gnu@0.2.1': + resolution: {integrity: sha512-7rO5NC6HKjl+rh8zK/bayKBP3MMQs/Ib5eyzZJi4U8Jb2Vke1pU9yta+XjKzVU2rQZB8ErntISTxczbUfZfhFQ==} engines: {node: '>= 20'} cpu: [arm64] os: [linux] libc: [glibc] - '@crawlee/fs-storage-native-linux-arm64-musl@0.2.1-beta.0': - resolution: {integrity: sha512-X8U+aS00umhQ+8gHJdQg7j40J20hLvbb9pvOP9gb0Y6PdZt/jAbivIyGP871wGViJ+fwF1QQQtFO1HP5IzE8+w==} + '@crawlee/fs-storage-native-linux-arm64-musl@0.2.1': + resolution: {integrity: sha512-+oNToqNyXkcaT7oi8WBcKaQxJNZD4oXv1QWyUq+z69h5H6GfzcQe/nqslXALY5SNx9yTCzTtVS5AgrgqvGzyTg==} engines: {node: '>= 20'} cpu: [arm64] os: [linux] libc: [musl] - '@crawlee/fs-storage-native-linux-x64-gnu@0.2.1-beta.0': - resolution: {integrity: sha512-u0yUDlk5rg51c4eTOyk6lnWiYaoUw1Z53VnlmXYDMFSUW7tGQHLdy0aQBL+RWBmD0+mLMRiBDJjYoxsrKYYCLg==} + '@crawlee/fs-storage-native-linux-x64-gnu@0.2.1': + resolution: {integrity: sha512-qHQTBNFlBHajKfcwvzN8u68mLBWsEbaZQb7yNBmocUcBSYXm0U6JAFn1BcpNJyFb+CfpERR3QaCIUxBVdbRBDw==} engines: {node: '>= 20'} cpu: [x64] os: [linux] libc: [glibc] - '@crawlee/fs-storage-native-linux-x64-musl@0.2.1-beta.0': - resolution: {integrity: sha512-T/9ZPPt6eGm3qUwhnEy7SGbilH9Bvku+afdsXxIoDDza5Uib7PnwFTjI+stnY4Y5xQ+GAcmfwkRRM3oZ2UK6ZA==} + '@crawlee/fs-storage-native-linux-x64-musl@0.2.1': + resolution: {integrity: sha512-IjrfVP7mnyQqAgoRKTLxWmjYRyktVI/cBfXg+RIEtjss9a9OIt4jrHhjLhUo8cQr5IT6wyEXRto0AOxZth96SA==} engines: {node: '>= 20'} cpu: [x64] os: [linux] libc: [musl] - '@crawlee/fs-storage-native-win32-arm64-msvc@0.2.1-beta.0': - resolution: {integrity: sha512-p8D+XSskEZ2FMGXlrzVQrkFozkS3Md6invEvTiWEPC6/Z6mvNbwJ9z8NVSRNx4Trwb1l85i0+rLJI9OqzYOa4Q==} + '@crawlee/fs-storage-native-win32-arm64-msvc@0.2.1': + resolution: {integrity: sha512-GsVD+XbiL5tTQlmIPtiThyGFp7B3vg1l8ERXyCHR6Gsz+85ZeMQ9jg3/ILFBKFY15I8v5FoDV0cQtdH7D5+FKg==} engines: {node: '>= 20'} cpu: [arm64] os: [win32] - '@crawlee/fs-storage-native-win32-x64-msvc@0.2.1-beta.0': - resolution: {integrity: sha512-ChmVNKYjlskX9WKPsQumbF3H0MEU1MumfkZxou6FjFLtgQ14rt03qld8n1EUzQXBApyqg5RSQCKdDoTIiehmCA==} + '@crawlee/fs-storage-native-win32-x64-msvc@0.2.1': + resolution: {integrity: sha512-H8uigGqYChXySUDpnDhwl9JrDbp/rZMFpjuoSDSssF1gr+83d7qRjISYKVC0TUlKiYAnT7SqYCBqNrsuSbJBUw==} engines: {node: '>= 20'} cpu: [x64] os: [win32] - '@crawlee/fs-storage-native@0.2.1-beta.0': - resolution: {integrity: sha512-SpQQj/4Jb7NT4mPL4e97loOixjg3Sb2UZPQUtZ3558I94BmKUh2+P35VKpzkMj3wnGU1QCcUOYIeNGPoH80Xgw==} + '@crawlee/fs-storage-native@0.2.1': + resolution: {integrity: sha512-x6umw+kVjuA6k+Qhx9RwJSjl/DRyeo1jlCiQH04ZKz5DP54NhQjMlZSQD1Y0v4SYuN0u+wqDVINQQQsarsFkeg==} engines: {node: '>= 20'} '@crawlee/types@3.16.0': @@ -14590,40 +14590,40 @@ snapshots: '@conventional-changelog/template@1.2.1': {} - '@crawlee/fs-storage-native-darwin-arm64@0.2.1-beta.0': + '@crawlee/fs-storage-native-darwin-arm64@0.2.1': optional: true - '@crawlee/fs-storage-native-darwin-x64@0.2.1-beta.0': + '@crawlee/fs-storage-native-darwin-x64@0.2.1': optional: true - '@crawlee/fs-storage-native-linux-arm64-gnu@0.2.1-beta.0': + '@crawlee/fs-storage-native-linux-arm64-gnu@0.2.1': optional: true - '@crawlee/fs-storage-native-linux-arm64-musl@0.2.1-beta.0': + '@crawlee/fs-storage-native-linux-arm64-musl@0.2.1': optional: true - '@crawlee/fs-storage-native-linux-x64-gnu@0.2.1-beta.0': + '@crawlee/fs-storage-native-linux-x64-gnu@0.2.1': optional: true - '@crawlee/fs-storage-native-linux-x64-musl@0.2.1-beta.0': + '@crawlee/fs-storage-native-linux-x64-musl@0.2.1': optional: true - '@crawlee/fs-storage-native-win32-arm64-msvc@0.2.1-beta.0': + '@crawlee/fs-storage-native-win32-arm64-msvc@0.2.1': optional: true - '@crawlee/fs-storage-native-win32-x64-msvc@0.2.1-beta.0': + '@crawlee/fs-storage-native-win32-x64-msvc@0.2.1': optional: true - '@crawlee/fs-storage-native@0.2.1-beta.0': + '@crawlee/fs-storage-native@0.2.1': optionalDependencies: - '@crawlee/fs-storage-native-darwin-arm64': 0.2.1-beta.0 - '@crawlee/fs-storage-native-darwin-x64': 0.2.1-beta.0 - '@crawlee/fs-storage-native-linux-arm64-gnu': 0.2.1-beta.0 - '@crawlee/fs-storage-native-linux-arm64-musl': 0.2.1-beta.0 - '@crawlee/fs-storage-native-linux-x64-gnu': 0.2.1-beta.0 - '@crawlee/fs-storage-native-linux-x64-musl': 0.2.1-beta.0 - '@crawlee/fs-storage-native-win32-arm64-msvc': 0.2.1-beta.0 - '@crawlee/fs-storage-native-win32-x64-msvc': 0.2.1-beta.0 + '@crawlee/fs-storage-native-darwin-arm64': 0.2.1 + '@crawlee/fs-storage-native-darwin-x64': 0.2.1 + '@crawlee/fs-storage-native-linux-arm64-gnu': 0.2.1 + '@crawlee/fs-storage-native-linux-arm64-musl': 0.2.1 + '@crawlee/fs-storage-native-linux-x64-gnu': 0.2.1 + '@crawlee/fs-storage-native-linux-x64-musl': 0.2.1 + '@crawlee/fs-storage-native-win32-arm64-msvc': 0.2.1 + '@crawlee/fs-storage-native-win32-x64-msvc': 0.2.1 '@crawlee/types@3.16.0': dependencies: diff --git a/test/core/storages/storage_purge.test.ts b/test/core/storages/storage_purge.test.ts index 7f11890a8e67..991d4e5a2caf 100644 --- a/test/core/storages/storage_purge.test.ts +++ b/test/core/storages/storage_purge.test.ts @@ -127,8 +127,9 @@ describe('FileSystemStorageBackend.purge over a pre-existing storage directory', await backend.purge(); + // Not the default store, so the file is adopted under its own name rather than as `INPUT`. const store = await backend.createKeyValueStoreBackend({ name: 'hand-placed' }); - expect(await readInput(store)).toBe('{"hand":"placed"}'); + expect((await store.getValue('INPUT.json'))?.value.toString()).toBe('{"hand":"placed"}'); }); // An unnamed storage whose directory is named after its own id can only be reached through