diff --git a/packages/project/lib/build/cache/ResourceRequestManager.js b/packages/project/lib/build/cache/ResourceRequestManager.js index f7a211f1e4b..bb29c90c1bd 100644 --- a/packages/project/lib/build/cache/ResourceRequestManager.js +++ b/packages/project/lib/build/cache/ResourceRequestManager.js @@ -1,3 +1,4 @@ +import crypto from "node:crypto"; import micromatch from "micromatch"; import ResourceRequestGraph, {Request} from "./ResourceRequestGraph.js"; import ResourceIndex from "./index/ResourceIndex.js"; @@ -5,6 +6,32 @@ import TreeRegistry from "./index/TreeRegistry.js"; import {getLogger} from "@ui5/logger"; const log = getLogger("build:cache:ResourceRequestManager"); +/** + * Restores an unresolved-requests marker from its serialized (array) form onto a metadata object. + * No-op for a missing or empty array, keeping the marker absent rather than an empty Set. + * + * @param {object} metadata Request-set node metadata (mutated in place) + * @param {string[]|undefined} serialized Serialized unresolved request keys + */ +function deserializeUnresolvedRequests(metadata, serialized) { + if (serialized?.length) { + metadata.unresolvedRequests = new Set(serialized); + } +} + +/** + * Serializes an unresolved-requests marker to its array form onto a cache entry. + * No-op for a missing or empty Set, keeping the marker out of the serialized output. + * + * @param {object} entry Cache entry (mutated in place) + * @param {Set|undefined} unresolvedRequests Set of unresolved request keys + */ +function serializeUnresolvedRequests(entry, unresolvedRequests) { + if (unresolvedRequests?.size) { + entry.unresolvedRequests = Array.from(unresolvedRequests); + } +} + /** * Manages resource requests and their associated indices for a single task * @@ -73,15 +100,16 @@ class ResourceRequestManager { projectName, taskName, useDifferentialUpdate, requestGraph, unusedAtLeastOnce); const registries = new Map(); // Restore root resource indices - for (const {nodeId, resourceIndex: serializedIndex} of rootIndices) { + for (const {nodeId, resourceIndex: serializedIndex, unresolvedRequests} of rootIndices) { const metadata = requestGraph.getMetadata(nodeId); const registry = resourceRequestManager.#newTreeRegistry(); registries.set(nodeId, registry); metadata.resourceIndex = ResourceIndex.fromCacheShared(serializedIndex, registry); + deserializeUnresolvedRequests(metadata, unresolvedRequests); } // Restore delta resource indices if (deltaIndices) { - for (const {nodeId, addedResourceIndex} of deltaIndices) { + for (const {nodeId, addedResourceIndex, unresolvedRequests} of deltaIndices) { const node = requestGraph.getNode(nodeId); const {resourceIndex: parentResourceIndex} = requestGraph.getMetadata(node.getParentId()); const registry = registries.get(node.getParentId()); @@ -91,9 +119,9 @@ class ResourceRequestManager { } const resourceIndex = parentResourceIndex.deriveTreeWithIndex(addedResourceIndex); - requestGraph.setMetadata(nodeId, { - resourceIndex, - }); + const metadata = {resourceIndex}; + deserializeUnresolvedRequests(metadata, unresolvedRequests); + requestGraph.setMetadata(nodeId, metadata); } } return resourceRequestManager; @@ -113,11 +141,11 @@ class ResourceRequestManager { getIndexSignatures() { const requestSetIds = this.#requestGraph.getAllNodeIds(); const signatures = requestSetIds.map((requestSetId) => { - const {resourceIndex} = this.#requestGraph.getMetadata(requestSetId); + const {resourceIndex, unresolvedRequests} = this.#requestGraph.getMetadata(requestSetId); if (!resourceIndex) { throw new Error(`Resource index missing for request set ID ${requestSetId}`); } - return resourceIndex.getSignature(); + return this.#computeNodeSignature(resourceIndex, unresolvedRequests); }); if (this.#unusedAtLeastOnce) { signatures.push("X"); // Signature for when no requests were made @@ -262,7 +290,8 @@ class ResourceRequestManager { // Phase 3: Process each request set from cache for (const requestSetId of matchingRequestSetIds) { - const {resourceIndex} = this.#requestGraph.getMetadata(requestSetId); + const metadata = this.#requestGraph.getMetadata(requestSetId); + const {resourceIndex} = metadata; if (!resourceIndex) { throw new Error(`Missing resource index for request set ID ${requestSetId}`); } @@ -284,6 +313,16 @@ class ResourceRequestManager { if (resourcesToUpdate.length) { await resourceIndex.upsertResources(resourcesToUpdate); } + // Drain unresolved markers: any recorded added-request that now resolves + // to at least one resource is no longer unresolved. Only inspect + // added-requests (the ones this node contributes); inherited requests + // are the parent's responsibility. + if (metadata.unresolvedRequests?.size && + resourcesToUpdate.length) { + this.#drainUnresolvedRequests( + metadata, this.#requestGraph.getNode(requestSetId).getAddedRequests(), + resourcesToUpdate.map((res) => res.getOriginalPath())); + } } let hasChanges; if (this.#useDifferentialUpdate) { @@ -555,10 +594,13 @@ class ResourceRequestManager { // Try to find an existing request set that we can reuse let setId = this.#requestGraph.findExactMatch(requests); let resourceIndex; + let unresolvedRequests; if (setId) { // Reuse existing resource index. // Note: This index has already been updated before the task executed, so no update is necessary here - resourceIndex = this.#requestGraph.getMetadata(setId).resourceIndex; + const existingMetadata = this.#requestGraph.getMetadata(setId); + resourceIndex = existingMetadata.resourceIndex; + unresolvedRequests = existingMetadata.unresolvedRequests; } else { // New request set, check whether we can create a delta const metadata = {}; // Will populate with resourceIndex below @@ -572,27 +614,138 @@ class ResourceRequestManager { const addedRequests = requestSet.getAddedRequests(); const resourcesToAdd = await this.#getResourcesForRequests(addedRequests, reader); - if (!resourcesToAdd.length) { - throw new Error(`Unexpected empty added resources for request set ID ${setId} ` + - `of task '${this.#taskName}' of project '${this.#projectName}'`); - } log.verbose(`Task '${this.#taskName}' of project '${this.#projectName}' ` + `created derived resource index for request set ID ${setId} ` + `based on parent ID ${parentId} with ${resourcesToAdd.length} additional resources`); resourceIndex = await parentResourceIndex.deriveTree(resourcesToAdd); + // Some added requests may resolve to no resource (e.g. a byPath probe for an + // optional file, or every request after a branch switch deletes probed files). + // Those reads still influence task output, so record the unresolved requests to + // keep the exposed signature distinct from the parent's until they resolve. + unresolvedRequests = this.#collectUnresolvedRequests(addedRequests, resourcesToAdd); } else { const resourcesRead = await this.#getResourcesForRequests(requests, reader); resourceIndex = await ResourceIndex.createShared(resourcesRead, Date.now(), this.#newTreeRegistry()); + // Same handling for the root case: distinguish independent recordings that + // happen to resolve to zero resources today. + unresolvedRequests = this.#collectUnresolvedRequests(requests, resourcesRead); } metadata.resourceIndex = resourceIndex; + if (unresolvedRequests?.size) { + metadata.unresolvedRequests = unresolvedRequests; + } } return { setId, - signature: resourceIndex.getSignature(), + signature: this.#computeNodeSignature(resourceIndex, unresolvedRequests), }; } + /** + * Computes the exposed signature for a request-set node + * + * With no unresolved requests, returns the underlying tree signature. When some + * recorded requests resolved to no resources (byPath probes for files that don't + * exist yet, or byGlob patterns matching nothing), those reads still influence + * task output, so the exposed signature must stay distinct from the tree hash of + * an otherwise-identical index. The signature therefore hashes the tree signature + * together with the sorted unresolved keys. + * + * Once all unresolved requests drain (a later updateIndices upserts each probed + * resource as it appears), the composite falls back to the tree signature, + * matching what a fresh recording with the resource present would produce. + * + * @param {ResourceIndex} resourceIndex Resource index of the node + * @param {Set|undefined} unresolvedRequests Set of unresolved request keys + * @returns {string} Exposed signature + */ + #computeNodeSignature(resourceIndex, unresolvedRequests) { + const treeSignature = resourceIndex.getSignature(); + if (!unresolvedRequests?.size) { + return treeSignature; + } + const sortedKeys = Array.from(unresolvedRequests).sort(); + // Separate the fields with NUL so the concatenation is unambiguous: the tree + // signature is hex and request keys are UTF-8 paths/patterns, so neither can + // contain a NUL byte. Without it, sha256("ab" + ["c"]) and sha256("a" + ["bc"]) + // would collide. sortedKeys is joined on NUL for the same reason. + return crypto.createHash("sha256") + .update(treeSignature) + .update("\0") + .update(sortedKeys.join("\0")) + .digest("hex"); + } + + /** + * Determines which recorded requests did not resolve to any resource + * + * A 'path' request is unresolved if no fetched resource has that exact path. + * A 'patterns' request is unresolved if no fetched resource path matches any + * pattern in the array. + * + * @param {Request[]} recordedRequests Requests as recorded by the MonitoredReader + * @param {Array} fetchedResources Resources actually returned + * @returns {Set} Set of unresolved request keys (Request.toKey() form) + */ + #collectUnresolvedRequests(recordedRequests, fetchedResources) { + const fetchedPaths = fetchedResources.map((res) => res.getOriginalPath()); + const fetchedPathSet = new Set(fetchedPaths); + const unresolved = new Set(); + for (const request of recordedRequests) { + if (!this.#requestResolvesAgainstPaths(request, fetchedPathSet, fetchedPaths)) { + unresolved.add(request.toKey()); + } + } + return unresolved; + } + + /** + * Drops keys from metadata.unresolvedRequests whose recorded request now resolves to + * at least one of the given paths. + * + * Called from updateIndices after upserting newly-appeared resources into a + * node's index. Once every key drains, the node's exposed signature collapses + * back to the pure tree hash, matching what a fresh recording with the + * resource present would have produced. + * + * @param {object} metadata Request-set node metadata (mutated in place) + * @param {Request[]} addedRequests Recorded added-requests for this node + * @param {string[]} resolvedPaths Paths that now resolve to a resource + */ + #drainUnresolvedRequests(metadata, addedRequests, resolvedPaths) { + const resolvedPathSet = new Set(resolvedPaths); + for (const request of addedRequests) { + const key = request.toKey(); + if (!metadata.unresolvedRequests.has(key)) { + continue; + } + if (this.#requestResolvesAgainstPaths(request, resolvedPathSet, resolvedPaths)) { + metadata.unresolvedRequests.delete(key); + } + } + if (metadata.unresolvedRequests.size === 0) { + delete metadata.unresolvedRequests; + } + } + + /** + * Whether a recorded request would resolve against the given set of resource paths. + * 'path' requests hit the Set for O(1) lookup; 'patterns' requests fall back to + * micromatch, which needs the array form. + * + * @param {Request} request + * @param {Set} pathSet Set of resource paths + * @param {string[]} pathList Same paths in array form, for micromatch + * @returns {boolean} + */ + #requestResolvesAgainstPaths(request, pathSet, pathList) { + if (request.type === "path") { + return pathSet.has(request.value); + } + return micromatch(pathList, request.value, {dot: true}).length > 0; + } + /** * Associates a request set from this manager with one from another manager * @@ -691,15 +844,17 @@ class ResourceRequestManager { const rootIndices = []; const deltaIndices = []; for (const {nodeId, parentId} of this.#requestGraph.traverseByDepth()) { - const {resourceIndex} = this.#requestGraph.getMetadata(nodeId); + const {resourceIndex, unresolvedRequests} = this.#requestGraph.getMetadata(nodeId); if (!resourceIndex) { throw new Error(`Missing resource index for node ID ${nodeId}`); } if (!parentId) { - rootIndices.push({ + const entry = { nodeId, resourceIndex: resourceIndex.toCacheObject(), - }); + }; + serializeUnresolvedRequests(entry, unresolvedRequests); + rootIndices.push(entry); } else { const {resourceIndex: rootResourceIndex} = this.#requestGraph.getMetadata(parentId); if (!rootResourceIndex) { @@ -708,10 +863,12 @@ class ResourceRequestManager { // Store the metadata for all added resources. Note: Those resources might not be available // in the current tree. In that case we store an empty array. const addedResourceIndex = resourceIndex.getAddedResourceIndex(rootResourceIndex); - deltaIndices.push({ + const entry = { nodeId, addedResourceIndex, - }); + }; + serializeUnresolvedRequests(entry, unresolvedRequests); + deltaIndices.push(entry); } } return { diff --git a/packages/project/test/lib/build/cache/BuildTaskCache.js b/packages/project/test/lib/build/cache/BuildTaskCache.js index 85f4052e3cf..948f7753ad5 100644 --- a/packages/project/test/lib/build/cache/BuildTaskCache.js +++ b/packages/project/test/lib/build/cache/BuildTaskCache.js @@ -493,3 +493,34 @@ test("Handles non-existent resource paths", async (t) => { t.is(typeof projectSig, "string", "Still returns signature"); t.is(typeof depSig, "string", "Still returns dependency signature"); }); + +test("recordRequests with unresolved probe in delta position returns a distinct signature", async (t) => { + // Shape observed in OpenUI5 after a branch switch: the first recording anchors + // a resolvable parent request set, then a subsequent recording adds a byPath + // probe for a file that no longer exists. + const cache = new BuildTaskCache("test.project", "testTask", false); + + const projectReader = createMockReader([ + createMockResource("/a.js"), + ]); + const dependencyReader = createMockReader([]); + + const firstRequests = { + paths: new Set(["/a.js"]), + patterns: new Set(), + }; + const [firstProjSig] = await cache.recordRequests( + firstRequests, undefined, projectReader, dependencyReader); + + const probingRequests = { + paths: new Set(["/a.js", "/optional.json"]), + patterns: new Set(), + }; + const [probingProjSig] = await cache.recordRequests( + probingRequests, undefined, projectReader, dependencyReader); + + t.is(typeof probingProjSig, "string", + "Probing recording completes without throwing"); + t.not(probingProjSig, firstProjSig, + "Probing recording gets a cache key distinct from the parent's; the probed absence matters for output"); +}); diff --git a/packages/project/test/lib/build/cache/ResourceRequestManager.js b/packages/project/test/lib/build/cache/ResourceRequestManager.js index 161b292ac2e..5e9ad65a443 100644 --- a/packages/project/test/lib/build/cache/ResourceRequestManager.js +++ b/packages/project/test/lib/build/cache/ResourceRequestManager.js @@ -659,6 +659,203 @@ test("ResourceRequestManager: Serialization round-trip with multiple request set t.false(manager2.hasNewOrModifiedCacheEntries(), "Restored manager has no new entries"); }); +// ===== UNRESOLVED-READS TESTS ===== +// +// A task's MonitoredReader records every byPath/byGlob call, including those that +// return null or []. When a later addRequests recording adds a delta whose +// requests all resolve to zero resources against the current reader (typical +// after a branch switch that deletes probed files), the manager still records +// the request set: its shape influences task output, so it cannot be collapsed +// onto the parent's cache key. +// +// The tests below assert: +// - #addRequestSet does not throw for empty-resolving deltas. +// - The exposed signature is distinct from the parent's and from any other +// empty-resolving delta. +// - Once a probed resource appears and updateIndices upserts it, the composite +// signature collapses to the pure tree hash, matching the signature a +// same-shape recording would have produced with the resource present. +// - The composite survives toCacheObject / fromCache round-trip. + +test("ResourceRequestManager: #addRequestSet with empty-resolving path delta produces distinct signature", + async (t) => { + const resourcesBefore = new Map([ + ["/a.js", createMockResource("/a.js", "hash-a")], + ]); + const reader = createMockReader(resourcesBefore); + + const manager = new ResourceRequestManager("test.project", "myTask", false); + // Parent recording: {/a.js} + const parent = await manager.addRequests({paths: ["/a.js"], patterns: []}, reader); + + // Probing recording: {/a.js, /c.js}. /c.js does not exist, but MonitoredReader + // still recorded the byPath call. + const probe = await manager.addRequests({ + paths: ["/a.js", "/c.js"], + patterns: [], + }, reader); + + t.truthy(probe.signature, "Empty-resolving delta yields a signature (no throw)"); + t.not(probe.signature, parent.signature, + "Delta signature is distinct from parent's; the probe influences task output"); + + const signatures = manager.getIndexSignatures(); + t.is(signatures.length, 2, "Both request sets are recorded"); + t.is(signatures[0], parent.signature); + t.is(signatures[1], probe.signature); + }); + +test("ResourceRequestManager: #addRequestSet with empty-resolving pattern delta produces distinct signature", + async (t) => { + const resourcesBefore = new Map([ + ["/src/a.js", createMockResource("/src/a.js", "hash-a")], + ]); + const reader = createMockReader(resourcesBefore); + + const manager = new ResourceRequestManager("test.project", "myTask", false); + const parent = await manager.addRequests({paths: ["/src/a.js"], patterns: []}, reader); + + // Recorded byGlob("/test/**/*.js") that matches nothing today. + const probe = await manager.addRequests({ + paths: ["/src/a.js"], + patterns: [["/test/**/*.js"]], + }, reader); + + t.truthy(probe.signature, "Empty-resolving pattern delta yields a signature"); + t.not(probe.signature, parent.signature, "Pattern probe changes cache identity"); + }); + +test("ResourceRequestManager: two independent empty-resolving deltas produce different signatures", + async (t) => { + const reader = createMockReader(new Map([ + ["/a.js", createMockResource("/a.js", "hash-a")], + ])); + + const manager = new ResourceRequestManager("test.project", "myTask", false); + await manager.addRequests({paths: ["/a.js"], patterns: []}, reader); + + const probeC = await manager.addRequests({ + paths: ["/a.js", "/c.js"], + patterns: [], + }, reader); + const probeD = await manager.addRequests({ + paths: ["/a.js", "/d.js"], + patterns: [], + }, reader); + + t.not(probeC.signature, probeD.signature, + "Different unresolved probes must map to different cache keys"); + }); + +test("ResourceRequestManager: signature collapses to tree hash once probed resource resolves", + async (t) => { + // A: full run against a state where /c.js exists, producing the natural + // derived-tree signature the recovering run should converge on. + const readerWithC = createMockReader(new Map([ + ["/a.js", createMockResource("/a.js", "hash-a")], + ["/c.js", createMockResource("/c.js", "hash-c")], + ])); + const managerA = new ResourceRequestManager("test.project", "myTask", false); + await managerA.addRequests({paths: ["/a.js"], patterns: []}, readerWithC); + const referenceRun = await managerA.addRequests({ + paths: ["/a.js", "/c.js"], + patterns: [], + }, readerWithC); + + // B: same recording against a reader where /c.js is missing, then + // updateIndices reintroduces it. + const readerWithoutC = createMockReader(new Map([ + ["/a.js", createMockResource("/a.js", "hash-a")], + ])); + const managerB = new ResourceRequestManager("test.project", "myTask", false); + await managerB.addRequests({paths: ["/a.js"], patterns: []}, readerWithoutC); + const probingRun = await managerB.addRequests({ + paths: ["/a.js", "/c.js"], + patterns: [], + }, readerWithoutC); + + t.not(probingRun.signature, referenceRun.signature, + "Probing recording differs while /c.js is missing"); + + // /c.js now appears. updateIndices upserts it into the probing node. + const readerNowWithC = createMockReader(new Map([ + ["/a.js", createMockResource("/a.js", "hash-a")], + ["/c.js", createMockResource("/c.js", "hash-c")], + ])); + await managerB.updateIndices(readerNowWithC, ["/c.js"]); + + const signaturesB = managerB.getIndexSignatures(); + t.is(signaturesB[1], referenceRun.signature, + "Once the probe resolves, the composite signature collapses to the natural tree hash"); + }); + +test("ResourceRequestManager: unresolved-requests marker survives toCacheObject / fromCache", + async (t) => { + const reader = createMockReader(new Map([ + ["/a.js", createMockResource("/a.js", "hash-a")], + ])); + + const manager1 = new ResourceRequestManager("test.project", "myTask", false); + await manager1.addRequests({paths: ["/a.js"], patterns: []}, reader); + const probe = await manager1.addRequests({ + paths: ["/a.js", "/c.js"], + patterns: [], + }, reader); + const signaturesBefore = manager1.getIndexSignatures(); + + const cacheObj = manager1.toCacheObject(); + const manager2 = ResourceRequestManager.fromCache("test.project", "myTask", false, cacheObj); + + const signaturesAfter = manager2.getIndexSignatures(); + t.deepEqual(signaturesAfter, signaturesBefore, + "Signatures survive the cache round-trip bit-for-bit"); + t.is(signaturesAfter[1], probe.signature, + "Probing signature is preserved"); + }); + +test("ResourceRequestManager: fully empty root recording gets a distinguished signature", + async (t) => { + // Two independent tasks that both recorded reads to files that don't exist + // must not collide on the same "dir::empty" tree constant. + const emptyReader = createMockReader(new Map()); + + const managerA = new ResourceRequestManager("test.project", "taskA", false); + const runA = await managerA.addRequests({paths: ["/only-in-A.js"], patterns: []}, emptyReader); + + const managerB = new ResourceRequestManager("test.project", "taskB", false); + const runB = await managerB.addRequests({paths: ["/only-in-B.js"], patterns: []}, emptyReader); + + t.not(runA.signature, runB.signature, + "Distinct root recordings with all-unresolved reads produce distinct signatures"); + }); + +test("ResourceRequestManager: BuildTaskCache-shape flow with unresolved probe (integration-ish)", + async (t) => { + // Mirrors what BuildTaskCache.recordRequests does with two consecutive + // addRequests recordings that share a parent but differ by one probed path. + const readerBefore = createMockReader(new Map([ + ["/a.js", createMockResource("/a.js", "hash-a")], + ["/b.js", createMockResource("/b.js", "hash-b")], + ])); + + const manager = new ResourceRequestManager("test.project", "myTask", true); + + // First recording produces a valid parent. + const first = await manager.addRequests({paths: ["/a.js", "/b.js"], patterns: []}, readerBefore); + + // Second recording adds an unresolvable byPath. + const second = await manager.addRequests({ + paths: ["/a.js", "/b.js", "/optional.json"], + patterns: [], + }, readerBefore); + + t.truthy(second.signature, "Second recording completes without throwing"); + t.not(second.signature, first.signature, "Cache key reflects the extra probe"); + + // hasNewOrModifiedCacheEntries stays true for a fresh manager. + t.true(manager.hasNewOrModifiedCacheEntries()); + }); + test("ResourceRequestManager: Serialization round-trip with multiple request sets and following update", async (t) => { const resources = new Map([ ["/a.js", createMockResource("/a.js", "hash-a")],