diff --git a/bun.lock b/bun.lock index 76e4334a78..b1b7395f99 100644 --- a/bun.lock +++ b/bun.lock @@ -29,6 +29,7 @@ "name": "@hyperframes/aws-lambda", "version": "0.8.169", "dependencies": { + "@aws-sdk/client-ecr": "^3.1146.0", "@aws-sdk/client-lambda": "^3.1146.0", "@aws-sdk/client-s3": "^3.1146.0", "@aws-sdk/client-sfn": "^3.1146.0", @@ -444,6 +445,8 @@ "@aws-sdk/checksums": ["@aws-sdk/checksums@3.1001.1", "", { "dependencies": { "@aws-sdk/core": "^3.978.1", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-x12Q17KYlJAd3nKf8LV5LV0vt8sh8/6YfQLGPtrGnQf/tW4jqxPGq5GPpuVitpQYM3eUR4XB7CbxZf751NMbLw=="], + "@aws-sdk/client-ecr": ["@aws-sdk/client-ecr@3.1146.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.1", "@aws-sdk/credential-provider-node": "^3.972.84", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", "@smithy/fetch-http-handler": "^5.8.0", "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-6SxQK33Jye0WHjOld95b9S12eHIYtPfpZhuAv9HYCMQYJJyVDhsV8BNWwsNjLEchTc/yjS8ksmzc8MhApDcD3g=="], + "@aws-sdk/client-lambda": ["@aws-sdk/client-lambda@3.1146.0", "", { "dependencies": { "@aws-sdk/core": "^3.978.1", "@aws-sdk/credential-provider-node": "^3.972.84", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", "@smithy/fetch-http-handler": "^5.8.0", "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-qCeVHGhkI5aw0Lvexhwuf1H41L2sCm+9C4K+IOUhErY/nx5ioCU0dFyXF4+P4m4Nk23uiTbON7lI7nmlPNncHw=="], "@aws-sdk/client-s3": ["@aws-sdk/client-s3@3.1146.0", "", { "dependencies": { "@aws-sdk/checksums": "^3.1001.1", "@aws-sdk/core": "^3.978.1", "@aws-sdk/credential-provider-node": "^3.972.84", "@aws-sdk/middleware-sdk-s3": "^3.972.77", "@aws-sdk/signature-v4-multi-region": "^3.996.47", "@aws-sdk/types": "^3.974.6", "@smithy/core": "^3.35.0", "@smithy/fetch-http-handler": "^5.8.0", "@smithy/node-http-handler": "^4.12.1", "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-WY0YCBzxc4muFfY6UbGpA+oib0nGT/Px2aEPGYX7pp1la2OmDLChnK81mu8H7LdYvwGU1pLAsLIa/qgad0Wd7g=="], diff --git a/packages/aws-lambda/build.mjs b/packages/aws-lambda/build.mjs index 58c48dbb22..577faea145 100644 --- a/packages/aws-lambda/build.mjs +++ b/packages/aws-lambda/build.mjs @@ -32,6 +32,7 @@ const sharedOpts = { format: "esm", minify: false, external: [ + "@aws-sdk/client-ecr", "@aws-sdk/client-lambda", "@aws-sdk/client-s3", "@aws-sdk/client-sfn", diff --git a/packages/aws-lambda/package.json b/packages/aws-lambda/package.json index 0c6eaf7f4f..ebbf3b497c 100644 --- a/packages/aws-lambda/package.json +++ b/packages/aws-lambda/package.json @@ -72,6 +72,7 @@ "verify:zip-size": "tsx scripts/verify-zip-size.ts" }, "dependencies": { + "@aws-sdk/client-ecr": "^3.1146.0", "@aws-sdk/client-lambda": "^3.1146.0", "@aws-sdk/client-s3": "^3.1146.0", "@aws-sdk/client-sfn": "^3.1146.0", diff --git a/packages/aws-lambda/src/sdk/index.ts b/packages/aws-lambda/src/sdk/index.ts index 7fe1460600..264fdaa884 100644 --- a/packages/aws-lambda/src/sdk/index.ts +++ b/packages/aws-lambda/src/sdk/index.ts @@ -45,5 +45,12 @@ export { prebuiltVideoImage, type ReleaseAssetManifest, } from "./prebuiltHandler.js"; +export { + ensureEcrRepository, + type EnsureEcrRepositoryOptions, + type PushedImage, + pushImageToEcr, + type PushImageToEcrOptions, +} from "./pushImageToEcr.js"; export type { SerializableDistributedRenderConfig } from "../events.js"; export type { DistributedFormat } from "../formatExtension.js"; diff --git a/packages/aws-lambda/src/sdk/pushImageToEcr.test.ts b/packages/aws-lambda/src/sdk/pushImageToEcr.test.ts new file mode 100644 index 0000000000..1398b550e7 --- /dev/null +++ b/packages/aws-lambda/src/sdk/pushImageToEcr.test.ts @@ -0,0 +1,367 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { ECRClient } from "@aws-sdk/client-ecr"; +import { ensureEcrRepository, pushImageToEcr } from "./pushImageToEcr.js"; + +const OCI_MANIFEST = "application/vnd.oci.image.manifest.v1+json"; +const OCI_INDEX = "application/vnd.oci.image.index.v1+json"; +const REGISTRY = "123456789012.dkr.ecr.us-east-1.amazonaws.com"; +const PART_SIZE = 7; +// 21 bytes is exactly three parts; 17 bytes ends with a short part. +const LAYER_A = Buffer.from("layer-a: 21 bytes!!!!"); +const LAYER_B = Buffer.from("layer-b: 17 bytes"); + +interface Descriptor { + mediaType: string; + digest: string; + size: number; +} + +function sha256(bytes: Buffer | string): string { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +} + +function awsError(name: string): Error { + return Object.assign(new Error(name), { name }); +} + +/** In-memory ECR that enforces the upload rules real ECR enforces. */ +class FakeEcr { + calls: string[] = []; + blobs = new Map(); + tags = new Map(); + repositories = new Map(); + repositoryTags = new Map(); + /** Digests BatchCheckLayerAvailability reports missing even though stored (a concurrent push). */ + hiddenFromCheck = new Set(); + private uploads = new Map(); + + async send(command: unknown): Promise { + const name = (command as { constructor: { name: string } }).constructor.name; + const input = (command as { input: unknown }).input; + this.calls.push(name); + const handler = (this as unknown as Record unknown>)[name]; + if (!handler) throw new Error(`FakeEcr: unexpected command ${name}`); + return handler.call(this, input); + } + + CreateRepositoryCommand(input: { repositoryName: string; tags?: unknown }): unknown { + if (this.repositories.has(input.repositoryName)) { + throw awsError("RepositoryAlreadyExistsException"); + } + const repositoryUri = `${REGISTRY}/${input.repositoryName}`; + this.repositories.set(input.repositoryName, repositoryUri); + this.repositoryTags.set(input.repositoryName, input.tags); + return { repository: { repositoryUri } }; + } + + DescribeRepositoriesCommand(input: { repositoryNames: string[] }): unknown { + return { + repositories: input.repositoryNames.map((name) => ({ + repositoryArn: `arn:aws:ecr:r:1:repository/${name}`, + repositoryUri: this.repositories.get(name), + })), + }; + } + + ListTagsForResourceCommand(input: { resourceArn: string }): unknown { + return { tags: this.repositoryTags.get(input.resourceArn.split("/").pop() ?? "") ?? [] }; + } + + BatchCheckLayerAvailabilityCommand(input: { layerDigests: string[] }): unknown { + if (input.layerDigests.length > 100) throw awsError("InvalidParameterException"); + const stored = (d: string) => this.blobs.has(d) && !this.hiddenFromCheck.has(d); + return { + layers: input.layerDigests + .filter(stored) + .map((layerDigest) => ({ layerDigest, layerAvailability: "AVAILABLE" })), + failures: input.layerDigests + .filter((d) => !stored(d)) + .map((layerDigest) => ({ layerDigest, failureCode: "MissingLayerDigest" })), + }; + } + + InitiateLayerUploadCommand(): unknown { + const uploadId = `upload-${this.uploads.size}`; + this.uploads.set(uploadId, []); + return { uploadId, partSize: PART_SIZE }; + } + + UploadLayerPartCommand(input: { + uploadId: string; + partFirstByte: number; + partLastByte: number; + layerPartBlob: Uint8Array; + }): unknown { + const parts = this.uploads.get(input.uploadId); + if (!parts) throw awsError("UploadNotFoundException"); + const received = parts.reduce((sum, part) => sum + part.length, 0); + const size = input.layerPartBlob.length; + const contiguous = input.partFirstByte === received; + const lastByteMatches = input.partLastByte === input.partFirstByte + size - 1; + const previousWasFull = parts.every((part) => part.length === PART_SIZE); + if (!contiguous || !lastByteMatches || size > PART_SIZE || !previousWasFull) { + throw awsError("InvalidLayerPartException"); + } + parts.push(Buffer.from(input.layerPartBlob)); + return { uploadId: input.uploadId, lastByteReceived: input.partLastByte }; + } + + CompleteLayerUploadCommand(input: { uploadId: string; layerDigests: string[] }): unknown { + const parts = this.uploads.get(input.uploadId); + if (!parts) throw awsError("UploadNotFoundException"); + this.uploads.delete(input.uploadId); + const bytes = Buffer.concat(parts); + const [digest] = input.layerDigests; + if (input.layerDigests.length !== 1 || sha256(bytes) !== digest) { + throw awsError("InvalidLayerException"); + } + if (this.blobs.has(digest)) throw awsError("LayerAlreadyExistsException"); + this.blobs.set(digest, bytes); + return { layerDigest: digest }; + } + + PutImageCommand(input: { + imageManifest: string; + imageTag: string; + imageDigest: string; + }): unknown { + if (sha256(input.imageManifest) !== input.imageDigest) + throw awsError("InvalidParameterException"); + const manifest = JSON.parse(input.imageManifest) as { + config: Descriptor; + layers: Descriptor[]; + }; + const referenced = [manifest.config, ...manifest.layers].map((blob) => blob.digest); + if (referenced.some((digest) => !this.blobs.has(digest))) { + throw awsError("LayersNotFoundException"); + } + if (this.tags.get(input.imageTag) === input.imageDigest) { + throw awsError("ImageAlreadyExistsException"); + } + this.tags.set(input.imageTag, input.imageDigest); + return { image: { imageId: { imageDigest: input.imageDigest, imageTag: input.imageTag } } }; + } +} + +function asEcr(fake: FakeEcr): ECRClient { + return fake as unknown as ECRClient; +} + +let layoutDir: string; + +beforeEach(() => { + layoutDir = mkdtempSync(join(tmpdir(), "hf-ecr-layout-")); +}); + +afterEach(() => { + rmSync(layoutDir, { recursive: true, force: true }); +}); + +function putBlob(mediaType: string, bytes: Buffer): Descriptor { + mkdirSync(join(layoutDir, "blobs", "sha256"), { recursive: true }); + const digest = sha256(bytes); + writeFileSync(join(layoutDir, "blobs", "sha256", digest.slice("sha256:".length)), bytes); + return { mediaType, digest, size: bytes.length }; +} + +function writeIndex(manifests: Descriptor[]): void { + writeFileSync( + join(layoutDir, "index.json"), + JSON.stringify({ schemaVersion: 2, mediaType: OCI_INDEX, manifests }), + ); +} + +function writeLayout( + layers: Buffer[] = [LAYER_A, LAYER_B], + platform: Record = { architecture: "arm64", os: "linux" }, +) { + const config = putBlob( + "application/vnd.oci.image.config.v1+json", + Buffer.from(JSON.stringify({ ...platform, rootfs: { type: "layers" } })), + ); + const layerDescriptors = layers.map((bytes) => + putBlob("application/vnd.oci.image.layer.v1.tar+gzip", bytes), + ); + const manifest = putBlob( + OCI_MANIFEST, + Buffer.from( + JSON.stringify({ + schemaVersion: 2, + mediaType: OCI_MANIFEST, + config, + layers: layerDescriptors, + }), + ), + ); + writeIndex([manifest]); + return { config, manifest, layers: layerDescriptors }; +} + +function push(ecr: FakeEcr) { + return pushImageToEcr({ + ecr: asEcr(ecr), + layoutDir, + repositoryName: "hyperframes-render", + repositoryUri: `${REGISTRY}/hyperframes-render`, + tag: "v1", + }); +} + +describe("pushImageToEcr", () => { + it("uploads every blob in partSize parts and puts the manifest under the tag", async () => { + const { config, manifest, layers } = writeLayout(); + const ecr = new FakeEcr(); + + const result = await push(ecr); + + expect(result).toEqual({ + imageUri: `${REGISTRY}/hyperframes-render@${manifest.digest}`, + digest: manifest.digest, + architecture: "arm64", + os: "linux", + }); + expect(ecr.blobs.get(layers[0]!.digest)).toEqual(LAYER_A); + expect(ecr.blobs.get(layers[1]!.digest)).toEqual(LAYER_B); + expect(ecr.blobs.has(config.digest)).toBe(true); + expect(ecr.tags.get("v1")).toBe(manifest.digest); + const parts = ecr.calls.filter((call) => call === "UploadLayerPartCommand").length; + expect(parts).toBe(Math.ceil(config.size / PART_SIZE) + 3 + 3); + expect(ecr.calls[0]).toBe("BatchCheckLayerAvailabilityCommand"); + expect(ecr.calls.at(-1)).toBe("PutImageCommand"); + }); + + it("skips blobs ECR already has", async () => { + const { config, layers } = writeLayout(); + const ecr = new FakeEcr(); + ecr.blobs.set(config.digest, Buffer.from("stored")); + ecr.blobs.set(layers[0]!.digest, LAYER_A); + + await push(ecr); + + expect(ecr.calls.filter((call) => call === "InitiateLayerUploadCommand")).toHaveLength(1); + expect(ecr.blobs.get(layers[1]!.digest)).toEqual(LAYER_B); + }); + + it("checks availability in batches of at most 100 digests", async () => { + writeLayout(Array.from({ length: 101 }, (_, i) => Buffer.from(`layer ${i}`))); + const ecr = new FakeEcr(); + + await push(ecr); + + expect(ecr.calls.filter((call) => call === "BatchCheckLayerAvailabilityCommand")).toHaveLength( + 2, + ); + expect(ecr.blobs.size).toBe(102); + }); + + it("refuses a tampered blob before making any ECR call", async () => { + const { layers } = writeLayout(); + const tampered = Buffer.from(LAYER_B); + tampered[0] = tampered[0]! ^ 1; + writeFileSync(join(layoutDir, "blobs", "sha256", layers[1]!.digest.slice(7)), tampered); + const ecr = new FakeEcr(); + + await expect(push(ecr)).rejects.toThrow(/does not match its descriptor/); + expect(ecr.calls).toEqual([]); + }); + + it("refuses an index listing two manifests", async () => { + const { manifest } = writeLayout(); + const attestation = putBlob(OCI_MANIFEST, Buffer.from("{}")); + writeIndex([manifest, attestation]); + const ecr = new FakeEcr(); + + await expect(push(ecr)).rejects.toThrow(/exactly one single-platform image manifest; found 2/); + expect(ecr.calls).toEqual([]); + }); + + it("refuses a multi-platform image index", async () => { + const { manifest } = writeLayout(); + writeIndex([{ ...manifest, mediaType: OCI_INDEX }]); + const ecr = new FakeEcr(); + + await expect(push(ecr)).rejects.toThrow(`found 1 (${OCI_INDEX})`); + expect(ecr.calls).toEqual([]); + }); + + it("refuses an image config with no architecture", async () => { + writeLayout([LAYER_A], { os: "linux" }); + const ecr = new FakeEcr(); + + await expect(push(ecr)).rejects.toThrow(/names no architecture\/os/); + expect(ecr.calls).toEqual([]); + }); + + it("treats LayerAlreadyExists on complete as uploaded", async () => { + const { layers, manifest } = writeLayout(); + const ecr = new FakeEcr(); + ecr.blobs.set(layers[0]!.digest, LAYER_A); + ecr.hiddenFromCheck.add(layers[0]!.digest); + + const result = await push(ecr); + + expect(result.digest).toBe(manifest.digest); + expect(ecr.tags.get("v1")).toBe(manifest.digest); + }); + + it("redeploying the same image uploads nothing and succeeds", async () => { + const { manifest } = writeLayout(); + const ecr = new FakeEcr(); + await push(ecr); + ecr.calls = []; + + const result = await push(ecr); + + expect(result.digest).toBe(manifest.digest); + expect(ecr.calls).toEqual(["BatchCheckLayerAvailabilityCommand", "PutImageCommand"]); + }); +}); + +describe("ensureEcrRepository", () => { + it("creates the repository with its tags", async () => { + const ecr = new FakeEcr(); + const tags = [{ Key: "app", Value: "hyperframes" }]; + + const { repositoryUri } = await ensureEcrRepository({ + ecr: asEcr(ecr), + repositoryName: "hyperframes-render", + tags, + }); + + expect(repositoryUri).toBe(`${REGISTRY}/hyperframes-render`); + expect(ecr.calls).toEqual(["CreateRepositoryCommand"]); + expect(ecr.repositoryTags.get("hyperframes-render")).toEqual(tags); + }); + + it("returns the existing repository's uri", async () => { + const ecr = new FakeEcr(); + ecr.repositories.set("hyperframes-render", `${REGISTRY}/hyperframes-render`); + + const { repositoryUri } = await ensureEcrRepository({ + ecr: asEcr(ecr), + repositoryName: "hyperframes-render", + }); + + expect(repositoryUri).toBe(`${REGISTRY}/hyperframes-render`); + expect(ecr.calls).toEqual(["CreateRepositoryCommand", "DescribeRepositoriesCommand"]); + }); + + it("adopts an existing repository only when it carries the tags", async () => { + const ecr = new FakeEcr(); + const tags = [{ Key: "hyperframes:stack", Value: "s" }]; + await ensureEcrRepository({ ecr: asEcr(ecr), repositoryName: "s-video", tags }); + + const again = await ensureEcrRepository({ ecr: asEcr(ecr), repositoryName: "s-video", tags }); + expect(again.repositoryUri).toBe(`${REGISTRY}/s-video`); + + ecr.repositories.set("other-video", `${REGISTRY}/other-video`); + ecr.repositoryTags.set("other-video", [{ Key: "hyperframes:stack", Value: "other" }]); + await expect( + ensureEcrRepository({ ecr: asEcr(ecr), repositoryName: "other-video", tags }), + ).rejects.toThrow("other-video already exists without hyperframes:stack=s"); + }); +}); diff --git a/packages/aws-lambda/src/sdk/pushImageToEcr.ts b/packages/aws-lambda/src/sdk/pushImageToEcr.ts new file mode 100644 index 0000000000..9c585d0330 --- /dev/null +++ b/packages/aws-lambda/src/sdk/pushImageToEcr.ts @@ -0,0 +1,254 @@ +/** + * Push a prebuilt single-platform OCI image layout into ECR through the ECR API alone, + * so deploying the container-image function needs neither Docker nor registry HTTP calls. + */ + +import { open, readFile, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { + BatchCheckLayerAvailabilityCommand, + CompleteLayerUploadCommand, + CreateRepositoryCommand, + DescribeRepositoriesCommand, + ECRClient, + InitiateLayerUploadCommand, + ListTagsForResourceCommand, + PutImageCommand, + type Repository, + type Tag, + UploadLayerPartCommand, +} from "@aws-sdk/client-ecr"; +import { sha256File } from "../s3Transport.js"; + +const MANIFEST_MEDIA_TYPES = new Set([ + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +]); +// ECR's BatchCheckLayerAvailability accepts at most 100 digests per call. +const CHECK_BATCH_SIZE = 100; + +interface Descriptor { + mediaType: string; + digest: string; + size: number; +} + +/** Options for {@link ensureEcrRepository}. */ +export interface EnsureEcrRepositoryOptions { + ecr?: ECRClient; + region?: string; + repositoryName: string; + /** Set on create. An existing repository is adopted only if it already carries every one. */ + tags?: Tag[]; +} + +/** Create the repository, or adopt an existing one that carries `tags`. */ +export async function ensureEcrRepository( + opts: EnsureEcrRepositoryOptions, +): Promise<{ repositoryUri: string }> { + const { repositoryName, tags } = opts; + const ecr = opts.ecr ?? new ECRClient({ region: opts.region }); + try { + const created = await ecr.send(new CreateRepositoryCommand({ repositoryName, tags })); + return { repositoryUri: repositoryUriOf(created.repository, repositoryName) }; + } catch (error) { + if ((error as Error).name !== "RepositoryAlreadyExistsException") throw error; + } + const found = await ecr.send( + new DescribeRepositoriesCommand({ repositoryNames: [repositoryName] }), + ); + const repository = found.repositories?.[0]; + if (tags?.length) { + const listed = await ecr.send( + new ListTagsForResourceCommand({ resourceArn: repository?.repositoryArn }), + ); + const has = new Map((listed.tags ?? []).map((tag) => [tag.Key, tag.Value])); + const missing = tags.filter((tag) => has.get(tag.Key) !== tag.Value); + if (missing.length > 0) { + const wanted = missing.map((tag) => `${tag.Key}=${tag.Value}`).join(", "); + throw new Error( + `[ensureEcrRepository] ${repositoryName} already exists without ${wanted}; refusing to adopt a repository this caller did not create`, + ); + } + } + return { repositoryUri: repositoryUriOf(repository, repositoryName) }; +} + +function repositoryUriOf(repository: Repository | undefined, repositoryName: string): string { + if (!repository?.repositoryUri) { + throw new Error(`[ensureEcrRepository] ECR returned no repositoryUri for ${repositoryName}`); + } + return repository.repositoryUri; +} + +/** Options for {@link pushImageToEcr}. */ +export interface PushImageToEcrOptions { + ecr?: ECRClient; + region?: string; + /** OCI image layout directory: `index.json` plus `blobs/sha256/`. */ + layoutDir: string; + repositoryName: string; + /** From {@link ensureEcrRepository}. */ + repositoryUri: string; + tag: string; + log?: (message: string) => void; +} + +/** Result of {@link pushImageToEcr}. */ +export interface PushedImage { + /** `@sha256:`, pinned to exactly the pushed bytes. */ + imageUri: string; + digest: string; + /** From the image config, so callers can assert the platform they expect. */ + architecture: string; + os: string; +} + +/** Upload the layout's missing blobs, then put its manifest under `tag`. Safe to re-run. */ +export async function pushImageToEcr(opts: PushImageToEcrOptions): Promise { + const { layoutDir, repositoryName, log = () => {} } = opts; + const ecr = opts.ecr ?? new ECRClient({ region: opts.region }); + const manifestDescriptor = await readManifestDescriptor(layoutDir); + await verifyBlob(layoutDir, manifestDescriptor); + const manifestText = await readFile(blobPath(layoutDir, manifestDescriptor.digest), "utf-8"); + const manifest = JSON.parse(manifestText) as { config?: Descriptor; layers?: Descriptor[] }; + if (!manifest.config || !Array.isArray(manifest.layers)) { + throw new Error( + `[pushImageToEcr] manifest ${manifestDescriptor.digest} has no config or layers`, + ); + } + const blobs = [manifest.config, ...manifest.layers]; + for (const blob of blobs) await verifyBlob(layoutDir, blob); + const config = JSON.parse( + await readFile(blobPath(layoutDir, manifest.config.digest), "utf-8"), + ) as { architecture?: string; os?: string }; + if (!config.architecture || !config.os) { + throw new Error( + `[pushImageToEcr] image config ${manifest.config.digest} names no architecture/os; ` + + "build the image for one platform", + ); + } + + const missing = await unavailableDigests(ecr, repositoryName, blobs); + // ponytail: sequential uploads; upload time on big layers is the ceiling, bounded parallel layers is the upgrade. + for (const blob of blobs) { + if (!missing.delete(blob.digest)) continue; + log(`uploading ${blob.digest} (${blob.size} bytes)`); + await uploadBlob(ecr, repositoryName, blobPath(layoutDir, blob.digest), blob); + } + + try { + await ecr.send( + new PutImageCommand({ + repositoryName, + imageManifest: manifestText, + imageManifestMediaType: manifestDescriptor.mediaType, + imageTag: opts.tag, + imageDigest: manifestDescriptor.digest, + }), + ); + } catch (error) { + if ((error as Error).name !== "ImageAlreadyExistsException") throw error; + } + return { + imageUri: `${opts.repositoryUri}@${manifestDescriptor.digest}`, + digest: manifestDescriptor.digest, + architecture: config.architecture, + os: config.os, + }; +} + +async function readManifestDescriptor(layoutDir: string): Promise { + const index = JSON.parse(await readFile(join(layoutDir, "index.json"), "utf-8")) as { + manifests?: Descriptor[]; + }; + const manifests = index.manifests ?? []; + const [only] = manifests; + if (manifests.length !== 1 || !only || !MANIFEST_MEDIA_TYPES.has(only.mediaType)) { + const found = manifests.map((m) => m.mediaType).join(", ") || "none"; + throw new Error( + `[pushImageToEcr] ${layoutDir}/index.json must list exactly one single-platform image ` + + `manifest; found ${manifests.length} (${found}). Build one platform with ` + + "--provenance=false --sbom=false.", + ); + } + return only; +} + +function blobPath(layoutDir: string, digest: string): string { + const hex = /^sha256:([a-f0-9]{64})$/.exec(digest)?.[1]; + if (!hex) throw new Error(`[pushImageToEcr] unsupported blob digest ${JSON.stringify(digest)}`); + return join(layoutDir, "blobs", "sha256", hex); +} + +async function verifyBlob(layoutDir: string, blob: Descriptor): Promise { + const path = blobPath(layoutDir, blob.digest); + const size = (await stat(path)).size; + if (size !== blob.size || `sha256:${await sha256File(path)}` !== blob.digest) { + throw new Error( + `[pushImageToEcr] blob ${blob.digest} (${size} bytes, expected ${blob.size}) does not ` + + "match its descriptor; the image layout is corrupt", + ); + } +} + +async function unavailableDigests( + ecr: ECRClient, + repositoryName: string, + blobs: Descriptor[], +): Promise> { + const missing = new Set(blobs.map((blob) => blob.digest)); + const digests = [...missing]; + for (let i = 0; i < digests.length; i += CHECK_BATCH_SIZE) { + const { layers = [] } = await ecr.send( + new BatchCheckLayerAvailabilityCommand({ + repositoryName, + layerDigests: digests.slice(i, i + CHECK_BATCH_SIZE), + }), + ); + for (const layer of layers) { + if (layer.layerAvailability === "AVAILABLE" && layer.layerDigest) { + missing.delete(layer.layerDigest); + } + } + } + return missing; +} + +async function uploadBlob( + ecr: ECRClient, + repositoryName: string, + path: string, + blob: Descriptor, +): Promise { + const { uploadId, partSize } = await ecr.send(new InitiateLayerUploadCommand({ repositoryName })); + if (!uploadId || !partSize) { + throw new Error(`[pushImageToEcr] InitiateLayerUpload returned no uploadId/partSize`); + } + const file = await open(path); + try { + for (let first = 0; first < blob.size; first += partSize) { + const part = Buffer.alloc(Math.min(partSize, blob.size - first)); + const { bytesRead } = await file.read(part, 0, part.length, first); + if (bytesRead !== part.length) throw new Error(`[pushImageToEcr] ${path} changed on disk`); + await ecr.send( + new UploadLayerPartCommand({ + repositoryName, + uploadId, + partFirstByte: first, + partLastByte: first + part.length - 1, + layerPartBlob: part, + }), + ); + } + } finally { + await file.close(); + } + try { + await ecr.send( + new CompleteLayerUploadCommand({ repositoryName, uploadId, layerDigests: [blob.digest] }), + ); + } catch (error) { + if ((error as Error).name !== "LayerAlreadyExistsException") throw error; + } +} diff --git a/scripts/package-size-budgets.json b/scripts/package-size-budgets.json index 28f31d115c..77d3467a6e 100644 --- a/scripts/package-size-budgets.json +++ b/scripts/package-size-budgets.json @@ -1,7 +1,7 @@ { "@hyperframes/aws-lambda": { - "packed": 70874, - "unpacked": 258166 + "packed": 73369, + "unpacked": 266805 }, "@hyperframes/cli": { "packed": 7902966,