diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c58e47f3..c260fa39 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,6 +3,7 @@ /* @celonis/astro /src @celonis/astro /src/core @celonis/astro +/src/commands/fragments/ @celonis/astro /src/commands/configuration-management/ @celonis/astro /tests/commands/configuration-management/ @celonis/astro /src/commands/t2tc/ @celonis/astro diff --git a/README.md b/README.md index 56ae8bdb..bcc8ca86 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,30 @@ content-cli -h content-cli pull package -h ``` +## Publish a custom app build + +```sh +content-cli sdk publish --app-id my-custom-app --dist ./dist --profile dev +``` + +The selected profile's tenant URL and Bearer authentication are reused. The app ID is used +directly as the Fragment Gateway app ID; it does not create or link a Studio asset. +The directory must contain browser-ready files and a regular `index.html` at its root. +Build Angular, React or TypeScript sources before running this command. + +The command validates the complete directory before creating an upload and automatically +compresses the build locally into a ZIP and sends it in one request. +It then finalizes the release and waits for admission to become ready. It prints the +upload ID and the server-generated release ID. The API requires a producer `version` label; +the command generates this automatically. + +Builds are limited to 500 files, 16 MiB per file and 64 MiB total. Symlinks, unsupported +extensions, reserved gateway metadata and invalid paths are rejected. +ZIP uploads require gateway support for `PUT /uploads/:uploadId/archive`. +Finalization does not activate or launch the app. A failed upload exits nonzero; after +session creation, its ID can be used to inspect the upload. +Rerunning the command creates a new upload; cross-process resume is not implemented. + ## Command Graph [**Open the interactive Command Graph**](https://raw.githack.com/celonis/content-cli/main/docs/command-graph.html) — a diff --git a/docs/command-graph.html b/docs/command-graph.html index 2645c540..5b87ac62 100644 --- a/docs/command-graph.html +++ b/docs/command-graph.html @@ -145,6 +145,11 @@ // group: root | area | subarea | subsubarea | command // --------------------------------------------------------------------------- const NODES = [ + { id: "area_sdk", label: "sdk", group: "area", path: "sdk", + description: "Publish custom app builds to the Fragment Gateway", options: ["-h, --help"] }, + { id: "sdk_publish", label: "publish", group: "command", path: "sdk publish", + description: "Upload a browser bundle with automatic ZIP selection and finalize a release; requires index.html", + options: ["--app-id ", "--dist ", "-p, --profile ", "-h, --help"] }, { id: "root", label: "content-cli", group: "root", path: "content-cli", description: "Root CLI. Global options apply to every command.", options: ["-V, --version", "-q, --quietmode (default: false)", "-p, --profile [profile]", "--gitProfile [gitProfile]", "--debug (default: false)", "--dev (default: false)", "-h, --help"] }, @@ -471,6 +476,7 @@ ]; const EDGES = [ + ["root","area_sdk"],["area_sdk","sdk_publish"], ["root","area_list"],["root","area_analyze"],["root","area_export"],["root","area_import"], ["root","area_pull"],["root","area_push"],["root","area_asset_registry"],["root","area_config"], ["root","area_t2tc"], diff --git a/src/commands/fragments/fragments-api.ts b/src/commands/fragments/fragments-api.ts new file mode 100644 index 00000000..72e36980 --- /dev/null +++ b/src/commands/fragments/fragments-api.ts @@ -0,0 +1,93 @@ +import { HttpClient } from "../../core/http/http-client"; +import { check, FragmentFile } from "./fragments-inventory"; +import { MAX_ARCHIVE_BYTES } from "./fragments-archive"; + +export interface UploadFile extends FragmentFile { + fileId: string; + variantIndex: number; + uploadPath: string; +} + +export interface UploadStatus { + schemaVersion: 1; + appId: string; + uploadId: string; + releaseId: string; + version: string; + files: UploadFile[]; + verifiedFileIds: string[]; + admission: { state: "uploading" | "validating" | "ready" | "rejected" | "expired"; errorCode?: string }; +} + +function record(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function identifier(value: unknown): value is string { + return typeof value === "string" && /^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(value); +} + +export class FragmentsApi { + private readonly base: string; + + constructor(private readonly http: HttpClient, private readonly appId: string) { + check(identifier(appId), "Invalid projectId: use 1–128 letters, digits, underscores, dots or hyphens, starting with a letter or digit."); + this.base = `/fragments/apis/v1/apps/${appId}/uploads`; + } + + public create(files: FragmentFile[], key: string): Promise { + // The API requires a producer version, independently of its generated releaseId. + return this.request("POST", this.base, { + schemaVersion: 1, version: `build-${key}`, variants: [{ entryPoint: "index.html", files }], + }, key); + } + + public status(uploadId: string): Promise { + return this.request("GET", `${this.base}/${uploadId}`); + } + + public transfer(upload: UploadStatus, file: UploadFile, bytes: Buffer): Promise { + const path = `${this.base}/${upload.uploadId}/files/${file.fileId}`; + check(identifier(file.fileId) && file.uploadPath === path, "Gateway returned an unexpected upload path."); + return this.request("PUT", path, bytes); + } + + public finalize(uploadId: string): Promise { + return this.request("POST", `${this.base}/${uploadId}/finalize`, { schemaVersion: 1 }); + } + + public transferArchive(uploadId: string, bytes: Buffer): Promise { + return this.request("PUT", `${this.base}/${uploadId}/archive`, bytes, undefined, "application/zip"); + } + + private async request(method: "GET" | "POST" | "PUT", path: string, body?: Buffer | object, key?: string, + contentType = Buffer.isBuffer(body) ? "application/octet-stream" : "application/json"): Promise { + const response = await this.http.requestRaw(method, path, body, { + "X-Fragment-Publication": "1", + "Content-Type": contentType, + "Accept": "application/json", + ...(key ? { "Idempotency-Key": key } : {}), + }, contentType === "application/zip" ? MAX_ARCHIVE_BYTES : undefined); + const data = response.data; + if (response.status < 200 || response.status >= 300) { + const error = record(data) && record(data.error) ? data.error : {}; + const code = identifier(error.code) ? error.code : "UNKNOWN_ERROR"; + const requestId = identifier(error.requestId) ? `; requestId=${error.requestId}` : ""; + throw new Error(`Fragment API HTTP ${response.status}: ${code}${requestId}`); + } + check(record(data) && data.schemaVersion === 1 && data.appId === this.appId + && identifier(data.uploadId) && identifier(data.releaseId) && identifier(data.version) + && Array.isArray(data.files) && Array.isArray(data.verifiedFileIds) + && data.verifiedFileIds.every(identifier) && record(data.admission) + && ["uploading", "validating", "ready", "rejected", "expired"].includes(String(data.admission.state)), + "Invalid fragment upload response."); + for (const file of data.files) { + check(record(file) && identifier(file.fileId) && file.variantIndex === 0 + && typeof file.path === "string" && typeof file.sha256 === "string" + && typeof file.mediaType === "string" && Number.isSafeInteger(file.size) + && file.uploadPath === `${this.base}/${data.uploadId}/files/${file.fileId}`, + "Invalid fragment upload file response."); + } + return data as unknown as UploadStatus; + } +} diff --git a/src/commands/fragments/fragments-archive.ts b/src/commands/fragments/fragments-archive.ts new file mode 100644 index 00000000..672d997a --- /dev/null +++ b/src/commands/fragments/fragments-archive.ts @@ -0,0 +1,19 @@ +import * as AdmZip from "adm-zip"; +import { check, FragmentFile, readBuildFile, sha256 } from "./fragments-inventory"; + +export const MAX_ARCHIVE_BYTES = 66 * 1024 * 1024; + +export async function buildArchive(root: string, files: FragmentFile[]): Promise<{ archive: Buffer; limit: number }> { + const zip = new AdmZip(); + for (const file of files) { + const bytes = await readBuildFile(root, file.path); + check(bytes.length === file.size && sha256(bytes) === file.sha256, + `Build changed since inventory: ${file.path}`); + // The gateway requires variant-prefixed files, without directory entries or metadata. + zip.addFile(`0/${file.path}`, bytes); + } + const archive = zip.toBuffer(); + const total = files.reduce((sum, file) => sum + file.size, 0); + const limit = Math.min(MAX_ARCHIVE_BYTES, total + Math.ceil(total / 1000) + files.length * 2200 + 22); + return { archive, limit }; +} diff --git a/src/commands/fragments/fragments-inventory.ts b/src/commands/fragments/fragments-inventory.ts new file mode 100644 index 00000000..48aebac8 --- /dev/null +++ b/src/commands/fragments/fragments-inventory.ts @@ -0,0 +1,105 @@ +import { createHash } from "crypto"; +import { constants } from "fs"; +import { lstat, open, readdir } from "fs/promises"; +import { extname, join, parse, resolve, sep } from "path"; + +export interface FragmentFile { + path: string; + size: number; + sha256: string; + mediaType: string; +} + +export const MAX_FILE_BYTES = 16 * 1024 * 1024; +const MAX_RELEASE_BYTES = 64 * 1024 * 1024; +const MEDIA_TYPES: Readonly> = { + ".html": "text/html", ".js": "text/javascript", ".mjs": "text/javascript", + ".css": "text/css", ".json": "application/json", ".map": "application/json", + ".wasm": "application/wasm", ".png": "image/png", ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", ".svg": "image/svg+xml", ".ico": "image/x-icon", + ".gif": "image/gif", ".webp": "image/webp", ".avif": "image/avif", + ".woff": "font/woff", ".woff2": "font/woff2", ".ttf": "font/ttf", ".txt": "text/plain", +}; +const RESERVED = new Set(["fragment-artifact.json", "fragment-release.json", "fragment-deployment.json"]); + +export function check(value: unknown, message: string): asserts value { + if (!value) { throw new Error(message); } +} + +export function sha256(bytes: Buffer): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +async function rejectSymlinks(filePath: string): Promise { + const absolute = resolve(filePath); + let current = parse(absolute).root; + for (const part of absolute.slice(current.length).split(sep).filter(Boolean)) { + current = join(current, part); + check(!(await lstat(current)).isSymbolicLink(), "Build paths must not contain symlinks."); + } +} + +/** Bound reads even if a file grows between stat and read. */ +export async function readBuildFile(root: string, path: string): Promise { + const absolute = join(root, path); + await rejectSymlinks(absolute); + const file = await open(absolute, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stat = await file.stat(); + check(stat.isFile() && stat.size <= MAX_FILE_BYTES, `Invalid or oversized build file: ${path}`); + const chunks: Buffer[] = []; + let size = 0; + for await (const chunk of file.createReadStream({ autoClose: false, highWaterMark: 65536 })) { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + size += bytes.length; + check(size <= MAX_FILE_BYTES, `Build file exceeds 16 MiB: ${path}`); + chunks.push(bytes); + } + return Buffer.concat(chunks); + } finally { + await file.close(); + } +} + +export async function inventoryBuild(directory: string): Promise<{ root: string; files: FragmentFile[] }> { + const root = resolve(directory); + await rejectSymlinks(root); + check((await lstat(root)).isDirectory(), "Build root must be a directory."); + // Check the fixed entry point before any network request or directory traversal. + try { + await readBuildFile(root, "index.html"); + } catch { + throw new Error("The build directory must contain a regular, non-symlink index.html (at most 16 MiB)."); + } + const files: FragmentFile[] = []; + const paths = new Set(); + let total = 0; + let examined = 0; + const walk = async (prefix: string): Promise => { + const entries = await readdir(join(root, prefix), { withFileTypes: true }); + entries.sort((a, b) => a.name.localeCompare(b.name)); + for (const entry of entries) { + const path = prefix + entry.name; + check(++examined <= 1000, "Build exceeds the directory traversal limit."); + check(path.length <= 1024 && /^[a-zA-Z0-9_@][a-zA-Z0-9_.@-]*$/.test(entry.name) + && !RESERVED.has(entry.name), `Invalid or reserved build path: ${path}`); + check(!entry.isSymbolicLink(), `Build contains a symlink: ${path}`); + check(!paths.has(path.toLowerCase()), `Case-insensitive duplicate build path: ${path}`); + paths.add(path.toLowerCase()); + if (entry.isDirectory()) { + await walk(path + "/"); + } else { + check(entry.isFile(), `Unsupported filesystem entry: ${path}`); + check(files.length < 500, "Build exceeds 500 files."); + const mediaType = MEDIA_TYPES[extname(path)]; + check(mediaType, `Unsupported build file extension: ${path}`); + const bytes = await readBuildFile(root, path); + total += bytes.length; + check(total <= MAX_RELEASE_BYTES, "Build exceeds 64 MiB."); + files.push({ path, size: bytes.length, sha256: sha256(bytes), mediaType }); + } + } + }; + await walk(""); + return { root, files }; +} diff --git a/src/commands/fragments/fragments.service.ts b/src/commands/fragments/fragments.service.ts new file mode 100644 index 00000000..06b86a49 --- /dev/null +++ b/src/commands/fragments/fragments.service.ts @@ -0,0 +1,76 @@ +import { randomUUID } from "crypto"; +import { setTimeout as delay } from "timers/promises"; +import { Context } from "../../core/command/cli-context"; +import { logger } from "../../core/utils/logger"; +import { FragmentsApi, identifier, UploadStatus } from "./fragments-api"; +import { check, inventoryBuild } from "./fragments-inventory"; +import { buildArchive, MAX_ARCHIVE_BYTES } from "./fragments-archive"; + +export class FragmentsService { + constructor(private readonly context: Context) {} + + public async upload(projectId: string, directory: string): Promise { + check(identifier(projectId), "Invalid projectId."); + const { root, files } = await inventoryBuild(directory); + const profile = this.context.profile; + check(profile && profile.apiToken, "Select an authenticated profile with --profile."); + check(!profile.authenticationType || profile.authenticationType === "Bearer", + "Fragments require a Bearer profile; AppKey authentication is not supported."); + const origin = new URL(profile.team); + check(origin.protocol === "https:" && !origin.username && !origin.password + && origin.pathname === "/" && !origin.search && !origin.hash, "Fragments require an HTTPS tenant origin."); + const api = new FragmentsApi(this.context.httpClient, projectId); + const key = randomUUID(); + logger.info(`Uploading ${files.length} files (${files.reduce((sum, file) => sum + file.size, 0)} bytes).`); + const created = await api.create(files, key); + logger.info(`Upload ID: ${created.uploadId}`); + const validate = (upload: UploadStatus): UploadStatus => { + check(upload.uploadId === created.uploadId && upload.releaseId === created.releaseId + && upload.version === `build-${key}` && upload.files.length === files.length, + "Gateway upload identity or inventory changed."); + const remaining = new Map(files.map(file => [file.path, file])); + const ids = new Set(); + for (const remote of upload.files) { + const local = remaining.get(remote.path); + const original = created.files.find(file => file.path === remote.path); + check(local && remote.size === local.size && remote.sha256 === local.sha256 + && remote.mediaType === local.mediaType && !ids.has(remote.fileId) + && original && remote.fileId === original.fileId && remote.uploadPath === original.uploadPath, + "Gateway returned files outside the declared inventory."); + remaining.delete(remote.path); + ids.add(remote.fileId); + } + check(upload.verifiedFileIds.every(id => ids.has(id)), "Gateway returned unknown verified file IDs."); + return upload; + }; + let upload = validate(created); + try { + logger.info("Compressing the build into a ZIP archive."); + const { archive, limit } = await buildArchive(root, files); + check(archive.length <= limit, "ZIP exceeds the gateway's compressed byte limit."); + upload = validate(await api.transferArchive(created.uploadId, archive)); + check(upload.files.every(file => upload.verifiedFileIds.includes(file.fileId)), + "Gateway did not verify every file in the archive."); + logger.info(`Uploaded ZIP (${archive.length} bytes).`); + try { + upload = validate(await api.finalize(created.uploadId)); + } catch (error) { + // A lost finalize response may still have committed the release. + upload = validate(await api.status(created.uploadId)); + if (upload.admission.state === "uploading") { throw error; } + } + const deadline = Date.now() + 60000; + while (upload.admission.state === "validating" && Date.now() < deadline) { + await delay(1000); + upload = validate(await api.status(created.uploadId)); + } + check(upload.admission.state === "ready", `Release admission is ${upload.admission.state}.` + + (identifier(upload.admission.errorCode) ? ` ${upload.admission.errorCode}` : "")); + logger.info(`Release ready: appId=${projectId}, uploadId=${upload.uploadId}, releaseId=${upload.releaseId}, version=${upload.version}. Not activated.`); + return upload; + } catch (error) { + const detail = error instanceof Error ? error.message : "Upload failed."; + throw new Error(`${detail} Inspect /fragments/apis/v1/apps/${projectId}/uploads/${created.uploadId}`); + } + } +} diff --git a/src/commands/fragments/module.ts b/src/commands/fragments/module.ts new file mode 100644 index 00000000..fa44d57b --- /dev/null +++ b/src/commands/fragments/module.ts @@ -0,0 +1,19 @@ +import { Configurator, IModule } from "../../core/command/module-handler"; +import { Context } from "../../core/command/cli-context"; +import { FragmentsService } from "./fragments.service"; + +class Module extends IModule { + public register(context: Context, configurator: Configurator): void { + configurator.command("sdk") + .description("Publish custom app builds to the Fragment Gateway") + .command("publish") + .description("Upload a browser bundle with automatic ZIP selection and finalize a release; requires index.html") + .requiredOption("--app-id ", "Fragment app ID") + .requiredOption("--dist ", "Browser-ready build directory containing index.html") + .action(async (ctx, command, options) => { + await new FragmentsService(ctx).upload(options.appId, options.dist); + }); + } +} + +export = Module; diff --git a/src/core/http/http-client.ts b/src/core/http/http-client.ts index 23e4c86a..6a896498 100644 --- a/src/core/http/http-client.ts +++ b/src/core/http/http-client.ts @@ -1,4 +1,4 @@ -import {AxiosResponse, RawAxiosRequestHeaders} from "axios"; +import {AxiosResponse, RawAxiosRequestHeaders, isAxiosError} from "axios"; import * as FormData from "form-data"; import { Context } from "../command/cli-context"; import {AxiosInitializer} from "./axios-initializer"; @@ -17,6 +17,39 @@ export class HttpClient { constructor(private context: Context) {} + /** Raw authenticated transport for APIs that require binary bodies and custom headers. */ + public async requestRaw( + method: "GET" | "POST" | "PUT", + url: string, + body?: Buffer | object, + headers: RawAxiosRequestHeaders = {}, + maxBodyLength = 16 * 1024 * 1024, + ): Promise<{ status: number; data: unknown }> { + const origin = new URL(this.context.profile.team); + const target = new URL(url, origin); + if (!url.startsWith("/") || target.origin !== origin.origin || target.search || target.hash) { + throw new Error("Expected an API path on the profile's tenant origin."); + } + try { + const response = await this.axios.request({ + method, + url: target.href, + data: body, + headers: { ...this.buildHeaders(), ...headers }, + timeout: 120000, + maxRedirects: 0, + maxBodyLength, + maxContentLength: 4 * 1024 * 1024, + validateStatus: () => true, + }); + return { status: response.status, data: response.data }; + } catch (error) { + // Never expose Axios errors: their config contains the Authorization header. + const code = isAxiosError(error) ? error.code : undefined; + throw new Error(code === "ECONNABORTED" ? "Fragment request timed out." : "Fragment request failed in transport."); + } + } + public async get(url: string): Promise { const fullUrl = this.resolveUrl(url); logger.debug(`HttpClient - GET ${fullUrl}`);