From 70c92c1b08a97c4dc789479ea2060e78a1641d60 Mon Sep 17 00:00:00 2001 From: Dan Barr <6922515+danbarr@users.noreply.github.com> Date: Wed, 21 Oct 2026 13:30:40 -0700 Subject: [PATCH] Validate Helm descriptions before generation --- .github/upstream-projects.yaml | 2 + .github/workflows/_static-checks.yaml | 3 + .../generate-helm-reference.mjs | 83 +++++++++++----- .../generate-helm-reference.test.mjs | 98 +++++++++++++++++++ 4 files changed, 160 insertions(+), 26 deletions(-) diff --git a/.github/upstream-projects.yaml b/.github/upstream-projects.yaml index bf1aa7a4..3370b82d 100644 --- a/.github/upstream-projects.yaml +++ b/.github/upstream-projects.yaml @@ -243,6 +243,8 @@ projects: title: AI Gateway inference Helm values description: Values, types, and defaults for the AI Gateway inference chart in Stacklok Enterprise. intro: Configure the AI Gateway inference chart with these Helm values. + description_overrides: + guardrails.adapter.metrics.port: Port for the guardrails adapter's Prometheus metrics listener when metrics serving is enabled. related: - label: Platform Helm chart references href: ./index.mdx diff --git a/.github/workflows/_static-checks.yaml b/.github/workflows/_static-checks.yaml index 44870286..807fcfa2 100644 --- a/.github/workflows/_static-checks.yaml +++ b/.github/workflows/_static-checks.yaml @@ -33,3 +33,6 @@ jobs: - name: Test release PR body renderer run: node --test scripts/upstream-release/render-pr-body.test.mjs + + - name: Test Helm reference generator + run: node --test scripts/upstream-release/generate-helm-reference.test.mjs diff --git a/scripts/upstream-release/generate-helm-reference.mjs b/scripts/upstream-release/generate-helm-reference.mjs index 31a122a9..debc3890 100644 --- a/scripts/upstream-release/generate-helm-reference.mjs +++ b/scripts/upstream-release/generate-helm-reference.mjs @@ -2,8 +2,9 @@ // SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. // SPDX-License-Identifier: Apache-2.0 -// Run against the release checkout, never the moving upstream default branch: +// Generate reference pages from a release checkout: // node scripts/upstream-release/generate-helm-reference.mjs --id toolhive --clone --tag +// Add --check to validate a source checkout without writing pages. // Requires helm-docs 1.14.2 on PATH. Other projects without `helm` entries are a no-op. import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; @@ -183,6 +184,28 @@ function renderDefault(value) { return `
\nDefault value\n\n${fence}yaml\n${content}\n${fence}\n\n
`; } +function descriptionFor(row) { + return row.Description || row.SourceDescription || row.AutoDescription; +} + +export function validateDescriptions(charts) { + const errors = []; + for (const { chart, rows } of charts) { + if (!Array.isArray(rows) || rows.length === 0) { + errors.push(`No Helm values found for ${chart.source}`); + continue; + } + for (const row of rows) { + if (!descriptionFor(row)?.trim()) { + errors.push( + `Missing description for ${chart.source}/values.yaml: ${row.Key}` + ); + } + } + } + if (errors.length) throw new Error(errors.join('\n')); +} + export function renderPage( chart, metadata, @@ -191,15 +214,9 @@ export function renderPage( tag, publicSources = true ) { - if (!Array.isArray(rows) || rows.length === 0) { - throw new Error(`No Helm values found for ${chart.source}`); - } + validateDescriptions([{ chart, rows }]); const fields = rows.map((row) => { - const description = - row.Description || row.SourceDescription || row.AutoDescription; - if (!description?.trim()) { - throw new Error(`Missing description for ${chart.source}: ${row.Key}`); - } + const description = descriptionFor(row); return `### \`${row.Key}\`\n\n**Type:** \`${row.Type}\`\n\n${renderDefault(row.Default || row.AutoDefault)}\n\n${renderDescription(description)}`; }); const source = `https://github.com/${repo}/blob/${encodeURIComponent(tag)}/${chart.source}/values.yaml`; @@ -232,13 +249,17 @@ ${chart.related.map(({ label, href }) => `- [${label}](${href})`).join('\n')} function main(argv) { const args = {}; - for (let i = 0; i < argv.length; i += 2) { + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--check') { + args.check = true; + continue; + } if (!['--id', '--clone', '--tag'].includes(argv[i]) || !argv[i + 1]) { throw new Error( - 'Usage: generate-helm-reference.mjs --id --clone --tag ' + 'Usage: generate-helm-reference.mjs --id --clone [--tag ] [--check]' ); } - args[argv[i].slice(2)] = argv[i + 1]; + args[argv[i].slice(2)] = argv[++i]; } const { projects } = yaml.parse( fs.readFileSync( @@ -250,14 +271,14 @@ function main(argv) { if (!project) throw new Error(`Unknown project: ${args.id}`); if (!project.helm?.length) return; const tag = args.tag || project.version; - if (!args.clone || !tag) + if (!args.clone || (!args.check && !tag)) throw new Error('Helm generation requires --clone and a release tag'); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'helm-reference-')); try { // Copy only documentation inputs: helm-docs must never overwrite upstream READMEs. // Prepare every page before replacing outputs, so incomplete annotations fail early. - const pages = project.helm.map((chart, index) => { + const inputs = project.helm.map((chart, index) => { const dir = path.join(tmp, String(index)); fs.mkdirSync(dir); for (const file of ['Chart.yaml', 'values.yaml']) { @@ -296,20 +317,30 @@ function main(argv) { fs.readFileSync(path.join(dir, 'Chart.yaml'), 'utf8') ); return { - page: chart.page, - content: renderPage( - chart, - metadata, - JSON.parse(output).map((row) => ({ - ...row, - SourceDescription: descriptions.get(row.Key), - })), - project.repo, - tag, - project.public_sources !== false - ), + chart, + metadata, + rows: JSON.parse(output).map((row) => ({ + ...row, + SourceDescription: descriptions.get(row.Key), + })), }; }); + validateDescriptions(inputs); + if (args.check) { + console.log(`Helm descriptions validated for ${inputs.length} chart(s).`); + return; + } + const pages = inputs.map(({ chart, metadata, rows }) => ({ + page: chart.page, + content: renderPage( + chart, + metadata, + rows, + project.repo, + tag, + project.public_sources !== false + ), + })); for (const { page, content } of pages) { const destination = path.join(repoRoot, page); fs.mkdirSync(path.dirname(destination), { recursive: true }); diff --git a/scripts/upstream-release/generate-helm-reference.test.mjs b/scripts/upstream-release/generate-helm-reference.test.mjs index 3029f859..c41cf80e 100644 --- a/scripts/upstream-release/generate-helm-reference.test.mjs +++ b/scripts/upstream-release/generate-helm-reference.test.mjs @@ -1,7 +1,12 @@ // SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. // SPDX-License-Identifier: Apache-2.0 import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import test from 'node:test'; +import { fileURLToPath } from 'node:url'; import { compile } from '@mdx-js/mdx'; import yaml from 'yaml'; import { @@ -9,6 +14,7 @@ import { sourceDescriptions, renderPage, renderDescription, + validateDescriptions, } from './generate-helm-reference.mjs'; const chart = { @@ -69,6 +75,98 @@ test('incomplete chart documentation fails instead of publishing empty reference ); }); +test('coverage validation reports every missing value across charts', () => { + assert.throws( + () => + validateDescriptions([ + { chart, rows: [{ Key: 'first' }, { Key: 'second' }] }, + { chart: { source: 'another/chart' }, rows: [{ Key: 'third' }] }, + ]), + (error) => { + assert.match(error.message, /operator\/values.yaml: first/); + assert.match(error.message, /operator\/values.yaml: second/); + assert.match(error.message, /another\/chart\/values.yaml: third/); + return true; + } + ); +}); + +test('check mode validates all charts without writing pages or formatting', () => { + const root = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), 'helm-reference-check-')) + ); + try { + const scripts = path.join(root, 'scripts/upstream-release'); + fs.mkdirSync(scripts, { recursive: true }); + for (const file of ['generate-helm-reference.mjs', 'helm-values.gotmpl']) { + fs.copyFileSync(new URL(file, import.meta.url), path.join(scripts, file)); + } + fs.symlinkSync( + fileURLToPath(new URL('../../node_modules', import.meta.url)), + path.join(root, 'node_modules'), + 'dir' + ); + fs.mkdirSync(path.join(root, '.github')); + const charts = ['first', 'second'].map((name) => ({ + source: name, + page: `pages/${name}.mdx`, + })); + fs.writeFileSync( + path.join(root, '.github/upstream-projects.yaml'), + yaml.stringify({ + projects: [{ id: 'fixture', helm: charts }], + }) + ); + for (const { source } of charts) { + fs.mkdirSync(path.join(root, source)); + fs.writeFileSync( + path.join(root, source, 'Chart.yaml'), + `name: ${source}\nversion: 1.0.0\n` + ); + fs.writeFileSync(path.join(root, source, 'values.yaml'), 'port: 9090\n'); + } + const bin = path.join(root, 'bin'); + fs.mkdirSync(bin); + fs.writeFileSync( + path.join(bin, 'helm-docs'), + `#!${process.execPath} +process.stdout.write(JSON.stringify([{ Key: 'port', AutoDescription: process.env.FIXTURE_DESCRIPTION }])); +`, + { mode: 0o755 } + ); + const run = (description) => + spawnSync( + process.execPath, + [ + path.join(scripts, 'generate-helm-reference.mjs'), + '--id', + 'fixture', + '--clone', + root, + '--check', + ], + { + encoding: 'utf8', + env: { ...process.env, PATH: bin, FIXTURE_DESCRIPTION: description }, + } + ); + const failed = run(''); + assert.equal(failed.status, 1, failed.stderr); + assert.match(failed.stderr, /first\/values.yaml: port/); + assert.match(failed.stderr, /second\/values.yaml: port/); + const passed = run('Metrics listener port.'); + assert.equal(passed.status, 0, passed.stderr); + assert.match(passed.stdout, /validated for 2 chart/); + assert.equal(fs.existsSync(path.join(root, 'pages')), false); + assert.equal( + fs.readFileSync(path.join(root, 'first/values.yaml'), 'utf8'), + 'port: 9090\n' + ); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + test('raw continuation comments preserve paragraphs, YAML examples, and lists', async () => { const source = `operator: # -- Default Secrets for workloads.