Validate Helm descriptions before generation

.github/upstream-projects.yaml

@@ -243,6 +243,8 @@ projects:
         title: AI Gateway inference Helm values
         description: Values, types, and defaults for the AI Gateway inference chart in Stacklok Enterprise.
         intro: Configure the AI Gateway inference chart with these Helm values.
+        description_overrides:
+          guardrails.adapter.metrics.port: Port for the guardrails adapter's Prometheus metrics listener when metrics serving is enabled.
         related:
           - label: Platform Helm chart references
             href: ./index.mdx

.github/workflows/_static-checks.yaml

@@ -33,3 +33,6 @@ jobs:
 
       - name: Test release PR body renderer
         run: node --test scripts/upstream-release/render-pr-body.test.mjs
+
+      - name: Test Helm reference generator
+        run: node --test scripts/upstream-release/generate-helm-reference.test.mjs

scripts/upstream-release/generate-helm-reference.mjs

@@ -2,8 +2,9 @@
 // SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-// Run against the release checkout, never the moving upstream default branch:
+// Generate reference pages from a release checkout:
 // node scripts/upstream-release/generate-helm-reference.mjs --id toolhive --clone <dir> --tag <tag>
+// Add --check to validate a source checkout without writing pages.
 // Requires helm-docs 1.14.2 on PATH. Other projects without `helm` entries are a no-op.
 import { execFileSync } from 'node:child_process';
 import fs from 'node:fs';
@@ -183,6 +184,28 @@ function renderDefault(value) {
   return `<details>\n<summary>Default value</summary>\n\n${fence}yaml\n${content}\n${fence}\n\n</details>`;
 }
 
+function descriptionFor(row) {
+  return row.Description || row.SourceDescription || row.AutoDescription;
+}
+
+export function validateDescriptions(charts) {
+  const errors = [];
+  for (const { chart, rows } of charts) {
+    if (!Array.isArray(rows) || rows.length === 0) {
+      errors.push(`No Helm values found for ${chart.source}`);
+      continue;
+    }
+    for (const row of rows) {
+      if (!descriptionFor(row)?.trim()) {
+        errors.push(
+          `Missing description for ${chart.source}/values.yaml: ${row.Key}`
+        );
+      }
+    }
+  }
+  if (errors.length) throw new Error(errors.join('\n'));
+}
+
 export function renderPage(
   chart,
   metadata,
@@ -191,15 +214,9 @@ export function renderPage(
   tag,
   publicSources = true
 ) {
-  if (!Array.isArray(rows) || rows.length === 0) {
-    throw new Error(`No Helm values found for ${chart.source}`);
-  }
+  validateDescriptions([{ chart, rows }]);
   const fields = rows.map((row) => {
-    const description =
-      row.Description || row.SourceDescription || row.AutoDescription;
-    if (!description?.trim()) {
-      throw new Error(`Missing description for ${chart.source}: ${row.Key}`);
-    }
+    const description = descriptionFor(row);
     return `### \`${row.Key}\`\n\n**Type:** \`${row.Type}\`\n\n${renderDefault(row.Default || row.AutoDefault)}\n\n${renderDescription(description)}`;
   });
   const source = `https://github.com/${repo}/blob/${encodeURIComponent(tag)}/${chart.source}/values.yaml`;
@@ -232,13 +249,17 @@ ${chart.related.map(({ label, href }) => `- [${label}](${href})`).join('\n')}
 
 function main(argv) {
   const args = {};
-  for (let i = 0; i < argv.length; i += 2) {
+  for (let i = 0; i < argv.length; i++) {
+    if (argv[i] === '--check') {
+      args.check = true;
+      continue;
+    }
     if (!['--id', '--clone', '--tag'].includes(argv[i]) || !argv[i + 1]) {
       throw new Error(
-        'Usage: generate-helm-reference.mjs --id <project> --clone <dir> --tag <tag>'
+        'Usage: generate-helm-reference.mjs --id <project> --clone <dir> [--tag <tag>] [--check]'
       );
     }
-    args[argv[i].slice(2)] = argv[i + 1];
+    args[argv[i].slice(2)] = argv[++i];
   }
   const { projects } = yaml.parse(
     fs.readFileSync(
@@ -250,14 +271,14 @@ function main(argv) {
   if (!project) throw new Error(`Unknown project: ${args.id}`);
   if (!project.helm?.length) return;
   const tag = args.tag || project.version;
-  if (!args.clone || !tag)
+  if (!args.clone || (!args.check && !tag))
     throw new Error('Helm generation requires --clone and a release tag');
 
   const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'helm-reference-'));
   try {
     // Copy only documentation inputs: helm-docs must never overwrite upstream READMEs.
     // Prepare every page before replacing outputs, so incomplete annotations fail early.
-    const pages = project.helm.map((chart, index) => {
+    const inputs = project.helm.map((chart, index) => {
       const dir = path.join(tmp, String(index));
       fs.mkdirSync(dir);
       for (const file of ['Chart.yaml', 'values.yaml']) {
@@ -296,20 +317,30 @@ function main(argv) {
         fs.readFileSync(path.join(dir, 'Chart.yaml'), 'utf8')
       );
       return {
-        page: chart.page,
-        content: renderPage(
-          chart,
-          metadata,
-          JSON.parse(output).map((row) => ({
-            ...row,
-            SourceDescription: descriptions.get(row.Key),
-          })),
-          project.repo,
-          tag,
-          project.public_sources !== false
-        ),
+        chart,
+        metadata,
+        rows: JSON.parse(output).map((row) => ({
+          ...row,
+          SourceDescription: descriptions.get(row.Key),
+        })),
       };
     });
+    validateDescriptions(inputs);
+    if (args.check) {
+      console.log(`Helm descriptions validated for ${inputs.length} chart(s).`);
+      return;
+    }
+    const pages = inputs.map(({ chart, metadata, rows }) => ({
+      page: chart.page,
+      content: renderPage(
+        chart,
+        metadata,
+        rows,
+        project.repo,
+        tag,
+        project.public_sources !== false
+      ),
+    }));
     for (const { page, content } of pages) {
       const destination = path.join(repoRoot, page);
       fs.mkdirSync(path.dirname(destination), { recursive: true });

scripts/upstream-release/generate-helm-reference.test.mjs

@@ -1,14 +1,20 @@
 // SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc.
 // SPDX-License-Identifier: Apache-2.0
 import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
 import test from 'node:test';
+import { fileURLToPath } from 'node:url';
 import { compile } from '@mdx-js/mdx';
 import yaml from 'yaml';
 import {
   annotateValues,
   sourceDescriptions,
   renderPage,
   renderDescription,
+  validateDescriptions,
 } from './generate-helm-reference.mjs';
 
 const chart = {
@@ -69,6 +75,98 @@ test('incomplete chart documentation fails instead of publishing empty reference
   );
 });
 
+test('coverage validation reports every missing value across charts', () => {
+  assert.throws(
+    () =>
+      validateDescriptions([
+        { chart, rows: [{ Key: 'first' }, { Key: 'second' }] },
+        { chart: { source: 'another/chart' }, rows: [{ Key: 'third' }] },
+      ]),
+    (error) => {
+      assert.match(error.message, /operator\/values.yaml: first/);
+      assert.match(error.message, /operator\/values.yaml: second/);
+      assert.match(error.message, /another\/chart\/values.yaml: third/);
+      return true;
+    }
+  );
+});
+
+test('check mode validates all charts without writing pages or formatting', () => {
+  const root = fs.realpathSync(
+    fs.mkdtempSync(path.join(os.tmpdir(), 'helm-reference-check-'))
+  );
+  try {
+    const scripts = path.join(root, 'scripts/upstream-release');
+    fs.mkdirSync(scripts, { recursive: true });
+    for (const file of ['generate-helm-reference.mjs', 'helm-values.gotmpl']) {
+      fs.copyFileSync(new URL(file, import.meta.url), path.join(scripts, file));
+    }
+    fs.symlinkSync(
+      fileURLToPath(new URL('../../node_modules', import.meta.url)),
+      path.join(root, 'node_modules'),
+      'dir'
+    );
+    fs.mkdirSync(path.join(root, '.github'));
+    const charts = ['first', 'second'].map((name) => ({
+      source: name,
+      page: `pages/${name}.mdx`,
+    }));
+    fs.writeFileSync(
+      path.join(root, '.github/upstream-projects.yaml'),
+      yaml.stringify({
+        projects: [{ id: 'fixture', helm: charts }],
+      })
+    );
+    for (const { source } of charts) {
+      fs.mkdirSync(path.join(root, source));
+      fs.writeFileSync(
+        path.join(root, source, 'Chart.yaml'),
+        `name: ${source}\nversion: 1.0.0\n`
+      );
+      fs.writeFileSync(path.join(root, source, 'values.yaml'), 'port: 9090\n');
+    }
+    const bin = path.join(root, 'bin');
+    fs.mkdirSync(bin);
+    fs.writeFileSync(
+      path.join(bin, 'helm-docs'),
+      `#!${process.execPath}
+process.stdout.write(JSON.stringify([{ Key: 'port', AutoDescription: process.env.FIXTURE_DESCRIPTION }]));
+`,
+      { mode: 0o755 }
+    );
+    const run = (description) =>
+      spawnSync(
+        process.execPath,
+        [
+          path.join(scripts, 'generate-helm-reference.mjs'),
+          '--id',
+          'fixture',
+          '--clone',
+          root,
+          '--check',
+        ],
+        {
+          encoding: 'utf8',
+          env: { ...process.env, PATH: bin, FIXTURE_DESCRIPTION: description },
+        }
+      );
+    const failed = run('');
+    assert.equal(failed.status, 1, failed.stderr);
+    assert.match(failed.stderr, /first\/values.yaml: port/);
+    assert.match(failed.stderr, /second\/values.yaml: port/);
+    const passed = run('Metrics listener port.');
+    assert.equal(passed.status, 0, passed.stderr);
+    assert.match(passed.stdout, /validated for 2 chart/);
+    assert.equal(fs.existsSync(path.join(root, 'pages')), false);
+    assert.equal(
+      fs.readFileSync(path.join(root, 'first/values.yaml'), 'utf8'),
+      'port: 9090\n'
+    );
+  } finally {
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
+
 test('raw continuation comments preserve paragraphs, YAML examples, and lists', async () => {
   const source = `operator:
   # -- Default Secrets for workloads.