Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/publish-workflow-catalog.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ on:
- main
paths:
- 'workflow-templates/**'
- 'workflow-catalog.json'
- 'scripts/sync_catalog.py'
- '.github/workflows/publish-workflow-catalog.yml'

Expand Down Expand Up @@ -66,6 +67,7 @@ jobs:
python3 scripts/sync_catalog.py sync
workflow-templates
catalog/workflow-templates
--manifest workflow-catalog.json
--report catalog-sync-report.json

- name: Create catalog update pull request
Expand Down
103 changes: 94 additions & 9 deletions scripts/sync_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,85 @@ def is_publishable(path: Path) -> bool:
return path.is_file() and path.name.endswith(PUBLISHABLE_SUFFIXES)


def collect_publishable(directory: Path) -> dict[str, Path]:
def load_catalog_manifest(path: Path) -> tuple[str, ...]:
payload = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(payload, dict):
raise ValueError("catalog manifest must be a JSON object")

templates = payload.get("templates")
if (
not isinstance(templates, list)
or not templates
or not all(isinstance(item, str) and item for item in templates)
):
raise ValueError("catalog manifest must contain a non-empty templates array")

if len(set(templates)) != len(templates):
raise ValueError("catalog manifest contains duplicate template names")

for template in templates:
if Path(template).name != template:
raise ValueError(f"invalid catalog template name: {template}")

return tuple(sorted(templates))


def collect_publishable(
directory: Path,
template_names: tuple[str, ...] | None = None,
) -> dict[str, Path]:
if not directory.is_dir():
raise ValueError(f"directory does not exist: {directory}")

files = {
all_files = {
path.name: path
for path in directory.iterdir()
if is_publishable(path)
}

if template_names is None:
files = all_files
else:
files: dict[str, Path] = {}

for template in template_names:
yml_name = f"{template}.yml"
yaml_name = f"{template}.yaml"
candidates = [
name
for name in (yml_name, yaml_name)
if name in all_files
]
if len(candidates) != 1:
raise ValueError(
f"catalog template must have exactly one workflow file: {template}"
)

metadata_name = f"{template}.properties.json"
if metadata_name not in all_files:
raise ValueError(f"missing template metadata: {metadata_name}")

files[candidates[0]] = all_files[candidates[0]]
files[metadata_name] = all_files[metadata_name]

for template in template_names:
metadata_name = f"{template}.properties.json"
metadata = json.loads(
files[metadata_name].read_text(encoding="utf-8")
)
icon_name = metadata.get("iconName") if isinstance(metadata, dict) else None
if (
isinstance(icon_name, str)
and icon_name
and not icon_name.startswith("octicon ")
):
icon_file = f"{icon_name}.svg"
if icon_file not in all_files:
raise ValueError(
f"metadata references missing icon {icon_file}: {metadata_name}"
)
files[icon_file] = all_files[icon_file]

validate_catalog(files)
return files

Expand Down Expand Up @@ -74,8 +144,12 @@ def validate_catalog(files: dict[str, Path]) -> None:
)


def sync_catalog(source: Path, target: Path) -> dict[str, list[str]]:
source_files = collect_publishable(source)
def sync_catalog(
source: Path,
target: Path,
template_names: tuple[str, ...] | None = None,
) -> dict[str, list[str]]:
source_files = collect_publishable(source, template_names)
target.mkdir(parents=True, exist_ok=True)

target_files = {
Expand Down Expand Up @@ -112,8 +186,12 @@ def sync_catalog(source: Path, target: Path) -> dict[str, list[str]]:
}


def check_catalog(source: Path, target: Path) -> None:
source_files = collect_publishable(source)
def check_catalog(
source: Path,
target: Path,
template_names: tuple[str, ...] | None = None,
) -> None:
source_files = collect_publishable(source, template_names)
target_files = {
path.name: path
for path in target.iterdir()
Expand Down Expand Up @@ -141,12 +219,19 @@ def main() -> int:
parser.add_argument("command", choices=("sync", "check"))
parser.add_argument("source", type=Path)
parser.add_argument("target", type=Path)
parser.add_argument("--manifest", type=Path)
parser.add_argument("--report", type=Path)
args = parser.parse_args()

try:
template_names = (
load_catalog_manifest(args.manifest)
if args.manifest is not None
else None
)

if args.command == "sync":
report = sync_catalog(args.source, args.target)
report = sync_catalog(args.source, args.target, template_names)
if args.report:
args.report.write_text(
json.dumps(report, indent=2, sort_keys=True) + "\n",
Expand All @@ -155,8 +240,8 @@ def main() -> int:
print(json.dumps(report, indent=2, sort_keys=True))
return 0

check_catalog(args.source, args.target)
except (OSError, ValueError) as error:
check_catalog(args.source, args.target, template_names)
except (OSError, ValueError, json.JSONDecodeError) as error:
parser.error(str(error))

return 0
Expand Down
58 changes: 57 additions & 1 deletion tests/test_sync_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,12 @@
import unittest
from pathlib import Path

from scripts.sync_catalog import check_catalog, collect_publishable, sync_catalog
from scripts.sync_catalog import (
check_catalog,
collect_publishable,
load_catalog_manifest,
sync_catalog,
)


class SyncCatalogTest(unittest.TestCase):
Expand Down Expand Up @@ -124,6 +129,57 @@ def test_invalid_source_does_not_partially_modify_catalog(self) -> None:
"name: Published\n",
)

def test_manifest_limits_catalog_to_approved_templates(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
source = root / "source"
target = root / "target"
self.write_template(source, "reuse")
self.write_template(source, "sensitive")

target.mkdir()
(target / "sensitive.yml").write_text(
"name: Published by mistake\n",
encoding="utf-8",
)
(target / "sensitive.properties.json").write_text(
'{"name":"Sensitive"}\n',
encoding="utf-8",
)

report = sync_catalog(source, target, ("reuse",))

self.assertTrue((target / "reuse.yml").is_file())
self.assertTrue((target / "reuse.properties.json").is_file())
self.assertFalse((target / "sensitive.yml").exists())
self.assertFalse((target / "sensitive.properties.json").exists())
self.assertEqual(
report["removed"],
["sensitive.properties.json", "sensitive.yml"],
)

def test_load_catalog_manifest_rejects_duplicates(self) -> None:
with tempfile.TemporaryDirectory() as directory:
manifest = Path(directory) / "workflow-catalog.json"
manifest.write_text(
'{"templates":["reuse","reuse"]}\n',
encoding="utf-8",
)

with self.assertRaisesRegex(ValueError, "duplicate template"):
load_catalog_manifest(manifest)

def test_manifest_requires_declared_template_to_exist(self) -> None:
with tempfile.TemporaryDirectory() as directory:
source = Path(directory)
self.write_template(source, "reuse")

with self.assertRaisesRegex(
ValueError,
"exactly one workflow file: missing",
):
collect_publishable(source, ("reuse", "missing"))

def test_check_detects_drift(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
Expand Down
14 changes: 14 additions & 0 deletions workflow-catalog.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"templates": [
"block-unconventional-commits",
"lint-eslint",
"lint-info-xml",
"lint-php",
"lint-php-cs",
"lint-stylelint",
"npm-build",
"openapi",
"psalm",
"reuse"
]
}
2 changes: 2 additions & 0 deletions workflow-catalog.json.license
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
SPDX-FileCopyrightText: 2026 LibreCode coop and contributors
SPDX-License-Identifier: AGPL-3.0-or-later
Loading