From fa5cb78cf6e08eb582b3b29c0871a1ce0da74d40 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 02:35:44 +0800 Subject: [PATCH 1/9] fix(ci): submit trusted Maven dependency baselines - submit canonical master snapshots in a separate job - limit write permission to trusted push and dispatch - pin privileged actions and verify the event revision - document repository scope and baseline acceptance --- .github/workflows/submit-dependencies.yml | 95 +++++++++++++++++++++++ docs/ci.md | 24 ++++++ 2 files changed, 119 insertions(+) create mode 100644 .github/workflows/submit-dependencies.yml diff --git a/.github/workflows/submit-dependencies.yml b/.github/workflows/submit-dependencies.yml new file mode 100644 index 0000000000..18d534a484 --- /dev/null +++ b/.github/workflows/submit-dependencies.yml @@ -0,0 +1,95 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Dependency baseline + +on: + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: dependency-baseline-${{ github.repository }}-master + cancel-in-progress: false + +jobs: + validate: + if: >- + (github.repository == 'apache/hugegraph' || github.repository == 'hugegraph/hugegraph') && + github.ref == 'refs/heads/master' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + PYTHONDONTWRITEBYTECODE: '1' + steps: + - name: Checkout event commit + uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Test CI policy and dependency inventory + run: | + python3 -m unittest discover -s .github/scripts -p 'test_ci_policy.py' + python3 -m unittest discover -s install-dist/scripts/dependency -p 'test_*.py' + + submit: + needs: validate + if: >- + github.repository == 'apache/hugegraph' && + github.event.repository.fork == false && + github.ref == 'refs/heads/master' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: write + steps: + - name: Checkout event commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Set up project JDK + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 + with: + java-version-file: .github/workflows/.java-version + distribution: 'temurin' + - name: Verify checkout and current master + env: + EXPECTED_SHA: ${{ github.sha }} + run: | + test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" + test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" + - name: Install the complete Maven reactor + run: mvn -B -ntp install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true --fail-at-end + - name: Refuse a stale master submission + env: + EXPECTED_SHA: ${{ github.sha }} + run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" + - name: Submit resolved reactor dependencies + uses: advanced-security/maven-dependency-submission-action@a64327a7329c9939cf675e458452febe1894a70c # v6.0.1 + with: + directory: '.' + ignore-maven-wrapper: true + maven-args: '-ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true' + correlator: '.github/workflows/submit-dependencies.yml:submit' + snapshot-sha: ${{ github.sha }} + snapshot-ref: ${{ github.ref }} diff --git a/docs/ci.md b/docs/ci.md index a6dd9fc1e5..3505c53fef 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -105,3 +105,27 @@ Each separate job or matrix item requests a runner; steps within a job share one but [plan-dependent total and macOS concurrency limits](https://docs.github.com/en/actions/reference/limits#job-concurrency-limits-for-github-hosted-runners) still apply. Reducing job allocations and long executions lowers shared-pool demand; it does not guarantee a specific queue delay. + +## Dependency baseline + +The separate [dependency submission workflow](../.github/workflows/submit-dependencies.yml) +installs the complete reactor with Java 17, then uses the pinned official +[Maven dependency submission action](https://github.com/advanced-security/maven-dependency-submission-action/tree/a64327a7329c9939cf675e458452febe1894a70c) +to submit each module's resolved dependencies. It runs on master pushes or manual +dispatches. Only `apache/hugegraph` on `refs/heads/master` can run the submission +job, which alone has `contents: write`; the org repository runs read-only policy +and inventory validation. PRs, forks and other refs cannot run the writer. + +Checkout and snapshot use the event commit SHA. Master is checked before install +and again before invoking the submission action; a run already stale at either +check fails. Snapshots retain the checked event SHA even if master advances during +graph generation. A fixed correlator keeps successive master snapshots in the same stream. +The [snapshot API](https://docs.github.com/en/rest/dependency-graph/dependency-submission#create-a-snapshot-of-dependencies-for-a-repository) +requires Contents write permission; no additional metadata permission is needed. + +This producer supplements the existing dependency review and shipped-inventory +checks without changing their severity, license rules or inventory. Merging the +workflow does not prove that a canonical baseline exists: after the first ASF +master run, verify the submitted SHA/ref, non-empty module manifests and a real +dependency-review comparison against that baseline. Until then, baseline acceptance +remains open. No ASF repository setting is changed by this workflow. From eed65ac5c9a4a23d8925ff192c59dd39a9231a2d Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 04:45:12 +0800 Subject: [PATCH 2/9] fix(ci): preserve both master dependency baselines - allow the designated org master producer - keep fork and pull request events read-only - document isolated baseline acceptance checks --- .github/workflows/submit-dependencies.yml | 4 +- docs/ci.md | 54 +++++++++++++++++------ 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/.github/workflows/submit-dependencies.yml b/.github/workflows/submit-dependencies.yml index 18d534a484..45fc0557fa 100644 --- a/.github/workflows/submit-dependencies.yml +++ b/.github/workflows/submit-dependencies.yml @@ -53,8 +53,8 @@ jobs: submit: needs: validate if: >- - github.repository == 'apache/hugegraph' && - github.event.repository.fork == false && + ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || + github.repository == 'hugegraph/hugegraph') && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest diff --git a/docs/ci.md b/docs/ci.md index 3505c53fef..6da30987bf 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -109,23 +109,49 @@ it does not guarantee a specific queue delay. ## Dependency baseline The separate [dependency submission workflow](../.github/workflows/submit-dependencies.yml) -installs the complete reactor with Java 17, then uses the pinned official -[Maven dependency submission action](https://github.com/advanced-security/maven-dependency-submission-action/tree/a64327a7329c9939cf675e458452febe1894a70c) -to submit each module's resolved dependencies. It runs on master pushes or manual -dispatches. Only `apache/hugegraph` on `refs/heads/master` can run the submission -job, which alone has `contents: write`; the org repository runs read-only policy -and inventory validation. PRs, forks and other refs cannot run the writer. +installs the complete reactor with Java 17 in a `contents: read` job, then generates +each module's resolved graph with Ferstl 4.0.3. The repository's +[stdlib serializer](../.github/scripts/dependency-snapshot.py) preserves the +[official Maven parser 6.0.1](https://github.com/actions/maven-dependency-submission-action/tree/1f666f7ac2ab7b72cd5e22eaf7f23c8a581db1db) +package URLs, scopes and relationships. Every tracked POM must have a graph; empty +aggregator manifests are allowed, but a completely empty inventory fails. It runs on master pushes or manual +dispatches. Only `apache/hugegraph` (not a fork) and the explicitly allowed +`hugegraph/hugegraph` test fork on `refs/heads/master` can run the submission job, +which alone has `contents: write`. PRs, other repositories and other refs cannot +run the writer. Maven inherits no submission token, and checkout does not persist +credentials. A pinned `actions/upload-artifact` transfers only `snapshot.json`. +The fresh writer runner uses fixed inline stdlib Python to verify artifact/run +identity before downloading. It bounds ZIP and JSON sizes, checks the archive digest, +and reads only one regular `snapshot.json` member in memory. Duplicate members, +paths and symlinks fail; no archive members are extracted to disk. A signed storage +redirect is fetched without forwarding GitHub Authorization, and further redirects +are refused. The writer does not check out the repository, run Maven, or execute +downloaded code. The first-party upload action is permitted by the +[ASF Actions policy](https://infra.apache.org/github-actions-policy.html). Checkout and snapshot use the event commit SHA. Master is checked before install -and again before invoking the submission action; a run already stale at either -check fails. Snapshots retain the checked event SHA even if master advances during -graph generation. A fixed correlator keeps successive master snapshots in the same stream. +and again after graph generation and serialization; a stale run fails. The writer +checks the run attempt, successful producer, artifact ID/name, JSON schema, +detector/correlator and source SHA/ref. It verifies all manifest paths against the +event commit's source tree, rejects empty or mismatched inventory, and checks master +immediately before and after submission. Snapshots retain the checked event SHA +if master advances during submission; that run fails its final freshness check. + +If submission fails, use **Re-run all jobs** or start a fresh `workflow_dispatch` +on master. **Re-run failed jobs** keeps the older generation artifact, which the +writer rejects because the snapshot must be produced in the same run attempt. + +A fixed master-only correlator keeps successive baseline snapshots in the +same stream within each repository; ASF and org inventories are independent. The writer's +identity is separate from automatic PR submissions so they cannot replace its baseline. The [snapshot API](https://docs.github.com/en/rest/dependency-graph/dependency-submission#create-a-snapshot-of-dependencies-for-a-repository) -requires Contents write permission; no additional metadata permission is needed. +requires Contents write permission. The writer also has Actions read permission +to verify run and artifact metadata. Snapshot metadata identifies the repository +adapter rather than claiming that the combined generation/submission action ran. This producer supplements the existing dependency review and shipped-inventory checks without changing their severity, license rules or inventory. Merging the -workflow does not prove that a canonical baseline exists: after the first ASF -master run, verify the submitted SHA/ref, non-empty module manifests and a real -dependency-review comparison against that baseline. Until then, baseline acceptance -remains open. No ASF repository setting is changed by this workflow. +workflow does not prove that either repository has a baseline: verify each repository's +first master submission, including its SHA/ref, non-empty module manifests and a real +dependency-review comparison against that baseline. Until then, that repository's baseline +acceptance remains open. This workflow changes no repository settings. From 4d6c07df2c46c4eeae37c704c6ee59d676215c32 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 09:26:19 +0800 Subject: [PATCH 3/9] fix(ci): use ASF-approved submission action - use the pinned GitHub-maintained upstream fork - preserve master-only writes and freshness guards - document the permitted publisher and source reference --- .github/workflows/submit-dependencies.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/submit-dependencies.yml b/.github/workflows/submit-dependencies.yml index 45fc0557fa..de40a11a27 100644 --- a/.github/workflows/submit-dependencies.yml +++ b/.github/workflows/submit-dependencies.yml @@ -85,7 +85,7 @@ jobs: EXPECTED_SHA: ${{ github.sha }} run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" - name: Submit resolved reactor dependencies - uses: advanced-security/maven-dependency-submission-action@a64327a7329c9939cf675e458452febe1894a70c # v6.0.1 + uses: actions/maven-dependency-submission-action@1f666f7ac2ab7b72cd5e22eaf7f23c8a581db1db # GitHub-maintained fork with: directory: '.' ignore-maven-wrapper: true From d9ca13cab2e7fd10b61a0304d41767149dc49d2d Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 12:26:09 +0800 Subject: [PATCH 4/9] fix(ci): isolate dependency snapshot token - build and serialize Maven graphs in a read-only job - validate a bounded JSON ZIP in a fresh data-only writer - preserve dependency identity and master freshness - document boundaries and add archive redirect regressions --- .github/scripts/dependency-snapshot.py | 154 ++++++++++ .github/scripts/test_dependency_snapshot.py | 302 ++++++++++++++++++++ .github/workflows/submit-dependencies.yml | 242 +++++++++++++++- 3 files changed, 687 insertions(+), 11 deletions(-) create mode 100644 .github/scripts/dependency-snapshot.py create mode 100644 .github/scripts/test_dependency_snapshot.py diff --git a/.github/scripts/dependency-snapshot.py b/.github/scripts/dependency-snapshot.py new file mode 100644 index 0000000000..ed71cef60a --- /dev/null +++ b/.github/scripts/dependency-snapshot.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Serialize native Ferstl 4.0.3 graphs; no token or submission code. + +Matches maven-dependency-submission-action 6.0.1's Maven parser/toolkit: +first classifier/type, test scope, shared PURL packages, direct precedence, +child edge order and cycle handling. See docs/ci.md for the producer boundary. +""" + +import datetime +import json +import os +from pathlib import Path +import subprocess +import sys +from urllib.parse import quote + +REPORT = 'maven-dependency-submission-action-depgraph.json' +DETECTOR = {'name': 'maven-dependency-submission-action', 'version': '6.0.1', + 'url': 'https://github.com/advanced-security/maven-dependency-submission-action'} +CORRELATOR = '.github/workflows/submit-dependencies.yml:submit' +SERIALIZER = 'Ferstl 4.0.3; repository stdlib adapter for Maven parser 6.0.1' + + +def unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f'Duplicate JSON key: {key}') + result[key] = value + return result + + +def package_url(artifact): + # packageurl-js uses encodeURIComponent for components and URLSearchParams + # for qualifiers; notably '+' in versions and '/' in namespaces survive. + namespace = '/'.join(part for part in artifact['groupId'].split('/') if part) + name, version = artifact['artifactId'].strip(), artifact['version'].strip() + if not namespace or not name or not version: + raise ValueError('Incomplete Maven coordinates') + encoded_namespace = quote(namespace, safe="!~*'():/") + encoded_name = quote(name, safe="!~*'()") + encoded_version = quote(version, safe="!~*'():+") + purl = f'pkg:maven/{encoded_namespace}/{encoded_name}@{encoded_version}' + qualifiers = {} + for key, field in [('classifier', 'classifiers'), ('type', 'types')]: + values = artifact.get(field) or [] + if values and values[0].strip(): + qualifiers[key] = values[0].strip() + if qualifiers: + purl += '?' + '&'.join(f'{key}={quote(value, safe="*").replace("~", "%7E")}' + for key, value in sorted(qualifiers.items())) + return purl + + +def manifest(graph, pom): + if not isinstance(graph, dict) or not isinstance(graph.get('graphName'), str) or not graph['graphName'].strip(): + raise ValueError(f'Malformed graph name: {pom}') + artifacts, edges = graph.get('artifacts'), graph.get('dependencies') or [] + if not isinstance(artifacts, list) or not artifacts or not isinstance(edges, list): + raise ValueError(f'Empty or malformed graph: {pom}') + if graph.get('dependencies') is not None and not isinstance(graph['dependencies'], list): + raise ValueError(f'Malformed dependency list: {pom}') + ids, packages, scopes = {}, {}, {} + for artifact in artifacts: + if not isinstance(artifact, dict) or any(not isinstance(artifact.get(key), str) or not artifact[key].strip() + for key in ['id', 'groupId', 'artifactId', 'version']): + raise ValueError(f'Malformed artifact: {pom}') + for key in ['scopes', 'types', 'classifiers']: + values = artifact.get(key) + if values is not None and (not isinstance(values, list) or any(not isinstance(x, str) for x in values)): + raise ValueError(f'Malformed artifact {key}: {pom}') + identifier, purl = artifact['id'], package_url(artifact) + if identifier in ids: + raise ValueError(f'Duplicate artifact ID: {identifier}') + ids[identifier] = purl + packages.setdefault(purl, []) + scopes[purl] = 'development' if 'test' in (artifact.get('scopes') or []) else 'runtime' + children, incoming = {}, set() + for edge in edges: + if not isinstance(edge, dict) or any(not isinstance(edge.get(key), str) for key in ['from', 'to']): + raise ValueError(f'Malformed dependency edge: {pom}') + source, target = edge['from'], edge['to'] + if source not in ids or target not in ids: + raise ValueError(f'Unknown dependency edge: {pom}') + children.setdefault(source, []).append(target) + incoming.add(target) + # Aliased artifact IDs share one package and append all their child edges. + for source, targets in children.items(): + packages[ids[source]].extend(ids[target] for target in targets) + direct = dict.fromkeys(target for source in ids if source not in incoming + for target in children.get(source, [])) + resolved = {} + + def entry(purl, relationship): + return {'package_url': purl, 'relationship': relationship, + 'scope': scopes[purl], 'dependencies': packages[purl]} + + for identifier in direct: + purl = ids[identifier] + resolved[purl] = entry(purl, 'direct') + seen, pending = set(), list(reversed(packages[purl])) + while pending: + child = pending.pop() + if child in seen: + continue + seen.add(child) + resolved.setdefault(child, entry(child, 'indirect')) + pending.extend(reversed(packages[child])) + return {'name': graph['graphName'], 'file': {'source_location': pom}, 'resolved': resolved} + + +def create_snapshot(workspace, poms, sha, repository, run_id, run_attempt): + manifests = {} + for pom in poms: + report = workspace / Path(pom).parent / 'target' / REPORT + if report.is_symlink() or report.stat().st_size > 25 * 1024 * 1024: + raise ValueError(f'Invalid report: {pom}') + item = manifest(json.loads(report.read_text(), object_pairs_hook=unique_object), pom) + if item['name'] in manifests: + raise ValueError(f'Duplicate graph name: {item["name"]}') + manifests[item['name']] = item + if not manifests or not any(item['resolved'] for item in manifests.values()): + raise ValueError('Refusing empty dependency inventory') + return {'version': 0, 'sha': sha, 'ref': 'refs/heads/master', + 'job': {'id': run_id, 'correlator': CORRELATOR, + 'html_url': f'https://github.com/{repository}/actions/runs/{run_id}'}, + 'detector': DETECTOR, 'metadata': {'serializer': SERIALIZER, 'producer': 'generate', + 'run_attempt': run_attempt}, + 'scanned': datetime.datetime.now(datetime.timezone.utc).isoformat().replace('+00:00', 'Z'), + 'manifests': manifests} + + +if __name__ == '__main__': + poms = subprocess.check_output(['git', 'ls-files', '-z', '*pom.xml']).decode().split('\0') + poms = [pom for pom in poms if Path(pom).name == 'pom.xml'] + snapshot = create_snapshot(Path.cwd(), poms, os.environ['GITHUB_SHA'], + os.environ['GITHUB_REPOSITORY'], os.environ['GITHUB_RUN_ID'], + os.environ['GITHUB_RUN_ATTEMPT']) + Path(sys.argv[1]).write_text(json.dumps(snapshot, indent=2) + '\n') diff --git a/.github/scripts/test_dependency_snapshot.py b/.github/scripts/test_dependency_snapshot.py new file mode 100644 index 0000000000..18cb868c59 --- /dev/null +++ b/.github/scripts/test_dependency_snapshot.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native-parser regression fixtures and local-only writer protocol controls.""" + +import copy +from contextlib import redirect_stdout +import datetime +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import tempfile +import textwrap +import unittest +import urllib.error +import warnings +import zipfile +from unittest.mock import patch +from urllib.parse import urlsplit + + +def load_module(path, name): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +producer = load_module(Path(__file__).with_name('dependency-snapshot.py'), 'producer') + + +def artifact(identifier, **fields): + return dict(id=identifier, groupId='org.example', artifactId=identifier, version='1.0', types=['jar'], **fields) + + +class SnapshotTest(unittest.TestCase): + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + workflow = Path(__file__).resolve().parents[1] / 'workflows/submit-dependencies.yml' + # Only the fixed workflow program is loaded as code. Artifacts remain JSON data. + program = workflow.read_text().split(" python3 -I - <<'PY'\n", 1)[1].split('\n PY', 1)[0] + script = self.root / 'fixed-writer.py' + script.write_text(textwrap.dedent(program)) + self.writer = load_module(script, 'writer') + self.env = {'EXPECTED_REPOSITORY': 'apache/hugegraph', 'EXPECTED_SHA': 'a' * 40, + 'RUN_ID': '321', 'RUN_ATTEMPT': '2', 'ARTIFACT_ID': '789', 'RUNNER_TEMP': str(self.root), + 'SNAPSHOT_TOKEN': 'public-synthetic-sentinel'} + directory = self.root / 'dependency-snapshot' + directory.mkdir() + self.source = directory / 'snapshot.json' + graph = {'graphName': 'module', 'artifacts': [artifact('root'), artifact('one'), artifact('two')], + 'dependencies': [{'from': 'root', 'to': 'one'}, {'from': 'one', 'to': 'two'}]} + now = datetime.datetime.now(datetime.timezone.utc) + self.snapshot = {'version': 0, 'sha': self.env['EXPECTED_SHA'], 'ref': 'refs/heads/master', + 'job': {'id': '321', 'correlator': producer.CORRELATOR, + 'html_url': 'https://github.com/apache/hugegraph/actions/runs/321'}, + 'detector': producer.DETECTOR, 'metadata': {'serializer': producer.SERIALIZER, + 'producer': 'generate', 'run_attempt': '2'}, + 'scanned': now.isoformat(), 'manifests': {'module': producer.manifest(graph, 'pom.xml')}} + self.responses = { + 'actions/runs/321/attempts/2': {'id': 321, 'run_attempt': 2, 'head_sha': 'a' * 40, + 'head_branch': 'master', 'event': 'push', 'repository': {'full_name': 'apache/hugegraph'}, + 'path': '.github/workflows/submit-dependencies.yml', + 'run_started_at': (now - datetime.timedelta(minutes=2)).isoformat()}, + 'actions/runs/321/attempts/2/jobs?per_page=100': {'total_count': 3, + 'jobs': [{'name': 'generate', 'conclusion': 'success', 'status': 'completed', + 'run_id': 321, 'head_sha': 'a' * 40}]}, + 'actions/artifacts/789': {'id': 789, 'expired': False, 'name': 'dependency-snapshot-321-2', + 'created_at': now.isoformat(), + 'workflow_run': {'id': 321, 'head_sha': 'a' * 40, 'head_branch': 'master'}}, + 'git/trees/' + 'a' * 40 + '?recursive=1': {'truncated': False, + 'tree': [{'path': 'pom.xml', 'type': 'blob', 'mode': '100644'}]}, + 'git/ref/heads/master': {'object': {'sha': 'a' * 40}}, + 'dependency-graph/snapshots': {'result': 'SUCCESS'}, + } + self.posts = [] + self.blob_headers = [] + + def run_writer(self, snapshot=None, raw=None, archive=None, artifact_changes=None): + if archive is None: + payload = raw if raw is not None else json.dumps(snapshot or self.snapshot) + archive = self.zip_bytes([('snapshot.json', payload)]) + self.responses['actions/artifacts/789'].update(size_in_bytes=len(archive), + digest='sha256:' + hashlib.sha256(archive).hexdigest()) + self.responses['actions/artifacts/789'].update(artifact_changes or {}) + + def open_request(request, timeout): + url = urlsplit(request.full_url) + self.assertEqual('https', url.scheme) + if url.netloc == 'storage.blob.core.windows.net': + self.blob_headers.append(dict(request.header_items())) + self.assertIsNone(request.get_header('Authorization')) + response = io.BytesIO(archive) + response.status = 200 + return response + self.assertEqual('api.github.com', url.netloc) + self.assertTrue(url.path.startswith('/repos/apache/hugegraph/')) + self.assertEqual('Bearer public-synthetic-sentinel', request.get_header('Authorization')) + path = url.path.removeprefix('/repos/apache/hugegraph/') + ('?' + url.query if url.query else '') + if path == 'actions/artifacts/789/zip': + raise urllib.error.HTTPError(request.full_url, 302, 'Found', + {'Location': 'https://storage.blob.core.windows.net/archive.zip?sig=public'}, io.BytesIO()) + if request.data is not None: + self.assertEqual('dependency-graph/snapshots', path) + self.posts.append(json.loads(request.data)) + response = io.BytesIO(json.dumps(self.responses[path]).encode()) + response.status = 201 if request.data is not None else 200 + return response + + opener = self.writer.urllib.request.OpenerDirector() + opener.open = open_request + with patch.dict(os.environ, self.env), patch.object(self.writer.urllib.request, + 'build_opener', return_value=opener): + with redirect_stdout(io.StringIO()): + self.writer.main() + + @staticmethod + def zip_bytes(entries, mode=None): + output = io.BytesIO() + with warnings.catch_warnings(): + warnings.simplefilter('ignore', UserWarning) + with zipfile.ZipFile(output, 'w', compression=zipfile.ZIP_DEFLATED) as archive: + for name, data in entries: + entry = zipfile.ZipInfo(name) + entry.compress_type = zipfile.ZIP_DEFLATED + if mode is not None: + entry.create_system = 3 + entry.external_attr = mode << 16 + archive.writestr(entry, data) + return output.getvalue() + + def test_native_package_url_encoding(self): + # Expected strings were captured from the pinned action's actual parser. + cases = [ + (dict(groupId='org.图//组/', artifactId='测试/名@', version='1+2/slash~', + classifiers=['测试 c/+~*'], types=['jar']), + 'pkg:maven/org.%E5%9B%BE/%E7%BB%84/%E6%B5%8B%E8%AF%95%2F%E5%90%8D%40@1+2%2Fslash~' + '?classifier=%E6%B5%8B%E8%AF%95%20c%2F%2B%7E*&type=jar'), + (dict(groupId="g:!/x~*'()", artifactId="n:!~*'()", version="v:!~*'()+", + classifiers=[' c/+~* '], types=[' jar ', 'war']), + "pkg:maven/g:!/x~*'()/n%3A!~*'()@v:!~*'()+?classifier=c%2F%2B%7E*&type=jar"), + ] + for value, expected in cases: + self.assertEqual(expected, producer.package_url(value)) + + def test_native_cycle_alias_scope_and_direct_precedence(self): + graph = {'graphName': 'module', 'artifacts': [artifact('root'), artifact('one', scopes=['test']), + dict(artifact('one'), id='alias', scopes=['provided']), artifact('two'), artifact('three')], + 'dependencies': [{'from': 'root', 'to': 'one'}, {'from': 'one', 'to': 'two'}, + {'from': 'two', 'to': 'one'}, {'from': 'alias', 'to': 'three'}, + {'from': 'root', 'to': 'two'}]} + resolved = producer.manifest(graph, 'pom.xml')['resolved'] + one, two, three = [f'pkg:maven/org.example/{name}@1.0?type=jar' for name in ['one', 'two', 'three']] + self.assertEqual({'package_url': one, 'relationship': 'direct', 'scope': 'runtime', + 'dependencies': [two, three]}, resolved[one]) + self.assertEqual({'direct'}, {entry['relationship'] for entry in resolved.values()}) + self.assertEqual({}, producer.manifest({'graphName': 'parent', 'artifacts': [artifact('root')]}, + 'pom.xml')['resolved']) + + def test_generation_rejects_missing_empty_duplicate_and_malformed_graphs(self): + with self.assertRaises(FileNotFoundError): + producer.create_snapshot(self.root, ['pom.xml'], 'a' * 40, 'apache/hugegraph', '321', '2') + report = self.root / 'target' / producer.REPORT + report.parent.mkdir() + report.write_text(json.dumps({'graphName': 'parent', 'artifacts': [artifact('root')]})) + with self.assertRaisesRegex(ValueError, 'empty dependency inventory'): + producer.create_snapshot(self.root, ['pom.xml'], 'a' * 40, 'apache/hugegraph', '321', '2') + for graph in [{'graphName': 'bad', 'artifacts': [artifact('root'), artifact('root')]}, + {'graphName': 'bad', 'artifacts': [artifact('root')], 'dependencies': False}, + {'graphName': 'bad', 'artifacts': [artifact('root', scopes='test')]}]: + with self.assertRaises(ValueError): + producer.manifest(graph, 'pom.xml') + with self.assertRaises(ValueError): + json.loads('{"sha":"a","sha":"b"}', object_pairs_hook=producer.unique_object) + + def test_writer_preserves_snapshot_and_never_executes_data(self): + marker = self.root / 'must-not-exist' + name = f"__import__('pathlib').Path({str(marker)!r}).touch()" + self.snapshot['manifests'][name] = self.snapshot['manifests'].pop('module') + self.snapshot['manifests'][name]['name'] = name + self.run_writer() + self.assertEqual([self.snapshot], self.posts) + self.assertFalse(marker.exists()) + + def test_writer_accepts_master_path_suffix_and_rejects_other_refs(self): + run = self.responses['actions/runs/321/attempts/2'] + path = '.github/workflows/submit-dependencies.yml' + run['path'] = path + '@master' + self.run_writer() + self.assertEqual([self.snapshot], self.posts) + self.posts.clear() + for ref in ['main', 'feature', 'refs/heads/master', '']: + run['path'] = path + '@' + ref + with self.subTest(ref=ref), self.assertRaisesRegex(ValueError, 'Run identity'): + self.run_writer() + self.assertEqual([], self.posts) + + def test_writer_rejects_snapshot_changes_before_post(self): + mutations = [('sha', 'b' * 40), ('ref', 'refs/heads/feature'), ('version', False), + ('job', {}), ('detector', {}), ('metadata', {}), ('manifests', {}), + ('scanned', '2000-01-01T00:00:00Z'), ('scanned', '2999-01-01T00:00:00Z')] + for key, value in mutations: + with self.subTest(key=key, value=value), self.assertRaises((ValueError, KeyError)): + snapshot = copy.deepcopy(self.snapshot) + snapshot[key] = value + self.run_writer(snapshot) + for field, value in [('source_location', 'other/pom.xml'), ('resolved', {}), ('scope', 'optional'), + ('dependencies', ['pkg:maven/missing/name@1']), + ('package_url', 'pkg:maven/invalid%Q0/a@1')]: + snapshot = copy.deepcopy(self.snapshot) + manifest = snapshot['manifests']['module'] + target = manifest['file'] if field == 'source_location' else manifest if field == 'resolved' else next( + iter(manifest['resolved'].values())) + target[field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + self.run_writer(snapshot) + with self.assertRaises(ValueError): + self.run_writer(raw=json.dumps(self.snapshot)[:-1] + ',"sha":"b"}') + self.assertEqual([], self.posts) + + def test_writer_rejects_failed_producer_wrong_artifact_and_stale_master(self): + for path, field, value in [('actions/runs/321/attempts/2', 'run_attempt', 1), + ('actions/artifacts/789', 'name', 'dependency-snapshot-321-1'), + ('actions/artifacts/789', 'expired', True), + ('actions/artifacts/789', 'workflow_run', {'id': 123}), + ('actions/runs/321/attempts/2/jobs?per_page=100', 'jobs', + [{'name': 'generate', 'conclusion': 'failure'}]), + ('git/trees/' + 'a' * 40 + '?recursive=1', 'truncated', True), + ('git/ref/heads/master', 'object', {'sha': 'b' * 40})]: + original = copy.deepcopy(self.responses[path]) + self.responses[path][field] = value + with self.subTest(path=path, field=field), self.assertRaises((ValueError, KeyError)): + self.run_writer() + self.responses[path] = original + self.assertEqual([], self.posts) + + def test_writer_rejects_zip_paths_duplicates_symlinks_and_bounds_without_writes(self): + body = json.dumps(self.snapshot) + fixtures = [self.zip_bytes([('snapshot.json', body), ('x/../../outside', 'must not write')]), + self.zip_bytes([('snapshot.json', body), ('snapshot.json', body)]), + self.zip_bytes([('snapshot.json', body)], 0o120777), + self.zip_bytes([('snapshot.json', '0' * (self.writer.LIMIT + 1))]), + b'0' * (self.writer.LIMIT + 1)] + fixtures += [self.zip_bytes([(path, body)]) for path in + ['../snapshot.json', 'x/../../snapshot.json', '/snapshot.json', 'x\\snapshot.json']] + before = set(self.root.rglob('*')) + for archive in fixtures: + with self.subTest(size=len(archive)), self.assertRaises(ValueError): + self.run_writer(archive=archive) + self.assertEqual(before, set(self.root.rglob('*'))) + self.assertFalse((self.root / 'outside').exists()) + self.assertEqual([], self.posts) + + def test_writer_download_has_no_cross_host_authorization_or_followup_redirect(self): + self.run_writer() + self.assertEqual([{}], self.blob_headers) + self.posts.clear() + for metadata in [{'digest': 'sha256:' + '0' * 64}, {'size_in_bytes': 1}]: + with self.assertRaisesRegex(ValueError, 'digest/size mismatch'): + self.run_writer(artifact_changes=metadata) + calls = [] + + def redirected(request, timeout): + calls.append(request) + if len(calls) == 1: + raise urllib.error.HTTPError(request.full_url, 302, 'Found', + {'Location': 'https://storage.blob.core.windows.net/archive.zip'}, io.BytesIO()) + self.assertIsNone(request.get_header('Authorization')) + raise urllib.error.HTTPError(request.full_url, 302, 'Found', + {'Location': 'https://other.example'}, io.BytesIO()) + + opener = self.writer.urllib.request.OpenerDirector() + opener.open = redirected + with patch.dict(os.environ, self.env), patch.object(self.writer.urllib.request, + 'build_opener', return_value=opener): + with self.assertRaises(urllib.error.HTTPError): + self.writer.download_snapshot('apache/hugegraph', '789', {}) + self.assertEqual(2, len(calls)) + self.assertEqual([], self.posts) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/submit-dependencies.yml b/.github/workflows/submit-dependencies.yml index de40a11a27..9b4a9edc46 100644 --- a/.github/workflows/submit-dependencies.yml +++ b/.github/workflows/submit-dependencies.yml @@ -48,9 +48,10 @@ jobs: - name: Test CI policy and dependency inventory run: | python3 -m unittest discover -s .github/scripts -p 'test_ci_policy.py' + python3 -m unittest discover -s .github/scripts -p 'test_dependency_snapshot.py' python3 -m unittest discover -s install-dist/scripts/dependency -p 'test_*.py' - submit: + generate: needs: validate if: >- ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || @@ -60,7 +61,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 permissions: - contents: write + contents: read + env: + PYTHONDONTWRITEBYTECODE: '1' + INPUT_TOKEN: '' + GITHUB_TOKEN: '' + GH_TOKEN: '' + outputs: + artifact_id: ${{ steps.snapshot.outputs.artifact-id }} steps: - name: Checkout event commit uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -80,16 +88,228 @@ jobs: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" - name: Install the complete Maven reactor run: mvn -B -ntp install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true --fail-at-end - - name: Refuse a stale master submission + - name: Generate resolved reactor snapshot + run: | + mvn -B -ntp com.github.ferstl:depgraph-maven-plugin:4.0.3:graph \ + -DgraphFormat=json -DoutputFileName=maven-dependency-submission-action-depgraph.json \ + -Dmaven.test.skip=true -Dmaven.javadoc.skip=true + python3 .github/scripts/dependency-snapshot.py "$RUNNER_TEMP/snapshot.json" + - name: Refuse a stale generated snapshot env: EXPECTED_SHA: ${{ github.sha }} run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" - - name: Submit resolved reactor dependencies - uses: actions/maven-dependency-submission-action@1f666f7ac2ab7b72cd5e22eaf7f23c8a581db1db # GitHub-maintained fork + - name: Upload snapshot data only + id: snapshot + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - directory: '.' - ignore-maven-wrapper: true - maven-args: '-ntp -Dmaven.test.skip=true -Dmaven.javadoc.skip=true' - correlator: '.github/workflows/submit-dependencies.yml:submit' - snapshot-sha: ${{ github.sha }} - snapshot-ref: ${{ github.ref }} + name: dependency-snapshot-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/snapshot.json + if-no-files-found: error + retention-days: 1 + + submit: + needs: [validate, generate] + if: >- + needs.validate.result == 'success' && needs.generate.result == 'success' && + ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || + github.repository == 'hugegraph/hugegraph') && + github.ref == 'refs/heads/master' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + actions: read + steps: + - name: Validate and submit snapshot data + env: + SNAPSHOT_TOKEN: ${{ github.token }} + EXPECTED_REPOSITORY: ${{ github.repository }} + EXPECTED_SHA: ${{ github.sha }} + RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + ARTIFACT_ID: ${{ needs.generate.outputs.artifact_id }} + run: | + python3 -I - <<'PY' + import datetime + import hashlib + import io + import json + import os + import re + import stat + import urllib.error + import urllib.parse + import urllib.request + import zipfile + + LIMIT = 25 * 1024 * 1024 + + def require(condition, message): + if not condition: + raise ValueError(message) + + def unique_object(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'Duplicate JSON key') + result[key] = value + return result + + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + def github_request(repository, path, data=None): + return urllib.request.Request('https://api.github.com/repos/' + repository + '/' + path, + data=None if data is None else json.dumps(data).encode(), + headers={'Authorization': 'Bearer ' + os.environ['SNAPSHOT_TOKEN'], + 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json', + 'X-GitHub-Api-Version': '2022-11-28'}) + + def bounded_body(response): + raw = response.read(LIMIT + 1) + require(len(raw) <= LIMIT, 'Response too large') + return raw + + def api(repository, path, data=None): + request = github_request(repository, path, data) + with urllib.request.build_opener(NoRedirect()).open(request, timeout=60) as response: + require(response.status == (200 if data is None else 201), 'Unexpected API status') + raw = bounded_body(response) + return json.loads(raw, object_pairs_hook=unique_object) + + def read_snapshot(raw): + require(0 < len(raw) <= LIMIT, 'Invalid ZIP size') + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + members = archive.infolist() + require(len(members) == 1, 'ZIP must contain one snapshot.json') + member = members[0] + require(member.orig_filename == 'snapshot.json' and not member.is_dir() and + not (member.external_attr & 0x10) and + stat.S_IFMT(member.external_attr >> 16) in [0, stat.S_IFREG] and + not (member.flag_bits & 1) and + member.compress_type in [zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED] and + 0 < member.file_size <= LIMIT, 'Invalid ZIP member') + with archive.open(member) as stream: + payload = bounded_body(stream) + require(len(payload) == member.file_size, 'Incomplete ZIP member') + return json.loads(payload, object_pairs_hook=unique_object) + + def download_snapshot(repository, artifact_id, artifact): + opener = urllib.request.build_opener(NoRedirect()) + try: + response = opener.open(github_request(repository, f'actions/artifacts/{artifact_id}/zip'), timeout=60) + except urllib.error.HTTPError as error: + require(error.code == 302, 'Expected artifact redirect') + location = error.headers.get('Location', '') + error.close() + else: + response.close() + raise ValueError('Expected artifact redirect') + target = urllib.parse.urlsplit(location) + require(target.scheme == 'https' and target.hostname and + target.hostname.endswith(('.blob.core.windows.net', '.githubusercontent.com')) and + target.username is None and target.password is None and + target.port in [None, 443] and not target.fragment, 'Invalid artifact redirect') + # A separate request carries no GitHub Authorization to blob storage. + with opener.open(urllib.request.Request(location), timeout=60) as response: + require(response.status == 200, 'Artifact download failed') + raw = bounded_body(response) + require(len(raw) == artifact['size_in_bytes'] and + 'sha256:' + hashlib.sha256(raw).hexdigest() == artifact['digest'], 'Artifact digest/size mismatch') + return read_snapshot(raw) + + def timestamp(value): + parsed = datetime.datetime.fromisoformat(value.replace('Z', '+00:00')) + require(parsed.tzinfo is not None, 'Timestamp lacks timezone') + return parsed + + def main(): + repository, sha = os.environ['EXPECTED_REPOSITORY'], os.environ['EXPECTED_SHA'] + run_id, attempt, artifact_id = (os.environ[key] for key in ['RUN_ID', 'RUN_ATTEMPT', 'ARTIFACT_ID']) + require(repository in ['apache/hugegraph', 'hugegraph/hugegraph'], 'Repository refused') + require(re.fullmatch('[0-9a-f]{40}', sha), 'Invalid commit') + require(all(re.fullmatch('[1-9][0-9]*', x) for x in [run_id, attempt, artifact_id]), 'Invalid ID') + run = api(repository, f'actions/runs/{run_id}/attempts/{attempt}') + require(run['id'] == int(run_id) and run['run_attempt'] == int(attempt) and + run['head_sha'] == sha and run['head_branch'] == 'master' and + run['event'] in ['push', 'workflow_dispatch'] and + run['repository']['full_name'] == repository and + run['path'] in ['.github/workflows/submit-dependencies.yml', + '.github/workflows/submit-dependencies.yml@master'], 'Run identity') + jobs = api(repository, f'actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100') + producers = [job for job in jobs['jobs'] if job['name'] == 'generate'] + require(jobs['total_count'] <= 100 and len(producers) == 1 and + producers[0]['conclusion'] == 'success' and producers[0]['status'] == 'completed' and + producers[0]['run_id'] == int(run_id) and producers[0]['head_sha'] == sha, + 'Generation did not succeed for this commit') + artifact = api(repository, f'actions/artifacts/{artifact_id}') + require(artifact['id'] == int(artifact_id) and artifact['expired'] is False and + artifact['name'] == f'dependency-snapshot-{run_id}-{attempt}' and + artifact['workflow_run']['id'] == int(run_id) and + artifact['workflow_run']['head_sha'] == sha and + artifact['workflow_run']['head_branch'] == 'master' and + type(artifact['size_in_bytes']) is int and 0 < artifact['size_in_bytes'] <= LIMIT and + isinstance(artifact['digest'], str) and + re.fullmatch('sha256:[0-9a-f]{64}', artifact['digest']), 'Artifact identity') + snapshot = download_snapshot(repository, artifact_id, artifact) + require(isinstance(snapshot, dict) and set(snapshot) == + {'version', 'sha', 'ref', 'job', 'detector', 'metadata', 'scanned', 'manifests'}, 'Snapshot schema') + require(type(snapshot['version']) is int and snapshot['version'] == 0 and + snapshot['sha'] == sha and snapshot['ref'] == 'refs/heads/master', 'Snapshot source') + require(snapshot['job'] == {'id': run_id, + 'correlator': '.github/workflows/submit-dependencies.yml:submit', + 'html_url': f'https://github.com/{repository}/actions/runs/{run_id}'}, 'Snapshot job') + require(snapshot['detector'] == {'name': 'maven-dependency-submission-action', 'version': '6.0.1', + 'url': 'https://github.com/advanced-security/maven-dependency-submission-action'}, 'Detector') + require(snapshot['metadata'] == {'serializer': + 'Ferstl 4.0.3; repository stdlib adapter for Maven parser 6.0.1', + 'producer': 'generate', 'run_attempt': attempt}, 'Producer identity') + now = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=1) + require(timestamp(run['run_started_at']) <= timestamp(snapshot['scanned']) <= + timestamp(artifact['created_at']) + datetime.timedelta(minutes=1) and + timestamp(artifact['created_at']) <= now and timestamp(snapshot['scanned']) <= now, + 'Snapshot freshness') + tree = api(repository, f'git/trees/{sha}?recursive=1') + require(tree['truncated'] is False, 'Incomplete source tree') + poms = {node['path'] for node in tree['tree'] if node['type'] == 'blob' and + node['path'].split('/')[-1] == 'pom.xml' and node['mode'] != '120000'} + manifests, paths = snapshot['manifests'], set() + require(isinstance(manifests, dict) and manifests and poms, 'Empty inventory') + component = r"(?:[A-Za-z0-9._!~*'():+-]|%[0-9A-F]{2})+" + qualifier = r'(?:[A-Za-z0-9._*\-]|%[0-9A-F]{2})+' + purl_pattern = (r'pkg:maven/' + component + '(?:/' + component + ')+@' + component + + r'(?:\?classifier=' + qualifier + '(?:&type=' + qualifier + r')?|\?type=' + qualifier + ')?') + for name, manifest in manifests.items(): + require(isinstance(name, str) and name.strip() and isinstance(manifest, dict) and + set(manifest) == {'name', 'file', 'resolved'} and manifest['name'] == name and + isinstance(manifest['file'], dict) and set(manifest['file']) == {'source_location'}, + 'Manifest schema') + location = manifest['file']['source_location'] + require(isinstance(location, str) and location in poms and location not in paths, 'Manifest source') + paths.add(location) + resolved = manifest['resolved'] + require(isinstance(resolved, dict), 'Resolved schema') + for purl, package in resolved.items(): + require(isinstance(purl, str) and len(purl) <= 4096 and + re.fullmatch(purl_pattern, purl), 'Package URL') + require(isinstance(package, dict) and set(package) == + {'package_url', 'relationship', 'scope', 'dependencies'} and + package['package_url'] == purl and package['relationship'] in ['direct', 'indirect'] and + package['scope'] in ['runtime', 'development'] and + isinstance(package['dependencies'], list) and + all(isinstance(child, str) and child in resolved for child in package['dependencies']), + 'Package schema') + require(not resolved or any(p['relationship'] == 'direct' for p in resolved.values()), + 'Missing direct dependencies') + require(paths == poms and any(m['resolved'] for m in manifests.values()), 'Incomplete inventory') + require(api(repository, 'git/ref/heads/master')['object']['sha'] == sha, 'Stale master before submission') + result = api(repository, 'dependency-graph/snapshots', snapshot) + require(result['result'] == 'SUCCESS', 'Snapshot not accepted') + require(api(repository, 'git/ref/heads/master')['object']['sha'] == sha, 'Master advanced during submission') + print('Dependency baseline submitted for ' + sha) + + if __name__ == '__main__': + main() + PY From d78ad6c8f509aa1f69beed3d117969e679c69c72 Mon Sep 17 00:00:00 2001 From: imbajin Date: Tue, 13 Oct 2026 23:32:54 +0800 Subject: [PATCH 5/9] fix(ci): clarify dependency rerun recovery - document all-job reruns or fresh master dispatch - test dependency paths containing only indirect edges - verify direct edges take precedence over indirect paths --- .github/scripts/test_dependency_snapshot.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/scripts/test_dependency_snapshot.py b/.github/scripts/test_dependency_snapshot.py index 18cb868c59..9c8eae3440 100644 --- a/.github/scripts/test_dependency_snapshot.py +++ b/.github/scripts/test_dependency_snapshot.py @@ -176,6 +176,27 @@ def test_native_cycle_alias_scope_and_direct_precedence(self): self.assertEqual({}, producer.manifest({'graphName': 'parent', 'artifacts': [artifact('root')]}, 'pom.xml')['resolved']) + def test_transitive_only_package_is_indirect(self): + graph = {'graphName': 'module', 'artifacts': [artifact('root'), artifact('parent'), artifact('leaf')], + 'dependencies': [{'from': 'root', 'to': 'parent'}, {'from': 'parent', 'to': 'leaf'}]} + parent = 'pkg:maven/org.example/parent@1.0?type=jar' + leaf = 'pkg:maven/org.example/leaf@1.0?type=jar' + self.assertEqual({ + parent: {'package_url': parent, 'relationship': 'direct', 'scope': 'runtime', 'dependencies': [leaf]}, + leaf: {'package_url': leaf, 'relationship': 'indirect', 'scope': 'runtime', 'dependencies': []}, + }, producer.manifest(graph, 'pom.xml')['resolved']) + + def test_direct_relationship_overrides_indirect_path(self): + graph = {'graphName': 'module', 'artifacts': [artifact('root'), artifact('parent'), artifact('shared')], + 'dependencies': [{'from': 'root', 'to': 'parent'}, {'from': 'parent', 'to': 'shared'}, + {'from': 'root', 'to': 'shared'}]} + parent = 'pkg:maven/org.example/parent@1.0?type=jar' + shared = 'pkg:maven/org.example/shared@1.0?type=jar' + self.assertEqual({ + parent: {'package_url': parent, 'relationship': 'direct', 'scope': 'runtime', 'dependencies': [shared]}, + shared: {'package_url': shared, 'relationship': 'direct', 'scope': 'runtime', 'dependencies': []}, + }, producer.manifest(graph, 'pom.xml')['resolved']) + def test_generation_rejects_missing_empty_duplicate_and_malformed_graphs(self): with self.assertRaises(FileNotFoundError): producer.create_snapshot(self.root, ['pom.xml'], 'a' * 40, 'apache/hugegraph', '321', '2') From 2fa78c13d1a2a21b9f27e61ce705d54341a8cbcb Mon Sep 17 00:00:00 2001 From: imbajin Date: Wed, 14 Oct 2026 04:35:48 +0800 Subject: [PATCH 6/9] fix(ci): scope dependency snapshot validation - select dependency checks for producer and workflow inputs - keep fixture tests in the existing entry job - verify mixed changes retain runtime coverage --- .github/scripts/ci-policy.py | 5 ++++- .github/scripts/test_ci_policy.py | 12 ++++++++++++ docs/ci.md | 2 ++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index dd2fd27331..c6059742ca 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -43,7 +43,8 @@ "codeql-analysis.yml": {"security"}, "server-memory-ci.yml": {"server"}, "server-tests.yml": {"server", "hbase"}, "server-compatibility-ci.yml": {"macos", "macos_full", "hbase", "riscv64"}, - "riscv64-ci.yml": {"riscv64"}, "check-dependencies.yml": {"dependency_license"}}, + "riscv64-ci.yml": {"riscv64"}, "check-dependencies.yml": {"dependency_license"}, + "submit-dependencies.yml": {"dependency_license"}}, "toolchain": {"client-ci.yml": {"client"}, "client-go-ci.yml": {"go"}, "loader-ci.yml": {"loader"}, "tools-ci.yml": {"tools"}, "spark-connector-ci.yml": {"spark"}, "hubble-ci.yml": {"hubble"}, @@ -60,6 +61,8 @@ ".asf.yaml": set(), ".editorconfig": set(), ".licenserc.yaml": set(), ".github/PULL_REQUEST_TEMPLATE.md": set(), ".github/dependabot.yml": set(), ".github/workflows/.java-version": MAINLINE | {"macos_full", "hbase", "riscv64"}, + ".github/scripts/dependency-snapshot.py": {"dependency_license"}, + ".github/scripts/test_dependency_snapshot.py": set(), # Covered by preflight discovery. ".github/scripts/check-docker-images.sh": {"docker"}, ".github/scripts/docker-deployment.py": {"docker"}, ".github/scripts/test_docker_deployment.py": {"docker"}, diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index a60486ebec..83b75337cd 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -121,6 +121,18 @@ def test_shared_runtime_consumers_and_protocols(self): self.assertTrue({"pd", "store", "hstore", "cluster"}.issubset(policy.select( "server", ["hugegraph-pd/hg-pd-grpc/src/main/proto/common.proto"]))) + def test_dependency_snapshot_inputs_do_not_fan_out_to_runtime_suites(self): + for path in [".github/scripts/dependency-snapshot.py", ".github/workflows/submit-dependencies.yml"]: + self.assertEqual({"dependency_license"}, set(self.plan_for([path])["selected"])) + self.assertEqual({"dependency_license"}, self.coordinator_jobs(self.plan_for([path]))) + fixture = ".github/scripts/test_dependency_snapshot.py" + self.assertEqual([], self.plan_for([fixture])["expected"]) + runtime = "hugegraph-server/hugegraph-core/src/main/A.java" + self.assertEqual(self.plan_for([runtime])["selected"], self.plan_for([runtime, fixture])["selected"]) + # Classifier changes still receive conservative mainline validation. + self.assertEqual(policy.select("server", [".github/scripts/ci-policy.py"]), + policy.select("server", [".github/scripts/ci-policy.py", fixture])) + def test_noncore_platforms_only_follow_direct_inputs(self): for path in ["hugegraph-server/hugegraph-hbase/src/main/A.java", "hugegraph-server/hugegraph-dist/src/assembly/travis/install-hbase.sh", diff --git a/docs/ci.md b/docs/ci.md index 6da30987bf..a7beebcec4 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -155,3 +155,5 @@ workflow does not prove that either repository has a baseline: verify each repos first master submission, including its SHA/ref, non-empty module manifests and a real dependency-review comparison against that baseline. Until then, that repository's baseline acceptance remains open. This workflow changes no repository settings. + +Dependency snapshot producer and submission-workflow changes select dependency checks. Snapshot fixture tests run in the entry job; they do not select runtime suites. Changes to the CI classifier still receive conservative mainline validation. From 397414764253e10efb41a2050e6f0f26b43ab13c Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 15 Oct 2026 01:54:43 +0800 Subject: [PATCH 7/9] fix(ci): verify dependency inventory - resolve an independent per-module Maven inventory - reject incomplete snapshots before token-bearing submission - cover inventory omissions and document acceptance gates --- .github/scripts/ci-policy.py | 1 + .github/scripts/dependency-inventory.py | 99 +++++++++++++ .github/scripts/test_ci_policy.py | 3 +- .github/scripts/test_dependency_snapshot.py | 148 +++++++++++++++++++- .github/workflows/submit-dependencies.yml | 125 ++++++++++++++--- docs/ci.md | 22 ++- 6 files changed, 364 insertions(+), 34 deletions(-) create mode 100644 .github/scripts/dependency-inventory.py diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index dfa19a952e..33a1d4ea1e 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -62,6 +62,7 @@ ".github/PULL_REQUEST_TEMPLATE.md": set(), ".github/dependabot.yml": set(), ".github/workflows/.java-version": MAINLINE | {"macos_full", "hbase", "riscv64"}, ".github/scripts/dependency-snapshot.py": {"dependency_license"}, + ".github/scripts/dependency-inventory.py": {"dependency_license"}, ".github/scripts/test_dependency_snapshot.py": set(), # Covered by preflight discovery. ".github/scripts/check-docker-images.sh": {"docker"}, ".github/scripts/docker-deployment.py": {"docker"}, diff --git a/.github/scripts/dependency-inventory.py b/.github/scripts/dependency-inventory.py new file mode 100644 index 0000000000..9010041017 --- /dev/null +++ b/.github/scripts/dependency-inventory.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Canonical inventory from independent Apache Maven dependency:tree JSON.""" + +import datetime +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import sys + +spec = importlib.util.spec_from_file_location('snapshot', Path(__file__).with_name('dependency-snapshot.py')) +snapshot = importlib.util.module_from_spec(spec) +spec.loader.exec_module(snapshot) + +REPORT = 'dependency-inventory.json' +RESOLVER = 'org.apache.maven.plugins:maven-dependency-plugin:3.8.1:tree' +LIMIT = 25 * 1024 * 1024 +FIELDS = {'groupId', 'artifactId', 'version', 'type', 'scope', 'classifier', 'optional', 'children'} +SCOPES = {'compile', 'runtime', 'provided', 'system', 'test'} + + +def packages(tree): + result = {} + pending = [(tree, 0)] + while pending: + node, depth = pending.pop() + if not isinstance(node, dict) or set(node) - FIELDS: + raise ValueError('Invalid inventory node') + for field in ['groupId', 'artifactId', 'version', 'type']: + if not isinstance(node.get(field), str) or not node[field].strip(): + raise ValueError('Incomplete inventory coordinates') + if not isinstance(node.get('classifier'), str) or node.get('optional') not in ['true', 'false']: + raise ValueError('Invalid inventory classifier/optional') + scope = node.get('scope') + if (depth == 0 and scope != '') or (depth > 0 and scope not in SCOPES): + raise ValueError('Invalid inventory scope') + children = node.get('children', []) + if not isinstance(children, list): + raise ValueError('Invalid inventory children') + pending.extend((child, depth + 1) for child in children) + if depth == 0: + continue + purl = snapshot.package_url(dict(node, types=[node['type']], classifiers=[node['classifier']])) + value = {'scope': 'development' if scope == 'test' else 'runtime', + 'relationship': 'direct' if depth == 1 else 'indirect'} + # Maven may expose the same selected package along more than one path. + previous = result.get(purl) + if previous: + if previous['scope'] == 'runtime': + value['scope'] = 'runtime' + if previous['relationship'] == 'direct': + value['relationship'] = 'direct' + result[purl] = value + return result + + +def create_inventory(workspace, poms, sha, repository, run_id, run_attempt): + manifests = {} + for pom in poms: + if pom in manifests: + raise ValueError('Duplicate inventory POM') + report = workspace / Path(pom).parent / 'target' / REPORT + if report.is_symlink(): + raise ValueError('Invalid inventory report') + with report.open('rb') as stream: + raw = stream.read(LIMIT + 1) + if not raw or len(raw) > LIMIT: + raise ValueError('Invalid inventory report size') + manifests[pom] = packages(json.loads(raw, object_pairs_hook=snapshot.unique_object)) + if not manifests or not any(manifests.values()): + raise ValueError('Empty independent dependency inventory') + return {'version': 1, 'sha': sha, 'repository': repository, 'run_id': run_id, + 'run_attempt': run_attempt, 'producer': 'inventory', 'resolver': RESOLVER, + 'scanned': datetime.datetime.now(datetime.timezone.utc).isoformat().replace('+00:00', 'Z'), + 'manifests': manifests} + + +if __name__ == '__main__': + poms = subprocess.check_output(['git', 'ls-files', '-z', '*pom.xml']).decode().split('\0') + poms = [pom for pom in poms if Path(pom).name == 'pom.xml'] + inventory = create_inventory(Path.cwd(), poms, os.environ['GITHUB_SHA'], os.environ['GITHUB_REPOSITORY'], + os.environ['GITHUB_RUN_ID'], os.environ['GITHUB_RUN_ATTEMPT']) + Path(sys.argv[1]).write_text(json.dumps(inventory, indent=2) + '\n') diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index ca566d5d70..c43130b904 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -122,7 +122,8 @@ def test_shared_runtime_consumers_and_protocols(self): "server", ["hugegraph-pd/hg-pd-grpc/src/main/proto/common.proto"]))) def test_dependency_snapshot_inputs_do_not_fan_out_to_runtime_suites(self): - for path in [".github/scripts/dependency-snapshot.py", ".github/workflows/submit-dependencies.yml"]: + for path in [".github/scripts/dependency-snapshot.py", ".github/scripts/dependency-inventory.py", + ".github/workflows/submit-dependencies.yml"]: self.assertEqual({"dependency_license"}, set(self.plan_for([path])["selected"])) self.assertEqual({"dependency_license"}, self.coordinator_jobs(self.plan_for([path]))) fixture = ".github/scripts/test_dependency_snapshot.py" diff --git a/.github/scripts/test_dependency_snapshot.py b/.github/scripts/test_dependency_snapshot.py index 9c8eae3440..7885ed46d5 100644 --- a/.github/scripts/test_dependency_snapshot.py +++ b/.github/scripts/test_dependency_snapshot.py @@ -43,6 +43,7 @@ def load_module(path, name): producer = load_module(Path(__file__).with_name('dependency-snapshot.py'), 'producer') +inventory_producer = load_module(Path(__file__).with_name('dependency-inventory.py'), 'inventory_producer') def artifact(identifier, **fields): @@ -62,7 +63,8 @@ def setUp(self): script.write_text(textwrap.dedent(program)) self.writer = load_module(script, 'writer') self.env = {'EXPECTED_REPOSITORY': 'apache/hugegraph', 'EXPECTED_SHA': 'a' * 40, - 'RUN_ID': '321', 'RUN_ATTEMPT': '2', 'ARTIFACT_ID': '789', 'RUNNER_TEMP': str(self.root), + 'RUN_ID': '321', 'RUN_ATTEMPT': '2', 'ARTIFACT_ID': '789', 'INVENTORY_ARTIFACT_ID': '987', + 'RUNNER_TEMP': str(self.root), 'SNAPSHOT_TOKEN': 'public-synthetic-sentinel'} directory = self.root / 'dependency-snapshot' directory.mkdir() @@ -76,17 +78,27 @@ def setUp(self): 'detector': producer.DETECTOR, 'metadata': {'serializer': producer.SERIALIZER, 'producer': 'generate', 'run_attempt': '2'}, 'scanned': now.isoformat(), 'manifests': {'module': producer.manifest(graph, 'pom.xml')}} + self.inventory = {'version': 1, 'sha': 'a' * 40, 'repository': 'apache/hugegraph', 'run_id': '321', + 'run_attempt': '2', 'producer': 'inventory', 'resolver': inventory_producer.RESOLVER, + 'scanned': now.isoformat(), 'manifests': {'pom.xml': { + key: {'scope': value['scope'], 'relationship': value['relationship']} + for key, value in self.snapshot['manifests']['module']['resolved'].items()}}} self.responses = { 'actions/runs/321/attempts/2': {'id': 321, 'run_attempt': 2, 'head_sha': 'a' * 40, 'head_branch': 'master', 'event': 'push', 'repository': {'full_name': 'apache/hugegraph'}, 'path': '.github/workflows/submit-dependencies.yml', 'run_started_at': (now - datetime.timedelta(minutes=2)).isoformat()}, - 'actions/runs/321/attempts/2/jobs?per_page=100': {'total_count': 3, + 'actions/runs/321/attempts/2/jobs?per_page=100': {'total_count': 4, 'jobs': [{'name': 'generate', 'conclusion': 'success', 'status': 'completed', + 'run_id': 321, 'head_sha': 'a' * 40}, + {'name': 'inventory', 'conclusion': 'success', 'status': 'completed', 'run_id': 321, 'head_sha': 'a' * 40}]}, 'actions/artifacts/789': {'id': 789, 'expired': False, 'name': 'dependency-snapshot-321-2', 'created_at': now.isoformat(), 'workflow_run': {'id': 321, 'head_sha': 'a' * 40, 'head_branch': 'master'}}, + 'actions/artifacts/987': {'id': 987, 'expired': False, 'name': 'dependency-inventory-321-2', + 'created_at': now.isoformat(), + 'workflow_run': {'id': 321, 'head_sha': 'a' * 40, 'head_branch': 'master'}}, 'git/trees/' + 'a' * 40 + '?recursive=1': {'truncated': False, 'tree': [{'path': 'pom.xml', 'type': 'blob', 'mode': '100644'}]}, 'git/ref/heads/master': {'object': {'sha': 'a' * 40}}, @@ -95,13 +107,19 @@ def setUp(self): self.posts = [] self.blob_headers = [] - def run_writer(self, snapshot=None, raw=None, archive=None, artifact_changes=None): + def run_writer(self, snapshot=None, raw=None, archive=None, artifact_changes=None, inventory=None, + inventory_archive=None, inventory_artifact_changes=None): if archive is None: payload = raw if raw is not None else json.dumps(snapshot or self.snapshot) archive = self.zip_bytes([('snapshot.json', payload)]) self.responses['actions/artifacts/789'].update(size_in_bytes=len(archive), digest='sha256:' + hashlib.sha256(archive).hexdigest()) self.responses['actions/artifacts/789'].update(artifact_changes or {}) + if inventory_archive is None: + inventory_archive = self.zip_bytes([('inventory.json', json.dumps(inventory or self.inventory))]) + self.responses['actions/artifacts/987'].update(size_in_bytes=len(inventory_archive), + digest='sha256:' + hashlib.sha256(inventory_archive).hexdigest()) + self.responses['actions/artifacts/987'].update(inventory_artifact_changes or {}) def open_request(request, timeout): url = urlsplit(request.full_url) @@ -109,16 +127,17 @@ def open_request(request, timeout): if url.netloc == 'storage.blob.core.windows.net': self.blob_headers.append(dict(request.header_items())) self.assertIsNone(request.get_header('Authorization')) - response = io.BytesIO(archive) + response = io.BytesIO(inventory_archive if url.path == '/987.zip' else archive) response.status = 200 return response self.assertEqual('api.github.com', url.netloc) self.assertTrue(url.path.startswith('/repos/apache/hugegraph/')) self.assertEqual('Bearer public-synthetic-sentinel', request.get_header('Authorization')) path = url.path.removeprefix('/repos/apache/hugegraph/') + ('?' + url.query if url.query else '') - if path == 'actions/artifacts/789/zip': + if path in ['actions/artifacts/789/zip', 'actions/artifacts/987/zip']: raise urllib.error.HTTPError(request.full_url, 302, 'Found', - {'Location': 'https://storage.blob.core.windows.net/archive.zip?sig=public'}, io.BytesIO()) + {'Location': 'https://storage.blob.core.windows.net/' + path.split('/')[2] + '.zip?sig=public'}, + io.BytesIO()) if request.data is not None: self.assertEqual('dependency-graph/snapshots', path) self.posts.append(json.loads(request.data)) @@ -274,6 +293,67 @@ def test_writer_rejects_failed_producer_wrong_artifact_and_stale_master(self): self.responses[path] = original self.assertEqual([], self.posts) + def test_writer_rejects_incomplete_package_inventory_before_post(self): + resolved = self.snapshot['manifests']['module']['resolved'] + for key in list(resolved): + candidate = copy.deepcopy(self.snapshot) + packages = candidate['manifests']['module']['resolved'] + packages.pop(key) + for package in packages.values(): + package['dependencies'] = [child for child in package['dependencies'] if child != key] + with self.subTest(missing=key), self.assertRaises(ValueError): + self.run_writer(snapshot=candidate) + for field, value in [('scope', 'development'), ('relationship', 'direct')]: + candidate = copy.deepcopy(self.snapshot) + key = next(key for key, value in resolved.items() if value['relationship'] == 'indirect') + candidate['manifests']['module']['resolved'][key][field] = value + with self.subTest(field=field), self.assertRaisesRegex(ValueError, 'inventory mismatch'): + self.run_writer(snapshot=candidate) + candidate = copy.deepcopy(self.snapshot) + packages = candidate['manifests']['module']['resolved'] + key = next(key for key, value in packages.items() if value['relationship'] == 'indirect') + altered = key.replace('@1.0?', '@2.0?') + packages[altered] = packages.pop(key) + packages[altered]['package_url'] = altered + for package in packages.values(): + package['dependencies'] = [altered if child == key else child for child in package['dependencies']] + with self.assertRaisesRegex(ValueError, 'inventory mismatch'): + self.run_writer(snapshot=candidate) + self.assertEqual([], self.posts) + + def test_writer_rejects_foreign_or_incomplete_inventory(self): + for key, value in [('sha', 'b' * 40), ('repository', 'other/repo'), ('run_id', '123'), + ('run_attempt', '1'), ('producer', 'generate'), ('resolver', 'unknown'), + ('version', False), ('scanned', '2000-01-01T00:00:00Z'), ('manifests', {})]: + candidate = copy.deepcopy(self.inventory) + candidate[key] = value + with self.subTest(field=key), self.assertRaises(ValueError): + self.run_writer(inventory=candidate) + for fields in [{'expired': True}, {'name': 'dependency-inventory-321-1'}, + {'workflow_run': {'id': 321, 'head_sha': 'b' * 40, 'head_branch': 'master'}}, + {'digest': 'sha256:' + '0' * 64}]: + original = copy.deepcopy(self.responses['actions/artifacts/987']) + with self.subTest(fields=fields), self.assertRaises((ValueError, KeyError)): + self.run_writer(inventory_artifact_changes=fields) + self.responses['actions/artifacts/987'] = original + jobs = self.responses['actions/runs/321/attempts/2/jobs?per_page=100']['jobs'] + for field, value in [('conclusion', 'failure'), ('head_sha', 'b' * 40), ('run_id', 123)]: + original = jobs[1][field] + jobs[1][field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + self.run_writer() + jobs[1][field] = original + self.assertEqual([], self.posts) + + def test_writer_rejects_inventory_zip_wrong_member_without_extraction(self): + before = set(self.root.rglob('*')) + for entries in [[('snapshot.json', '{}')], [('inventory.json', '{}'), ('inventory.json', '{}')], + [('../inventory.json', '{}')]]: + with self.subTest(entries=entries), self.assertRaises(ValueError): + self.run_writer(inventory_archive=self.zip_bytes(entries)) + self.assertEqual(before, set(self.root.rglob('*'))) + self.assertEqual([], self.posts) + def test_writer_rejects_zip_paths_duplicates_symlinks_and_bounds_without_writes(self): body = json.dumps(self.snapshot) fixtures = [self.zip_bytes([('snapshot.json', body), ('x/../../outside', 'must not write')]), @@ -293,7 +373,7 @@ def test_writer_rejects_zip_paths_duplicates_symlinks_and_bounds_without_writes( def test_writer_download_has_no_cross_host_authorization_or_followup_redirect(self): self.run_writer() - self.assertEqual([{}], self.blob_headers) + self.assertEqual([{}, {}], self.blob_headers) self.posts.clear() for metadata in [{'digest': 'sha256:' + '0' * 64}, {'size_in_bytes': 1}]: with self.assertRaisesRegex(ValueError, 'digest/size mismatch'): @@ -319,5 +399,59 @@ def redirected(request, timeout): self.assertEqual([], self.posts) +class InventoryTest(unittest.TestCase): + + @staticmethod + def node(name, scope='', children=None, **fields): + return dict(groupId='org.example', artifactId=name, version='1.0', type='jar', scope=scope, + classifier='', optional='false', children=children or [], **fields) + + def test_independent_tree_scopes_classifier_and_direct_precedence(self): + leaf = self.node('leaf', 'test') + parent = self.node('parent', 'compile', [leaf]) + classified = self.node('native', 'provided') + classified['classifier'] = 'linux-aarch_64' + tree = self.node('root', children=[parent, self.node('leaf', 'compile'), classified]) + prefix = 'pkg:maven/org.example/' + self.assertEqual({ + prefix + 'parent@1.0?type=jar': {'relationship': 'direct', 'scope': 'runtime'}, + prefix + 'leaf@1.0?type=jar': {'relationship': 'direct', 'scope': 'runtime'}, + prefix + 'native@1.0?classifier=linux-aarch_64&type=jar': + {'relationship': 'direct', 'scope': 'runtime'}, + }, inventory_producer.packages(tree)) + self.assertEqual({}, inventory_producer.packages(self.node('aggregate'))) + + def test_inventory_refuses_invalid_nodes(self): + for field, value in [('scope', 'compile'), ('children', {}), ('optional', None), + ('classifier', None), ('version', '')]: + node = self.node('root') + node[field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + inventory_producer.packages(node) + with self.assertRaises(ValueError): + inventory_producer.packages(self.node('root', children=[self.node('bad', 'unknown')])) + + def test_inventory_requires_every_report_and_refuses_empty_or_duplicate_poms(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / 'target' + target.mkdir() + report = target / inventory_producer.REPORT + report.write_text(json.dumps(self.node('root', children=[self.node('leaf', 'test')]))) + result = inventory_producer.create_inventory(root, ['pom.xml'], 'a' * 40, 'apache/hugegraph', '1', '1') + self.assertEqual('development', next(iter(result['manifests']['pom.xml'].values()))['scope']) + with self.assertRaises(ValueError): + inventory_producer.create_inventory(root, ['pom.xml', 'pom.xml'], 'a' * 40, 'repo', '1', '1') + with self.assertRaises(FileNotFoundError): + inventory_producer.create_inventory(root, ['module/pom.xml'], 'a' * 40, 'repo', '1', '1') + report.write_text(json.dumps(self.node('root'))) + with self.assertRaisesRegex(ValueError, 'Empty independent'): + inventory_producer.create_inventory(root, ['pom.xml'], 'a' * 40, 'repo', '1', '1') + report.unlink() + report.symlink_to(target / 'outside') + with self.assertRaises(ValueError): + inventory_producer.create_inventory(root, ['pom.xml'], 'a' * 40, 'repo', '1', '1') + + if __name__ == '__main__': unittest.main() diff --git a/.github/workflows/submit-dependencies.yml b/.github/workflows/submit-dependencies.yml index 9b4a9edc46..c29d561ec5 100644 --- a/.github/workflows/submit-dependencies.yml +++ b/.github/workflows/submit-dependencies.yml @@ -107,10 +107,68 @@ jobs: if-no-files-found: error retention-days: 1 + inventory: + needs: validate + if: >- + ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || + github.repository == 'hugegraph/hugegraph') && + github.ref == 'refs/heads/master' && + (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + env: + PYTHONDONTWRITEBYTECODE: '1' + INPUT_TOKEN: '' + GITHUB_TOKEN: '' + GH_TOKEN: '' + outputs: + artifact_id: ${{ steps.inventory.outputs.artifact-id }} + steps: + - name: Checkout event commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Set up project JDK + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 + with: + java-version-file: .github/workflows/.java-version + distribution: 'temurin' + - name: Verify checkout and current master + env: + EXPECTED_SHA: ${{ github.sha }} + run: | + test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" + test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" + - name: Install the complete Maven reactor + run: mvn -B -ntp install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true --fail-at-end + - name: Resolve independent dependency inventory + run: | + mvn -B -ntp org.apache.maven.plugins:maven-dependency-plugin:3.8.1:tree \ + -DoutputType=json -DoutputFile=target/dependency-inventory.json \ + -Dverbose=false -DappendOutput=false \ + -Dmaven.test.skip=true -Dmaven.javadoc.skip=true + python3 .github/scripts/dependency-inventory.py "$RUNNER_TEMP/inventory.json" + - name: Refuse a stale inventory + env: + EXPECTED_SHA: ${{ github.sha }} + run: test "$(git ls-remote origin refs/heads/master | cut -f1)" = "$EXPECTED_SHA" + - name: Upload inventory data only + id: inventory + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: dependency-inventory-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/inventory.json + if-no-files-found: error + retention-days: 1 + submit: - needs: [validate, generate] + needs: [validate, generate, inventory] if: >- needs.validate.result == 'success' && needs.generate.result == 'success' && + needs.inventory.result == 'success' && ((github.repository == 'apache/hugegraph' && github.event.repository.fork == false) || github.repository == 'hugegraph/hugegraph') && github.ref == 'refs/heads/master' && @@ -129,6 +187,7 @@ jobs: RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} ARTIFACT_ID: ${{ needs.generate.outputs.artifact_id }} + INVENTORY_ARTIFACT_ID: ${{ needs.inventory.outputs.artifact_id }} run: | python3 -I - <<'PY' import datetime @@ -179,13 +238,13 @@ jobs: raw = bounded_body(response) return json.loads(raw, object_pairs_hook=unique_object) - def read_snapshot(raw): + def read_snapshot(raw, member_name='snapshot.json'): require(0 < len(raw) <= LIMIT, 'Invalid ZIP size') with zipfile.ZipFile(io.BytesIO(raw)) as archive: members = archive.infolist() - require(len(members) == 1, 'ZIP must contain one snapshot.json') + require(len(members) == 1, 'ZIP must contain one data file') member = members[0] - require(member.orig_filename == 'snapshot.json' and not member.is_dir() and + require(member.orig_filename == member_name and not member.is_dir() and not (member.external_attr & 0x10) and stat.S_IFMT(member.external_attr >> 16) in [0, stat.S_IFREG] and not (member.flag_bits & 1) and @@ -196,7 +255,7 @@ jobs: require(len(payload) == member.file_size, 'Incomplete ZIP member') return json.loads(payload, object_pairs_hook=unique_object) - def download_snapshot(repository, artifact_id, artifact): + def download_snapshot(repository, artifact_id, artifact, member_name='snapshot.json'): opener = urllib.request.build_opener(NoRedirect()) try: response = opener.open(github_request(repository, f'actions/artifacts/{artifact_id}/zip'), timeout=60) @@ -218,7 +277,7 @@ jobs: raw = bounded_body(response) require(len(raw) == artifact['size_in_bytes'] and 'sha256:' + hashlib.sha256(raw).hexdigest() == artifact['digest'], 'Artifact digest/size mismatch') - return read_snapshot(raw) + return read_snapshot(raw, member_name) def timestamp(value): parsed = datetime.datetime.fromisoformat(value.replace('Z', '+00:00')) @@ -239,20 +298,28 @@ jobs: run['path'] in ['.github/workflows/submit-dependencies.yml', '.github/workflows/submit-dependencies.yml@master'], 'Run identity') jobs = api(repository, f'actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100') - producers = [job for job in jobs['jobs'] if job['name'] == 'generate'] - require(jobs['total_count'] <= 100 and len(producers) == 1 and - producers[0]['conclusion'] == 'success' and producers[0]['status'] == 'completed' and - producers[0]['run_id'] == int(run_id) and producers[0]['head_sha'] == sha, - 'Generation did not succeed for this commit') - artifact = api(repository, f'actions/artifacts/{artifact_id}') - require(artifact['id'] == int(artifact_id) and artifact['expired'] is False and - artifact['name'] == f'dependency-snapshot-{run_id}-{attempt}' and - artifact['workflow_run']['id'] == int(run_id) and - artifact['workflow_run']['head_sha'] == sha and - artifact['workflow_run']['head_branch'] == 'master' and - type(artifact['size_in_bytes']) is int and 0 < artifact['size_in_bytes'] <= LIMIT and - isinstance(artifact['digest'], str) and - re.fullmatch('sha256:[0-9a-f]{64}', artifact['digest']), 'Artifact identity') + require(jobs['total_count'] <= 100, 'Too many jobs') + for name in ['generate', 'inventory']: + producers = [job for job in jobs['jobs'] if job['name'] == name] + require(len(producers) == 1 and producers[0]['conclusion'] == 'success' and + producers[0]['status'] == 'completed' and producers[0]['run_id'] == int(run_id) and + producers[0]['head_sha'] == sha, 'Producer did not succeed for this commit') + inventory_id = os.environ['INVENTORY_ARTIFACT_ID'] + require(re.fullmatch('[1-9][0-9]*', inventory_id) and inventory_id != artifact_id, 'Inventory ID') + + def checked_artifact(identifier, kind): + item = api(repository, f'actions/artifacts/{identifier}') + require(item['id'] == int(identifier) and item['expired'] is False and + item['name'] == f'dependency-{kind}-{run_id}-{attempt}' and + item['workflow_run']['id'] == int(run_id) and item['workflow_run']['head_sha'] == sha and + item['workflow_run']['head_branch'] == 'master' and + type(item['size_in_bytes']) is int and 0 < item['size_in_bytes'] <= LIMIT and + isinstance(item['digest'], str) and re.fullmatch('sha256:[0-9a-f]{64}', item['digest']), + 'Artifact identity') + return item + + artifact = checked_artifact(artifact_id, 'snapshot') + inventory_artifact = checked_artifact(inventory_id, 'inventory') snapshot = download_snapshot(repository, artifact_id, artifact) require(isinstance(snapshot, dict) and set(snapshot) == {'version', 'sha', 'ref', 'job', 'detector', 'metadata', 'scanned', 'manifests'}, 'Snapshot schema') @@ -271,10 +338,25 @@ jobs: timestamp(artifact['created_at']) + datetime.timedelta(minutes=1) and timestamp(artifact['created_at']) <= now and timestamp(snapshot['scanned']) <= now, 'Snapshot freshness') + inventory = download_snapshot(repository, inventory_id, inventory_artifact, 'inventory.json') + require(isinstance(inventory, dict) and set(inventory) == + {'version', 'sha', 'repository', 'run_id', 'run_attempt', 'producer', 'resolver', + 'scanned', 'manifests'}, 'Inventory schema') + require(type(inventory['version']) is int and inventory['version'] == 1 and + inventory['sha'] == sha and inventory['repository'] == repository and + inventory['run_id'] == run_id and inventory['run_attempt'] == attempt and + inventory['producer'] == 'inventory' and inventory['resolver'] == + 'org.apache.maven.plugins:maven-dependency-plugin:3.8.1:tree', 'Inventory source') + require(timestamp(run['run_started_at']) <= timestamp(inventory['scanned']) <= + timestamp(inventory_artifact['created_at']) + datetime.timedelta(minutes=1) and + timestamp(inventory_artifact['created_at']) <= now and timestamp(inventory['scanned']) <= now, + 'Inventory freshness') tree = api(repository, f'git/trees/{sha}?recursive=1') require(tree['truncated'] is False, 'Incomplete source tree') poms = {node['path'] for node in tree['tree'] if node['type'] == 'blob' and node['path'].split('/')[-1] == 'pom.xml' and node['mode'] != '120000'} + require(isinstance(inventory['manifests'], dict) and set(inventory['manifests']) == poms, + 'Incomplete independent inventory') manifests, paths = snapshot['manifests'], set() require(isinstance(manifests, dict) and manifests and poms, 'Empty inventory') component = r"(?:[A-Za-z0-9._!~*'():+-]|%[0-9A-F]{2})+" @@ -303,6 +385,9 @@ jobs: 'Package schema') require(not resolved or any(p['relationship'] == 'direct' for p in resolved.values()), 'Missing direct dependencies') + observed = {purl: {'scope': package['scope'], 'relationship': package['relationship']} + for purl, package in resolved.items()} + require(observed == inventory['manifests'][location], 'Dependency inventory mismatch') require(paths == poms and any(m['resolved'] for m in manifests.values()), 'Incomplete inventory') require(api(repository, 'git/ref/heads/master')['object']['sha'] == sha, 'Stale master before submission') result = api(repository, 'dependency-graph/snapshots', snapshot) diff --git a/docs/ci.md b/docs/ci.md index de63f40411..118917926a 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -156,10 +156,20 @@ dispatches. Only `apache/hugegraph` (not a fork) and the explicitly allowed `hugegraph/hugegraph` test fork on `refs/heads/master` can run the submission job, which alone has `contents: write`. PRs, other repositories and other refs cannot run the writer. Maven inherits no submission token, and checkout does not persist -credentials. A pinned `actions/upload-artifact` transfers only `snapshot.json`. -The fresh writer runner uses fixed inline stdlib Python to verify artifact/run -identity before downloading. It bounds ZIP and JSON sizes, checks the archive digest, -and reads only one regular `snapshot.json` member in memory. Duplicate members, +credentials. An independent `contents: read` job resolves all module dependency +scopes with Apache Maven Dependency Plugin 3.8.1 `tree` JSON. Its +[stdlib inventory adapter](../.github/scripts/dependency-inventory.py) records each POM's +package URLs, scopes and direct/indirect relationships independently of the Ferstl +parser. The writer requires exact equality with that inventory, including empty +aggregators; a graph retaining one direct package while omitting another fails. +The global runtime/shipped jar filename inventory has a different scope and does +not replace this per-module check. + +A pinned `actions/upload-artifact` transfers `snapshot.json` and `inventory.json` +as separate artifacts. The fresh writer runner uses fixed inline stdlib Python to +verify both successful producer jobs and both artifact/run/attempt/SHA identities +before downloading. It bounds ZIP and JSON sizes, checks each archive digest, +and reads only its one expected regular JSON member in memory. Duplicate members, paths and symlinks fail; no archive members are extracted to disk. A signed storage redirect is fetched without forwarding GitHub Authorization, and further redirects are refused. The writer does not check out the repository, run Maven, or execute @@ -167,7 +177,7 @@ downloaded code. The first-party upload action is permitted by the [ASF Actions policy](https://infra.apache.org/github-actions-policy.html). Checkout and snapshot use the event commit SHA. Master is checked before install -and again after graph generation and serialization; a stale run fails. The writer +and again after generation in both read-only jobs; a stale run fails. The writer checks the run attempt, successful producer, artifact ID/name, JSON schema, detector/correlator and source SHA/ref. It verifies all manifest paths against the event commit's source tree, rejects empty or mismatched inventory, and checks master @@ -176,7 +186,7 @@ if master advances during submission; that run fails its final freshness check. If submission fails, use **Re-run all jobs** or start a fresh `workflow_dispatch` on master. **Re-run failed jobs** keeps the older generation artifact, which the -writer rejects because the snapshot must be produced in the same run attempt. +writer rejects because both reports must be produced in the same run attempt. A fixed master-only correlator keeps successive baseline snapshots in the same stream within each repository; ASF and org inventories are independent. The writer's From a75e16be16395b394698fae425e9dd6a12a52f91 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 15 Oct 2026 05:29:17 +0800 Subject: [PATCH 8/9] fix(ci): preserve mixed dependency scopes - classify mixed test and non-test scopes as runtime - verify scope parity through the existing inventory writer - document precedence without changing submission flow --- .github/scripts/dependency-snapshot.py | 9 +++++---- .github/scripts/test_dependency_snapshot.py | 17 +++++++++++++++++ docs/ci.md | 6 ++++-- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/scripts/dependency-snapshot.py b/.github/scripts/dependency-snapshot.py index ed71cef60a..55a38b2627 100644 --- a/.github/scripts/dependency-snapshot.py +++ b/.github/scripts/dependency-snapshot.py @@ -16,9 +16,10 @@ """Serialize native Ferstl 4.0.3 graphs; no token or submission code. -Matches maven-dependency-submission-action 6.0.1's Maven parser/toolkit: -first classifier/type, test scope, shared PURL packages, direct precedence, -child edge order and cycle handling. See docs/ci.md for the producer boundary. +Uses maven-dependency-submission-action 6.0.1's Maven parser/toolkit conventions: +first classifier/type, shared PURL packages, direct precedence, child edge order +and cycle handling. Non-test scopes take precedence over test to match the +independent inventory. See docs/ci.md for the producer boundary. """ import datetime @@ -89,7 +90,7 @@ def manifest(graph, pom): raise ValueError(f'Duplicate artifact ID: {identifier}') ids[identifier] = purl packages.setdefault(purl, []) - scopes[purl] = 'development' if 'test' in (artifact.get('scopes') or []) else 'runtime' + scopes[purl] = 'development' if set(artifact.get('scopes') or []) == {'test'} else 'runtime' children, incoming = {}, set() for edge in edges: if not isinstance(edge, dict) or any(not isinstance(edge.get(key), str) for key in ['from', 'to']): diff --git a/.github/scripts/test_dependency_snapshot.py b/.github/scripts/test_dependency_snapshot.py index 7885ed46d5..72dd80df67 100644 --- a/.github/scripts/test_dependency_snapshot.py +++ b/.github/scripts/test_dependency_snapshot.py @@ -195,6 +195,23 @@ def test_native_cycle_alias_scope_and_direct_precedence(self): self.assertEqual({}, producer.manifest({'graphName': 'parent', 'artifacts': [artifact('root')]}, 'pom.xml')['resolved']) + def test_native_mixed_scopes_match_independent_inventory(self): + for scope in ['compile', 'runtime', 'provided', 'system']: + for scopes in [[scope, 'test'], ['test', scope]]: + with self.subTest(scopes=scopes): + graph = {'graphName': 'module', 'artifacts': [ + artifact('root'), artifact('parent', scopes=['test']), artifact('one', scopes=scopes)], + 'dependencies': [{'from': 'root', 'to': 'parent'}, {'from': 'parent', 'to': 'one'}, + {'from': 'root', 'to': 'one'}]} + self.snapshot['manifests']['module'] = producer.manifest(graph, 'pom.xml') + tree = InventoryTest.node('root', children=[ + InventoryTest.node('parent', 'test', [InventoryTest.node('one', 'test')]), + InventoryTest.node('one', scope)]) + self.inventory['manifests']['pom.xml'] = inventory_producer.packages(tree) + self.posts.clear() + self.run_writer() + self.assertEqual([self.snapshot], self.posts) + def test_transitive_only_package_is_indirect(self): graph = {'graphName': 'module', 'artifacts': [artifact('root'), artifact('parent'), artifact('leaf')], 'dependencies': [{'from': 'root', 'to': 'parent'}, {'from': 'parent', 'to': 'leaf'}]} diff --git a/docs/ci.md b/docs/ci.md index 118917926a..0e80499ed8 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -150,8 +150,10 @@ installs the complete reactor with Java 17 in a `contents: read` job, then gener each module's resolved graph with Ferstl 4.0.3. The repository's [stdlib serializer](../.github/scripts/dependency-snapshot.py) preserves the [official Maven parser 6.0.1](https://github.com/actions/maven-dependency-submission-action/tree/1f666f7ac2ab7b72cd5e22eaf7f23c8a581db1db) -package URLs, scopes and relationships. Every tracked POM must have a graph; empty -aggregator manifests are allowed, but a completely empty inventory fails. It runs on master pushes or manual +package URLs and relationships. Non-test scopes take precedence when Ferstl merges +scopes onto one artifact; only test-only artifacts are development dependencies. +Every tracked POM must have a graph; empty aggregator manifests are allowed, but a +completely empty inventory fails. It runs on master pushes or manual dispatches. Only `apache/hugegraph` (not a fork) and the explicitly allowed `hugegraph/hugegraph` test fork on `refs/heads/master` can run the submission job, which alone has `contents: write`. PRs, other repositories and other refs cannot From e821d8e79e74666435d1d6fa73fa1432ebbd1cca Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 15 Oct 2026 08:50:45 +0800 Subject: [PATCH 9/9] docs(ci): clarify baseline review reruns - require a successful baseline for the exact PR event base SHA - distinguish the same repository from other baseline streams - explain missing or skipped baselines are not fixed by review reruns --- docs/ci.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/ci.md b/docs/ci.md index 0e80499ed8..49b471c4c5 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -190,6 +190,12 @@ If submission fails, use **Re-run all jobs** or start a fresh `workflow_dispatch on master. **Re-run failed jobs** keeps the older generation artifact, which the writer rejects because both reports must be produced in the same run attempt. +If a PR `dependency-review` failure reports 0 base snapshots, rerun that job only +once the same repository's **Dependency baseline** run has succeeded for the event's +exact `pull_request.base.sha` (not the branch's merge base or a newer master tip). +If that SHA has no successful baseline, including a skipped run, rerunning the +review alone cannot supply it. + A fixed master-only correlator keeps successive baseline snapshots in the same stream within each repository; ASF and org inventories are independent. The writer's identity is separate from automatic PR submissions so they cannot replace its baseline.