From be8056c90f38c6198d9ec08d03ae6a1a7e8f892d Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Fri, 28 Aug 2026 11:42:33 +0100 Subject: [PATCH 1/2] Add debug option to package update workflow Previously, we would print a bunch of debug output every time we ran the version query playbook. That ouput is super slow, so this change hides it behind an input, so it's still an option but not the default. Added a general release_train_debug variable, so the pattern can be repeated in the future. --- .github/workflows/package-update-kayobe.yml | 7 ++++++- ansible/inventory/group_vars/all/meta | 3 +++ ansible/kayobe-repo-version-query.yml | 2 ++ ansible/test-pulp-repo-version-query.yml | 12 ++++++++++-- 4 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 ansible/inventory/group_vars/all/meta diff --git a/.github/workflows/package-update-kayobe.yml b/.github/workflows/package-update-kayobe.yml index f007bd03..511a4775 100644 --- a/.github/workflows/package-update-kayobe.yml +++ b/.github/workflows/package-update-kayobe.yml @@ -12,6 +12,10 @@ on: required: false description: Branch of StackHPC Kayobe configuration to use default: stackhpc/2026.1 + debug: + description: "Enable debug logging" + type: boolean + default: false env: ANSIBLE_FORCE_COLOR: true @@ -44,7 +48,8 @@ jobs: ansible/test-kayobe-repo-version-generate.yml \ -e deb_package_repo_filter="'$FILTER'" \ -e rpm_package_repo_filter="'$FILTER'" \ - -e kayobe_config_repo_path=./stackhpc-kayobe-config/ + -e kayobe_config_repo_path=./stackhpc-kayobe-config/ \ + -e release_train_debug=${{ github.event.inputs.debug }} env: FILTER: ${{ github.event.inputs.filter }} diff --git a/ansible/inventory/group_vars/all/meta b/ansible/inventory/group_vars/all/meta new file mode 100644 index 00000000..ebdaebf0 --- /dev/null +++ b/ansible/inventory/group_vars/all/meta @@ -0,0 +1,3 @@ +--- +# Enable to print debug logs in release train workflows +release_train_debug: false diff --git a/ansible/kayobe-repo-version-query.yml b/ansible/kayobe-repo-version-query.yml index 454a2af1..ee745c33 100644 --- a/ansible/kayobe-repo-version-query.yml +++ b/ansible/kayobe-repo-version-query.yml @@ -46,3 +46,5 @@ - name: Display Kayobe Pulp repository versions ansible.builtin.debug: var: kayobe_pulp_repo_versions + when: + - release_train_debug | bool diff --git a/ansible/test-pulp-repo-version-query.yml b/ansible/test-pulp-repo-version-query.yml index 22c68d64..09d12d2e 100644 --- a/ansible/test-pulp-repo-version-query.yml +++ b/ansible/test-pulp-repo-version-query.yml @@ -117,7 +117,9 @@ loop: "{{ dev_pulp_distribution_deb }}" loop_control: label: "{{ item.repository }}" - when: pub | length > 0 + when: + - release_train_debug | bool + - pub | length > 0 - name: Set a fact about latest Deb versions ansible.builtin.set_fact: @@ -133,6 +135,8 @@ - name: Display latest versions fact ansible.builtin.debug: var: test_pulp_repository_deb_repo_versions + when: + - release_train_debug | bool - vars: repo_list: >- @@ -188,7 +192,9 @@ loop: "{{ dev_pulp_distribution_rpm }}" loop_control: label: "{{ item.repository }}" - when: pub | length > 0 + when: + - release_train_debug | bool + - pub | length > 0 - name: Set a fact about latest RPM versions ansible.builtin.set_fact: @@ -204,3 +210,5 @@ - name: Display latest versions fact ansible.builtin.debug: var: test_pulp_repository_rpm_repo_versions + when: + - release_train_debug | bool From 7458d0463abf70ad2ffbd4cbc2c08cc8054e2663 Mon Sep 17 00:00:00 2001 From: Alex Welsh Date: Fri, 28 Aug 2026 15:06:58 +0100 Subject: [PATCH 2/2] Move pulp version query to python filters --- ansible/filter_plugins/filters.py | 62 ++++++ ansible/test-pulp-repo-version-query.yml | 248 +++++++++-------------- 2 files changed, 161 insertions(+), 149 deletions(-) diff --git a/ansible/filter_plugins/filters.py b/ansible/filter_plugins/filters.py index f3045d4e..054886e1 100644 --- a/ansible/filter_plugins/filters.py +++ b/ansible/filter_plugins/filters.py @@ -12,9 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. +import posixpath import re +_VERSION_SUFFIX = re.compile(r"-[0-9]{8}T[0-9]{6}$") + + def select_repos(repos, filter_string, package_sync_group): """Select repositories that match a filter string and package sync group. @@ -50,6 +54,62 @@ def select_images(images, filter_string): patterns = re.compile(r"|".join(regexes).join('()')) return [image for image in images if re.search(patterns, image)] +def _repo_version(pub): + # Orders by version, not timestamp. + return int(pub["repository_version"].rstrip("/").rsplit("/", 1)[-1]) + + +def _resolve(dist_specs, repositories, publications, distributions): + """Resolve each desired distribution to its repo, latest publication and distribution. + + Builds the lookup tables once, rather than re-scanning every list for each + repository as the equivalent Jinja did. + """ + repo_by_name = {repo["name"]: repo for repo in repositories} + + # Newest publication per repository href. + newest_pub = {} + for pub in sorted(publications, key=_repo_version, reverse=True): + newest_pub.setdefault(pub["repository"], pub) + + dists_by_pub = {} + for dist in distributions: + dists_by_pub.setdefault(dist.get("publication"), []).append(dist) + + for spec in dist_specs: + repo = repo_by_name.get(spec["repository"]) + pub = newest_pub.get(repo["pulp_href"]) if repo else None + dist = None + if pub: + base_name = _VERSION_SUFFIX.sub("", spec["name"]) + pattern = re.compile( + r"^%s(-[0-9]{8}T[0-9]{6})?$" % re.escape(base_name)) + dist = next((d for d in dists_by_pub.get(pub["pulp_href"], []) + if pattern.match(d["name"])), None) + yield spec, pub, dist + +def latest_repo_versions(dist_specs, repositories, publications, distributions): + """Map repository short_name to the version of its latest distribution.""" + return { + spec["short_name"]: posixpath.basename(dist["base_path"]) + for spec, _pub, dist in _resolve(dist_specs, repositories, + publications, distributions) + if dist + } + +def distributions_to_create(dist_specs, repositories, publications, + distributions): + """Return specs whose latest publication has no distribution serving it. + + Each returned spec gains a 'publication' key holding that publication's href. + """ + return [ + dict(spec, publication=pub["pulp_href"]) + for spec, pub, dist in _resolve(dist_specs, repositories, + publications, distributions) + if pub and not dist + ] + class FilterModule(object): @@ -57,4 +117,6 @@ def filters(self): return { "select_repos": select_repos, "select_images": select_images, + "latest_repo_versions": latest_repo_versions, + "distributions_to_create": distributions_to_create, } diff --git a/ansible/test-pulp-repo-version-query.yml b/ansible/test-pulp-repo-version-query.yml index 09d12d2e..ac8197d4 100644 --- a/ansible/test-pulp-repo-version-query.yml +++ b/ansible/test-pulp-repo-version-query.yml @@ -63,152 +63,102 @@ password: "{{ pulp_password }}" register: pulp_rpm_dists_list - - vars: - repo_list: >- - {{ pulp_deb_repos_list.repositories - | selectattr('name', 'equalto', item.repository) - | list }} - repo: "{{ repo_list[0] if (repo_list | length > 0) else {} }}" - pubs: >- - {{ (pulp_deb_pubs_list.publications + pulp_deb_verbatim_pubs_list.publications) - | selectattr('repository', 'equalto', repo.get('pulp_href', '')) - | stackhpc.pulp.sort_publications | list }} - pub: "{{ pubs[0] if (pubs | length > 0) else {} }}" - base_name: "{{ item.name | regex_replace('-[0-9]{8}T[0-9]{6}$', '') }}" - safe_pattern: "^{{ base_name | regex_escape }}(-[0-9]{8}T[0-9]{6})?$" - existing_dists: >- - {{ pulp_deb_dists_list.distributions - | selectattr('publication', 'equalto', pub.get('pulp_href', '')) - | selectattr('name', 'match', safe_pattern) - | list }} - dist: "{{ existing_dists[0] if (existing_dists | length > 0) else {} }}" - - block: - - name: Ensure Deb distributions exist for latest publication - pulp.squeezer.deb_distribution: - pulp_url: "{{ pulp_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - name: "{{ item.name }}" - base_path: "{{ item.base_path }}" - publication: "{{ pub.get('pulp_href') }}" - content_guard: "{{ item.content_guard | default(omit) }}" - state: present - loop: "{{ dev_pulp_distribution_deb }}" - loop_control: - label: "{{ item.repository }}" - when: (pub | length > 0) and (existing_dists | length == 0) - - - name: Re-query Deb distributions (to capture newly created ones) - pulp.squeezer.deb_distribution: - pulp_url: "{{ pulp_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - register: pulp_deb_dists_list - - - name: Display latest Deb distributions - vars: - info: - repo: "{{ repo }}" - pub: "{{ pub }}" - dist: "{{ dist }}" - ansible.builtin.debug: - var: info - loop: "{{ dev_pulp_distribution_deb }}" - loop_control: - label: "{{ item.repository }}" - when: - - release_train_debug | bool - - pub | length > 0 - - - name: Set a fact about latest Deb versions - ansible.builtin.set_fact: - test_pulp_repository_deb_repo_versions: >- - {{ test_pulp_repository_deb_repo_versions - | default({}) - | combine({item.short_name: dist.base_path | basename}) }} - loop: "{{ dev_pulp_distribution_deb }}" - loop_control: - label: "{{ item.repository }}" - when: dist | length > 0 - - - name: Display latest versions fact - ansible.builtin.debug: - var: test_pulp_repository_deb_repo_versions - when: - - release_train_debug | bool - - - vars: - repo_list: >- - {{ pulp_rpm_repos_list.repositories - | selectattr('name', 'equalto', item.repository) - | list }} - repo: "{{ repo_list[0] if (repo_list | length > 0) else {} }}" - pubs: >- - {{ pulp_rpm_pubs_list.publications - | selectattr('repository', 'equalto', repo.get('pulp_href', '')) - | stackhpc.pulp.sort_publications | list }} - pub: "{{ pubs[0] if (pubs | length > 0) else {} }}" - base_name: "{{ item.name | regex_replace('-[0-9]{8}T[0-9]{6}$', '') }}" - safe_pattern: "^{{ base_name | regex_escape }}(-[0-9]{8}T[0-9]{6})?$" - existing_dists: >- - {{ pulp_rpm_dists_list.distributions - | selectattr('publication', 'equalto', pub.get('pulp_href', '')) - | selectattr('name', 'match', safe_pattern) - | list }} - dist: "{{ existing_dists[0] if (existing_dists | length > 0) else {} }}" - - block: - - name: Ensure RPM distributions exist for latest publication - pulp.squeezer.rpm_distribution: - pulp_url: "{{ pulp_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - name: "{{ item.name }}" - base_path: "{{ item.base_path }}" - publication: "{{ pub.get('pulp_href') }}" - content_guard: "{{ item.content_guard | default(omit) }}" - state: present - loop: "{{ dev_pulp_distribution_rpm }}" - loop_control: - label: "{{ item.repository }}" - when: (pub | length > 0) and (existing_dists | length == 0) - - - name: Re-query RPM distributions (to capture newly created ones) - pulp.squeezer.rpm_distribution: - pulp_url: "{{ pulp_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - register: pulp_rpm_dists_list - - - name: Display latest RPM distributions - vars: - info: - repo: "{{ repo }}" - pub: "{{ pub }}" - dist: "{{ dist }}" - ansible.builtin.debug: - var: info - loop: "{{ dev_pulp_distribution_rpm }}" - loop_control: - label: "{{ item.repository }}" - when: - - release_train_debug | bool - - pub | length > 0 - - - name: Set a fact about latest RPM versions - ansible.builtin.set_fact: - test_pulp_repository_rpm_repo_versions: >- - {{ test_pulp_repository_rpm_repo_versions - | default({}) - | combine({item.short_name: dist.base_path | basename}) }} - loop: "{{ dev_pulp_distribution_rpm }}" - loop_control: - label: "{{ item.repository }}" - when: dist | length > 0 - - - name: Display latest versions fact - ansible.builtin.debug: - var: test_pulp_repository_rpm_repo_versions - when: - - release_train_debug | bool + - name: Create missing Deb distributions for the latest publication + pulp.squeezer.deb_distribution: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + name: "{{ item.name }}" + base_path: "{{ item.base_path }}" + publication: "{{ item.publication }}" + content_guard: "{{ item.content_guard | default(omit) }}" + state: present + loop: >- + {{ dev_pulp_distribution_deb + | distributions_to_create( + pulp_deb_repos_list.repositories, + pulp_deb_pubs_list.publications + + pulp_deb_verbatim_pubs_list.publications, + pulp_deb_dists_list.distributions) }} + loop_control: + label: "{{ item.repository }}" + register: deb_dists_created + + - name: Re-query Deb distributions (to capture newly created ones) + pulp.squeezer.deb_distribution: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + register: pulp_deb_dists_requery + when: deb_dists_created is changed # noqa: no-handler + + - name: Set a fact about latest Deb versions + ansible.builtin.set_fact: + test_pulp_repository_deb_repo_versions: >- + {{ dev_pulp_distribution_deb + | latest_repo_versions( + pulp_deb_repos_list.repositories, + pulp_deb_pubs_list.publications + + pulp_deb_verbatim_pubs_list.publications, + pulp_deb_dists_requery.distributions + | default(pulp_deb_dists_list.distributions, true)) }} + + - name: Display latest Deb versions + ansible.builtin.debug: + var: test_pulp_repository_deb_repo_versions + when: release_train_debug | bool + + - name: Create missing RPM distributions for the latest publication + pulp.squeezer.rpm_distribution: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + name: "{{ item.name }}" + base_path: "{{ item.base_path }}" + publication: "{{ item.publication }}" + content_guard: "{{ item.content_guard | default(omit) }}" + state: present + loop: >- + {{ dev_pulp_distribution_rpm + | distributions_to_create( + pulp_rpm_repos_list.repositories, + pulp_rpm_pubs_list.publications, + pulp_rpm_dists_list.distributions) }} + loop_control: + label: "{{ item.repository }}" + register: rpm_dists_created + + - name: Re-query RPM distributions (to capture newly created ones) + pulp.squeezer.rpm_distribution: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + register: pulp_rpm_dists_requery + when: rpm_dists_created is changed # noqa: no-handler + + - name: Set a fact about latest RPM versions + ansible.builtin.set_fact: + test_pulp_repository_rpm_repo_versions: >- + {{ dev_pulp_distribution_rpm + | latest_repo_versions( + pulp_rpm_repos_list.repositories, + pulp_rpm_pubs_list.publications, + pulp_rpm_dists_requery.distributions + | default(pulp_rpm_dists_list.distributions, true)) }} + + - name: Display latest RPM versions + ansible.builtin.debug: + var: test_pulp_repository_rpm_repo_versions + when: release_train_debug | bool + + - name: Fail if no RPM repository versions were resolved + ansible.builtin.assert: + that: test_pulp_repository_rpm_repo_versions | length > 0 + fail_msg: "No RPM repository versions resolved from {{ pulp_url }}" + when: dev_pulp_distribution_rpm | length > 0 + + - name: Fail if no Deb repository versions were resolved + ansible.builtin.assert: + that: test_pulp_repository_deb_repo_versions | length > 0 + fail_msg: "No Deb repository versions resolved from {{ pulp_url }}" + when: dev_pulp_distribution_deb | length > 0