diff --git a/.github/workflows/package-update-kayobe.yml b/.github/workflows/package-update-kayobe.yml index f007bd03..511a4775 100644 --- a/.github/workflows/package-update-kayobe.yml +++ b/.github/workflows/package-update-kayobe.yml @@ -12,6 +12,10 @@ on: required: false description: Branch of StackHPC Kayobe configuration to use default: stackhpc/2026.1 + debug: + description: "Enable debug logging" + type: boolean + default: false env: ANSIBLE_FORCE_COLOR: true @@ -44,7 +48,8 @@ jobs: ansible/test-kayobe-repo-version-generate.yml \ -e deb_package_repo_filter="'$FILTER'" \ -e rpm_package_repo_filter="'$FILTER'" \ - -e kayobe_config_repo_path=./stackhpc-kayobe-config/ + -e kayobe_config_repo_path=./stackhpc-kayobe-config/ \ + -e release_train_debug=${{ github.event.inputs.debug }} env: FILTER: ${{ github.event.inputs.filter }} diff --git a/ansible/filter_plugins/filters.py b/ansible/filter_plugins/filters.py index f3045d4e..054886e1 100644 --- a/ansible/filter_plugins/filters.py +++ b/ansible/filter_plugins/filters.py @@ -12,9 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. +import posixpath import re +_VERSION_SUFFIX = re.compile(r"-[0-9]{8}T[0-9]{6}$") + + def select_repos(repos, filter_string, package_sync_group): """Select repositories that match a filter string and package sync group. @@ -50,6 +54,62 @@ def select_images(images, filter_string): patterns = re.compile(r"|".join(regexes).join('()')) return [image for image in images if re.search(patterns, image)] +def _repo_version(pub): + # Orders by version, not timestamp. + return int(pub["repository_version"].rstrip("/").rsplit("/", 1)[-1]) + + +def _resolve(dist_specs, repositories, publications, distributions): + """Resolve each desired distribution to its repo, latest publication and distribution. + + Builds the lookup tables once, rather than re-scanning every list for each + repository as the equivalent Jinja did. + """ + repo_by_name = {repo["name"]: repo for repo in repositories} + + # Newest publication per repository href. + newest_pub = {} + for pub in sorted(publications, key=_repo_version, reverse=True): + newest_pub.setdefault(pub["repository"], pub) + + dists_by_pub = {} + for dist in distributions: + dists_by_pub.setdefault(dist.get("publication"), []).append(dist) + + for spec in dist_specs: + repo = repo_by_name.get(spec["repository"]) + pub = newest_pub.get(repo["pulp_href"]) if repo else None + dist = None + if pub: + base_name = _VERSION_SUFFIX.sub("", spec["name"]) + pattern = re.compile( + r"^%s(-[0-9]{8}T[0-9]{6})?$" % re.escape(base_name)) + dist = next((d for d in dists_by_pub.get(pub["pulp_href"], []) + if pattern.match(d["name"])), None) + yield spec, pub, dist + +def latest_repo_versions(dist_specs, repositories, publications, distributions): + """Map repository short_name to the version of its latest distribution.""" + return { + spec["short_name"]: posixpath.basename(dist["base_path"]) + for spec, _pub, dist in _resolve(dist_specs, repositories, + publications, distributions) + if dist + } + +def distributions_to_create(dist_specs, repositories, publications, + distributions): + """Return specs whose latest publication has no distribution serving it. + + Each returned spec gains a 'publication' key holding that publication's href. + """ + return [ + dict(spec, publication=pub["pulp_href"]) + for spec, pub, dist in _resolve(dist_specs, repositories, + publications, distributions) + if pub and not dist + ] + class FilterModule(object): @@ -57,4 +117,6 @@ def filters(self): return { "select_repos": select_repos, "select_images": select_images, + "latest_repo_versions": latest_repo_versions, + "distributions_to_create": distributions_to_create, } diff --git a/ansible/inventory/group_vars/all/meta b/ansible/inventory/group_vars/all/meta new file mode 100644 index 00000000..ebdaebf0 --- /dev/null +++ b/ansible/inventory/group_vars/all/meta @@ -0,0 +1,3 @@ +--- +# Enable to print debug logs in release train workflows +release_train_debug: false diff --git a/ansible/kayobe-repo-version-query.yml b/ansible/kayobe-repo-version-query.yml index 454a2af1..ee745c33 100644 --- a/ansible/kayobe-repo-version-query.yml +++ b/ansible/kayobe-repo-version-query.yml @@ -46,3 +46,5 @@ - name: Display Kayobe Pulp repository versions ansible.builtin.debug: var: kayobe_pulp_repo_versions + when: + - release_train_debug | bool diff --git a/ansible/test-pulp-repo-version-query.yml b/ansible/test-pulp-repo-version-query.yml index 22c68d64..ac8197d4 100644 --- a/ansible/test-pulp-repo-version-query.yml +++ b/ansible/test-pulp-repo-version-query.yml @@ -63,144 +63,102 @@ password: "{{ pulp_password }}" register: pulp_rpm_dists_list - - vars: - repo_list: >- - {{ pulp_deb_repos_list.repositories - | selectattr('name', 'equalto', item.repository) - | list }} - repo: "{{ repo_list[0] if (repo_list | length > 0) else {} }}" - pubs: >- - {{ (pulp_deb_pubs_list.publications + pulp_deb_verbatim_pubs_list.publications) - | selectattr('repository', 'equalto', repo.get('pulp_href', '')) - | stackhpc.pulp.sort_publications | list }} - pub: "{{ pubs[0] if (pubs | length > 0) else {} }}" - base_name: "{{ item.name | regex_replace('-[0-9]{8}T[0-9]{6}$', '') }}" - safe_pattern: "^{{ base_name | regex_escape }}(-[0-9]{8}T[0-9]{6})?$" - existing_dists: >- - {{ pulp_deb_dists_list.distributions - | selectattr('publication', 'equalto', pub.get('pulp_href', '')) - | selectattr('name', 'match', safe_pattern) - | list }} - dist: "{{ existing_dists[0] if (existing_dists | length > 0) else {} }}" - - block: - - name: Ensure Deb distributions exist for latest publication - pulp.squeezer.deb_distribution: - pulp_url: "{{ pulp_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - name: "{{ item.name }}" - base_path: "{{ item.base_path }}" - publication: "{{ pub.get('pulp_href') }}" - content_guard: "{{ item.content_guard | default(omit) }}" - state: present - loop: "{{ dev_pulp_distribution_deb }}" - loop_control: - label: "{{ item.repository }}" - when: (pub | length > 0) and (existing_dists | length == 0) - - - name: Re-query Deb distributions (to capture newly created ones) - pulp.squeezer.deb_distribution: - pulp_url: "{{ pulp_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - register: pulp_deb_dists_list - - - name: Display latest Deb distributions - vars: - info: - repo: "{{ repo }}" - pub: "{{ pub }}" - dist: "{{ dist }}" - ansible.builtin.debug: - var: info - loop: "{{ dev_pulp_distribution_deb }}" - loop_control: - label: "{{ item.repository }}" - when: pub | length > 0 - - - name: Set a fact about latest Deb versions - ansible.builtin.set_fact: - test_pulp_repository_deb_repo_versions: >- - {{ test_pulp_repository_deb_repo_versions - | default({}) - | combine({item.short_name: dist.base_path | basename}) }} - loop: "{{ dev_pulp_distribution_deb }}" - loop_control: - label: "{{ item.repository }}" - when: dist | length > 0 - - - name: Display latest versions fact - ansible.builtin.debug: - var: test_pulp_repository_deb_repo_versions - - - vars: - repo_list: >- - {{ pulp_rpm_repos_list.repositories - | selectattr('name', 'equalto', item.repository) - | list }} - repo: "{{ repo_list[0] if (repo_list | length > 0) else {} }}" - pubs: >- - {{ pulp_rpm_pubs_list.publications - | selectattr('repository', 'equalto', repo.get('pulp_href', '')) - | stackhpc.pulp.sort_publications | list }} - pub: "{{ pubs[0] if (pubs | length > 0) else {} }}" - base_name: "{{ item.name | regex_replace('-[0-9]{8}T[0-9]{6}$', '') }}" - safe_pattern: "^{{ base_name | regex_escape }}(-[0-9]{8}T[0-9]{6})?$" - existing_dists: >- - {{ pulp_rpm_dists_list.distributions - | selectattr('publication', 'equalto', pub.get('pulp_href', '')) - | selectattr('name', 'match', safe_pattern) - | list }} - dist: "{{ existing_dists[0] if (existing_dists | length > 0) else {} }}" - - block: - - name: Ensure RPM distributions exist for latest publication - pulp.squeezer.rpm_distribution: - pulp_url: "{{ pulp_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - name: "{{ item.name }}" - base_path: "{{ item.base_path }}" - publication: "{{ pub.get('pulp_href') }}" - content_guard: "{{ item.content_guard | default(omit) }}" - state: present - loop: "{{ dev_pulp_distribution_rpm }}" - loop_control: - label: "{{ item.repository }}" - when: (pub | length > 0) and (existing_dists | length == 0) - - - name: Re-query RPM distributions (to capture newly created ones) - pulp.squeezer.rpm_distribution: - pulp_url: "{{ pulp_url }}" - username: "{{ pulp_username }}" - password: "{{ pulp_password }}" - register: pulp_rpm_dists_list - - - name: Display latest RPM distributions - vars: - info: - repo: "{{ repo }}" - pub: "{{ pub }}" - dist: "{{ dist }}" - ansible.builtin.debug: - var: info - loop: "{{ dev_pulp_distribution_rpm }}" - loop_control: - label: "{{ item.repository }}" - when: pub | length > 0 - - - name: Set a fact about latest RPM versions - ansible.builtin.set_fact: - test_pulp_repository_rpm_repo_versions: >- - {{ test_pulp_repository_rpm_repo_versions - | default({}) - | combine({item.short_name: dist.base_path | basename}) }} - loop: "{{ dev_pulp_distribution_rpm }}" - loop_control: - label: "{{ item.repository }}" - when: dist | length > 0 - - - name: Display latest versions fact - ansible.builtin.debug: - var: test_pulp_repository_rpm_repo_versions + - name: Create missing Deb distributions for the latest publication + pulp.squeezer.deb_distribution: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + name: "{{ item.name }}" + base_path: "{{ item.base_path }}" + publication: "{{ item.publication }}" + content_guard: "{{ item.content_guard | default(omit) }}" + state: present + loop: >- + {{ dev_pulp_distribution_deb + | distributions_to_create( + pulp_deb_repos_list.repositories, + pulp_deb_pubs_list.publications + + pulp_deb_verbatim_pubs_list.publications, + pulp_deb_dists_list.distributions) }} + loop_control: + label: "{{ item.repository }}" + register: deb_dists_created + + - name: Re-query Deb distributions (to capture newly created ones) + pulp.squeezer.deb_distribution: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + register: pulp_deb_dists_requery + when: deb_dists_created is changed # noqa: no-handler + + - name: Set a fact about latest Deb versions + ansible.builtin.set_fact: + test_pulp_repository_deb_repo_versions: >- + {{ dev_pulp_distribution_deb + | latest_repo_versions( + pulp_deb_repos_list.repositories, + pulp_deb_pubs_list.publications + + pulp_deb_verbatim_pubs_list.publications, + pulp_deb_dists_requery.distributions + | default(pulp_deb_dists_list.distributions, true)) }} + + - name: Display latest Deb versions + ansible.builtin.debug: + var: test_pulp_repository_deb_repo_versions + when: release_train_debug | bool + + - name: Create missing RPM distributions for the latest publication + pulp.squeezer.rpm_distribution: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + name: "{{ item.name }}" + base_path: "{{ item.base_path }}" + publication: "{{ item.publication }}" + content_guard: "{{ item.content_guard | default(omit) }}" + state: present + loop: >- + {{ dev_pulp_distribution_rpm + | distributions_to_create( + pulp_rpm_repos_list.repositories, + pulp_rpm_pubs_list.publications, + pulp_rpm_dists_list.distributions) }} + loop_control: + label: "{{ item.repository }}" + register: rpm_dists_created + + - name: Re-query RPM distributions (to capture newly created ones) + pulp.squeezer.rpm_distribution: + pulp_url: "{{ pulp_url }}" + username: "{{ pulp_username }}" + password: "{{ pulp_password }}" + register: pulp_rpm_dists_requery + when: rpm_dists_created is changed # noqa: no-handler + + - name: Set a fact about latest RPM versions + ansible.builtin.set_fact: + test_pulp_repository_rpm_repo_versions: >- + {{ dev_pulp_distribution_rpm + | latest_repo_versions( + pulp_rpm_repos_list.repositories, + pulp_rpm_pubs_list.publications, + pulp_rpm_dists_requery.distributions + | default(pulp_rpm_dists_list.distributions, true)) }} + + - name: Display latest RPM versions + ansible.builtin.debug: + var: test_pulp_repository_rpm_repo_versions + when: release_train_debug | bool + + - name: Fail if no RPM repository versions were resolved + ansible.builtin.assert: + that: test_pulp_repository_rpm_repo_versions | length > 0 + fail_msg: "No RPM repository versions resolved from {{ pulp_url }}" + when: dev_pulp_distribution_rpm | length > 0 + + - name: Fail if no Deb repository versions were resolved + ansible.builtin.assert: + that: test_pulp_repository_deb_repo_versions | length > 0 + fail_msg: "No Deb repository versions resolved from {{ pulp_url }}" + when: dev_pulp_distribution_deb | length > 0