From a0022dd745bf4b314c00a28bd5d3959af4ad5a75 Mon Sep 17 00:00:00 2001 From: Itay Matza Date: Fri, 14 Aug 2026 14:20:40 +0300 Subject: [PATCH] [multiple] Enable Zuul auto-retry for infrastructure failures Remove explicit attempts: 1 overrides from Zuul base job definitions, restoring auto-retry for pre-run playbook failures and unreachable-node errors. Add timeout and pre-timeout to molecule CRC jobs and multinode base jobs to address CRC startup timeouts. Fix content-provider-bootc timeout and mark broken whitebox test non-voting. Changes: - zuul.d/base.yaml: remove attempts: 1 from 3 multinode base jobs (inherit attempts: 3 from parent base-crc-cloud). Add attempts: 3 to cifmw-base-crc (override base-simple-crc parent). Add pre-timeout: 3600 to cifmw-podified-multinode-edpm-base-crc (cap pre-run at 60 min so CRC hangs retry instead of consuming the full 3h timeout). - zuul.d/adoption.yaml: change attempts: 1 to attempts: 2 on 4 adoption base jobs (limit retries for long-running jobs). - zuul.d/molecule-base.yaml: add timeout: 3600 (60 min), pre-timeout: 2100 (35 min cap on CRC startup), and attempts: 3 to cifmw-molecule-base-crc. - zuul.d/content_provider.yaml: add timeout: 7200 and attempts: 2 to content-provider-bootc (parent timeout of 2700 is insufficient for bootc image builds, timed out at 47 min in buildset e2b7708f). - zuul.d/project-templates.yaml: set whitebox-neutron-tempest job non-voting on ci-framework (71% failure rate from broken test_igmp_snooping_after_openvswitch_restart, independent of PR changes, uses override-checkout: main). Assisted-By: Claude Code Signed-off-by: Itay Matza --- zuul.d/adoption.yaml | 8 ++++---- zuul.d/base.yaml | 9 ++++++--- zuul.d/content_provider.yaml | 2 ++ zuul.d/molecule-base.yaml | 3 +++ zuul.d/project-templates.yaml | 1 + 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/zuul.d/adoption.yaml b/zuul.d/adoption.yaml index 31aa9e3d7..9ea62c9dc 100644 --- a/zuul.d/adoption.yaml +++ b/zuul.d/adoption.yaml @@ -6,7 +6,7 @@ parent: base-crc-cloud abstract: true timeout: 14400 - attempts: 1 + attempts: 2 nodeset: centos-9-rhel-9-2-crc-cloud-ocp-4-22-1-3xl roles: - zuul: github.com/openstack-k8s-operators/ci-framework @@ -201,7 +201,7 @@ parent: base-crc-cloud abstract: true timeout: 14400 - attempts: 1 + attempts: 2 nodeset: centos-9-multinode-rhel-9-2-crc-cloud-ocp-4-22-1-3xl roles: &multinode-roles - zuul: github.com/openstack-k8s-operators/ci-framework @@ -412,7 +412,7 @@ abstract: true voting: false timeout: 14400 - attempts: 1 + attempts: 2 nodeset: centos-9-multinode-rhel-9-2-crc-cloud-ocp-4-22-1-3xl-novacells roles: - zuul: github.com/openstack-k8s-operators/ci-framework @@ -496,7 +496,7 @@ name: cifmw-adoption-base-multinode-networker parent: base-crc-cloud abstract: true - attempts: 1 + attempts: 2 roles: *multinode-roles pre-run: *multinode-prerun post-run: *multinode-postrun diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index db4974783..1f8f8da70 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -127,7 +127,7 @@ name: cifmw-podified-multinode-edpm-base-crc parent: base-crc-cloud timeout: 10800 - attempts: 1 + pre-timeout: 3600 nodeset: centos-9-medium-centos-9-crc-cloud-ocp-4-20-1-3xl irrelevant-files: *ir_files required-projects: &multinode_edpm_rp @@ -215,7 +215,8 @@ name: cifmw-podified-multinode-edpm-ci-bootstrap parent: base-extracted-crc-ci-bootstrap timeout: 10800 - attempts: 1 + attempts: 3 + pre-timeout: 3600 nodeset: centos-9-medium-centos-9-crc-cloud-ocp-4-20-1-3xl irrelevant-files: *ir_files required-projects: *multinode_edpm_rp @@ -235,7 +236,8 @@ name: cifmw-podified-multinode-edpm-ci-bootstrap-staging parent: base-extracted-crc-ci-bootstrap-staging timeout: 10800 - attempts: 1 + attempts: 3 + pre-timeout: 3600 nodeset: centos-9-medium-centos-9-crc-cloud-ocp-4-20-1-3xl irrelevant-files: *ir_files required-projects: *multinode_edpm_rp @@ -257,6 +259,7 @@ name: cifmw-base-crc nodeset: centos-9-crc-2-56-0-3xl timeout: 10800 + attempts: 3 abstract: true parent: base-simple-crc vars: diff --git a/zuul.d/content_provider.yaml b/zuul.d/content_provider.yaml index c963907d5..431fa2db8 100644 --- a/zuul.d/content_provider.yaml +++ b/zuul.d/content_provider.yaml @@ -6,6 +6,8 @@ - job: name: openstack-k8s-operators-content-provider-bootc parent: openstack-k8s-operators-content-provider + timeout: 7200 + attempts: 2 description: | A zuul job to build and publish bootc qcow2 and ironic-python-agent content for dependent jobs. diff --git a/zuul.d/molecule-base.yaml b/zuul.d/molecule-base.yaml index db666b251..9a035bbf3 100644 --- a/zuul.d/molecule-base.yaml +++ b/zuul.d/molecule-base.yaml @@ -23,6 +23,9 @@ - job: name: cifmw-molecule-base-crc nodeset: centos-9-crc-2-56-0-xxl + timeout: 3600 + pre-timeout: 2100 + attempts: 3 parent: base-simple-crc provides: - cifmw-molecule diff --git a/zuul.d/project-templates.yaml b/zuul.d/project-templates.yaml index 1439051a0..0bb3abf97 100644 --- a/zuul.d/project-templates.yaml +++ b/zuul.d/project-templates.yaml @@ -110,6 +110,7 @@ jobs: - openstack-k8s-operators-content-provider - whitebox-neutron-tempest-plugin-podified-multinode-edpm-deployment-crc-2comp: + voting: false dependencies: - openstack-k8s-operators-content-provider files: