Add E2E golden tests for SONiC config generation

.gitignore

@@ -1,4 +1,5 @@
 *.egg-info
+.venv-sonic-e2e/
 *.pyc
 *.swp
 __pycache__

.zuul.yaml

@@ -140,6 +140,34 @@
     pre-run: playbooks/pre.yml
     run: playbooks/test-integration.yml
 
+# End-to-end golden test for the SONiC config generator: provisions
+# NetBox on kind, seeds it with the netbox-manager example data, runs
+# sync_sonic() and compares the exported config_db files against
+# tests/e2e/golden/. In check the job only runs when files that can
+# change the generated output (or the harness itself) are touched;
+# periodic-daily runs it unconditionally and catches drift of the
+# netbox-manager seed data, which is consumed at tip-of-main via
+# required-projects (honoring Depends-On).
+- job:
+    name: python-osism-sonic-e2e
+    nodeset: ubuntu-noble
+    pre-run: playbooks/pre-sonic-e2e.yml
+    run: playbooks/test-sonic-e2e.yml
+    required-projects:
+      - osism/netbox-manager
+    timeout: 2400
+    files:
+      - ^\.zuul\.yaml$
+      - ^Makefile$
+      - ^Pipfile\.lock$
+      - ^files/sonic/.*
+      - ^osism/settings\.py$
+      - ^osism/tasks/conductor/.*
+      - ^osism/utils/.*
+      - ^playbooks/(pre-|test-)sonic-e2e\.yml$
+      - ^requirements\.txt$
+      - ^tests/e2e/.*
+
 - project:
     merge-mode: squash-merge
     default-branch: main
@@ -154,6 +182,7 @@
         - python-osism-test-setup
         - python-osism-unit-tests
         - python-osism-integration-tests
+        - python-osism-sonic-e2e
     periodic-daily:
       jobs:
         - flake8
@@ -163,6 +192,7 @@
         - python-osism-test-setup
         - python-osism-unit-tests
         - python-osism-integration-tests
+        - python-osism-sonic-e2e
     periodic-midnight:
       jobs:
         - container-image-python-osism-push

Makefile

@@ -0,0 +1,25 @@
+CLUSTER_NAME ?= sonic-e2e
+NETBOX_MANAGER_DIR ?= $(abspath ../netbox-manager)
+
+# SONiC config-generation E2E golden test (see tests/e2e/run.sh).
+
+# Full cycle: provision kind + NetBox (an existing cluster is reused and
+# left in place), seed, generate, compare against tests/e2e/golden/.
+sonic-e2e:
+	NETBOX_MANAGER_DIR=$(NETBOX_MANAGER_DIR) CLUSTER_NAME=$(CLUSTER_NAME) tests/e2e/run.sh
+
+# Regenerate the golden files after an intentional generator change,
+# then review and commit the diff.
+sonic-e2e-regen:
+	NETBOX_MANAGER_DIR=$(NETBOX_MANAGER_DIR) CLUSTER_NAME=$(CLUSTER_NAME) tests/e2e/run.sh --regenerate
+
+# Provision kind + NetBox and leave it running for debugging. Export a
+# NETBOX_TOKEN beforehand to get a known API token minted.
+sonic-e2e-up:
+	CLUSTER_NAME=$(CLUSTER_NAME) $(NETBOX_MANAGER_DIR)/tests/e2e/deploy_netbox.sh
+
+# Delete the kind cluster created for the E2E test.
+sonic-e2e-down:
+	kind delete cluster --name $(CLUSTER_NAME)
+
+.PHONY: sonic-e2e sonic-e2e-regen sonic-e2e-up sonic-e2e-down

osism/settings.py

@@ -70,6 +70,10 @@ def read_secret(secret_name):
 SONIC_EXPORT_SUFFIX = os.getenv("SONIC_EXPORT_SUFFIX", "_config_db.json")
 SONIC_EXPORT_IDENTIFIER = os.getenv("SONIC_EXPORT_IDENTIFIER", "serial-number")
 
+# Directory holding the per-HWSKU port_config .ini files (bundled in the
+# repo under files/sonic/port_config and installed by the Dockerfile)
+SONIC_PORT_CONFIG_PATH = os.getenv("SONIC_PORT_CONFIG_PATH", "/etc/sonic/port_config")
+
 # SONiC ZTP firmware configuration
 #
 # The ZTP firmware install uses a dynamic-url built from

osism/tasks/conductor/sonic/config_generator.py

@@ -2268,14 +2268,14 @@ def _add_snmp_configuration(config, device, oob_ip):
     in the config_context of the device.
     """
 
-    # Create vault instance for Custom Field decryption
-    vault = get_vault()
-
-    # Decrypt secrets Custom Field
+    # Decrypt the secrets Custom Field. The vault needs the key file and
+    # Redis, and get_vault() logs ERRORs where those are absent, so it is
+    # only built when there are secrets to decrypt.
     node_secrets = device.custom_fields.get("secrets", {})
     if node_secrets is None:
         node_secrets = {}
-    deep_decrypt(node_secrets, vault)
+    if node_secrets:
+        deep_decrypt(node_secrets, get_vault())
 
     # Configure SNMP location and contact
     location = device.config_context.get("_segment_snmp_server_location", "Data Center")

osism/tasks/conductor/sonic/constants.py

@@ -2,6 +2,8 @@
 
 """Constants and mappings for SONiC configuration."""
 
+from osism import settings
+
 # Tag to add AF L2VPN EVPN to BGP neighbor
 BGP_AF_L2VPN_EVPN_TAG = "bgp-af-l2vpn-evpn"
 
@@ -74,7 +76,7 @@
 HIGH_SPEED_PORTS = {100000, 200000, 400000, 800000}  # 100G, 200G, 400G, 800G in Mbps
 
 # Path to SONiC port configuration files
-PORT_CONFIG_PATH = "/etc/sonic/port_config"
+PORT_CONFIG_PATH = settings.SONIC_PORT_CONFIG_PATH
 
 # List of supported vendors
 SUPPORTED_VENDORS = [

playbooks/pre-sonic-e2e.yml

@@ -0,0 +1,119 @@
+---
+# Node preparation for the SONiC config-generation E2E golden test.
+#
+# Adapted from netbox-manager's playbooks/pre-e2e.yml (which prepares the
+# same kind + NetBox stack for its own E2E job); the tool pins and the
+# IPv6 workaround must stay in sync with that playbook until the setup is
+# factored into a shared zuul-jobs role.
+- name: Prepare the SONiC E2E node
+  hosts: all
+
+  vars:
+    # SHA256 digests of the linux/amd64 artifacts for the pinned tool
+    # versions below. They guard against a tampered download (CDN/DNS
+    # hijack, TLS-intercepting proxy) installing an attacker-controlled
+    # binary as root. Re-pin each digest whenever its *_version is bumped.
+    kind_version: v0.32.0
+    kind_sha256: 50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54
+    kubectl_version: v1.35.5
+    kubectl_sha256: 90f75ea6ecc9ea5633262e1c0b83a40560003b30fc94a04cb099404fcef0c224
+    helm_version: v3.16.4
+    helm_sha256: fc307327959aa38ed8f9f7e66d45492bb022a66c3e5da6063958254b9767d179
+
+  pre_tasks:
+    # This CI node is IPv6-only and learns its address and default route via
+    # SLAAC / Router Advertisements. Later, `kind create cluster` makes Docker
+    # create a dual-stack network, which sets net.ipv6.conf.all.forwarding=1;
+    # with the default accept_ra=1 the kernel then stops honouring RAs, so the
+    # SLAAC default route expires and the node drops off the network a few
+    # minutes into the run. accept_ra=2 keeps RAs honoured even while
+    # forwarding is on, preserving the default route. Set it here -- before
+    # Docker/kind enable forwarding -- so the route never lapses. See
+    # https://docs.docker.com/engine/daemon/ipv6/ and
+    # https://forums.docker.com/t/docker-removes-host-ipv6-default-route/83238
+    - name: Keep accepting IPv6 RAs after Docker enables forwarding (preserve default route)
+      become: true
+      ansible.builtin.copy:
+        dest: /etc/sysctl.d/99-sonic-e2e-accept-ra.conf
+        owner: root
+        group: root
+        mode: "0644"
+        content: |
+          net.ipv6.conf.all.accept_ra = 2
+          net.ipv6.conf.default.accept_ra = 2
+          {% if ansible_default_ipv6.interface is defined %}
+          net.ipv6.conf.{{ ansible_default_ipv6.interface }}.accept_ra = 2
+          {% endif %}
+
+    - name: Apply the accept_ra sysctl settings now
+      become: true
+      ansible.builtin.command:
+        cmd: sysctl -p /etc/sysctl.d/99-sonic-e2e-accept-ra.conf
+      changed_when: true
+
+  roles:
+    - ensure-pip
+    - ensure-pipenv
+    - ensure-docker
+
+  tasks:
+    - name: Ensure the Docker service is running
+      become: true
+      ansible.builtin.service:
+        name: docker
+        state: started
+        enabled: true
+
+    # curl and openssl are used by tests/e2e/run.sh directly; python3-venv
+    # provides the venv module the script uses for the seeding venv.
+    - name: Install required packages
+      become: true
+      ansible.builtin.apt:
+        name:
+          - curl
+          - openssl
+          - python3-venv
+
+    - name: Install kubectl
+      become: true
+      ansible.builtin.get_url:
+        url: "https://dl.k8s.io/release/{{ kubectl_version }}/bin/linux/amd64/kubectl"
+        dest: /usr/local/bin/kubectl
+        mode: "0755"
+        checksum: "sha256:{{ kubectl_sha256 }}"
+
+    - name: Install kind
+      become: true
+      ansible.builtin.get_url:
+        url: "https://kind.sigs.k8s.io/dl/{{ kind_version }}/kind-linux-amd64"
+        dest: /usr/local/bin/kind
+        mode: "0755"
+        checksum: "sha256:{{ kind_sha256 }}"
+
+    - name: Create a private temporary directory for the Helm archive
+      ansible.builtin.tempfile:
+        state: directory
+        suffix: helm
+      register: helm_tmp
+
+    - name: Download the Helm archive
+      ansible.builtin.get_url:
+        url: "https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz"
+        dest: "{{ helm_tmp.path }}/helm.tar.gz"
+        mode: "0644"
+        checksum: "sha256:{{ helm_sha256 }}"
+
+    - name: Install Helm
+      become: true
+      ansible.builtin.unarchive:
+        src: "{{ helm_tmp.path }}/helm.tar.gz"
+        dest: /usr/local/bin
+        remote_src: true
+        extra_opts:
+          - --strip-components=1
+          - linux-amd64/helm
+
+    - name: Remove the temporary Helm directory
+      ansible.builtin.file:
+        path: "{{ helm_tmp.path }}"
+        state: absent

playbooks/test-sonic-e2e.yml

@@ -0,0 +1,35 @@
+---
+- name: Run the SONiC config-generation E2E golden test
+  hosts: all
+
+  vars:
+    python_venv_dir: /tmp/venv
+
+  tasks:
+    - name: Install dependencies
+      ansible.builtin.shell:
+        executable: /bin/bash
+        chdir: "{{ zuul.project.src_dir }}"
+        cmd: |
+          set -e
+          set -o pipefail
+          set -x
+
+          {{ python_venv_dir }}/bin/pipenv install --dev --deploy
+          {{ python_venv_dir }}/bin/pipenv run pip install .
+
+    - name: Run the E2E golden test
+      ansible.builtin.shell:
+        executable: /bin/bash
+        chdir: "{{ zuul.project.src_dir }}"
+        cmd: |
+          set -e
+          set -o pipefail
+          set -x
+
+          # run.sh invokes bare `pipenv`; the netbox-manager checkout comes
+          # from Zuul's required-projects, so a Depends-On change to it is
+          # tested against the changed code and seed data.
+          export PATH="{{ python_venv_dir }}/bin:${PATH}"
+          export NETBOX_MANAGER_DIR="{{ ansible_user_dir }}/{{ zuul.projects['github.com/osism/netbox-manager'].src_dir }}"
+          tests/e2e/run.sh