diff --git a/.ci/manual/aro-disconnected/.env.example b/.ci/manual/aro-disconnected/.env.example new file mode 100644 index 0000000000..3015edf001 --- /dev/null +++ b/.ci/manual/aro-disconnected/.env.example @@ -0,0 +1,21 @@ +# ARO disconnected deployment variables. +# Copy this file to .env and edit it. The .env file is ignored by git. +# Used by install-aro-disconnected.sh and create-azure-firewall-rules.sh on the local machine. + +BASE_NAME=aro-disconnected +SUFFIX=6 +LOCATION=eastus +RESOURCEGROUP=${BASE_NAME}_${SUFFIX} +CLUSTER=${BASE_NAME}_${SUFFIX} +VNET_NAME=${BASE_NAME}-vnet_${SUFFIX} +FIREWALL_NAME=${BASE_NAME}-firewall_${SUFFIX} +FIREWALL_COLLECTION_NAME=azure_ms +FIREWALL_ALLOWED_LIST_BASE="management.azure.com mirror.openshift.com login.microsoftonline.com gcs.prod.monitoring.core.windows.net *.blob.core.windows.net *.servicebus.windows.net *.table.core.windows.net" +FIREWALL_ALLOWED_LIST_INSTALL="*.quay.io sso.redhat.com registry.redhat.io management.azure.com mirror.openshift.com api.openshift.com registry.access.redhat.com" +BASTION_IMAGE=RedHat:RHEL:10_1:latest +SSH_KEY_NAME=azure-disconnected-key_${SUFFIX} +WORKER_COUNT=4 +PULL_SECRET_FILE=pull-secret.txt +OPENSHIFT_VERSION=4.19.20 +BASTION_NAME=bastion-${SUFFIX} +BASTION_SOURCE_ADDRESS_PREFIX= diff --git a/.ci/manual/aro-disconnected/.gitignore b/.ci/manual/aro-disconnected/.gitignore new file mode 100644 index 0000000000..e3bf3aa7de --- /dev/null +++ b/.ci/manual/aro-disconnected/.gitignore @@ -0,0 +1,8 @@ +.env +.local/ +pull-secret.txt +*_access-information.txt +azure-disconnected-key_* +*.key +*.pem +*.pub diff --git a/.ci/manual/aro-disconnected/README.md b/.ci/manual/aro-disconnected/README.md new file mode 100644 index 0000000000..f65e866184 --- /dev/null +++ b/.ci/manual/aro-disconnected/README.md @@ -0,0 +1,94 @@ +# Manual ARO Disconnected Cluster + +These scripts provision an Azure Red Hat OpenShift (ARO) cluster in a disconnected Azure network for +manual RHDH testing. They are not connected to Prow. + +The workflow is based on the +[ARO disconnected installation guide](https://github.com/redhat-cop/ocp-disconnected-docs/blob/main/AROInstall.md). +Restricting outbound traffic can violate the ARO support policy; use this workflow only where that +trade-off is understood. + +## Prerequisites + +- Azure CLI with access to the target subscription +- `ssh-keygen` on the local workstation +- An OpenShift pull secret from +- A bastion VM image available in the selected Azure region + +## Configuration + +Run these commands in this directory: + +```bash +cp .env.example .env +``` + +Edit at least `SUFFIX`, `LOCATION`, `OPENSHIFT_VERSION`, and `BASTION_SOURCE_ADDRESS_PREFIX`. Set +the latter to the public IP address or CIDR of the workstation that will SSH to the bastion, for +example `198.51.100.10/32`. Place `pull-secret.txt` in this directory, or set `PULL_SECRET_FILE` to +an absolute path. The configuration contains names and network allow-lists only; it must not contain +registry passwords, tokens, or other secret values. + +## Workflow + +### 1. Create the ARO cluster and bastion + +Run on the local workstation: + +```bash +./install-aro-disconnected.sh +``` + +This creates the resource group, VNet, master and worker subnets, Azure Firewall, routes, private +ARO cluster, and bastion VM. The bastion allows SSH only from `BASTION_SOURCE_ADDRESS_PREFIX`. The +script creates an SSH key pair and writes the SSH command, API server, console URL, and kubeadmin +credentials to a `*_access-information.txt` file with mode `0600`. The file is ignored by git. + +The installer firewall rule is removed after ARO creation. Use the next step to add the outbound +rules needed by the workloads you run. + +### 2. Add workload firewall rules + +Run on the local workstation: + +```bash +./create-azure-firewall-rules.sh +``` + +The script creates the configured rule collection and adds each rule from its indexed rule list in +order. If the collection already exists, it asks before replacing it. + +### 3. Set up the bastion + +Copy the setup script to the bastion, then SSH to it using the protected access information file. +Replace the placeholders with the values from that file: + +```bash +scp -i setup-bastion.sh azureuser@:~/setup-bastion.sh +ssh -i azureuser@ +``` + +Export the cluster values in the bastion shell without committing or logging them: + +```bash +export API_SERVER='https://...' +export KUBEADMIN_PASSWORD='...' +export OPENSHIFT_VERSION='4.19.20' +chmod 700 ~/setup-bastion.sh +~/setup-bastion.sh +``` + +The script installs version-pinned `oc`, Helm, and `opm` binaries, verifies their published SHA-256 +manifests, installs the pinned `umoci` release after checksum verification, enables the OpenShift +internal registry, and resizes the bastion partitions. The default Helm and umoci versions can be +overridden with `HELM_VERSION` and `UMOCI_VERSION`. + +## Cleanup + +When testing is finished, delete the resource group from the local workstation: + +```bash +az group delete --name "$RESOURCEGROUP" --yes --no-wait +``` + +The command reads `RESOURCEGROUP` from the `.env` file if it is sourced in the current shell. diff --git a/.ci/manual/aro-disconnected/create-azure-firewall-rules.sh b/.ci/manual/aro-disconnected/create-azure-firewall-rules.sh new file mode 100755 index 0000000000..2cd30e7952 --- /dev/null +++ b/.ci/manual/aro-disconnected/create-azure-firewall-rules.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# +# Run on: LOCAL MACHINE (workstation with Azure CLI installed). +# Purpose: Create the application rules required by an ARO disconnected cluster. +# Based on: https://github.com/redhat-cop/ocp-disconnected-docs/blob/main/AROInstall.md +# Requires: An ARO cluster and Azure Firewall created by install-aro-disconnected.sh. +# + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) + +print_status() { + printf '[INFO] %s\n' "$1" +} + +print_warning() { + printf '[WARNING] %s\n' "$1" +} + +print_error() { + printf '[ERROR] %s\n' "$1" >&2 +} + +if [[ ! -f "${SCRIPT_DIR}/.env" ]]; then + print_error 'Missing .env. Copy .env.example to .env and edit it first.' + exit 1 +fi + +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/.env" + +if [[ -z "${RESOURCEGROUP:-}" || -z "${FIREWALL_NAME:-}" || -z "${FIREWALL_COLLECTION_NAME:-}" ]]; then + print_error 'RESOURCEGROUP, FIREWALL_NAME, and FIREWALL_COLLECTION_NAME are required.' + exit 1 +fi + +RESOURCE_GROUP="$RESOURCEGROUP" +RULE_COLLECTION_NAME="$FIREWALL_COLLECTION_NAME" +SOURCE_ADDRESSES=(10.0.0.0/24 10.0.1.0/24) +PROTOCOLS=(Http=80 Https=443) +RULE_NAMES=( + azure + redhat + ms-graph + github + gitlab + pagerduty + quay + okta + auth0 + atlassian + atlassian-third-party +) +RULE_TARGETS=( + 'management.azure.com mirror.openshift.com login.microsoftonline.com gcs.prod.monitoring.core.windows.net *.blob.core.windows.net *.servicebus.windows.net *.table.core.windows.net' + '*.redhat.com redhat.com redhat.io *.redhat.io' + 'graph.microsoft.com' + '*.github.com github.com *.githubusercontent.com' + 'gitlab.com *.gitlab.com *.gitlab.io' + '*.pagerduty.com pagerduty.com' + 'quay.io *.quay.io' + '*.okta.com *.mtls.okta.com *.oktapreview.com *.mtls.oktapreview.com *.oktacdn.com *.okta-emea.com *.mtls.okta-emea.com *.kerberos.okta.com *.kerberos.okta-emea.com *.kerberos.oktapreview.com *.okta-gov.com *.mtls.okta-gov.com *.okta.mil *.mtls.okta.mil *.awsglobalaccelerator.com' + 'auth0.com *.auth0.com' + '*.atlassian.com atlassian.com' + '*.pndsn.com *.cloudfront.net *.wp.com *.gravatar.com *.googleapis.com' +) + +check_prerequisites() { + print_status 'Checking prerequisites' + if ! command -v az > /dev/null 2>&1; then + print_error 'Azure CLI is not installed.' + exit 1 + fi + if ! az account show > /dev/null 2>&1; then + print_error "Not logged in to Azure. Run 'az login' first." + exit 1 + fi +} + +check_firewall() { + print_status "Checking firewall: ${FIREWALL_NAME}" + if ! az network firewall show \ + --resource-group "$RESOURCE_GROUP" \ + --name "$FIREWALL_NAME" > /dev/null; then + print_error "Firewall '${FIREWALL_NAME}' not found in resource group '${RESOURCE_GROUP}'." + exit 1 + fi +} + +check_existing_rule_collection() { + print_status "Checking rule collection: ${RULE_COLLECTION_NAME}" + if ! az network firewall application-rule collection show \ + --resource-group "$RESOURCE_GROUP" \ + --firewall-name "$FIREWALL_NAME" \ + --collection-name "$RULE_COLLECTION_NAME" > /dev/null 2>&1; then + print_status "Rule collection '${RULE_COLLECTION_NAME}' does not exist." + return + fi + + print_warning "Rule collection '${RULE_COLLECTION_NAME}' already exists." + read -r -p 'Overwrite it? (y/N): ' -n 1 REPLY || REPLY='' + printf '\n' + if [[ ! "$REPLY" =~ ^[Yy]$ ]]; then + print_status 'Operation cancelled.' + exit 0 + fi + + print_status "Removing existing rule collection: ${RULE_COLLECTION_NAME}" + az network firewall application-rule collection delete \ + --resource-group "$RESOURCE_GROUP" \ + --firewall-name "$FIREWALL_NAME" \ + --collection-name "$RULE_COLLECTION_NAME" +} + +create_application_rule() { + local rule_name="$1" + local target_definition="$2" + local rule_index="$3" + local -a target_fqdns + local -a command + + read -r -a target_fqdns <<< "$target_definition" + command=( + az network firewall application-rule create + --resource-group "$RESOURCE_GROUP" + --firewall-name "$FIREWALL_NAME" + --collection-name "$RULE_COLLECTION_NAME" + --name "$rule_name" + --target-fqdns "${target_fqdns[@]}" + --source-addresses "${SOURCE_ADDRESSES[@]}" + --protocols "${PROTOCOLS[@]}" + ) + + if [[ "$rule_index" -eq 0 ]]; then + command+=(--action Allow --priority 200) + fi + + print_status "Adding firewall rule: ${rule_name}" + "${command[@]}" +} + +create_rule_collection() { + print_status "Creating firewall rule collection: ${RULE_COLLECTION_NAME}" + if [[ "${#RULE_NAMES[@]}" -ne "${#RULE_TARGETS[@]}" ]]; then + print_error 'Firewall rule names and target definitions are out of sync.' + exit 1 + fi + + for rule_index in "${!RULE_NAMES[@]}"; do + create_application_rule \ + "${RULE_NAMES[$rule_index]}" \ + "${RULE_TARGETS[$rule_index]}" \ + "$rule_index" + done +} + +check_prerequisites +check_firewall +check_existing_rule_collection +create_rule_collection +print_status "Rule collection '${RULE_COLLECTION_NAME}' created successfully." diff --git a/.ci/manual/aro-disconnected/install-aro-disconnected.sh b/.ci/manual/aro-disconnected/install-aro-disconnected.sh new file mode 100755 index 0000000000..e189e1c95a --- /dev/null +++ b/.ci/manual/aro-disconnected/install-aro-disconnected.sh @@ -0,0 +1,312 @@ +#!/usr/bin/env bash +# +# Run on: LOCAL MACHINE (workstation with Azure CLI installed). +# Purpose: Create a resource group, VNet, Azure Firewall, ARO cluster, and bastion host. +# Based on: https://github.com/redhat-cop/ocp-disconnected-docs/blob/main/AROInstall.md +# + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) + +require_command() { + if ! command -v "$1" > /dev/null 2>&1; then + printf 'Missing required command: %s\n' "$1" >&2 + exit 1 + fi +} + +resolve_path() { + case "$1" in + /*) printf '%s\n' "$1" ;; + *) printf '%s/%s\n' "$SCRIPT_DIR" "$1" ;; + esac +} + +require_command az +require_command ssh-keygen + +if [[ ! -f "${SCRIPT_DIR}/.env" ]]; then + printf 'Missing .env. Copy .env.example to .env and edit it first.\n' >&2 + exit 1 +fi + +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/.env" + +required_variables=( + BASE_NAME + LOCATION + RESOURCEGROUP + CLUSTER + VNET_NAME + FIREWALL_NAME + FIREWALL_COLLECTION_NAME + FIREWALL_ALLOWED_LIST_BASE + FIREWALL_ALLOWED_LIST_INSTALL + BASTION_IMAGE + SSH_KEY_NAME + WORKER_COUNT + PULL_SECRET_FILE + OPENSHIFT_VERSION + BASTION_NAME + BASTION_SOURCE_ADDRESS_PREFIX +) +for variable_name in "${required_variables[@]}"; do + if [[ -z "${!variable_name:-}" ]]; then + printf 'Required variable is empty: %s\n' "$variable_name" >&2 + exit 1 + fi +done + +if [[ ! "$BASTION_NAME" =~ ^[A-Za-z0-9-]{1,64}$ ]]; then + printf 'BASTION_NAME must contain only letters, numbers, and hyphens: %s\n' "$BASTION_NAME" >&2 + exit 1 +fi + +if [[ ! "$BASTION_SOURCE_ADDRESS_PREFIX" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?$ ]]; then + printf 'BASTION_SOURCE_ADDRESS_PREFIX must be an IPv4 address or CIDR: %s\n' \ + "$BASTION_SOURCE_ADDRESS_PREFIX" >&2 + exit 1 +fi + +if [[ "$SSH_KEY_NAME" == */* || ! "$SSH_KEY_NAME" =~ ^[A-Za-z0-9._-]+$ ]]; then + printf 'SSH_KEY_NAME must be a simple filename; generated keys are stored under .local/: %s\n' \ + "$SSH_KEY_NAME" >&2 + exit 1 +fi + +PULL_SECRET_FILE=$(resolve_path "$PULL_SECRET_FILE") +SSH_KEY_NAME="${SCRIPT_DIR}/.local/${SSH_KEY_NAME}" +ACCESS_INFORMATION_FILE="${SCRIPT_DIR}/${BASE_NAME}_access-information.txt" +FIREWALL_ALLOWED_LIST_BASE_ARRAY=() +FIREWALL_ALLOWED_LIST_INSTALL_ARRAY=() +read -r -a FIREWALL_ALLOWED_LIST_BASE_ARRAY <<< "$FIREWALL_ALLOWED_LIST_BASE" +read -r -a FIREWALL_ALLOWED_LIST_INSTALL_ARRAY <<< "$FIREWALL_ALLOWED_LIST_INSTALL" + +if [[ ! -f "$PULL_SECRET_FILE" ]]; then + printf 'Pull secret file not found: %s\n' "$PULL_SECRET_FILE" >&2 + exit 1 +fi + +if [[ -e "$SSH_KEY_NAME" || -e "${SSH_KEY_NAME}.pub" ]]; then + printf 'SSH key already exists. Set SSH_KEY_NAME to a new path: %s\n' "$SSH_KEY_NAME" >&2 + exit 1 +fi + +umask 077 + +printf '%s\n' 'Installing Azure Red Hat OpenShift (ARO) in disconnected Microsoft Azure' + +printf '%s\n' 'Checking Azure CLI login' +if ! az account show > /dev/null 2>&1; then + printf '%s\n' 'Please log in to Azure CLI' + az login +fi + +printf '%s\n' 'Creating resource group' +az group create --location "$LOCATION" --name "$RESOURCEGROUP" + +printf '%s\n' 'Creating virtual network' +az network vnet create \ + --resource-group "$RESOURCEGROUP" \ + --name "$VNET_NAME" \ + --address-prefixes 10.0.0.0/16 + +printf '%s\n' 'Creating master subnet' +az network vnet subnet create \ + --resource-group "$RESOURCEGROUP" \ + --vnet-name "$VNET_NAME" \ + --name master-subnet \ + --address-prefixes 10.0.0.0/24 \ + --service-endpoints Microsoft.ContainerRegistry + +printf '%s\n' 'Creating worker subnet' +az network vnet subnet create \ + --resource-group "$RESOURCEGROUP" \ + --vnet-name "$VNET_NAME" \ + --name worker-subnet \ + --address-prefixes 10.0.1.0/24 \ + --service-endpoints Microsoft.ContainerRegistry + +printf '%s\n' 'Disabling master subnet private endpoint policies' +az network vnet subnet update \ + --name master-subnet \ + --resource-group "$RESOURCEGROUP" \ + --vnet-name "$VNET_NAME" \ + --disable-private-link-service-network-policies true + +printf '%s\n' 'Creating Azure Firewall subnet' +az network vnet subnet create \ + --resource-group "$RESOURCEGROUP" \ + --vnet-name "$VNET_NAME" \ + --name AzureFirewallSubnet \ + --address-prefixes 10.0.10.0/26 + +printf '%s\n' 'Creating public subnet for bastion host' +az network vnet subnet create \ + --resource-group "$RESOURCEGROUP" \ + --vnet-name "$VNET_NAME" \ + --name public-subnet \ + --address-prefixes 10.0.2.0/24 + +printf '%s\n' 'Creating Azure Firewall' +az network public-ip create \ + --name fw-pip \ + --resource-group "$RESOURCEGROUP" \ + --allocation-method static \ + --sku standard +az extension add --name azure-firewall --upgrade +az network firewall create \ + --resource-group "$RESOURCEGROUP" \ + --name "$FIREWALL_NAME" \ + --location "$LOCATION" +az network firewall ip-config create \ + --firewall-name "$FIREWALL_NAME" \ + --name FW-config \ + --public-ip-address fw-pip \ + --resource-group "$RESOURCEGROUP" \ + --vnet-name "$VNET_NAME" + +fw_private_address=$(az network firewall ip-config list \ + --resource-group "$RESOURCEGROUP" \ + --firewall-name "$FIREWALL_NAME" \ + --query "[?name=='FW-config'].privateIpAddress" \ + --output tsv) +if [[ -z "$fw_private_address" ]]; then + printf 'Unable to determine the Azure Firewall private IP address.\n' >&2 + exit 1 +fi + +printf '%s\n' 'Creating routing table' +az network route-table create \ + --name "${FIREWALL_NAME}-rt-table" \ + --resource-group "$RESOURCEGROUP" +az network route-table route create \ + --resource-group "$RESOURCEGROUP" \ + --name "${FIREWALL_NAME}-rt-table-route" \ + --route-table-name "${FIREWALL_NAME}-rt-table" \ + --address-prefix 0.0.0.0/0 \ + --next-hop-type VirtualAppliance \ + --next-hop-ip-address "$fw_private_address" + +printf '%s\n' 'Adding Azure Firewall application rules for ARO installation' +az network firewall application-rule create \ + --resource-group "$RESOURCEGROUP" \ + --firewall-name "$FIREWALL_NAME" \ + --collection-name "$FIREWALL_COLLECTION_NAME" \ + --name azure \ + --protocols http=80 https=443 \ + --target-fqdns "${FIREWALL_ALLOWED_LIST_BASE_ARRAY[@]}" \ + --source-addresses 10.0.0.0/24 10.0.1.0/24 \ + --priority 100 \ + --action Allow +az network firewall application-rule create \ + --resource-group "$RESOURCEGROUP" \ + --firewall-name "$FIREWALL_NAME" \ + --collection-name "$FIREWALL_COLLECTION_NAME" \ + --name azure_install \ + --protocols http=80 https=443 \ + --target-fqdns "${FIREWALL_ALLOWED_LIST_INSTALL_ARRAY[@]}" \ + --source-addresses 10.0.0.0/24 10.0.1.0/24 + +printf '%s\n' 'Routing master and worker traffic through Azure Firewall' +az network vnet subnet update \ + --resource-group "$RESOURCEGROUP" \ + --vnet-name "$VNET_NAME" \ + --name master-subnet \ + --route-table "${FIREWALL_NAME}-rt-table" +az network vnet subnet update \ + --resource-group "$RESOURCEGROUP" \ + --vnet-name "$VNET_NAME" \ + --name worker-subnet \ + --route-table "${FIREWALL_NAME}-rt-table" + +printf '%s\n' 'Creating the ARO cluster in the disconnected network' +az aro create \ + --resource-group "$RESOURCEGROUP" \ + --name "$CLUSTER" \ + --vnet "$VNET_NAME" \ + --master-subnet master-subnet \ + --worker-subnet worker-subnet \ + --apiserver-visibility Private \ + --ingress-visibility Private \ + --version "$OPENSHIFT_VERSION" \ + --worker-count "$WORKER_COUNT" \ + --pull-secret "$PULL_SECRET_FILE" + +printf '%s\n' 'Removing the temporary ARO installation firewall rule' +az network firewall application-rule delete \ + --resource-group "$RESOURCEGROUP" \ + --firewall-name "$FIREWALL_NAME" \ + --collection-name "$FIREWALL_COLLECTION_NAME" \ + --name azure_install + +printf '%s\n' 'Creating bastion host SSH key pair' +mkdir -p "$(dirname "$SSH_KEY_NAME")" +ssh-keygen -m PEM -t rsa -b 4096 -N '' -f "$SSH_KEY_NAME" +chmod 600 "$SSH_KEY_NAME" + +printf '%s\n' 'Creating bastion host VM' +BASTION_NSG_NAME="${BASTION_NAME}-nsg" +az network nsg create \ + --resource-group "$RESOURCEGROUP" \ + --name "$BASTION_NSG_NAME" \ + --location "$LOCATION" +az network nsg rule create \ + --resource-group "$RESOURCEGROUP" \ + --nsg-name "$BASTION_NSG_NAME" \ + --name allow-ssh-from-workstation \ + --priority 100 \ + --source-address-prefixes "$BASTION_SOURCE_ADDRESS_PREFIX" \ + --destination-port-ranges 22 \ + --access Allow \ + --protocol Tcp \ + --direction Inbound +az vm create \ + --name "$BASTION_NAME" \ + --resource-group "$RESOURCEGROUP" \ + --image "$BASTION_IMAGE" \ + --size Standard_D2s_v3 \ + --public-ip-address bastion-pub-ip \ + --vnet-name "$VNET_NAME" \ + --subnet public-subnet \ + --nsg "$BASTION_NSG_NAME" \ + --admin-username azureuser \ + --ssh-key-values "${SSH_KEY_NAME}.pub" + +printf '%s\n' 'Collecting cluster and bastion access information' +KUBEADMIN_PASSWORD=$(az aro list-credentials \ + --name "$CLUSTER" \ + --resource-group "$RESOURCEGROUP" \ + --query kubeadminPassword \ + --output tsv) +API_SERVER=$(az aro show \ + --resource-group "$RESOURCEGROUP" \ + --name "$CLUSTER" \ + --query apiserverProfile.url \ + --output tsv) +BASTION_PUBLIC_IP=$(az vm show \ + --show-details \ + --resource-group "$RESOURCEGROUP" \ + --name "$BASTION_NAME" \ + --query publicIps \ + --output tsv) +CONSOLE_URL=$(az aro show \ + --resource-group "$RESOURCEGROUP" \ + --name "$CLUSTER" \ + --query consoleProfile.url \ + --output tsv) + +: > "$ACCESS_INFORMATION_FILE" +chmod 600 "$ACCESS_INFORMATION_FILE" +{ + printf 'ssh -i %s azureuser@%s\n' "$SSH_KEY_NAME" "$BASTION_PUBLIC_IP" + printf 'oc login %s -u kubeadmin -p %s\n' "$API_SERVER" "$KUBEADMIN_PASSWORD" + printf 'CONSOLE_URL=%s\n' "$CONSOLE_URL" + printf 'API_SERVER=%s\n' "$API_SERVER" + printf 'KUBEADMIN_PASSWORD=%s\n' "$KUBEADMIN_PASSWORD" + printf 'OPENSHIFT_VERSION=%s\n' "$OPENSHIFT_VERSION" +} > "$ACCESS_INFORMATION_FILE" + +printf 'Access information written to protected file: %s\n' "$ACCESS_INFORMATION_FILE" diff --git a/.ci/manual/aro-disconnected/setup-bastion.sh b/.ci/manual/aro-disconnected/setup-bastion.sh new file mode 100755 index 0000000000..1392113da9 --- /dev/null +++ b/.ci/manual/aro-disconnected/setup-bastion.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# +# Run on: BASTION HOST (inside Azure, via SSH). +# Prerequisites: API_SERVER, KUBEADMIN_PASSWORD, and OPENSHIFT_VERSION supplied in the environment. +# Purpose: Install oc, Helm, Podman, Skopeo, opm, and umoci; enable the internal registry. +# + +set -euo pipefail + +if [[ -z "${API_SERVER:-}" || -z "${KUBEADMIN_PASSWORD:-}" || -z "${OPENSHIFT_VERSION:-}" ]]; then + printf 'API_SERVER, KUBEADMIN_PASSWORD, and OPENSHIFT_VERSION must be set in the environment.\n' >&2 + exit 1 +fi + +if [[ ! "$OPENSHIFT_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + printf 'OPENSHIFT_VERSION must be a stable semantic version: %s\n' "$OPENSHIFT_VERSION" >&2 + exit 1 +fi + +HELM_VERSION="${HELM_VERSION:-3.18.6}" +UMOCI_VERSION="${UMOCI_VERSION:-0.5.0}" + +if [[ ! "$HELM_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + printf 'HELM_VERSION must be a stable semantic version: %s\n' "$HELM_VERSION" >&2 + exit 1 +fi +if [[ ! "$UMOCI_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + printf 'UMOCI_VERSION must be a stable semantic version: %s\n' "$UMOCI_VERSION" >&2 + exit 1 +fi + +for command_name in curl sha256sum tar sudo; do + if ! command -v "$command_name" > /dev/null 2>&1; then + printf 'Missing required command: %s\n' "$command_name" >&2 + exit 1 + fi +done + +verify_checksum() { + local artifact_file="$1" + local checksum_file="$2" + local artifact_name="${artifact_file##*/}" + local expected_checksum='' + local checksum='' + local checksum_name='' + + while read -r checksum checksum_name; do + checksum_name="${checksum_name#\*}" + if [[ "$checksum" =~ ^[[:xdigit:]]{64}$ ]] \ + && [[ -z "$checksum_name" || "$checksum_name" == "$artifact_name" ]]; then + expected_checksum="$checksum" + break + fi + done < "$checksum_file" + + if [[ -z "$expected_checksum" ]]; then + printf 'No checksum found for %s in %s\n' "$artifact_name" "$checksum_file" >&2 + exit 1 + fi + + printf '%s %s\n' "$expected_checksum" "$artifact_file" | sha256sum --check --status - +} + +download_and_verify() { + local artifact_url="$1" + local checksum_url="$2" + local artifact_file="$3" + local checksum_file="${artifact_file}.sha256sum" + + curl -fsSL --output "$artifact_file" "$artifact_url" + curl -fsSL --output "$checksum_file" "$checksum_url" + verify_checksum "$artifact_file" "$checksum_file" +} + +temporary_directory=$(mktemp -d) +trap 'rm -rf "$temporary_directory"' EXIT + +printf '%s\n' 'Installing oc on the bastion host' +oc_archive="${temporary_directory}/openshift-client-linux-${OPENSHIFT_VERSION}.tar.gz" +download_and_verify \ + "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${OPENSHIFT_VERSION}/openshift-client-linux-${OPENSHIFT_VERSION}.tar.gz" \ + "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${OPENSHIFT_VERSION}/sha256sum.txt" \ + "$oc_archive" +mkdir -p "${temporary_directory}/openshift" +tar -xzf "$oc_archive" -C "${temporary_directory}/openshift" +sudo install -m 0755 "${temporary_directory}/openshift/oc" /usr/local/bin/oc + +printf '%s\n' 'Logging in to the cluster' +oc login "$API_SERVER" --username kubeadmin --password "$KUBEADMIN_PASSWORD" + +printf '%s\n' 'Testing outbound connection from the cluster' +oc exec -it alertmanager-main-0 -n openshift-monitoring -- curl redhat.com + +printf '%s\n' 'Enabling the internal registry' +oc patch config.imageregistry cluster \ + --namespace openshift-image-registry \ + --type merge \ + --patch '{"spec": {"defaultRoute": true}}' + +printf '%s\n' 'Installing Helm' +helm_archive="${temporary_directory}/helm-v${HELM_VERSION}-linux-amd64.tar.gz" +download_and_verify \ + "https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz" \ + "https://get.helm.sh/helm-v${HELM_VERSION}-linux-amd64.tar.gz.sha256sum" \ + "$helm_archive" +mkdir -p "${temporary_directory}/helm" +tar -xzf "$helm_archive" -C "${temporary_directory}/helm" +sudo install -m 0755 "${temporary_directory}/helm/linux-amd64/helm" /usr/local/bin/helm + +printf '%s\n' 'Installing Podman and Skopeo' +sudo dnf install --assumeyes podman skopeo + +printf '%s\n' 'Installing opm' +opm_archive="${temporary_directory}/opm-linux-${OPENSHIFT_VERSION}.tar.gz" +download_and_verify \ + "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${OPENSHIFT_VERSION}/opm-linux-${OPENSHIFT_VERSION}.tar.gz" \ + "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${OPENSHIFT_VERSION}/sha256sum.txt" \ + "$opm_archive" +mkdir -p "${temporary_directory}/opm" +tar -xzf "$opm_archive" -C "${temporary_directory}/opm" +sudo install -m 0755 "${temporary_directory}/opm/opm-rhel8" /usr/local/bin/opm + +printf '%s\n' 'Installing umoci' +umoci_file="${temporary_directory}/umoci.linux.amd64" +download_and_verify \ + "https://github.com/opencontainers/umoci/releases/download/v${UMOCI_VERSION}/umoci.linux.amd64" \ + "https://github.com/opencontainers/umoci/releases/download/v${UMOCI_VERSION}/umoci.sha256sum" \ + "$umoci_file" +sudo install -m 0755 "$umoci_file" /usr/local/bin/umoci + +printf '%s\n' 'Resizing /home and /tmp partitions' +sudo lvresize -r -L +9G /dev/mapper/rootvg-homelv +sudo lvresize -r -L +8G /dev/mapper/rootvg-tmplv diff --git a/.ci/manual/aro-disconnected/test-firewall-rules.sh b/.ci/manual/aro-disconnected/test-firewall-rules.sh new file mode 100755 index 0000000000..f537f63516 --- /dev/null +++ b/.ci/manual/aro-disconnected/test-firewall-rules.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +TEST_DIR=$(mktemp -d) +trap 'rm -rf "$TEST_DIR"' EXIT + +mkdir -p "$TEST_DIR/bin" +cp "$SCRIPT_DIR/create-azure-firewall-rules.sh" "$TEST_DIR/" + +printf '%s\n' \ + 'BASE_NAME=test' \ + 'RESOURCEGROUP=test-rg' \ + 'FIREWALL_NAME=test-firewall' \ + 'FIREWALL_COLLECTION_NAME=azure_ms' > "$TEST_DIR/.env" + +printf '%s\n' '#!/usr/bin/env bash' \ + 'printf "%s\n" "$*" >> "${AZ_LOG}"' \ + 'case "${1:-}:${2:-}:${3:-}:${4:-}:${5:-}" in' \ + ' account:show::::) exit 0 ;;' \ + ' network:firewall:show::::) exit 0 ;;' \ + ' network:firewall:application-rule:collection:show)' \ + ' [[ "${EXISTING_COLLECTION:-0}" -eq 1 ]] && exit 0 || exit 1 ;;' \ + ' network:firewall:application-rule:create:*) exit 0 ;;' \ + 'esac' \ + 'exit 0' > "$TEST_DIR/bin/az" +chmod +x "$TEST_DIR/bin/az" + +PATH="$TEST_DIR/bin:$PATH" AZ_LOG="$TEST_DIR/az.log" \ + bash "$TEST_DIR/create-azure-firewall-rules.sh" + +expected_rules=( + azure + redhat + ms-graph + github + gitlab + pagerduty + quay + okta + auth0 + atlassian + atlassian-third-party +) +create_count=0 +first_create='' +while IFS= read -r line; do + case "$line" in + network\ firewall\ application-rule\ create\ *) + create_count=$((create_count + 1)) + if [[ -z "$first_create" ]]; then + first_create="$line" + else + [[ "$line" != *'--action '* ]] + [[ "$line" != *'--priority '* ]] + fi + [[ "$line" == *'--collection-name azure_ms '* ]] + ;; + esac +done < "$TEST_DIR/az.log" + +[[ "$create_count" -eq "${#expected_rules[@]}" ]] +[[ "$first_create" == *'--action Allow'*'--priority 200'* ]] + +printf 'y' | PATH="$TEST_DIR/bin:$PATH" EXISTING_COLLECTION=1 AZ_LOG="$TEST_DIR/overwrite.log" \ + bash "$TEST_DIR/create-azure-firewall-rules.sh" + +delete_count=0 +while IFS= read -r line; do + case "$line" in + network\ firewall\ application-rule\ collection\ delete\ *) + delete_count=$((delete_count + 1)) + ;; + esac +done < "$TEST_DIR/overwrite.log" +[[ "$delete_count" -eq 1 ]] + +overwrite_create_count=0 +while IFS= read -r line; do + case "$line" in + network\ firewall\ application-rule\ create\ *) + overwrite_create_count=$((overwrite_create_count + 1)) + ;; + esac +done < "$TEST_DIR/overwrite.log" +[[ "$overwrite_create_count" -eq "${#expected_rules[@]}" ]] + +for rule_name in "${expected_rules[@]}"; do + found=0 + while IFS= read -r line; do + if [[ "$line" == *"--name $rule_name "* ]]; then + found=1 + break + fi + done < "$TEST_DIR/az.log" + [[ "$found" -eq 1 ]] +done + +printf '%s\n' 'firewall rule iteration test passed' diff --git a/.ci/manual/aro-disconnected/test-install-configuration.sh b/.ci/manual/aro-disconnected/test-install-configuration.sh new file mode 100644 index 0000000000..c6339aec3b --- /dev/null +++ b/.ci/manual/aro-disconnected/test-install-configuration.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) + +( + source "$SCRIPT_DIR/.env.example" + [[ "$BASTION_NAME" =~ ^[A-Za-z0-9-]+$ ]] +) + +config_contents=$(< "$SCRIPT_DIR/.env.example") +installer_contents=$(< "$SCRIPT_DIR/install-aro-disconnected.sh") +setup_contents=$(< "$SCRIPT_DIR/setup-bastion.sh") +readme_contents=$(< "$SCRIPT_DIR/README.md") +[[ "$config_contents" == *'BASTION_SOURCE_ADDRESS_PREFIX='* ]] +[[ "$installer_contents" == *'BASTION_SOURCE_ADDRESS_PREFIX'* ]] +[[ "$installer_contents" == *'--nsg'* ]] +[[ "$installer_contents" == *'/.local/'* ]] +[[ "$installer_contents" == *'--upgrade'* ]] +[[ "$setup_contents" == *'OPENSHIFT_VERSION'* ]] +[[ "$setup_contents" == *'HELM_VERSION'* ]] +[[ "$setup_contents" == *'sha256sum'* ]] +[[ "$readme_contents" == *'scp '* ]] + +printf '%s\n' 'manual ARO configuration test passed' diff --git a/.vscode/extensions.json b/.vscode/extensions.json index f30fcb03e3..4628ee0221 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -3,7 +3,6 @@ "esbenp.prettier-vscode", "dbaeumer.vscode-eslint", "ms-playwright.playwright", - "timonwong.shellcheck", "redhat.vscode-yaml", "mads-hartmann.bash-ide-vscode", ],