From b67f7e8cf2411392d32d9be6c1ca2012dcd9867d Mon Sep 17 00:00:00 2001 From: Bashir Partovi Date: Wed, 19 Aug 2026 23:46:36 -0400 Subject: [PATCH 1/7] FEAT: Add fixed egress deployment for CoPyRIT Deploy the GUI on a VNet-integrated public Container Apps environment with static NAT egress and optional Azure Front Door. Harden the Azure DevOps workflow around immutable image digests, fail-closed what-if validation, and preservation of the reserved public IP. Update isolated-instance lifecycle guidance and add focused infrastructure tests. --- docker/start.sh | 6 +- gui-deploy.yml | 309 +++++----- infra/DEPLOY_NEW_INSTANCE.md | 321 ++++------ infra/README.md | 596 ++++++++++--------- infra/deploy_instance.py | 505 +++++++++++----- infra/env.demo.template | 13 +- infra/main.bicep | 254 ++++---- infra/modules/aca_front_door.bicep | 93 +++ infra/modules/aca_nat_network.bicep | 108 ++++ infra/parameters.demo.json | 17 +- infra/parameters.example.json | 23 +- infra/pipelines/deploy_public_nat.sh | 327 ++++++++++ infra/pipelines/validate_what_if.py | 142 +++++ infra/teardown_instance.py | 244 ++++++-- tests/unit/infra/__init__.py | 2 + tests/unit/infra/test_bicep_topology.py | 180 ++++++ tests/unit/infra/test_instance_lifecycle.py | 216 +++++++ tests/unit/infra/test_pipeline_guardrails.py | 238 ++++++++ 18 files changed, 2601 insertions(+), 993 deletions(-) create mode 100644 infra/modules/aca_front_door.bicep create mode 100644 infra/modules/aca_nat_network.bicep create mode 100644 infra/pipelines/deploy_public_nat.sh create mode 100644 infra/pipelines/validate_what_if.py create mode 100644 tests/unit/infra/__init__.py create mode 100644 tests/unit/infra/test_bicep_topology.py create mode 100644 tests/unit/infra/test_instance_lifecycle.py create mode 100644 tests/unit/infra/test_pipeline_guardrails.py diff --git a/docker/start.sh b/docker/start.sh index 9bb394865d..20e431182d 100644 --- a/docker/start.sh +++ b/docker/start.sh @@ -37,10 +37,8 @@ fi echo "Checking PyRIT installation..." python -c "import pyrit; print(f'Running PyRIT version: {pyrit.__version__}')" -# Write .env file from PYRIT_ENV_CONTENTS (injected as the Container App's -# inline `env-file` secret; previously a Key Vault secretRef, but ACA isn't on -# Key Vault's "trusted services" list so SFI-locked-down KVs can't be read at -# runtime — see infra/main.bicep for details). +# Write .env from PYRIT_ENV_CONTENTS, which references the Container App's +# inline or Key Vault-backed `env-file` secret (see infra/main.bicep). if [ -n "$PYRIT_ENV_CONTENTS" ]; then mkdir -p ~/.pyrit echo "$PYRIT_ENV_CONTENTS" > ~/.pyrit/.env diff --git a/gui-deploy.yml b/gui-deploy.yml index ed12ab0cdd..fabe54c455 100644 --- a/gui-deploy.yml +++ b/gui-deploy.yml @@ -1,17 +1,7 @@ -# CI/CD pipeline for CoPyRIT GUI deployment. +# CI/CD pipeline for the CoPyRIT GUI. # -# Triggers on changes to GUI-relevant paths on the main branch. -# Builds Docker image, pushes to ACR, deploys to test ACA environment. -# Production deployment is opt-in via the deployToProd parameter. -# -# All infrastructure details (IDs, connection strings, resource names) are -# stored in ADO variable groups — nothing sensitive appears in this file. -# Required variable groups: -# - copyrit-gui-common (azureServiceConnection, acrName, acrLoginServer, imageName) -# - copyrit-gui-test (resourceGroup, appName, entraTenantId, entraClientId, -# allowedGroupObjectIds, sqlServerFqdn, sqlDatabaseName, -# keyVaultResourceId, acrResourceId, enablePrivateEndpoint, enableOtel) -# - copyrit-gui-prod (same keys as test, with production values) +# Every deployment uses the single topology defined by infra/main.bicep: +# public ACA-managed HTTPS ingress plus VNet-integrated fixed NAT egress. trigger: branches: @@ -32,47 +22,28 @@ parameters: type: boolean default: false -# Service connection must be a compile-time variable (not from a variable group) -# because azureSubscription is validated during YAML parsing. variables: azureServiceConnection: 'copyrit-gui-azure' stages: - # ────────────────────────────────────────────── - # Stage 0: Validate runtime inputs - # ────────────────────────────────────────────── - stage: ValidateInputs displayName: 'Validate Inputs' pool: vmImage: 'ubuntu-latest' jobs: - - job: ValidateProdBranchGate - displayName: 'Validate production branch gate' + - job: ValidateProdSource + displayName: 'Validate production source' steps: - checkout: none - - task: Bash@3 - displayName: 'Fail if prod deploy requested from non-release branch' - inputs: - targetType: 'inline' - script: | - set -euo pipefail - - DEPLOY_TO_PROD='${{ parameters.deployToProd }}' - DEPLOY_TO_PROD_LOWER=$(echo "$DEPLOY_TO_PROD" | tr '[:upper:]' '[:lower:]') - - echo "Build.SourceBranch=$BUILD_SOURCEBRANCH" - echo "deployToProd=$DEPLOY_TO_PROD" - - if [[ "$DEPLOY_TO_PROD_LOWER" == "true" && "$BUILD_SOURCEBRANCH" != refs/heads/releases/* ]]; then - echo "##vso[task.logissue type=error]deployToProd can only be used from refs/heads/releases/* branches. Current branch: $BUILD_SOURCEBRANCH" - exit 1 - fi - - echo "Validation passed" + - bash: | + set -euo pipefail + if [[ '${{ parameters.deployToProd }}' == 'true' \ + && "$BUILD_SOURCEBRANCH" != refs/heads/main ]]; then + echo "##vso[task.logissue type=error]Production deployment requires a commit merged to refs/heads/main" + exit 1 + fi + displayName: 'Validate production source' - # ────────────────────────────────────────────── - # Stage 1: Build Docker image and push to ACR - # ────────────────────────────────────────────── - stage: Build displayName: 'Build and Push Image' dependsOn: ValidateInputs @@ -82,13 +53,19 @@ stages: vmImage: 'ubuntu-latest' jobs: - job: BuildAndPush - displayName: 'Build Docker image and push to ACR' + displayName: 'Build and push immutable source image' steps: - checkout: self fetchDepth: 1 - task: AzureCLI@2 + name: BuildImage displayName: 'Build and push Docker image' + env: + PYRIT_ACR_NAME: $(acrName) + PYRIT_ACR_LOGIN_SERVER: $(acrLoginServer) + PYRIT_IMAGE_NAME: $(imageName) + PYRIT_SOURCE_VERSION: $(Build.SourceVersion) inputs: azureSubscription: '$(azureServiceConnection)' scriptType: 'bash' @@ -96,44 +73,67 @@ stages: inlineScript: | set -euo pipefail - echo "=== Building devcontainer base image ===" + required_variables=( + PYRIT_ACR_NAME + PYRIT_ACR_LOGIN_SERVER + PYRIT_IMAGE_NAME + PYRIT_SOURCE_VERSION + ) + for variable_name in "${required_variables[@]}"; do + if [[ -z "${!variable_name:-}" || "${!variable_name}" == '$('* ]]; then + echo "##vso[task.logissue type=error]Required build value is missing: $variable_name" + exit 1 + fi + done + + repository_pattern='^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*$' + if [[ ! "$PYRIT_ACR_NAME" =~ ^[a-z0-9]{5,50}$ \ + || "$PYRIT_ACR_LOGIN_SERVER" != "$PYRIT_ACR_NAME.azurecr.io" \ + || ! "$PYRIT_IMAGE_NAME" =~ $repository_pattern \ + || ! "$PYRIT_SOURCE_VERSION" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "##vso[task.logissue type=error]Invalid registry, repository, or source version" + exit 1 + fi + docker build \ -f .devcontainer/Dockerfile \ -t pyrit-devcontainer \ .devcontainer/ - echo "=== Building production image ===" + image="$PYRIT_ACR_LOGIN_SERVER/$PYRIT_IMAGE_NAME:$PYRIT_SOURCE_VERSION" docker build --no-cache \ -f docker/Dockerfile \ - -t $(acrLoginServer)/$(imageName):$(Build.SourceVersion) \ - -t $(acrLoginServer)/$(imageName):latest \ + -t "$image" \ --build-arg BASE_IMAGE=pyrit-devcontainer \ --build-arg PYRIT_SOURCE=local \ - --build-arg GIT_COMMIT=$(Build.SourceVersion) \ + --build-arg GIT_COMMIT="$PYRIT_SOURCE_VERSION" \ --build-arg GIT_MODIFIED=false \ . - echo "=== Pushing to ACR ===" - az acr login --name $(acrName) - docker push $(acrLoginServer)/$(imageName):$(Build.SourceVersion) - docker push $(acrLoginServer)/$(imageName):latest - - echo "✅ Image pushed: $(acrLoginServer)/$(imageName):$(Build.SourceVersion)" + az acr login --name "$PYRIT_ACR_NAME" + push_output=$(docker push "$image") + printf '%s\n' "$push_output" + digest=$(sed -nE 's/.*digest: (sha256:[0-9a-fA-F]{64}).*/\1/p' <<< "$push_output" | tail -n 1) + if [[ ! "$digest" =~ ^sha256:[0-9a-fA-F]{64}$ ]]; then + echo "##vso[task.logissue type=error]Docker push did not return an immutable digest" + exit 1 + fi + immutable_image="$PYRIT_ACR_LOGIN_SERVER/$PYRIT_IMAGE_NAME@$digest" + echo "##vso[task.setvariable variable=immutableImage;isOutput=true]$immutable_image" - # ────────────────────────────────────────────── - # Stage 2: Deploy to test environment - # ────────────────────────────────────────────── - stage: DeployTest displayName: 'Deploy to Test' dependsOn: Build variables: - group: copyrit-gui-common - group: copyrit-gui-test + - name: immutableImage + value: $[ stageDependencies.Build.BuildAndPush.outputs['BuildImage.immutableImage'] ] pool: vmImage: 'ubuntu-latest' jobs: - deployment: DeployToTest - displayName: 'Deploy to test environment' + displayName: 'Deploy test environment' environment: 'copyrit-test' strategy: runOnce: @@ -143,80 +143,74 @@ stages: fetchDepth: 1 - task: AzureCLI@2 - displayName: 'Deploy Bicep to test' + displayName: 'Preview, deploy, and verify test' + env: + PYRIT_SLOT: test + PYRIT_BUILD_ID: $(Build.BuildId) + PYRIT_SOURCE_DIRECTORY: $(Build.SourcesDirectory) + PYRIT_AGENT_TEMP_DIRECTORY: $(Agent.TempDirectory) + PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup) + PYRIT_APP_NAME: $(deploymentAppName) + PYRIT_CONTAINER_IMAGE: $(immutableImage) + PYRIT_VNET_ADDRESS_PREFIX: $(deploymentVnetAddressPrefix) + PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX: $(deploymentInfrastructureSubnetAddressPrefix) + PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr) + PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId) + PYRIT_ENTRA_TENANT_ID: $(entraTenantId) + PYRIT_ENTRA_CLIENT_ID: $(entraClientId) + PYRIT_ALLOWED_GROUP_OBJECT_IDS: $(allowedGroupObjectIds) + PYRIT_SQL_SERVER_FQDN: $(sqlServerFqdn) + PYRIT_SQL_DATABASE_NAME: $(sqlDatabaseName) + PYRIT_KEY_VAULT_RESOURCE_ID: $(keyVaultResourceId) + PYRIT_ACR_RESOURCE_ID: $(acrResourceId) + PYRIT_ENABLE_OTEL: $(enableOtel) + PYRIT_ENV_SECRET_NAME: $(envSecretName) inputs: azureSubscription: '$(azureServiceConnection)' scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - set -euo pipefail - - az deployment group create \ - --resource-group $(resourceGroup) \ - --template-file $(Build.SourcesDirectory)/infra/main.bicep \ - --parameters appName=$(appName) \ - --parameters containerImage=$(acrLoginServer)/$(imageName):$(Build.SourceVersion) \ - --parameters entraTenantId=$(entraTenantId) \ - --parameters entraClientId=$(entraClientId) \ - --parameters allowedGroupObjectIds="$(allowedGroupObjectIds)" \ - --parameters sqlServerFqdn=$(sqlServerFqdn) \ - --parameters sqlDatabaseName=$(sqlDatabaseName) \ - --parameters keyVaultResourceId=$(keyVaultResourceId) \ - --parameters acrResourceId=$(acrResourceId) \ - --parameters enablePrivateEndpoint=$(enablePrivateEndpoint) \ - --parameters enableOtel=$(enableOtel) \ - --parameters envSecretName=$(envSecretName) - - - task: AzureCLI@2 - displayName: 'Health check' - inputs: - azureSubscription: '$(azureServiceConnection)' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - set -euo pipefail - - IMAGE="$(acrLoginServer)/$(imageName):$(Build.SourceVersion)" - HEALTH="" - - for i in {1..5}; do - HEALTH="$(az containerapp revision list \ - --resource-group "$(resourceGroup)" \ - --name "$(appName)" \ - --query "[?properties.template.containers[0].image=='$IMAGE'] | sort_by(@,&properties.createdTime)[-1].properties.healthState" \ - -o tsv)" + scriptLocation: 'scriptPath' + scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh' - echo "Attempt $i/5 - Image: $IMAGE - Health: ${HEALTH:-}" - - if [[ "$HEALTH" == "Healthy" ]]; then - echo "✅ Test environment health check passed" - exit 0 - fi - - if [[ "$i" -lt 5 ]]; then - echo "Revision is not healthy yet. Waiting 2 minutes before retry..." - sleep 120 - fi - done - - echo "❌ Test environment health check failed after 5 attempts" - exit 1 + - stage: ApproveProd + displayName: 'Approve Production Deployment' + dependsOn: DeployTest + condition: and(succeeded('DeployTest'), eq('${{ parameters.deployToProd }}', 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) + variables: + - group: copyrit-gui-prod + jobs: + - job: WaitForProdApproval + displayName: 'Validate production readiness' + pool: server + timeoutInMinutes: 1440 + steps: + - task: ManualValidation@1 + timeoutInMinutes: 1440 + inputs: + notifyUsers: '$(prodApprovers)' + approvers: '$(prodApprovers)' + allowApproversToApproveTheirOwnRuns: false + instructions: | + Confirm test is healthy, Entra and backend group authorization work, + the static test egress IP is allow-listed, and production variables + describe the same public ACA plus fixed NAT topology. + onTimeout: reject - # ────────────────────────────────────────────── - # Stage 3: Deploy to production (manual trigger) - # ────────────────────────────────────────────── - stage: DeployProd displayName: 'Deploy to Production' - dependsOn: DeployTest - condition: and(succeeded('DeployTest'), eq('${{ parameters.deployToProd }}', 'true'), startsWith(variables['Build.SourceBranch'], 'refs/heads/releases/')) + dependsOn: + - ApproveProd + - Build + condition: succeeded('ApproveProd') variables: - group: copyrit-gui-common - group: copyrit-gui-prod + - name: immutableImage + value: $[ stageDependencies.Build.BuildAndPush.outputs['BuildImage.immutableImage'] ] pool: vmImage: 'ubuntu-latest' jobs: - deployment: DeployToProd - displayName: 'Deploy to production environment' + displayName: 'Deploy production environment' environment: 'copyrit-prod' strategy: runOnce: @@ -226,61 +220,30 @@ stages: fetchDepth: 1 - task: AzureCLI@2 - displayName: 'Deploy Bicep to prod' - inputs: - azureSubscription: '$(azureServiceConnection)' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - set -euo pipefail - - az deployment group create \ - --resource-group $(resourceGroup) \ - --template-file $(Build.SourcesDirectory)/infra/main.bicep \ - --parameters appName=$(appName) \ - --parameters containerImage=$(acrLoginServer)/$(imageName):$(Build.SourceVersion) \ - --parameters entraTenantId=$(entraTenantId) \ - --parameters entraClientId=$(entraClientId) \ - --parameters allowedGroupObjectIds="$(allowedGroupObjectIds)" \ - --parameters sqlServerFqdn=$(sqlServerFqdn) \ - --parameters sqlDatabaseName=$(sqlDatabaseName) \ - --parameters keyVaultResourceId=$(keyVaultResourceId) \ - --parameters acrResourceId=$(acrResourceId) \ - --parameters enablePrivateEndpoint=$(enablePrivateEndpoint) \ - --parameters enableOtel=$(enableOtel) \ - --parameters envSecretName=$(envSecretName) - - - task: AzureCLI@2 - displayName: 'Health check' + displayName: 'Preview, deploy, and verify production' + env: + PYRIT_SLOT: prod + PYRIT_BUILD_ID: $(Build.BuildId) + PYRIT_SOURCE_DIRECTORY: $(Build.SourcesDirectory) + PYRIT_AGENT_TEMP_DIRECTORY: $(Agent.TempDirectory) + PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup) + PYRIT_APP_NAME: $(deploymentAppName) + PYRIT_CONTAINER_IMAGE: $(immutableImage) + PYRIT_VNET_ADDRESS_PREFIX: $(deploymentVnetAddressPrefix) + PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX: $(deploymentInfrastructureSubnetAddressPrefix) + PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr) + PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId) + PYRIT_ENTRA_TENANT_ID: $(entraTenantId) + PYRIT_ENTRA_CLIENT_ID: $(entraClientId) + PYRIT_ALLOWED_GROUP_OBJECT_IDS: $(allowedGroupObjectIds) + PYRIT_SQL_SERVER_FQDN: $(sqlServerFqdn) + PYRIT_SQL_DATABASE_NAME: $(sqlDatabaseName) + PYRIT_KEY_VAULT_RESOURCE_ID: $(keyVaultResourceId) + PYRIT_ACR_RESOURCE_ID: $(acrResourceId) + PYRIT_ENABLE_OTEL: $(enableOtel) + PYRIT_ENV_SECRET_NAME: $(envSecretName) inputs: azureSubscription: '$(azureServiceConnection)' scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - set -euo pipefail - - IMAGE="$(acrLoginServer)/$(imageName):$(Build.SourceVersion)" - HEALTH="" - - for i in {1..5}; do - HEALTH="$(az containerapp revision list \ - --resource-group "$(resourceGroup)" \ - --name "$(appName)" \ - --query "[?properties.template.containers[0].image=='$IMAGE'] | sort_by(@,&properties.createdTime)[-1].properties.healthState" \ - -o tsv)" - - echo "Attempt $i/5 - Image: $IMAGE - Health: ${HEALTH:-}" - - if [[ "$HEALTH" == "Healthy" ]]; then - echo "✅ Production health check passed" - exit 0 - fi - - if [[ "$i" -lt 5 ]]; then - echo "Revision is not healthy yet. Waiting 2 minutes before retry..." - sleep 120 - fi - done - - echo "❌ Production health check failed after 5 attempts" - exit 1 + scriptLocation: 'scriptPath' + scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh' \ No newline at end of file diff --git a/infra/DEPLOY_NEW_INSTANCE.md b/infra/DEPLOY_NEW_INSTANCE.md index 489f557368..99aede0831 100644 --- a/infra/DEPLOY_NEW_INSTANCE.md +++ b/infra/DEPLOY_NEW_INSTANCE.md @@ -1,24 +1,17 @@ # Deploy a New CoPyRIT GUI Instance -Deploy an isolated CoPyRIT GUI instance for an external team (CELA, model ops, -partners). Each instance gets its own database, secrets, and Entra app -registration — fully isolated from the AIRT instance and from other instances. -Access is controlled via existing Entra security groups that you provide at -deploy time. +Deploy an isolated CoPyRIT GUI instance for an external team (CELA, model ops, partners). Each instance gets its own database, secrets, and Entra app registration, Container Apps environment, virtual network, and fixed egress IP. Instances share only the selected subscription and ACR. Access is controlled via existing Entra security groups that you provide at deploy time. ## Security Model -All authenticated users on a GUI instance are **fully trusted**. Any user with -Entra group membership can view and modify all targets, attack history, and -query anything on the database connection. There is no per-user data isolation -within an instance. The trust boundary is Entra group membership. +All authenticated users on a GUI instance are **fully trusted**. Any user with Entra group membership can view and modify all targets, attack history, and query anything on the database connection. There is no per-user data isolation within an instance. The trust boundary is Entra group membership. **Deploy separate instances for separate trust groups.** ## What You Need | Prerequisite | Notes | -|---|---| +| --- | --- | | [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) 2.84+ | Version 2.77 has a known `content-already-consumed` bug | | Python 3.10+ | For running the deployment script | | `az login` with Graph permissions | The script creates Entra app registrations, which requires Graph API access. Run `az login --scope https://graph.microsoft.com//.default` | @@ -26,13 +19,15 @@ within an instance. The trust boundary is Entra group membership. | Container image pushed to ACR | Build and push before deploying (see [Building the Image](#building-the-image)) | | A `.env` file with runtime config | Copy and fill in `infra/env.demo.template`. Contains target endpoints and content safety config. `AZURE_SQL_DB_CONNECTION_STRING` and `AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL` are auto-injected by the script — you can omit them. Required for the default `target` initializer. Targets can also be created manually in the GUI if deploying with the `target` initializer only | -### What the script creates (per-instance) +### What the deployment creates (script + Bicep) | Resource | Naming Convention | -|---|---| +| --- | --- | | Resource Group | `copyrit-{instance-name}` | | Container App | `copyrit-{instance-name}` | | Container App Environment | `copyrit-{instance-name}-env` | +| Virtual Network + delegated ACA subnet | `copyrit-{instance-name}-vnet` / `copyrit-{instance-name}-aca-subnet` | +| NAT Gateway + static egress Public IP | `copyrit-{instance-name}-nat` / `copyrit-{instance-name}-egress-pip` | | User-Assigned Managed Identity | `copyrit-{instance-name}-identity` | | Azure SQL Server + Database | `copyrit-{instance-name}-sql` / `pyrit-{instance-name}` | | Storage Account + Blob Container | `copyrit{instance-name-no-hyphens}sa` / `dbdata` | @@ -40,20 +35,18 @@ within an instance. The trust boundary is Entra group membership. | Entra App Registration | `CoPyRIT GUI ({instance-name})` | | Log Analytics Workspace | `copyrit-{instance-name}-logs` | +All per-instance resources receive `Service`, `Instance`, `ManagedBy`, and `DataClass` ownership tags plus `Owner` when `--owner-tag` is supplied. Bicep creates the Container App, environment, network, static IP, and Log Analytics; the Python script provisions the other resources before invoking Bicep. + ### What is shared across instances -| Resource | Notes | -|---|---| -| Azure Container Registry | Same image, different config per instance | -| Subscription | All instances deploy to the same subscription | +| Resource | Notes | +| ------------------------ | --------------------------------------------- | +| Azure Container Registry | Same image, different config per instance | +| Subscription | All instances deploy to the same subscription | -> **Time estimate:** A new instance takes approximately 15–20 minutes end-to-end -> (script runtime + manual SQL user creation). Plan for this cadence if deploying -> new instances monthly. +> **Time estimate:** A new instance takes approximately 15–20 minutes end-to-end (script runtime + manual SQL user creation). Plan for this cadence if deploying new instances monthly. -The script configures delegated Microsoft Graph `User.Read`; the backend uses the -token with `/me` and `/me/checkMemberGroups` and requires at least one allowed group. -See [README.md](README.md#security) for the full authentication model. +The script configures delegated Microsoft Graph `User.Read`; the backend uses the token with `/me` and `/me/checkMemberGroups` and requires at least one allowed group. See [README.md](README.md#security) for the full authentication model. ## Quick Deploy @@ -87,21 +80,21 @@ python infra/deploy_instance.py \ ``` | Flag | Required | Description | -|---|---|---| +| --- | --- | --- | | `--instance-name` | Yes | Short name for this instance (max 13 chars) | | `--env-file` | Yes | Path to the `.env` file with target endpoints | | `--subscription` | Yes | Azure subscription ID | | `--location` | No | Azure region (default: `eastus2`) | | `--acr-name` | Yes | Shared ACR name | -| `--container-image` | Yes | Full image reference (ACR + tag) | -| `--allowed-groups` | Yes | Comma-separated Entra group OIDs | +| `--container-image` | Yes | Image in `--acr-name` using a non-`latest` tag or SHA-256 digest | +| `--allowed-groups` | Yes | Comma-separated Entra group object IDs (GUIDs) | +| `--allowed-cidr` | No | Optional public ingress IPv4 network in canonical CIDR notation; empty permits all source IPs while Entra and backend group authorization remain enabled | | `--owner-tag` | Conditional | `Owner` tag value applied to all per-instance resources. **Required when the target subscription enforces a "Require a tag on resources" Azure Policy** (this is the case for the AI Red Team Tooling subscription — deployments without it fail with `RequestDisallowedByPolicy`). Optional only on subscriptions without such a policy | | `--service-management-reference` | No | Service Tree ID (required by some tenants for Entra app creation) | | `--aoai-resource-names` | No | Comma-separated Cognitive Services account names for automatic AOAI RBAC. Grants `Cognitive Services OpenAI User` to the MI on each resource. Does **not** cover Content Safety — see step 3 for that. If omitted, all AOAI roles must be granted manually | | `--dry-run` | No | Preview what will be created without executing | -> **Instance name constraints:** Max 13 characters (lowercase letters, numbers, -> hyphens). The Key Vault name `copyrit-{name}-kv` has a 24-character limit. +> **Instance name constraints:** 1–13 lowercase letters, numbers, or internal hyphens; the name must start and end with a letter or number. The Key Vault name `copyrit-{name}-kv` has a 24-character limit. Use `--dry-run` to preview what will be created without making changes: @@ -129,15 +122,11 @@ ALTER ROLE db_ddladmin ADD MEMBER [copyrit-{instance-name}-identity]; **Grant Cognitive Services roles** (if using managed identity auth for Azure OpenAI): -If you passed `--aoai-resource-names` during deployment, the script granted -`Cognitive Services OpenAI User` on each specified AOAI resource. Check the -deploy output for the `AOAI RBAC: X/Y resources granted` line. Verify all -requested resources were granted (X should equal Y). +If you passed `--aoai-resource-names` during deployment, the script granted `Cognitive Services OpenAI User` on each specified AOAI resource. Check the deploy output for the `AOAI RBAC: X/Y resources granted` line. Verify all requested resources were granted (X should equal Y). + +An inaccessible or unknown AOAI name is logged and skipped rather than failing the deployment. Treat `X != Y` as incomplete setup and grant the missing roles manually. -**Content Safety requires a separate role** (`Cognitive Services User`, not -`OpenAI User`). The `--aoai-resource-names` flag does not cover this. If your -`.env` uses managed identity auth for Content Safety (blank API key), grant -the role manually: +**Content Safety requires a separate role** (`Cognitive Services User`, not `OpenAI User`). The `--aoai-resource-names` flag does not cover this. If your `.env` uses managed identity auth for Content Safety (blank API key), grant the role manually: ```bash MI_ID= @@ -168,8 +157,7 @@ az role assignment create --assignee-object-id $MI_ID \ ### 4. Restart the container app -After creating the SQL user, restart the container so it picks up the database -permissions: +After creating the SQL user, restart the container so it picks up the database permissions. This command targets the current latest revision immediately after deployment; use an explicit revision name for older revisions: ```bash az containerapp revision restart \ @@ -183,8 +171,7 @@ az containerapp revision restart \ ### 5. Validate -Do **not** rely solely on `/api/health` — it can pass on an old revision while -the new one is crashing. Run through this checklist: +Do **not** rely solely on `/api/health` — it can pass on an old revision while the new one is crashing. Run through this checklist: - [ ] Latest ACA revision is `Healthy`: ```bash @@ -194,6 +181,8 @@ the new one is crashing. Run through this checklist: --query "[0].{name:name, healthState:properties.healthState}" -o table ``` - [ ] App loads in browser at `https://` +- [ ] The static egress IP printed by the script is allowlisted by external providers +- [ ] SQL and Storage network rules contain that egress IP and no `0.0.0.0` SQL rule - [ ] Entra login works - [ ] Signed-in user name appears in the top bar - [ ] Operator label auto-populates from signed-in username @@ -206,21 +195,19 @@ the new one is crashing. Run through this checklist: ## Customizing the .env -The `.env` file controls which targets appear in the GUI. You can point to any -Azure OpenAI or OpenAI endpoints — they don't need to match the AIRT instance. +The `.env` file controls which targets appear in the GUI. You can point to any Azure OpenAI or OpenAI endpoints — they don't need to match the AIRT instance. **Minimum viable** (just chat + converters): + - `AZURE_OPENAI_GPT4O_*` — one chat target - `AZURE_OPENAI_GPT4O_UNSAFE_CHAT_*` — converter target - `AZURE_OPENAI_GPT4O_UNSAFE_CHAT_*2` — scorer target - `AZURE_CONTENT_SAFETY_*` — harm detection -> **Note:** `AZURE_SQL_DB_CONNECTION_STRING` and -> `AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL` are auto-injected by the deploy -> script from the SQL server and storage account it creates. You do not need -> to set them manually. +> **Note:** `AZURE_SQL_DB_CONNECTION_STRING` and `AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL` are auto-injected by the deploy script from the SQL server and storage account it creates. You do not need to set them manually. **Full modality demo** (uncomment optional sections in the template): + - Image (DALL-E 3) - TTS - Video (Sora-2) @@ -229,113 +216,56 @@ Azure OpenAI or OpenAI endpoints — they don't need to match the AIRT instance. ## Updating Secrets -The Container App reads its `.env` contents from an **inline secret** named -`env-file`. To rotate it safely, use `az containerapp secret set` with the -`@file` form (the file path is on the command line, not the secret value, so -nothing leaks via `ps`). +The Container App reads its `.env` contents from an **inline secret** named `env-file`. `deploy_instance.py` is a create-only workflow and must not be rerun for rotation. -> **`updated.env` is your local file** — same format as `infra/env.demo.template` -> and the file you passed to `--env-file` during initial deployment, but with -> the new values you want to deploy. The filename is just a convention; you -> can name it anything. +Prepare a complete local file in the same format as `infra/env.demo.template`. The filename `updated.env` is only a convention. It must include the two values that the deployment script injected during initial deployment: -> ⚠️ **Verify the file exists before running `az`.** The Azure CLI's `@file` -> expansion is silent: if the path doesn't exist or has a typo, `az` falls -> back to the literal string `@./your-typo.env` and stores **that** as the -> secret value. The container will then read garbage and chat will break with -> no obvious error in the deploy step. +- `AZURE_SQL_DB_CONNECTION_STRING` +- `AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL` -**bash:** +Use one of these approved update paths: -```bash -# Pre-flight: fail fast if the file is missing -test -f ./updated.env || { echo "ERROR: ./updated.env not found"; exit 1; } +1. Update the `env-file` Container App secret in the Azure portal without retrieving or printing its current value. +2. Redeploy `main.bicep` with a complete parameter file containing the current resource values and override only the secure parameter from disk: -az containerapp secret set \ - -n copyrit-{instance-name} \ - -g copyrit-{instance-name} \ - --secrets "env-file=@./updated.env" -``` + ```bash + test -f ./updated.env || { echo "ERROR: ./updated.env not found"; exit 1; } + test -f ./current.parameters.json || { echo "ERROR: complete parameter file not found"; exit 1; } + az deployment group create \ + --resource-group copyrit-{instance-name} \ + --template-file infra/main.bicep \ + --parameters @./current.parameters.json \ + --parameters envFileContents=@./updated.env + ``` -**PowerShell:** + `current.parameters.json` must describe the existing deployment exactly; start from `infra/parameters.example.json` and fill it from the deployed resources. Review `what-if` first. Azure CLI file expansion is silent when a path is wrong, so both existence checks are mandatory. -```powershell -# Pre-flight: fail fast if the file is missing -if (-not (Test-Path .\updated.env)) { throw 'ERROR: .\updated.env not found' } +Application-scoped secret updates do not update an existing revision. Restart the active revision after either path: -az containerapp secret set ` - -n copyrit-{instance-name} ` - -g copyrit-{instance-name} ` - --secrets "env-file=@./updated.env" +```bash +APP_NAME=copyrit-{instance-name} +RESOURCE_GROUP=copyrit-{instance-name} +REVISION=$(az containerapp show -n "$APP_NAME" -g "$RESOURCE_GROUP" \ + --query properties.latestRevisionName -o tsv) +az containerapp revision restart -n "$APP_NAME" -g "$RESOURCE_GROUP" \ + --revision "$REVISION" ``` -> **Important — `.env` content requirements when rotating manually:** -> -> The deploy script auto-injects two values during the **initial** deployment -> that you must include in `updated.env` if you rotate manually: -> - `AZURE_SQL_DB_CONNECTION_STRING` -> - `AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL` -> -> Without them the container will fail to connect to SQL or read blob storage -> on the next restart. Use the values that the deploy script logged on initial -> deploy (or look them up from the SQL/Storage resources). The KV -> `env-global` backup is the authoritative copy of what was last deployed via -> the script. - -After updating the secret, **force a new revision** so the running container -picks up the new value. Per Microsoft Container Apps docs, secret updates do -**not** auto-restart existing revisions — you must either deploy a new -revision or restart an existing one: +These paths update the inline ACA secret, not the `env-global` Key Vault backup. To keep the backup synchronized, update it only from a host with an approved vault network path, or through a temporary network rule when policy permits: ```bash -# bash -az containerapp update \ - -n copyrit-{instance-name} \ - -g copyrit-{instance-name} \ - --set-env-vars "SECRET_UPDATED=$(date +%s)" +az keyvault secret set --vault-name copyrit-{instance-name}-kv \ + --name env-global --file ./updated.env ``` -```powershell -# PowerShell -az containerapp update ` - -n copyrit-{instance-name} ` - -g copyrit-{instance-name} ` - --set-env-vars "SECRET_UPDATED=$([DateTimeOffset]::Now.ToUnixTimeSeconds())" -``` - -> **Note: this rotation path updates the inline ACA secret, not the KV -> `env-global` backup.** The KV copy will drift until the next full deploy. -> If you want to keep KV in sync, also run: -> ```bash -> az keyvault secret set --vault-name copyrit-{instance-name}-kv \ -> --name env-global --file ./updated.env -> ``` -> The KV is locked down (`publicNetworkAccess=Disabled`); this command must -> run from Azure Cloud Shell or with public access temporarily re-enabled. - > **Anti-patterns to avoid:** > -> - `az containerapp secret set --secrets "env-file=$ENV_CONTENT"` — exposes -> the value via process arguments (visible in `ps` while the command runs). -> The `@file` form above passes only the path, not the value. -> - `az containerapp secret show --secret-name env-file` — returns the full -> plaintext to your terminal / shell history. Inspect the KV backup -> instead, or use `az containerapp secret list -o table` to confirm the -> secret exists without revealing its value. -> - `python infra/deploy_instance.py ... --env-file ./updated.env` — the -> deploy script is **not** rotation-safe. It runs unconditional `create` -> operations on the Entra app, SQL server, Key Vault, and managed identity, -> most of which fail with "already exists" errors when re-run against an -> existing instance. The Entra app create succeeds and produces a -> duplicate registration, which is worse than a hard failure. - -> **Why inline instead of Key Vault reference?** Azure Container Apps is not on -> Key Vault's "trusted services" allowlist, so a locked-down KV -> (`publicNetworkAccess=Disabled`, required for SFI / NS221 compliance) blocks -> ACA's runtime secret resolver. Passing the secret inline at deploy time -> sidesteps the issue: the value is stored encrypted in the Container App's -> own secrets store, and the Key Vault is locked down with no runtime -> dependency. +> - `az containerapp secret set --secrets "env-file=$ENV_CONTENT"` — exposes the value via process arguments (visible in `ps` while the command runs). +> - `az containerapp secret set --secrets "env-file=@./updated.env"` — the Container Apps CLI does not document file expansion for secret values and can store the literal path marker instead of the file contents. +> - `az containerapp secret show --secret-name env-file` — returns the full plaintext to your terminal / shell history. Inspect the KV backup instead, or use `az containerapp secret list -o table` to confirm the secret exists without revealing its value. +> - `python infra/deploy_instance.py ... --env-file ./updated.env` — the deploy script is **not** rotation-safe. It runs unconditional `create` operations on the Entra app, SQL server, Key Vault, and managed identity, most of which fail with "already exists" errors when re-run against an existing instance. The Entra app create succeeds and produces a duplicate registration, which is worse than a hard failure. + +> **Why inline instead of a Key Vault reference?** This workflow disables the vault's public network access and does not create a private endpoint, DNS, or peering path from the ACA VNet. Passing the value through a secure Bicep parameter stores it in the Container App secret store without creating that runtime network dependency. ## Adding or Removing Users @@ -358,17 +288,22 @@ az ad group member list --group "" --query "[].displayName" python infra/teardown_instance.py \ --instance-name partners-demo \ --subscription "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" \ + --resource-group-id "/subscriptions//resourceGroups/copyrit-partners-demo" \ + --acknowledge-egress-ip-release \ --delete-entra-app \ + --entra-app-id "" \ --yes ``` This deletes: -- The resource group (Container App, SQL server, Key Vault, MI, networking, logs) -- The Entra app registration and service principal (with `--delete-entra-app`) -> **Note:** Key Vault uses purge protection. The vault name will be reserved -> for ~90 days after deletion. Use a different instance name if redeploying -> immediately. +- All role assignments held by the instance managed identity, including assignments on resources outside the instance resource group +- The verified, deployment-tagged resource group and all resources in it +- The exact Entra app registration and service principal when both `--delete-entra-app` and `--entra-app-id` are supplied + +Before teardown, remove the printed static egress IP from every external allowlist. The acknowledgement flag is mandatory even with `--yes`; the script then waits for resource-group deletion to finish. It refuses untagged legacy groups and groups not created by `deploy_instance.py`. For a legacy instance, inventory it manually rather than bypassing these checks. + +> **Note:** Key Vault uses purge protection. The vault name will be reserved for ~90 days after deletion. Use a different instance name if redeploying immediately. The static egress IP is released and must not remain trusted by downstream systems. ## Building the Image @@ -386,12 +321,9 @@ az acr login --name $ACR_NAME docker push $ACR_NAME.azurecr.io/pyrit:$COMMIT_SHA ``` -Use the resulting `$ACR_NAME.azurecr.io/pyrit:$COMMIT_SHA` as the -`--container-image` argument. +Use the resulting `$ACR_NAME.azurecr.io/pyrit:$COMMIT_SHA` as the `--container-image` argument. -> **Note:** The CI/CD pipeline handles this automatically for the AIRT -> instance. Manual builds are only needed for the initial bootstrap or -> deployments outside the pipeline. +> **Microsoft maintainer note:** The team-owned ADO workflow builds its own image. Community and isolated-instance deployments must build and push the image before running `deploy_instance.py`. ## Troubleshooting @@ -407,18 +339,15 @@ az containerapp revision list \ ``` Common causes: -- **AcrPull role not propagated yet** — RBAC can take a few minutes. The - container will retry automatically. -- **Inline `env-file` secret missing or malformed** — The Container App reads - the `.env` from its own inline secret, not from Key Vault. Verify it exists: + +- **AcrPull role not propagated yet** — RBAC can take a few minutes. The container will retry automatically. +- **Inline `env-file` secret missing or malformed** — The Container App reads the `.env` from its own inline secret, not from Key Vault. Verify it exists: ```bash az containerapp secret list \ -n copyrit-{instance-name} \ -g copyrit-{instance-name} -o table ``` -- **Missing `.pyrit_conf`** — Older container images (before the `.pyrit_conf` - guard was added) crash on startup because the `airt` initializer - unconditionally reads this file. Use an image built from current `main`. +- **Missing `.pyrit_conf`** — Older container images (before the `.pyrit_conf` guard was added) crash on startup because the legacy initializer unconditionally reads this file. Use an image built from current `main`. ### Entra login fails @@ -440,13 +369,7 @@ Common causes: -n copyrit-{instance-name} \ -g copyrit-{instance-name} -o table ``` -- If you suspect the env content is wrong, inspect the Key Vault backup - (`env-global`) instead of the inline secret. The Key Vault snapshot is - written by the deploy script alongside the Container App secret. Reading - from KV via `az keyvault secret show` requires either Cloud Shell or - temporarily re-opening KV public access (`--public-network-access Enabled`). - Avoid `az containerapp secret show --secret-name env-file` — it prints the - full plaintext to terminal/logs. +- If you suspect the env content is wrong, inspect the Key Vault backup (`env-global`) instead of the inline secret. The Key Vault snapshot is written by the deploy script alongside the Container App secret. Read it only through an approved vault network path or a temporary network rule when policy permits. Avoid `az containerapp secret show --secret-name env-file` — it prints the full plaintext to terminal/logs. - Check container logs for initializer errors: ```bash az containerapp logs show \ @@ -457,29 +380,25 @@ Common causes: ### Database connection errors -- Verify the SQL contained user was created (step 3) with all three roles - (`db_datareader`, `db_datawriter`, `db_ddladmin`). -- The deploy script auto-injects `AZURE_SQL_DB_CONNECTION_STRING` into the - `.env` before passing it to the Container App as an inline secret. If you - see a connection string mismatch, inspect the Key Vault backup - (`env-global`) — it holds the value that was last deployed via the script. - Reading from KV requires Cloud Shell or temporarily re-opening KV public - access. Avoid `az containerapp secret show` — it prints the full plaintext - to terminal/logs. -- Verify the Azure SQL firewall allows Azure services (the script configures - this, but verify with `az sql server firewall-rule list`). +- Verify the SQL contained user was created (step 3) with all three roles (`db_datareader`, `db_datawriter`, `db_ddladmin`). +- The deploy script auto-injects `AZURE_SQL_DB_CONNECTION_STRING` into the `.env` before passing it to the Container App as an inline secret. If you see a connection string mismatch, inspect the Key Vault backup (`env-global`) — it holds the value that was last deployed via the script. Read it only through an approved vault network path. Avoid `az containerapp secret show` — it prints the full plaintext to terminal/logs. +- Verify the Azure SQL firewall contains `AllowContainerAppEgress` with the static egress IP printed by the deployment. A `0.0.0.0` rule is not expected: + ```bash + az sql server firewall-rule list \ + -g copyrit-{instance-name} \ + -s copyrit-{instance-name}-sql -o table + ``` ### Blob storage errors -If the container logs show 403/AuthorizationPermissionMismatch when reading or -writing to blob storage: +If the container logs show 403/AuthorizationPermissionMismatch when reading or writing to blob storage: - Verify the storage account exists in the per-instance resource group: ```bash az storage account list -g copyrit-{instance-name} -o table ``` -- Verify the managed identity has `Storage Blob Data Contributor` on the - storage account scope (the script grants this automatically): +- Verify the managed identity has `Storage Blob Data Contributor` on the storage account scope (the script grants this automatically): + ```bash az role assignment list \ --assignee \ @@ -487,20 +406,23 @@ writing to blob storage: -g copyrit-{instance-name} --query id -o tsv) \ -o table ``` -- Verify the deployed env content has the correct - `AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL`. The safest way to check is - to inspect the Key Vault backup snapshot (after temporarily re-enabling - public access on the KV or running from Cloud Shell): + + - Verify the storage firewall default is `Deny`, bypass is `None`, and its IP rules contain the static egress IP: + ```bash + az storage account show \ + -g copyrit-{instance-name} \ + -n \ + --query networkRuleSet -o json + ``` + +- Verify the deployed env content has the correct `AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL`. The safest way to check is to inspect the Key Vault backup snapshot from an approved vault network path: ```bash az keyvault secret show --vault-name copyrit-{instance-name}-kv \ --name env-global --query value -o tsv | \ grep AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL ``` - Avoid `az containerapp secret show` — it prints the full plaintext to - terminal/logs. If the value is wrong, rotate the secret using the manual - procedure in [Updating Secrets](#updating-secrets). -- RBAC propagation takes ~60 seconds after a fresh deployment; if the role was - granted very recently, restart the container revision. + Avoid `az containerapp secret show` — it prints the full plaintext to terminal/logs. If the value is wrong, rotate the secret using the manual procedure in [Updating Secrets](#updating-secrets). +- RBAC propagation takes ~60 seconds after a fresh deployment; if the role was granted very recently, restart the container revision. ### Graph API / Entra commands fail @@ -510,28 +432,21 @@ If `az ad` commands fail with `AADSTS530084`, re-login with Graph scope: az login --scope https://graph.microsoft.com//.default ``` -This commonly happens in codespaces or non-corp-joined devices due to -conditional access policies. Run Entra-related commands from a local machine -with `az login`. +This commonly happens in codespaces or non-corp-joined devices due to conditional access policies. Run Entra-related commands from a local machine with `az login`. ### AOAI returns 401 PermissionDenied -If chat returns `The principal lacks the required data action`, the -managed identity doesn't have `Cognitive Services OpenAI User` on the AOAI -resource. Either: -- Re-run deployment with `--aoai-resource-names` to automate the grants, or -- Grant manually: - ```bash - az role assignment create --assignee-object-id \ - --assignee-principal-type ServicePrincipal \ - --role "Cognitive Services OpenAI User" \ - --scope - ``` - RBAC propagation takes ~60 seconds. No container restart needed. +If chat returns `The principal lacks the required data action`, the managed identity doesn't have `Cognitive Services OpenAI User` on the AOAI resource. Do not rerun the create-only deployment script. Grant the role manually: + +```bash +az role assignment create --assignee-object-id \ + --assignee-principal-type ServicePrincipal \ + --role "Cognitive Services OpenAI User" \ + --scope +``` + +RBAC propagation takes ~60 seconds. No container restart needed. ### Windows: `FileNotFoundError` when running scripts -The Azure CLI is installed as `az.cmd` on Windows. Both `deploy_instance.py` -and `teardown_instance.py` handle this automatically with `shell=True` when -running on Windows. If you encounter this error, ensure you are using the -latest version of the scripts. +The Azure CLI is installed as `az.cmd` on Windows. Both `deploy_instance.py` and `teardown_instance.py` handle this automatically with `shell=True` when running on Windows. If you encounter this error, ensure you are using the latest version of the scripts. diff --git a/infra/README.md b/infra/README.md index f4af84d426..cd4ee19fe0 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,29 +1,70 @@ # CoPyRIT GUI — Azure Deployment -Deploy the CoPyRIT GUI as an Azure Container App with -[MSAL](https://learn.microsoft.com/en-us/entra/msal/) PKCE authentication, -managed identity, security response headers, and no embedded secrets. +Deploy the CoPyRIT GUI as an Azure Container App with [MSAL](https://learn.microsoft.com/en-us/entra/msal/) PKCE authentication, managed identity, security response headers, and no secrets embedded in source or container images. ## Architecture -``` -Users ──→ MSAL PKCE auth ──→ Container App - ↓ - Graph-backed authentication - ↓ - Microsoft Graph /me + memberships - ↓ - User-Assigned MI - ↙ ↙ ↓ ↘ ↘ - Azure SQL ACR Azure OpenAI Key Vault Storage - (MI auth) (AcrPull) (RBAC) (secret refs) (Blob) +```mermaid +flowchart TB + user["User browser"] + entra["Microsoft Entra ID"] + msGraph["Microsoft Graph
/me + /me/checkMemberGroups"] + providers["External model providers"] + + subgraph azure["Azure subscription"] + frontDoor["Azure Front Door Premium
Optional managed HTTPS entry point"] + ingress["ACA-managed public HTTPS ingress
Optional allowedCidr restriction"] + + subgraph vnet["Virtual network"] + subgraph subnet["Delegated ACA infrastructure subnet"] + environment["Public ACA workload-profiles environment"] + app["Container App
React SPA + FastAPI API"] + environment --> app + end + nat["NAT Gateway"] + end + + egress["Static public egress IPv4"] + identity["User-assigned managed identity"] + acr["Azure Container Registry"] + keyVault["Key Vault"] + sql["Azure SQL"] + azureAi["Azure OpenAI / Azure AI"] + storage["Azure Storage"] + logAnalytics["Log Analytics"] + appInsights["Application Insights
Optional OpenTelemetry"] + end + + user -->|"HTTPS when Front Door enabled"| frontDoor + frontDoor -->|"HTTPS origin"| ingress + user -->|"Direct ACA public URL"| ingress + ingress --> environment + user -->|"MSAL PKCE sign-in"| entra + entra -->|"Delegated Graph token"| user + + app -.->|"Uses"| identity + identity -.->|"AcrPull"| acr + identity -.->|"Key Vault reference
(when configured)"| keyVault + identity -.->|"Passwordless access"| sql + identity -.->|"RBAC"| azureAi + identity -.->|"Blob data access"| storage + + app -->|"Application egress"| nat + environment -->|"Environment egress"| nat + nat --> egress + egress --> msGraph + egress --> acr + egress --> keyVault + egress --> sql + egress --> azureAi + egress --> storage + egress --> providers + + environment -->|"App logs"| logAnalytics + app -.->|"Traces after agent setup"| appInsights ``` -Logging & monitoring: -``` -ACA Environment → Log Analytics (app logs) -Container App → Application Insights (OTel traces, when enabled) -``` +The base topology is public ACA-managed HTTPS ingress plus VNet-integrated fixed NAT egress. `enableFrontDoor=true` adds Front Door Premium as the preferred managed HTTPS URL while the ACA origin remains concurrently public. Requests can therefore bypass Front Door through the ACA hostname; Front Door is a routing and reliability layer, not the exclusive ingress security boundary. The Microsoft team ADO workflow enables Front Door; community deployments leave it disabled by default. Front Door mode requires `allowedCidr` to be empty; Bicep rejects that combination instead of silently dropping a requested client restriction. Front Door changes inbound routing only: outbound connections from ACA continue to use the NAT Gateway's static IPv4. ## Development Workflow @@ -35,103 +76,65 @@ npm install # one-time: install frontend dependencies npm start # starts both backend (port 8000) and frontend (port 3000) ``` -`npm start` runs `dev.py`, which launches the FastAPI backend and Vite dev server -together, waits for the health check, and prints URLs when ready. Press Ctrl+C to -stop both. +`npm start` runs `dev.py`, which launches the FastAPI backend and Vite dev server together, waits for the health check, and prints URLs when ready. Press Ctrl+C to stop both. -When `ENTRA_TENANT_ID` and `ENTRA_CLIENT_ID` are not set, auth is disabled — -all requests are allowed. Swagger UI is available at `http://localhost:8000/docs`. +When `ENTRA_TENANT_ID`, `ENTRA_CLIENT_ID`, and `ENTRA_ALLOWED_GROUP_IDS` are all unset, auth is disabled and all requests are allowed. Setting only a subset is a startup configuration error. Swagger UI is available at `http://localhost:8000/docs`. -> ⚠️ Auth-disabled mode is for **local development only**. Never deploy to a -> network-accessible environment without both env vars set. +> ⚠️ Auth-disabled mode is for **local development only**. Never deploy to a network-accessible environment without all three authentication settings. -### Deployment Workflow +### Deployment workflows ``` -Local dev → Push to branch → Trigger pipeline in ADO → Deploys to test → Opt-in prod deploy +Local dev → Build and push image → Preview Bicep changes → Deploy → Complete post-deployment steps ``` -The CI/CD pipeline (`gui-deploy.yml`) automates build → push → deploy. -Production is opt-in via `deployToProd: true`. +Community users can deploy `main.bicep` directly using the instructions below. For a fully provisioned isolated instance, use `deploy_instance.py` and [DEPLOY_NEW_INSTANCE.md](DEPLOY_NEW_INSTANCE.md). `gui-deploy.yml` is the Microsoft team's internal Azure DevOps workflow; it depends on team-owned ADO configuration and is not the community deployment interface. + +`deploy_instance.py` uses the default `enableFrontDoor=false` path. Use direct Bicep when a community deployment needs Front Door. The script still automates resource creation, Entra setup, inline secrets, and selected RBAC, but its documented SQL and provider post-deployment steps remain required. ## Security -- **Authentication**: [MSAL](https://learn.microsoft.com/en-us/entra/msal/) - [PKCE](https://oauth.net/2/pkce/) on the frontend (`@azure/msal-browser`) + - Microsoft Graph-backed middleware on the backend. The frontend sends a delegated - Graph token, and the backend authenticates it through Graph `/me`. PKCE (public - client) requires no client secrets or certificates. -- **Authorization**: Entra group check via `allowedGroupObjectIds` param. Requires - delegated Graph `User.Read`; the backend calls `/me/checkMemberGroups` and compares - the returned transitive memberships with the configured group IDs. Each security - group must also be assigned to the enterprise app (see Prerequisites §3). Authenticated - deployments require at least one allowed group and fail to start without one. - Successful identity and membership results are cached in-process for 60 seconds by - token digest to reduce Graph latency and throttling; bearer tokens are never cached. -- **Identity**: User-assigned managed identity (UAMI) — created before the container - app so [RBAC](https://learn.microsoft.com/en-us/azure/role-based-access-control/) - roles are active before the first revision starts. `AZURE_CLIENT_ID` is set to the - UAMI's client ID so `DefaultAzureCredential` uses the correct identity. -- **Network** (opt-in hardening): - - **Private Endpoint** (`enablePrivateEndpoint=true`): disables public access - entirely — the app is only reachable via the PE's private network. Requires VNet - peering or VPN to reach. When PE is enabled, `allowedCidr` is ignored. - - **IP restriction** (`allowedCidr`): restricts ingress to a - [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) range. - Only applies when PE is disabled. Empty = no IP-level restriction (auth still - required). - - Neither is required — MSAL auth + group checks are the primary access controls. -- **Response headers**: `SecurityHeadersMiddleware` adds - [CSP](https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP), - HTTP Strict Transport Security (HSTS, production only), X-Frame-Options, X-Content-Type-Options, Referrer-Policy, - Permissions-Policy, and Cache-Control (`no-store` on API routes). Swagger/OpenAPI - disabled in production. +- **Authentication**: [MSAL](https://learn.microsoft.com/en-us/entra/msal/) [PKCE](https://oauth.net/2/pkce/) on the frontend (`@azure/msal-browser`) + Microsoft Graph-backed middleware on the backend. The frontend sends a delegated Graph token, and the backend authenticates it through Graph `/me`. PKCE (public client) requires no client secrets or certificates. +- **Authorization**: Entra group check via `allowedGroupObjectIds` param. Requires delegated Graph `User.Read`; the backend calls `/me/checkMemberGroups` and compares the returned transitive memberships with the configured group IDs. Each security group must also be assigned to the enterprise app (see Prerequisites §3). Authenticated deployments require at least one allowed group and fail to start without one. `/api/health`, `/api/auth/config`, and `/api/media` are intentional public exceptions; other `/api` routes require authentication when auth is enabled. Successful identity and membership results are cached in-process for 60 seconds, keyed by a SHA-256 token digest, to reduce Graph latency and throttling. Bearer tokens themselves are not stored in the cache. +- **Identity**: `deploy_instance.py` creates its user-assigned managed identity (UAMI) and grants AcrPull and Storage Blob Data Contributor before deploying Bicep. A direct Bicep deployment can create `-identity`, but the template creates no role assignments, so its first revision can remain unhealthy until required roles are granted and the revision is restarted. A healthy one-pass direct deployment uses an existing, pre-authorized UAMI. `AZURE_CLIENT_ID` is set to the UAMI's client ID so `DefaultAzureCredential` selects the correct identity. +- **Network**: The template always creates a VNet-integrated public Container Apps environment, one delegated ACA infrastructure subnet, a Standard NAT Gateway, and a static outbound IPv4. ACA supplies the generated HTTPS hostname and trusted certificate. In direct-ACA mode, `allowedCidr` optionally restricts public ingress to one IPv4 CIDR; an empty value permits public ingress. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door backend addresses, not the original client; Bicep and the internal pipeline reject the invalid combination. Entra sign-in, enterprise-app assignment, and backend group checks remain mandatory application access controls. +- **Front Door**: `enableFrontDoor=true` creates a Premium profile, managed `azurefd.net` endpoint, HTTPS ACA origin, `/api/health` probe, and uncached catch-all route. The module does not create a WAF policy or isolate the public ACA origin, so application authentication and authorization remain mandatory on both hostnames. +- **Routing**: Public inbound requests reach ACA either directly or through Front Door and do not traverse the NAT Gateway. Outbound connections from the ACA environment that leave the virtual network use the NAT Gateway's static public IPv4. +- **Response headers**: `SecurityHeadersMiddleware` adds [CSP](https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP), HTTP Strict Transport Security (HSTS, production only), X-Frame-Options, X-Content-Type-Options, Referrer-Policy, Permissions-Policy, and Cache-Control (`no-store` on API routes). Swagger/OpenAPI disabled in production. - **Data**: Azure SQL with managed identity authentication (no passwords) -- **Secrets**: The `.env` file contents are passed inline to the Container App - as a `@secure()` Bicep parameter (`envFileContents`) and stored encrypted in - the Container App's secret store. Azure Container Apps is not on Key Vault's - "trusted services" list, so a locked-down KV would block runtime secret - resolution — passing inline avoids that. The Key Vault is still created and - populated with the `.env` content as `env-global` for backup/audit, then - fully locked down (`publicNetworkAccess=Disabled`). -- **Images**: Unique tags or digests required — `:latest` is detected by a soft guardrail -- **Supply chain**: [ACR](https://learn.microsoft.com/en-us/azure/container-registry/) - pull via managed identity RBAC (must be granted manually; see Post-Deployment §2). - `frontend/.npmrc` pins the npm registry. `docker/Dockerfile` declares - `ARG BASE_IMAGE` with no default — all callers pass it explicitly to avoid container - supply chain security scanner warnings. -- **Tags**: All resources tagged with Service/Owner/DataClass for governance -- **Logging**: Log Analytics (app logs) + optional - [OTel](https://opentelemetry.io/) via Application Insights +- **Secrets**: When `envFileContents` is nonempty, Bicep stores it as an inline ACA secret. Otherwise, Bicep creates a versionless Key Vault reference to `envSecretName` using the app UAMI; that path requires `Key Vault Secrets User` and network access to the vault. `deploy_instance.py` uses the inline path. +- **Images**: Direct Bicep deployments must supply a unique tag or digest; the template does not reject `:latest`. +- **Supply chain**: [ACR](https://learn.microsoft.com/en-us/azure/container-registry/) pull uses managed identity RBAC. `deploy_instance.py` grants AcrPull, while direct Bicep callers manage it themselves. `frontend/.npmrc` pins the npm registry. `docker/Dockerfile` declares `ARG BASE_IMAGE` with no default — all callers pass it explicitly to avoid container supply chain security scanner warnings. +- **Tags**: Bicep applies the supplied `tags` object to every resource it creates; the default object includes Service/Owner/DataClass governance tags. +- **Logging**: Log Analytics (app logs) + optional [OTel](https://opentelemetry.io/) via Application Insights ## Prerequisites -> **Before you begin**: Run `az login` and confirm your subscription with -> `az account show`. You need permissions to create Entra app registrations, -> security groups, and Azure resource deployments. +> **Before you begin**: Run `az login` and confirm your subscription with `az account show`. You need permissions to create Entra app registrations, security groups, and Azure resource deployments. -The Bicep template creates most infrastructure automatically (ACR, Log Analytics, -managed identity). Entra ID resources must be created -separately (Microsoft Graph, not ARM). Key Vault must be an existing vault -(avoids purge-protection issues on redeployment). RBAC role assignments must -be created manually — see [Post-Deployment §2](#post-deployment). +The Bicep template creates the Container Apps resources, dedicated network, NAT Gateway, static egress IP, and (unless supplied) Log Analytics workspace. It can also declare an ACR and UAMI, but it does not push an image or create RBAC role assignments. The supported one-pass workflows therefore use an existing ACR; a healthy one-pass direct Bicep deployment also uses an existing, pre-authorized UAMI. Entra resources must be created separately through Microsoft Graph. Bicep requires an existing Key Vault. See [Post-Deployment §2](#post-deployment) for direct-deployment RBAC. + +Front Door is optional. When enabled, the subscription must have the `Microsoft.Cdn` resource provider registered. + +> **Migration boundary:** This template owns a dedicated VNet and creates a VNet-integrated workload-profiles environment. ACA environment network type is creation-time configuration. Do not apply this template in place to a legacy environment created without this VNet or with the former private-endpoint parameters. Deploy a parallel resource group/app/environment, validate it, and then migrate users and redirect URIs. **Requirements:** -- [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) **2.84+** -(version 2.77 has a known `content-already-consumed` bug) -- Container image must be pushed to ACR **before** deployment (see [§5 -below](#5-container-image-must-be-pushed-to-acr-before-deployment)) + +- [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) **2.84+** (version 2.77 has a known `content-already-consumed` bug) +- `jq` for the Bash redirect-URI set-union example +- Container image must be pushed to an existing ACR **before** deployment (see [§5 below](#5-container-image-must-be-pushed-to-acr-before-deployment)) **Quick reference** — what you need before running `az deployment group create`: | # | What | How | Key Output | -|---|------|-----|------------| +| --- | --- | --- | --- | | 1 | Resource group | `az group create` | `` name | | 2 | Entra app registration | Portal or CLI (Graph API) | `entraClientId`, `entraTenantId` | | 3 | Security group + SP assignment | Portal or CLI | `allowedGroupObjectIds` | | 4 | SQL server with Entra admin | Existing server | `sqlServerFqdn`, `sqlDatabaseName` | | 5 | Container image in ACR | Docker build + push | `containerImage` | | 6 | Key Vault | Existing vault | `keyVaultResourceId` | +| 7 | Pre-authorized UAMI | Existing identity | `existingManagedIdentityResourceId` | ### 1. Resource group @@ -145,25 +148,41 @@ No secrets or certificates needed — MSAL PKCE uses only the client ID (public ```bash # Create app registration (--service-management-reference may be required by your org) -az ad app create --display-name pyrit-gui --sign-in-audience AzureADMyOrg \ - --service-management-reference "" - -# Get the client ID (use this as entraClientId) -APP_ID=$(az ad app list --display-name pyrit-gui --query '[0].appId' -o tsv) +APP_ID=$(az ad app create \ + --display-name pyrit-gui \ + --sign-in-audience AzureADMyOrg \ + --service-management-reference "" \ + --query appId -o tsv) echo "entraClientId: $APP_ID" +# Create the service principal (enterprise app) used for group assignments +az ad sp create --id "$APP_ID" --output none + # Get the tenant ID (use this as entraTenantId) az account show --query tenantId -o tsv ``` -> **Note**: The redirect URI requires the app FQDN, which is only known after -> the first deployment. After deploying, set the SPA redirect URI: +> **Fresh app registrations only**: The ACA and optional Front Door hostnames are known after deployment. For a newly created app with no existing SPA redirects, register both the direct ACA URL and selected public URL (they are identical when Front Door is disabled): +> > ```bash -> FQDN=$(az deployment group show -g -n main \ +> ACA_FQDN=$(az deployment group show -g -n \ > --query properties.outputs.appFqdn.value -o tsv) -> az ad app update --id $APP_ID \ -> --spa-redirect-uris "https://$FQDN" +> PUBLIC_FQDN=$(az deployment group show -g -n \ +> --query properties.outputs.publicFqdn.value -o tsv) +> APP_OBJECT_ID=$(az ad app show --id "$APP_ID" --query id -o tsv) +> REDIRECT_URIS=$(jq -cn \ +> --arg aca "https://$ACA_FQDN" \ +> --arg public "https://$PUBLIC_FQDN" \ +> '[$aca, $public] | unique') +> PATCH_BODY=$(jq -cn --argjson uris "$REDIRECT_URIS" \ +> '{spa:{redirectUris:$uris}}') +> az rest --method PATCH \ +> --uri "https://graph.microsoft.com/v1.0/applications/$APP_OBJECT_ID" \ +> --headers 'Content-Type=application/json' \ +> --body "$PATCH_BODY" > ``` +> +> For replacement/migration deployments that reuse an app registration, do not run this command because it replaces the URI list. Use the set-union procedure in Post-Deployment instead. **Configure delegated Microsoft Graph access** (required): @@ -171,37 +190,33 @@ In Azure Portal → App registrations → your app → **API permissions**: 1. Select **Add a permission** → **Microsoft Graph** → **Delegated permissions**. 2. Add `User.Read`. -3. Grant consent according to your tenant policy. `User.Read` does not normally - require admin consent, but some tenants disable user consent. +3. Grant consent according to your tenant policy. `User.Read` does not normally require admin consent, but some tenants disable user consent. -The frontend requests `User.Read`. The backend treats the resulting Graph access -token as opaque and forwards it only to fixed or allowlisted Graph endpoints. Graph -validates the token when the backend calls `/me` and `/me/checkMemberGroups`. +The frontend requests `User.Read`. The backend treats the resulting Graph access token as opaque and forwards it only to fixed or allowlisted Graph endpoints. Graph validates the token when the backend calls `/me` and `/me/checkMemberGroups`. + +Or via CLI (this adds `User.Read` without replacing other API permissions): -Or via CLI: ```bash # Add delegated Microsoft Graph User.Read # Graph app ID: 00000003-0000-0000-c000-000000000000 # User.Read delegated permission ID: e1fe6dd8-ba31-4d61-89e7-88639da4683d -APP_OBJ_ID=$(az ad app show --id $APP_ID --query id -o tsv) -az rest --method PATCH \ - --url "https://graph.microsoft.com/v1.0/applications/$APP_OBJ_ID" \ - --body '{"requiredResourceAccess":[{"resourceAppId":"00000003-0000-0000-c000-000000000000","resourceAccess":[{"id":"e1fe6dd8-ba31-4d61-89e7-88639da4683d","type":"Scope"}]}]}' +az ad app permission add --id "$APP_ID" \ + --api 00000003-0000-0000-c000-000000000000 \ + --api-permissions e1fe6dd8-ba31-4d61-89e7-88639da4683d=Scope ``` ### 3. Entra security groups (required for group-based authorization) -Create one or more security groups for authorized users. Multiple groups can be -specified as comma-separated IDs in `allowedGroupObjectIds`. +Create one or more security groups for authorized users. Multiple groups can be specified as comma-separated IDs in `allowedGroupObjectIds`. ```bash # Create security group for authorized users # NOTE: This may require elevated permissions. If it fails, create the group # in Azure Portal → Entra ID → Groups → New group (Security type). -az ad group create --display-name "MyApp-Users" --mail-nickname myapp-users - -# Get the group Object ID (use this as allowedGroupObjectIds) -GROUP_ID=$(az ad group show --group "MyApp-Users" --query id -o tsv) +GROUP_ID=$(az ad group create \ + --display-name "MyApp-Users" \ + --mail-nickname myapp-users \ + --query id -o tsv) echo "allowedGroupObjectIds: $GROUP_ID" # Add users to the group @@ -211,9 +226,7 @@ az ad group member add --group "MyApp-Users" --member-id az ad group member list --group "MyApp-Users" --query '[].displayName' -o tsv ``` -**IMPORTANT: Assign each group to the enterprise application.** This enables the -recommended `appRoleAssignmentRequired` sign-in restriction in addition to the -backend's Graph-based group authorization: +**IMPORTANT: Assign each group to the enterprise application.** This enables the recommended `appRoleAssignmentRequired` sign-in restriction in addition to the backend's Graph-based group authorization: ```bash # Get the service principal (enterprise app) object ID @@ -221,7 +234,7 @@ SP_ID=$(az ad sp show --id $APP_ID --query id -o tsv) # Assign the security group (uses default access role) az rest --method POST \ - --url "https://graph.microsoft.com/v1.0/servicePrincipals/$SP_ID/appRoleAssignments" \ + --url "https://graph.microsoft.com/v1.0/servicePrincipals/$SP_ID/appRoleAssignedTo" \ --body "{\"principalId\": \"$GROUP_ID\", \"resourceId\": \"$SP_ID\", \"appRoleId\": \"00000000-0000-0000-0000-000000000000\"}" # Restrict token issuance to assigned users/groups only (recommended). @@ -230,20 +243,11 @@ az rest --method POST \ az ad sp update --id $SP_ID --set appRoleAssignmentRequired=true ``` -Enterprise-app assignment restricts sign-in through this SPA, but a Graph token is -not client-bound at this backend. The configured allowed groups are therefore the -backend's authoritative security boundary. Never deploy with an empty group list. +Enterprise-app assignment restricts sign-in through this SPA, but a Graph token is not client-bound at this backend. The configured allowed groups are therefore the backend's authoritative security boundary. Never deploy with an empty group list. -**Nested groups**: Entra enterprise app assignment does **not** cascade to nested -groups. If group A contains group B as a member, only direct members of A are -considered assigned. To grant access to members of B, assign B to the enterprise -app separately and include both group IDs in `allowedGroupObjectIds`. +**Nested groups**: Entra enterprise app assignment does **not** cascade to nested groups. If group A contains group B as a member, only direct members of A are considered assigned. To grant access to members of B, assign B to the enterprise app separately and include both group IDs in `allowedGroupObjectIds`. -**App roles** (optional): You can define custom app roles on the app registration -(e.g., `MyApp.User.All`) and assign groups to specific roles instead of the -default access role. The backend authorizes using memberships returned by Graph, -not token `groups` or `roles` claims, so app roles serve as organizational metadata -and for `appRoleAssignmentRequired` gating at the IdP level. +**App roles** (optional): You can define custom app roles on the app registration (e.g., `MyApp.User.All`) and assign groups to specific roles instead of the default access role. The backend authorizes using memberships returned by Graph, not token `groups` or `roles` claims, so app roles serve as organizational metadata and for `appRoleAssignmentRequired` gating at the IdP level. ### 4. Azure SQL server with Entra admin (existing) @@ -286,18 +290,14 @@ docker push $ACR_NAME.azurecr.io/pyrit:$COMMIT_SHA echo "containerImage: $ACR_NAME.azurecr.io/pyrit:$COMMIT_SHA" ``` -> **Note**: The CI/CD pipeline handles build + push automatically. Manual push is -> only needed for the initial bootstrap or if deploying outside the pipeline. +> `deploy_instance.py` and direct Bicep deployments both require the image to exist in ACR; neither path builds or pushes it. -### 6. Key Vault (existing — required for backup/audit only) +### 6. Key Vault (existing) -Use an existing Key Vault to avoid soft-delete/purge-protection naming conflicts -on redeployment. As of the inline-secret migration, the Container App does -**not** read secrets from Key Vault at runtime — the `.env` content is passed -inline via the `envFileContents` Bicep parameter. The vault is still required -because the deploy script writes the `.env` content as `env-global` for -backup/audit, but the managed identity does **not** need `Key Vault Secrets -User` (was previously required, no longer is). +`main.bicep` consumes an existing Key Vault reference; it never creates or deletes a vault. `deploy_instance.py` creates its vault before invoking Bicep. Secret behavior depends on the deployment path: + +- `deploy_instance.py` passes `.env` content through `envFileContents`; Key Vault is a locked-down backup/audit copy and runtime does not read it. +- Direct Bicep deployments with empty `envFileContents` resolve `envSecretName` from Key Vault through the app UAMI. The identity needs `Key Vault Secrets User`, and the vault network policy must permit the Container Apps environment. ```bash # Create a vault (if your org doesn't provide one) @@ -311,31 +311,27 @@ az keyvault create \ az keyvault show --name --query id -o tsv ``` -> **Note**: The vault should have `enableRbacAuthorization: true`. Diagnostic -> settings (AuditEvent logs) should be configured separately by the vault -> owner. The deploy script creates the vault with `defaultAction=Deny` and only -> the deployer's IP allowlisted, then removes the IP rule and sets -> `publicNetworkAccess=Disabled` after writing the backup secret (matches the -> team standard for SFI/NS221 compliance — no window of unrestricted public access). +> **Note**: The vault should have `enableRbacAuthorization: true`. Diagnostic settings (AuditEvent logs) should be configured separately by the vault owner. `deploy_instance.py` creates its vault with `defaultAction=Deny` and only the deployer's IP allowlisted, then removes that rule and sets `publicNetworkAccess=Disabled` after writing the backup secret. The generic `az keyvault create` example above does not configure that network policy; apply your organization's approved policy and preserve a runtime path whenever using Key Vault references. ## Preview changes before deploying (recommended) -Use `what-if` to see what Azure will create, modify, or delete -— without making any changes. Review the output before deploying. +Use `what-if` to see what Azure will create, modify, or delete — without making any changes. Review the output before deploying. ```bash az deployment group what-if \ + --name \ --resource-group \ --template-file infra/main.bicep \ - --parameters @infra/parameters.json + --parameters @infra/parameters.json \ + --parameters existingManagedIdentityResourceId="" ``` -The output shows a color-coded diff: green (+) for new resources, -orange (~) for modifications, red (-) for deletions, and purple (*) -for no change. +The output shows a color-coded diff: green (+) for new resources, orange (~) for modifications, red (-) for deletions, and purple (\*) for no change. ## Deploy +For a healthy one-pass direct deployment, set `acrName` or `acrResourceId` to an existing registry and set `existingManagedIdentityResourceId` to a UAMI that already has AcrPull and all required data-plane permissions. If `envFileContents` is empty, that identity also needs `Key Vault Secrets User` and a network path to the vault. If Bicep creates the identity instead, expect to grant its roles after resource creation and restart the failed revision. + ```bash # Copy and fill in parameters cp infra/parameters.example.json infra/parameters.json @@ -343,198 +339,252 @@ cp infra/parameters.example.json infra/parameters.json # Deploy az deployment group create \ + --name \ --resource-group \ --template-file infra/main.bicep \ - --parameters @infra/parameters.json + --parameters @infra/parameters.json \ + --parameters existingManagedIdentityResourceId="" +``` + +### Deployment outputs + +Use deployment outputs rather than reconstructing public hostnames: + +| Output | Meaning | +| --- | --- | +| `publicFqdn` | User-facing hostname: Front Door when enabled, otherwise ACA | +| `frontDoorFqdn`, `frontDoorUrl` | Managed Front Door hostname/URL; empty when disabled | +| `appFqdn` | Direct public ACA origin for diagnostics and rollback | +| `egressPublicIpAddress` | Static outbound NAT IPv4 for provider allowlists | +| `natGatewayId`, `acaInfrastructureSubnetId`, `vnetName` | Created network resources | +| `managedIdentityPrincipalId`, `managedIdentityResourceId` | UAMI identifiers for RBAC and SQL setup | +| `acrLoginServer`, `keyVaultName` | Effective existing/created service names | +| `appInsightsConnectionString` | Application Insights value when OTel is enabled | + +```bash +az deployment group show -g -n \ + --query properties.outputs ``` +### Microsoft team Azure DevOps deployment + +> This section documents the repository maintainers' internal pipeline. It depends on Microsoft team-owned ADO service connections, environments, and variable groups. It is not required or expected for community deployments; use direct Bicep or `deploy_instance.py` instead. + +`gui-deploy.yml` is an **update-only** workflow for the pre-created test-v2 and prod-v2 stacks: + +1. Build the source image and push a commit-SHA tag to ACR. +2. Capture the exact pushed digest and pass it across stages. +3. Require the existing app, environment, VNet, subnet, NAT, and reserved PIP; validate their IDs, prefixes, tags, SKU, allocation, and attachments. +4. Run a full ARM `what-if` through a fail-closed validator; reject malformed results, deletions, cross-resource-group writes, protected-network deltas other than the documented read-only NAT/PIP normalization, and core network, app, or Log Analytics workspace creates. The expected PIP protection lock may be created. +5. Preserve policy-managed PIP tags and deploy with `enableFrontDoor=true` and `protectEgressPublicIp=true`. +6. Verify the digest-pinned ACA revision, Front Door `/api/health`, and the same PIP resource ID/address after deployment. +7. Print the Front Door URL, direct ACA origin, and static egress IPv4. + +Qualifying merges to `main` automatically deploy test. Production deployment is independent of PyRIT package releases: manually queue a commit merged to `main` with `deployToProd=true`. The workflow deploys test first, then requires a timeout-rejecting manual approval whose requester cannot self-approve. + +`copyrit-gui-common` supplies the shared image settings: + +| Variable | Purpose | +| --------------------------- | ---------------------------------------------- | +| `acrName`, `acrLoginServer` | Existing shared registry name and login server | +| `imageName` | Repository name within the registry | + +Both `copyrit-gui-test` and `copyrit-gui-prod` supply: + +| Variable | Purpose | +| --- | --- | +| `deploymentResourceGroup` | Pre-created dedicated resource group | +| `deploymentAppName` | Container App and resource-name prefix | +| `deploymentVnetAddressPrefix` | IPAM-approved, nonoverlapping VNet CIDR | +| `deploymentInfrastructureSubnetAddressPrefix` | Dedicated ACA subnet CIDR, `/27` minimum (`/26` recommended) | +| `deploymentAllowedClientCidr` | Must be empty (enforced). ACA sees Front Door backend addresses, not original client IPs | +| `managedIdentityResourceId` | Existing UAMI with ACR, Key Vault, SQL, and provider permissions | +| `entraTenantId`, `entraClientId` | SPA authentication configuration | +| `allowedGroupObjectIds` | Backend-authorized Entra security groups | +| `sqlServerFqdn`, `sqlDatabaseName` | Existing SQL database | +| `keyVaultResourceId`, `envSecretName` | Existing runtime configuration secret | +| `acrResourceId`, `enableOtel` | Registry resource ID and observability setting | + +The container image is not a library variable. The Build stage publishes the exact pushed digest as `immutableImage`, and both deployment stages consume that output. Do not add the legacy `image`, `resourceGroup`, `appName`, or `enablePrivateEndpoint` variables; the current workflow does not consume them. + +Pipeline definition 139 reads `gui-deploy.yml` from the GitHub commit being queued. Treat YAML and variable-group contract changes as one release: do not remove old keys before the commit that consumes the replacement keys reaches the target branch. Otherwise ADO leaves unresolved `$(name)` text in Bash, where it is interpreted as command substitution. + +`copyrit-gui-prod` must additionally define `prodApprovers` as the users or ADO groups allowed to approve `ManualValidation@1`. Protect the production variable group with ADO permissions; the approver list is authorization configuration, not a secret. + +The resource group, registry, image-pull authorization, managed identity, Key Vault secret and access path, SQL user/roles and network path, and provider permissions must exist before the first pipeline run. The pipeline does not bootstrap those dependencies or update Entra redirect URIs. Setting `enableOtel=true` creates Application Insights and configures the app endpoint, but the managed agent still requires the post-deployment command in Notes. + +The internal workflow is update-only for networking: its app name and prefixes must resolve to the existing app/environment/VNet/subnet/NAT/PIP. It records the current PIP resource ID and address before preview, requires protected resources to remain unchanged except Azure read-only normalization, and verifies the same PIP/address after deployment. + +The workflow also creates a `CanNotDelete` lock scoped to the reserved PIP. The validated Front Door origin is the public ACA hostname; direct ACA access remains available and can bypass Front Door. + ## Post-Deployment -1. **Set SPA redirect URI** on the app registration (requires the FQDN from deploy output): +1. **Add the SPA redirect URI** without removing existing migration/rollback URIs: + ```bash - FQDN=$(az deployment group show -g -n main \ + ACA_FQDN=$(az deployment group show -g -n \ --query properties.outputs.appFqdn.value -o tsv) - az ad app update --id \ - --spa-redirect-uris "https://$FQDN" + PUBLIC_FQDN=$(az deployment group show -g -n \ + --query properties.outputs.publicFqdn.value -o tsv) + APP_OBJECT_ID=$(az ad app show --id --query id -o tsv) + CURRENT_URIS=$(az rest --method GET \ + --uri "https://graph.microsoft.com/v1.0/applications/$APP_OBJECT_ID?\$select=spa" \ + --query 'spa.redirectUris' -o json) + UPDATED_URIS=$(jq -cn \ + --argjson existing "$CURRENT_URIS" \ + --arg aca "https://$ACA_FQDN" \ + --arg public "https://$PUBLIC_FQDN" \ + '($existing // []) + [$aca, $public] | unique') + PATCH_BODY=$(jq -cn --argjson uris "$UPDATED_URIS" '{spa:{redirectUris:$uris}}') + az rest --method PATCH \ + --uri "https://graph.microsoft.com/v1.0/applications/$APP_OBJECT_ID" \ + --headers 'Content-Type=application/json' \ + --body "$PATCH_BODY" ``` -2. **Grant managed identity RBAC** (required — the Bicep template does **not** create - role assignments; the app will fail to start without AcrPull): + This requires an identity authorized to update the Entra application. Remove the old URI only after rollback is retired. + +2. **Grant managed identity RBAC** (required — the Bicep template does **not** create role assignments; the app will fail to start without AcrPull): + ```bash - MI_ID=$(az deployment group show -g -n main \ - --query properties.outputs.managedIdentityPrincipalId.value -o tsv) + MI_RESOURCE_ID=$(az containerapp show -g -n \ + --query 'keys(identity.userAssignedIdentities)[0]' -o tsv) + MI_ID=$(az resource show --ids "$MI_RESOURCE_ID" --api-version 2023-01-31 \ + --query properties.principalId -o tsv) # Required — app won't start without AcrPull # To find acrResourceId: az acr show --name --query id -o tsv az role assignment create --assignee-object-id $MI_ID \ --assignee-principal-type ServicePrincipal --role "AcrPull" --scope - # Note: Key Vault Secrets User is NOT required — the Container App reads - # its .env contents from an inline secret (envFileContents), not via a - # Key Vault reference. + # Required whenever envFileContents is empty and a Key Vault reference is used. + # deploy_instance.py uses an inline envFileContents secret instead. + az role assignment create --assignee-object-id $MI_ID \ + --assignee-principal-type ServicePrincipal --role "Key Vault Secrets User" \ + --scope # Grant based on which services you use (scope as narrowly as possible) az role assignment create --assignee-object-id $MI_ID \ --assignee-principal-type ServicePrincipal --role "Cognitive Services OpenAI User" \ --scope /subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts/ + az role assignment create --assignee-object-id $MI_ID \ --assignee-principal-type ServicePrincipal --role "Cognitive Services User" \ --scope /subscriptions//resourceGroups//providers/Microsoft.CognitiveServices/accounts/ + az role assignment create --assignee-object-id $MI_ID \ --assignee-principal-type ServicePrincipal --role "Storage Blob Data Contributor" \ --scope /subscriptions//resourceGroups//providers/Microsoft.Storage/storageAccounts/ + az role assignment create --assignee-object-id $MI_ID \ --assignee-principal-type ServicePrincipal --role "Azure ML Data Scientist" \ --scope /subscriptions//resourceGroups//providers/Microsoft.MachineLearningServices/workspaces/ ``` -3. **Create Azure SQL contained user** for the managed identity: +3. **Create Azure SQL contained user** for the managed identity. Use the actual UAMI resource name; it is not necessarily `-identity` when an existing identity is supplied: + ```sql -- Connect as Entra admin (Azure Portal Query Editor, Azure Data Studio, or sqlcmd) - CREATE USER [-identity] FROM EXTERNAL PROVIDER; - ALTER ROLE db_datareader ADD MEMBER [-identity]; - ALTER ROLE db_datawriter ADD MEMBER [-identity]; + CREATE USER [] FROM EXTERNAL PROVIDER; + ALTER ROLE db_datareader ADD MEMBER []; + ALTER ROLE db_datawriter ADD MEMBER []; + ALTER ROLE db_ddladmin ADD MEMBER []; ``` -4. **Manage access** — Add or remove users via Entra security groups - (`allowedGroupObjectIds`). Each group must also be assigned to the enterprise app. +4. **Manage access** — Add or remove users via Entra security groups (`allowedGroupObjectIds`). Each group must also be assigned to the enterprise app. ## Access the GUI ```bash -az deployment group show -g -n main --query properties.outputs.appFqdn.value -o tsv +PUBLIC_FQDN=$(az deployment group show -g -n \ + --query properties.outputs.publicFqdn.value -o tsv) +ACA_FQDN=$(az deployment group show -g -n \ + --query properties.outputs.appFqdn.value -o tsv) +echo "Public URL: https://$PUBLIC_FQDN" +echo "Direct ACA origin: https://$ACA_FQDN" ``` -Open `https://` in a browser. If `allowedCidr` is set, only traffic from -that CIDR range can reach the app. +Open the public URL and verify unauthenticated users are redirected to Entra and only assigned users in an allowed backend group can complete access. When Front Door is enabled, the direct ACA origin remains publicly reachable and bypasses Front Door; retain it only as an intentional diagnostic/rollback path. ## Configuration: .pyrit_conf and .env -The template replaces `.pyrit_conf` and `.env` with Bicep parameters — no files -needed in the container. +The deployment interface replaces local `.pyrit_conf` and `.env` inputs with Bicep parameters, so neither file is baked into the image. For `.env` content, the container entrypoint materializes `~/.pyrit/.env` at runtime. ### .pyrit_conf fields → Bicep params | .pyrit_conf field | Bicep param | Env var | Notes | -|-------------------|-------------|---------|-------| -| `initializers` | `pyritInitializer` | `PYRIT_INITIALIZER` | Default `target`: `target` populates the TargetRegistry (read by the GUI);| -| `operator` | — | Set per-user in the GUI | | -| `operation` | — | Set per-user in the GUI | | +| --- | --- | --- | --- | +| `initializers` | `pyritInitializer` | `PYRIT_INITIALIZER` | Default `target`: `target` populates the TargetRegistry (read by the GUI); | +| `operator` | — | Set per-user in the GUI | | +| `operation` | — | Set per-user in the GUI | | -### .env file → Container App inline secret +### .env file → Container App secret -The entire `.env` file is passed to the Bicep template as the `envFileContents` -`@secure()` parameter and stored as an inline `env-file` secret on the -Container App. The template injects it as the `PYRIT_ENV_CONTENTS` env var. -PyRIT parses this at startup to set all endpoint, model, and API key -environment variables. The Key Vault still receives the same content as -`env-global` for backup/audit, but it is **not** read at runtime. +The template injects the `env-file` ACA secret as `PYRIT_ENV_CONTENTS`. The container entrypoint writes it to `~/.pyrit/.env`, which PyRIT loads at startup. `deploy_instance.py` stores the secure `envFileContents` value inline. When `envFileContents` is empty, Bicep instead stores a versionless Key Vault reference to `envSecretName`. -To rotate the `.env` after deployment, the rotation path depends on which -deploy path you used: +To rotate the `.env` after deployment, the rotation path depends on which deploy path you used: **For instances deployed via `infra/deploy_instance.py`:** -Use `az containerapp secret set` with the `@file` form (the file path is on -the command line, not the secret value). +`updated.env` means a complete local configuration file in the same format as `infra/env.demo.template`; the filename is only a convention. Update the inline `env-file` secret through the Azure portal or an approved ARM deployment that reads `envFileContents` from a secure parameter file. The documented `az containerapp secret set --secrets` interface accepts literal `key=value` arguments; it has no documented `key=@file` form, and placing the `.env` value in that argument exposes it through process inspection. -> **`updated.env` is your local file** — same format as `infra/env.demo.template` -> and the file you passed to `--env-file` during initial deployment, but with -> the new values you want to deploy. The filename is just a convention; you -> can name it anything. - -> ⚠️ **Verify the file exists before running `az`.** The CLI's `@file` -> expansion is silent: if the path is wrong, `az` falls back to storing the -> literal string `@./your-typo.env` as the secret value, with no error. The -> container will then read garbage at startup. +Updating an application-scoped inline secret does not automatically update an existing revision. After changing the secret, restart the active revision: ```bash -# bash -test -f ./updated.env || { echo "ERROR: ./updated.env not found"; exit 1; } -az containerapp secret set \ - -n copyrit-{instance-name} \ - -g copyrit-{instance-name} \ - --secrets "env-file=@./updated.env" - -# Force a new revision (per Microsoft docs, secret updates do NOT auto-restart -# existing revisions — a revision-scoped change is required to pick them up) -az containerapp update \ - -n copyrit-{instance-name} \ - -g copyrit-{instance-name} \ - --set-env-vars "SECRET_UPDATED=$(date +%s)" +APP_NAME=copyrit-{instance-name} +RESOURCE_GROUP=copyrit-{instance-name} +REVISION=$(az containerapp show -n "$APP_NAME" -g "$RESOURCE_GROUP" \ + --query properties.latestRevisionName -o tsv) +az containerapp revision restart -n "$APP_NAME" -g "$RESOURCE_GROUP" \ + --revision "$REVISION" ``` -The `updated.env` file must include the auto-injected -`AZURE_SQL_DB_CONNECTION_STRING` and -`AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL` values from initial deploy -(check the KV `env-global` backup or the deploy script's log output). +The `updated.env` file must include the auto-injected `AZURE_SQL_DB_CONNECTION_STRING` and `AZURE_STORAGE_ACCOUNT_DB_DATA_CONTAINER_URL` values from initial deploy (check the KV `env-global` backup or the deploy script's log output). To keep the KV backup in sync, also run: + ```bash az keyvault secret set --vault-name copyrit-{instance-name}-kv \ --name env-global --file ./updated.env ``` -(The KV is locked down — this needs Cloud Shell or temporary public access.) -**For instances deployed via `gui-deploy.yml` (ADO pipeline):** +(Run this only from a host with an approved vault network path, or through a temporary network rule when policy permits.) -Update the `envFileContents` SECURE variable in the relevant ADO library -variable group (`copyrit-gui-test` or `copyrit-gui-prod`) with the new -content, then re-run the pipeline. +**For instances deployed via the Microsoft team `gui-deploy.yml` ADO workflow:** -> ⚠️ The pipeline runs `az deployment group create`, which is incremental: -> if only the secret value changed, no revision-scoped property changes, so -> ACA will **not** automatically start a new revision. After the pipeline -> succeeds, manually force a new revision so the running container picks up -> the new secret: -> ```bash -> az containerapp update \ -> -n -g \ -> --set-env-vars "SECRET_UPDATED=$(date +%s)" -> ``` -> Or fold this into a final pipeline step. +Update the Key Vault secret named by `envSecretName` (for example, `env-global` or `env-global-prod`) through the approved secret-management path. Do not add plaintext `.env` content to an ADO variable group. Verify the app UAMI retains `Key Vault Secrets User` and that the vault network policy permits runtime resolution. + +The Bicep reference is versionless. Azure Container Apps checks for a newer Key Vault version within 30 minutes and automatically restarts active revisions that consume it through an environment variable. Verify the new version and healthy revision after that window; do not rerun the pipeline merely to transport secret content. > ⚠️ **Anti-patterns to avoid:** -> - `az containerapp secret set --secrets "env-file=$ENV_CONTENT"` — -> passing the value as a literal CLI argument exposes it via `ps` while -> the command runs. Use the `@file` form above instead. -> - `az containerapp secret show --secret-name env-file` — returns the full -> plaintext to your terminal / shell history. -> - **Do not re-run `infra/deploy_instance.py` against an existing instance -> to rotate secrets.** The script's create steps (Entra app, SQL server, -> Key Vault, managed identity) are not idempotent and will either fail or -> produce duplicate Entra app registrations. - -> ⚠️ `PYRIT_ENV_CONTENTS` may contain API keys. Ensure application logging -> does **not** dump environment variables or process state. - -Azure services (OpenAI, Content Safety, Speech) support managed identity — when -API key env vars are not set, PyRIT auto-falls back to `DefaultAzureCredential`, -which picks up the container app's user-assigned MI. Non-Azure providers (OpenAI -Platform, Groq, Google Gemini) require API keys in the `.env`. +> +> - `az containerapp secret set --secrets "env-file=$ENV_CONTENT"` — passing the value as a literal CLI argument exposes it via `ps` while the command runs. Use the portal or secure ARM parameter-file path described above instead. +> - `az containerapp secret show --secret-name env-file` — returns the full plaintext to your terminal / shell history. +> - **Do not re-run `infra/deploy_instance.py` against an existing instance to rotate secrets.** The script's create steps (Entra app, SQL server, Key Vault, managed identity) are not idempotent and will either fail or produce duplicate Entra app registrations. + +> ⚠️ `PYRIT_ENV_CONTENTS` may contain API keys. Ensure application logging does **not** dump environment variables or process state. + +Supported Azure integrations, including OpenAI, Content Safety, and Speech, can use managed identity when their API-key settings are absent. Those paths use `DefaultAzureCredential`, which selects the container app UAMI through `AZURE_CLIENT_ID`; each service still requires its corresponding data-plane role. Non-Azure providers require their documented credentials in the `.env`. ## Notes -- **Network hardening** (opt-in): Both Private Endpoint and IP restriction are - optional. See the Security section for details. The CI/CD pipeline controls - `enablePrivateEndpoint` via the ADO variable group — check your pipeline variables - to confirm the current posture. -- **Log Analytics shared key**: `listKeys()` is the standard ACA pattern. The key is - used during deployment only, not exposed to the application. +- **Network topology**: Public ACA-managed HTTPS ingress with optional `allowedCidr` plus VNet-integrated fixed NAT egress is the base topology. Front Door Premium is an optional inbound layer and is enabled by the internal ADO workflow. `allowedCidr` must be empty when Front Door is enabled; Bicep rejects the combination. +- **Ingress vs. egress**: Front Door affects inbound requests only. The reserved NAT public IP remains the source for ACA-originated outbound connections. +- **NAT routing**: NAT Gateway supplies the outbound source IP only while the subnet's effective default route remains `Internet`. A UDR or propagated BGP `0.0.0.0/0` route to a firewall or gateway takes precedence; in that topology, allow-list the egress device's public IP instead. +- **Network outputs**: `egressPublicIpAddress`, `natGatewayId`, `acaInfrastructureSubnetId`, and `vnetName` describe the created network. +- **PIP lock**: `protectEgressPublicIp=true` creates a resource-scoped `CanNotDelete` lock. The internal ADO workflow enables it; community examples leave it disabled unless the operator explicitly opts in. +- **Log Analytics shared key**: `listKeys()` is the standard ACA pattern. The key is used during deployment only, not exposed to the application. - **Workload profiles**: Consumption tier. Defaults to 1 replica (no auto-scale). -- **Key Vault**: Must be an existing vault. Used for `.env` content backup/audit - only — the runtime secret comes from an inline ACA secret. RBAC for AcrPull is - still granted manually (see Post-Deployment §2); Key Vault Secrets User is no - longer required. +- **Key Vault**: Bicep requires a supplied vault resource ID. The vault is backup/audit-only for `deploy_instance.py`, but it is the runtime source for any deployment using `envSecretName`. Those app identities require `Key Vault Secrets User` and a permitted network path. AcrPull is still granted separately. - **OpenTelemetry**: When `enableOtel=true`, configure the agent post-deploy: ```bash - AI_CONN=$(az deployment group show -g -n main \ - --query properties.outputs.appInsightsConnectionString.value -o tsv) + AI_CONN=$(az resource show -g -n -ai \ + --resource-type Microsoft.Insights/components --api-version 2020-02-02 \ + --query properties.ConnectionString -o tsv) az containerapp env telemetry app-insights set \ --name -env -g --connection-string "$AI_CONN" ``` -- **Existing resources**: Log Analytics, VNet, and ACR can be provided as existing - resources to skip creation. +- **Existing resources**: Log Analytics, ACR, and a UAMI can be supplied as existing resources; Key Vault must be supplied. The template always creates its dedicated VNet, ACA subnet, NAT Gateway, and egress public IP. Although Bicep can declare an ACR when no registry is supplied, a separate bootstrap is required to push the image and authorize its identity before the app can run. - **Azure CLI**: Version 2.84+ required (2.77 has a known bug). ## Teardown and Redeployment @@ -543,7 +593,19 @@ Platform, Groq, Google Gemini) require API keys in the `.env`. az group delete --name --yes ``` -Key Vault is external to the RG — no purge-protection naming conflicts. +Use resource-group deletion only for an unlocked, dedicated community deployment. Do not use it for Microsoft internal test/prod or a migration that shares identity, logging, DNS, or network resources with another app. + +If `protectEgressPublicIp=true`, resource-group deletion is intentionally blocked. Before an approved egress migration, remove the IP from every downstream allowlist, record the replacement, and then remove the scoped lock explicitly: + +```bash +PIP_ID=$(az network public-ip show -g -n -egress-pip \ + --query id -o tsv) +az lock list --resource "$PIP_ID" -o table +az lock delete --name -egress-pip-lock --resource "$PIP_ID" +``` + +Removing the lock does not delete the PIP; it only permits a separately approved delete or resource-group teardown. + +Key Vault is external to Bicep ownership, but it can still reside in the deleted resource group. In particular, `deploy_instance.py` creates its purge-protected vault in the instance resource group. Deleting that group soft-deletes the vault and retains its name; recover the vault before redeploying the same instance name. -> **Note**: Entra ID resources (app registration, security groups) are **not** deleted -> by `az group delete`. Remove them manually if no longer needed. +> **Note**: Entra ID resources (app registration, security groups) are **not** deleted by `az group delete`. Remove them manually if no longer needed. diff --git a/infra/deploy_instance.py b/infra/deploy_instance.py index e038d518ab..bc69d8d760 100644 --- a/infra/deploy_instance.py +++ b/infra/deploy_instance.py @@ -14,8 +14,9 @@ applies SFI network lockdown — backup/audit only, NOT read at runtime) 7. Managed identity + RBAC role assignments (AcrPull, Storage Blob Data Contributor) 7b. AOAI RBAC (optional — Cognitive Services OpenAI User on specified resources) - 8. Bicep deployment (Container App, networking, logging) - 9. Post-deploy: SPA redirect URI + 8. Bicep deployment (Container App, VNet, NAT, static egress, logging) + 9. Restrict SQL and Storage network access to the static egress IP + 10. Post-deploy: SPA redirect URI Usage: python infra/deploy_instance.py \\ @@ -30,6 +31,7 @@ """ import argparse +import ipaddress import json import logging import platform @@ -39,6 +41,7 @@ import tempfile import time from pathlib import Path +from typing import cast logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") logger = logging.getLogger(__name__) @@ -47,11 +50,31 @@ BICEP_TEMPLATE = INFRA_DIR / "main.bicep" _MICROSOFT_GRAPH_APP_ID = "00000003-0000-0000-c000-000000000000" _GRAPH_USER_READ_SCOPE_ID = "e1fe6dd8-ba31-4d61-89e7-88639da4683d" +_INSTANCE_NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,11}[a-z0-9])?$") +_ACR_NAME_RE = re.compile(r"^[a-z0-9]{5,50}$") +_GROUP_ID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) +_IMAGE_REPOSITORY_RE = r"[a-z0-9]+(?:[._-][a-z0-9]+)*(?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*" +_IMAGE_VERSION_RE = r"(?:[A-Za-z0-9_][A-Za-z0-9_.-]*|sha256:[0-9a-fA-F]{64})" # On Windows, az CLI is a .cmd script that requires shell=True for subprocess to find it. _SHELL = platform.system() == "Windows" +def _deployment_tags(*, instance: str, owner: str) -> dict[str, str]: + """Build ownership and governance tags shared by all per-instance resources.""" + tags = { + "Service": "pyrit-gui", + "Instance": instance, + "ManagedBy": "infra/deploy_instance.py", + "DataClass": "Confidential", + } + if owner: + tags["Owner"] = owner + return tags + + def run_az( *, args: list[str], @@ -83,7 +106,28 @@ def run_az( ) -def run_az_json(*, args: list[str]) -> dict | list | str: +def _expect_json_object(value: object, *, context: str) -> dict[str, object]: + """Require a JSON object with string keys at an Azure CLI response boundary.""" + if not isinstance(value, dict): + raise RuntimeError(f"Azure CLI returned invalid {context} data") + return cast(dict[str, object], value) + + +def _expect_json_array(value: object, *, context: str) -> list[object]: + """Require a JSON array at an Azure CLI response boundary.""" + if not isinstance(value, list): + raise RuntimeError(f"Azure CLI returned invalid {context} data") + return cast(list[object], value) + + +def _expect_string(value: object, *, context: str) -> str: + """Require a nonempty string at an Azure CLI response boundary.""" + if not isinstance(value, str) or not value: + raise RuntimeError(f"Azure CLI returned invalid {context} data") + return value + + +def run_az_json(*, args: list[str]) -> object: """ Run an Azure CLI command and parse JSON output. @@ -91,10 +135,11 @@ def run_az_json(*, args: list[str]) -> dict | list | str: args (list[str]): The az CLI arguments (without the leading 'az'). Returns: - dict | list | str: The parsed JSON output. + object: The parsed JSON output. Callers validate its expected shape. """ result = run_az(args=args + ["-o", "json"]) - return json.loads(result.stdout) + parsed: object = json.loads(result.stdout) + return parsed def set_subscription(subscription: str) -> None: @@ -124,7 +169,7 @@ def create_resource_group(*, name: str, location: str, tags: list[str] | None = run_az(args=cmd) -def create_entra_app(*, display_name: str, service_management_reference: str = "") -> dict: +def create_entra_app(*, display_name: str, service_management_reference: str = "") -> dict[str, str]: """ Create an Entra app registration with delegated Microsoft Graph access. @@ -153,30 +198,36 @@ def create_entra_app(*, display_name: str, service_management_reference: str = " "--query", "{appId:appId, id:id}", ] - app_create_result = run_az_json(args=create_args) - app_id = app_create_result["appId"] - app_object_id = app_create_result["id"] + app_create_result = _expect_json_object(run_az_json(args=create_args), context="Entra application") + app_id = _expect_string(app_create_result.get("appId"), context="Entra application ID") + app_object_id = _expect_string(app_create_result.get("id"), context="Entra application object ID") - tenant_id = run_az_json(args=["account", "show", "--query", "tenantId"]) + tenant_id = _expect_string( + run_az_json(args=["account", "show", "--query", "tenantId"]), + context="tenant ID", + ) # Create service principal (enterprise app) logger.info("Creating service principal for app: %s", app_id) run_az(args=["ad", "sp", "create", "--id", app_id]) - sp_info = run_az_json( - args=[ - "ad", - "sp", - "show", - "--id", - app_id, - "--query", - "{id:id}", - ] + sp_info = _expect_json_object( + run_az_json( + args=[ + "ad", + "sp", + "show", + "--id", + app_id, + "--query", + "{id:id}", + ] + ), + context="service principal", ) - sp_id = sp_info["id"] + sp_id = _expect_string(sp_info.get("id"), context="service principal object ID") logger.info("Configuring delegated Microsoft Graph User.Read permission") - graph_access_body = { + graph_access_body: dict[str, object] = { "requiredResourceAccess": [ { "resourceAppId": _MICROSOFT_GRAPH_APP_ID, @@ -234,7 +285,7 @@ def assign_groups_to_app(*, sp_id: str, group_ids: list[str]) -> None: "--method", "POST", "--url", - f"https://graph.microsoft.com/v1.0/servicePrincipals/{sp_id}/appRoleAssignments", + f"https://graph.microsoft.com/v1.0/servicePrincipals/{sp_id}/appRoleAssignedTo", "--body", json.dumps(body), ] @@ -248,7 +299,7 @@ def create_sql_server_and_db( server_name: str, database_name: str, tags: list[str] | None = None, -) -> dict: +) -> dict[str, str]: """ Create an Azure SQL server with Entra-only auth and a database. @@ -263,17 +314,22 @@ def create_sql_server_and_db( dict: A dict with keys 'server_fqdn' and 'database_name'. """ # Get current user for Entra admin - current_user = run_az_json( - args=[ - "ad", - "signed-in-user", - "show", - "--query", - "{displayName:displayName, id:id}", - ] + current_user = _expect_json_object( + run_az_json( + args=[ + "ad", + "signed-in-user", + "show", + "--query", + "{displayName:displayName, id:id}", + ] + ), + context="signed-in user", ) + current_user_name = _expect_string(current_user.get("displayName"), context="signed-in user display name") + current_user_id = _expect_string(current_user.get("id"), context="signed-in user object ID") - logger.info("Creating SQL server: %s (Entra admin: %s)", server_name, current_user["displayName"]) + logger.info("Creating SQL server: %s (Entra admin: %s)", server_name, current_user_name) sql_server_cmd = [ "sql", "server", @@ -288,26 +344,29 @@ def create_sql_server_and_db( "--external-admin-principal-type", "User", "--external-admin-name", - current_user["displayName"], + current_user_name, "--external-admin-sid", - current_user["id"], + current_user_id, ] if tags: sql_server_cmd += ["--tags"] + tags run_az(args=sql_server_cmd) - server_fqdn = run_az_json( - args=[ - "sql", - "server", - "show", - "--name", - server_name, - "--resource-group", - resource_group, - "--query", - "fullyQualifiedDomainName", - ] + server_fqdn = _expect_string( + run_az_json( + args=[ + "sql", + "server", + "show", + "--name", + server_name, + "--resource-group", + resource_group, + "--query", + "fullyQualifiedDomainName", + ] + ), + context="SQL server FQDN", ) logger.info("Creating database: %s on server %s", database_name, server_name) @@ -330,27 +389,6 @@ def create_sql_server_and_db( sql_db_cmd += ["--tags"] + tags run_az(args=sql_db_cmd) - # Allow Azure services to access the SQL server - logger.info("Allowing Azure services to access SQL server") - run_az( - args=[ - "sql", - "server", - "firewall-rule", - "create", - "--resource-group", - resource_group, - "--server", - server_name, - "--name", - "AllowAzureServices", - "--start-ip-address", - "0.0.0.0", - "--end-ip-address", - "0.0.0.0", - ] - ) - return {"server_fqdn": server_fqdn, "database_name": database_name} @@ -471,7 +509,7 @@ def create_storage_account( account_name: str, container_name: str = _STORAGE_CONTAINER_NAME, tags: list[str] | None = None, -) -> dict: +) -> dict[str, str]: """ Create a per-instance storage account and a private blob container. @@ -490,7 +528,7 @@ def create_storage_account( Returns: dict: A dict with keys 'account_id' (resource ID) and 'container_url' - (the full HTTPS URL the AIRT initializer expects). + (the full HTTPS URL the target initializer expects). """ logger.info("Creating storage account: %s", account_name) sa_cmd = [ @@ -516,18 +554,21 @@ def create_storage_account( sa_cmd += ["--tags"] + tags run_az(args=sa_cmd) - account_id = run_az_json( - args=[ - "storage", - "account", - "show", - "--name", - account_name, - "--resource-group", - resource_group, - "--query", - "id", - ] + account_id = _expect_string( + run_az_json( + args=[ + "storage", + "account", + "show", + "--name", + account_name, + "--resource-group", + resource_group, + "--query", + "id", + ] + ), + context="storage account resource ID", ) logger.info("Creating blob container: %s/%s", account_name, container_name) @@ -560,6 +601,73 @@ def create_storage_account( return {"account_id": account_id, "container_url": container_url} +def configure_data_plane_network_access( + *, + resource_group: str, + sql_server_name: str, + storage_account_name: str, + egress_ip: str, +) -> None: + """Restrict per-instance SQL and Storage public endpoints to the static NAT IP.""" + try: + ipaddress.IPv4Address(egress_ip) + except ipaddress.AddressValueError as error: + raise RuntimeError("Bicep did not return a valid static egress IPv4 address") from error + + logger.info("Allowlisting static egress IP %s on Azure SQL", egress_ip) + run_az( + args=[ + "sql", + "server", + "firewall-rule", + "create", + "--resource-group", + resource_group, + "--server", + sql_server_name, + "--name", + "AllowContainerAppEgress", + "--start-ip-address", + egress_ip, + "--end-ip-address", + egress_ip, + ] + ) + + logger.info("Restricting Storage to static egress IP %s", egress_ip) + run_az( + args=[ + "storage", + "account", + "network-rule", + "add", + "--resource-group", + resource_group, + "--account-name", + storage_account_name, + "--ip-address", + egress_ip, + ] + ) + run_az( + args=[ + "storage", + "account", + "update", + "--resource-group", + resource_group, + "--name", + storage_account_name, + "--public-network-access", + "Enabled", + "--default-action", + "Deny", + "--bypass", + "None", + ] + ) + + def create_key_vault( *, resource_group: str, @@ -639,26 +747,32 @@ def create_key_vault( kv_cmd += ["--tags"] + tags run_az(args=kv_cmd) - kv_id = run_az_json( - args=[ - "keyvault", - "show", - "--name", - vault_name, - "--query", - "id", - ] + kv_id = _expect_string( + run_az_json( + args=[ + "keyvault", + "show", + "--name", + vault_name, + "--query", + "id", + ] + ), + context="Key Vault resource ID", ) # Grant current user Secrets Officer so we can write the secret - current_user_id = run_az_json( - args=[ - "ad", - "signed-in-user", - "show", - "--query", - "id", - ] + current_user_id = _expect_string( + run_az_json( + args=[ + "ad", + "signed-in-user", + "show", + "--query", + "id", + ] + ), + context="signed-in user object ID", ) logger.info("Granting Key Vault Secrets Officer to current user") run_az( @@ -762,13 +876,14 @@ def deploy_bicep( tenant_id: str, client_id: str, group_ids: str, + allowed_cidr: str, sql_server_fqdn: str, sql_database_name: str, kv_resource_id: str, acr_name: str, env_file_contents: str, - owner_tag: str = "", -) -> dict: + tags: dict[str, str], +) -> dict[str, object]: """ Deploy the Bicep template. @@ -785,6 +900,7 @@ def deploy_bicep( tenant_id (str): The Entra tenant ID. client_id (str): The Entra app registration client ID. group_ids (str): Comma-separated group object IDs. + allowed_cidr (str): Optional public ingress IPv4 CIDR. sql_server_fqdn (str): The SQL server FQDN. sql_database_name (str): The SQL database name. kv_resource_id (str): The Key Vault resource ID (kept for the @@ -792,30 +908,29 @@ def deploy_bicep( acr_name (str): The ACR name. env_file_contents (str): The prepared .env content to inject as the Container App's `env-file` secret. - owner_tag (str): Value for the Owner tag on Bicep-managed resources. + tags (dict[str, str]): Ownership and governance tags for Bicep-managed resources. Returns: dict: The deployment outputs. """ logger.info("Deploying Bicep template to resource group: %s", resource_group) - parameters: dict = { + parameters: dict[str, object] = { "appName": {"value": app_name}, "containerImage": {"value": container_image}, "entraTenantId": {"value": tenant_id}, "entraClientId": {"value": client_id}, "allowedGroupObjectIds": {"value": group_ids}, + "allowedCidr": {"value": allowed_cidr}, "sqlServerFqdn": {"value": sql_server_fqdn}, "sqlDatabaseName": {"value": sql_database_name}, "keyVaultResourceId": {"value": kv_resource_id}, "acrName": {"value": acr_name}, - "enablePrivateEndpoint": {"value": False}, "envFileContents": {"value": env_file_contents}, + "tags": {"value": tags}, } - if owner_tag: - parameters["tags"] = {"value": {"Service": "pyrit-gui", "Owner": owner_tag}} - parameters_doc = { + parameters_doc: dict[str, object] = { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", "contentVersion": "1.0.0.0", "parameters": parameters, @@ -827,20 +942,23 @@ def deploy_bicep( params_path = tmp.name json.dump(parameters_doc, tmp) - return run_az_json( - args=[ - "deployment", - "group", - "create", - "--resource-group", - resource_group, - "--template-file", - str(BICEP_TEMPLATE), - "--parameters", - f"@{params_path}", - "--query", - "properties.outputs", - ] + return _expect_json_object( + run_az_json( + args=[ + "deployment", + "group", + "create", + "--resource-group", + resource_group, + "--template-file", + str(BICEP_TEMPLATE), + "--parameters", + f"@{params_path}", + "--query", + "properties.outputs", + ] + ), + context="deployment outputs", ) finally: if params_path: @@ -926,29 +1044,35 @@ def create_managed_identity_and_grant_roles( mi_cmd += ["--tags"] + tags run_az(args=mi_cmd) - mi_principal_id = run_az_json( - args=[ - "identity", - "show", - "--name", - identity_name, - "--resource-group", - resource_group, - "--query", - "principalId", - ] + mi_principal_id = _expect_string( + run_az_json( + args=[ + "identity", + "show", + "--name", + identity_name, + "--resource-group", + resource_group, + "--query", + "principalId", + ] + ), + context="managed identity principal ID", ) # Grant AcrPull - acr_id = run_az_json( - args=[ - "acr", - "show", - "--name", - acr_name, - "--query", - "id", - ] + acr_id = _expect_string( + run_az_json( + args=[ + "acr", + "show", + "--name", + acr_name, + "--query", + "id", + ] + ), + context="ACR resource ID", ) logger.info("Granting AcrPull to managed identity on ACR: %s", acr_name) run_az( @@ -1008,9 +1132,8 @@ def _grant_aoai_roles( Returns: int: The number of successful role assignments. """ - granted = 0 - for name in aoai_resource_names: - resource_id = run_az_json( + accounts = _expect_json_array( + run_az_json( args=[ "cognitiveservices", "account", @@ -1018,9 +1141,21 @@ def _grant_aoai_roles( "--subscription", subscription, "--query", - f"[?name=='{name}'].id | [0]", + "[].{name:name,id:id}", ] - ) + ), + context="Cognitive Services accounts", + ) + resource_ids: dict[str, str] = {} + for value in accounts: + account = _expect_json_object(value, context="Cognitive Services account") + account_name = _expect_string(account.get("name"), context="Cognitive Services account name") + account_id = _expect_string(account.get("id"), context="Cognitive Services account resource ID") + resource_ids[account_name] = account_id + + granted = 0 + for name in aoai_resource_names: + resource_id = resource_ids.get(name) if not resource_id: logger.warning("AOAI resource '%s' not found in subscription — skipping", name) continue @@ -1096,6 +1231,11 @@ def parse_args(args: list[str] | None = None) -> argparse.Namespace: required=True, help="Comma-separated Entra group object IDs to grant access", ) + parser.add_argument( + "--allowed-cidr", + default="", + help="Optional public ingress IPv4 CIDR; empty allows traffic from any source", + ) parser.add_argument( "--owner-tag", default="", @@ -1143,6 +1283,36 @@ def main(args: list[str] | None = None) -> int: instance = parsed.instance_name env_file = parsed.env_file.resolve() + if not _INSTANCE_NAME_RE.fullmatch(instance): + logger.error( + "--instance-name must be 1-13 lowercase letters, numbers, or internal hyphens, " + "and must start and end with a letter or number" + ) + return 1 + + if not _ACR_NAME_RE.fullmatch(parsed.acr_name): + logger.error("--acr-name must be 5-50 lowercase alphanumeric characters") + return 1 + + expected_image = re.compile( + rf"^{re.escape(parsed.acr_name)}\.azurecr\.io/{_IMAGE_REPOSITORY_RE}(?::|@){_IMAGE_VERSION_RE}$" + ) + if not expected_image.fullmatch(parsed.container_image) or parsed.container_image.endswith(":latest"): + logger.error( + "--container-image must use --acr-name and an immutable tag or sha256 digest; :latest is not allowed" + ) + return 1 + + if parsed.allowed_cidr: + try: + allowed_network = ipaddress.ip_network(parsed.allowed_cidr, strict=True) + except ValueError: + logger.error("--allowed-cidr must be an IPv4 network in canonical CIDR notation") + return 1 + if allowed_network.version != 4: + logger.error("--allowed-cidr must be an IPv4 network") + return 1 + if not env_file.exists(): logger.error("Env file not found: %s", env_file) return 1 @@ -1161,8 +1331,8 @@ def main(args: list[str] | None = None) -> int: entra_app_name = f"CoPyRIT GUI ({instance})" group_ids = [g.strip() for g in parsed.allowed_groups.split(",") if g.strip()] - if not group_ids: - logger.error("--allowed-groups must contain at least one Entra group object ID") + if not group_ids or any(not _GROUP_ID_RE.fullmatch(group_id) for group_id in group_ids): + logger.error("--allowed-groups must contain one or more comma-separated Entra group object IDs") return 1 # Validate Azure resource name length constraints @@ -1203,6 +1373,7 @@ def main(args: list[str] | None = None) -> int: logger.info("Storage account: %s (container: %s)", storage_account_name, _STORAGE_CONTAINER_NAME) logger.info("Entra app: %s", entra_app_name) logger.info("Allowed groups: %s", group_ids) + logger.info("Allowed ingress CIDR: %s", parsed.allowed_cidr or "(unrestricted source IPs)") logger.info("Env file: %s", env_file) logger.info("Container image: %s", parsed.container_image) logger.info("ACR: %s", parsed.acr_name) @@ -1215,11 +1386,16 @@ def main(args: list[str] | None = None) -> int: return 0 try: - # Build tags list from --owner-tag - resource_tags = [f"Owner={parsed.owner_tag}"] if parsed.owner_tag else None + deployment_tags = _deployment_tags(instance=instance, owner=parsed.owner_tag) + resource_tags = [f"{key}={value}" for key, value in deployment_tags.items()] # Step 1: Set subscription set_subscription(parsed.subscription) + subscription_id = _expect_string( + run_az_json(args=["account", "show", "--query", "id"]), + context="active subscription ID", + ) + resource_group_id = f"/subscriptions/{subscription_id}/resourceGroups/{rg_name}" # Step 2: Create resource group create_resource_group(name=rg_name, location=parsed.location, tags=resource_tags) @@ -1297,15 +1473,30 @@ def main(args: list[str] | None = None) -> int: tenant_id=entra["tenant_id"], client_id=entra["app_id"], group_ids=",".join(group_ids), + allowed_cidr=parsed.allowed_cidr, sql_server_fqdn=sql["server_fqdn"], sql_database_name=sql["database_name"], kv_resource_id=kv_id, acr_name=parsed.acr_name, env_file_contents=env_content, - owner_tag=parsed.owner_tag, + tags=deployment_tags, ) - fqdn = outputs["appFqdn"]["value"] + app_fqdn_output = _expect_json_object(outputs.get("appFqdn"), context="appFqdn deployment output") + egress_ip_output = _expect_json_object( + outputs.get("egressPublicIpAddress"), + context="egressPublicIpAddress deployment output", + ) + fqdn = _expect_string(app_fqdn_output.get("value"), context="appFqdn output value") + egress_ip = _expect_string(egress_ip_output.get("value"), context="egress IP output value") + + # Step 9b: Restrict data-plane network access to the instance NAT IP. + configure_data_plane_network_access( + resource_group=rg_name, + sql_server_name=sql_server_name, + storage_account_name=storage_account_name, + egress_ip=egress_ip, + ) # Step 10: Post-deploy (SPA redirect) post_deploy( @@ -1321,6 +1512,8 @@ def main(args: list[str] | None = None) -> int: logger.info("Instance: %s", instance) logger.info("URL: https://%s", fqdn) logger.info("Resource group: %s", rg_name) + logger.info("Resource group ID: %s", resource_group_id) + logger.info("Static egress IP: %s", egress_ip) logger.info("Entra app ID: %s", entra["app_id"]) logger.info("SQL server: %s", sql["server_fqdn"]) logger.info("SQL database: %s", sql["database_name"]) @@ -1337,9 +1530,14 @@ def main(args: list[str] | None = None) -> int: logger.info(" 2. Add users to the Entra security group(s)") if not aoai_names: logger.info(" 3. Grant Cognitive Services roles if using MI-auth for AOAI:") + logger.info(" # Azure OpenAI") logger.info(" az role assignment create --assignee-object-id %s \\", mi_principal_id) logger.info(" --assignee-principal-type ServicePrincipal \\") logger.info(" --role 'Cognitive Services OpenAI User' --scope ") + logger.info(" # Azure AI Content Safety") + logger.info(" az role assignment create --assignee-object-id %s \\", mi_principal_id) + logger.info(" --assignee-principal-type ServicePrincipal \\") + logger.info(" --role 'Cognitive Services User' --scope ") else: logger.info( " 3. AOAI RBAC: %d/%d resources granted (via --aoai-resource-names)", aoai_granted, len(aoai_names) @@ -1353,6 +1551,9 @@ def main(args: list[str] | None = None) -> int: return 0 + except RuntimeError as error: + logger.error("%s", error) + return 1 except subprocess.CalledProcessError as e: logger.error("Command failed (exit code %d): %s", e.returncode, " ".join(e.cmd)) if e.stderr: diff --git a/infra/env.demo.template b/infra/env.demo.template index 4c5e7dac7e..190b5f504f 100644 --- a/infra/env.demo.template +++ b/infra/env.demo.template @@ -5,13 +5,14 @@ # # Edit my-demo.env with real endpoints/keys # python infra/deploy_instance.py --instance-name my-demo --env-file my-demo.env ... # -# The deployment script uploads this as a Key Vault secret. The container -# reads it via PYRIT_ENV_CONTENTS and writes it to ~/.pyrit/.env at startup. +# The deployment script stores this as an inline ACA secret and writes a +# locked-down Key Vault backup. The container receives it via +# PYRIT_ENV_CONTENTS and writes it to ~/.pyrit/.env at startup. # # Endpoints don't need to match the AIRT instance — point them at whatever # models make sense for your demo audience. # -# See pyrit/setup/initializers/components/targets.py for all supported env vars. +# See pyrit/setup/initializers/targets.py for all supported env vars. # ─── Chat Target (required — at least one chat model for the GUI to be useful) ─── AZURE_OPENAI_GPT4O_ENDPOINT=https://YOUR_ENDPOINT.openai.azure.com/openai/v1 @@ -19,19 +20,19 @@ AZURE_OPENAI_GPT4O_KEY= AZURE_OPENAI_GPT4O_MODEL=YOUR_DEPLOYMENT_NAME AZURE_OPENAI_GPT4O_UNDERLYING_MODEL=gpt-4o -# ─── Unsafe Chat (required for converters via airt initializer) ─── +# ─── Unsafe Chat (required for converters via target initializer) ─── AZURE_OPENAI_GPT4O_UNSAFE_CHAT_ENDPOINT=https://YOUR_UNSAFE_ENDPOINT.openai.azure.com/openai/v1 AZURE_OPENAI_GPT4O_UNSAFE_CHAT_KEY= AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL=YOUR_UNSAFE_DEPLOYMENT_NAME AZURE_OPENAI_GPT4O_UNSAFE_CHAT_UNDERLYING_MODEL=gpt-4o -# ─── Unsafe Chat 2 (required for scoring via airt initializer) ─── +# ─── Unsafe Chat 2 (required for scoring via target initializer) ─── AZURE_OPENAI_GPT4O_UNSAFE_CHAT_ENDPOINT2=https://YOUR_UNSAFE_ENDPOINT2.openai.azure.com/openai/v1 AZURE_OPENAI_GPT4O_UNSAFE_CHAT_KEY2= AZURE_OPENAI_GPT4O_UNSAFE_CHAT_MODEL2=YOUR_UNSAFE_DEPLOYMENT_NAME2 AZURE_OPENAI_GPT4O_UNSAFE_CHAT_UNDERLYING_MODEL2=gpt-4o -# ─── Content Safety (required for harm detection via airt initializer) ─── +# ─── Content Safety (required for harm detection via target initializer) ─── AZURE_CONTENT_SAFETY_API_ENDPOINT=https://YOUR_CONTENT_SAFETY.cognitiveservices.azure.com/ AZURE_CONTENT_SAFETY_API_KEY= diff --git a/infra/main.bicep b/infra/main.bicep index 367be3ea92..fa22bf559d 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -2,13 +2,14 @@ // PyRIT GUI — Azure Container Apps Deployment (Security-Hardened) // // Deploys the CoPyRIT GUI as an Azure Container App with: -// - Workload profiles environment with public ingress + optional IP restriction +// - Azure Front Door Premium public entry point +// - VNet-integrated public workload-profiles environment with fixed NAT egress // - MSAL PKCE authentication (frontend) + Microsoft Graph-backed auth (backend) // - User-assigned managed identity for Azure SQL, ACR, Azure OpenAI, Key Vault // - Azure SQL (existing) via managed identity — no passwords -// - Key Vault for secrets (referenced via ACA secretRef, not embedded) +// - ACA secret populated inline or backed by a Key Vault reference // - Centralized logging via Log Analytics (configurable retention) -// - No storage account keys, no embedded secrets, no :latest tags +// - No storage account keys or secrets embedded in source/container images // // Prerequisites: // 1. An Entra ID app registration (no secrets/certs needed — PKCE public client) @@ -33,6 +34,8 @@ // --- Parameters --- @description('Name for the Container App and related resources') +@minLength(2) +@maxLength(32) param appName string = 'pyrit-gui' @description('Azure region for all resources') @@ -48,8 +51,7 @@ param entraTenantId string @description('Entra ID app registration client ID (no secrets needed)') param entraClientId string -@description('Object ID of the Entra security group allowed to access the GUI') -@metadata({ description: 'Find this in Azure Portal → Entra ID → Groups → your group → Object ID' }) +@description('Comma-separated object IDs of Entra security groups allowed to access the GUI. Find each ID in Azure Portal → Entra ID → Groups → your group → Object ID.') @minLength(1) param allowedGroupObjectIds string @@ -58,7 +60,7 @@ var normalizedAllowedGroupObjectIds = filter( groupId => !empty(groupId) ) -@description('CIDR range allowed to reach the app (e.g., your corp VPN CIDR). Empty = no IP restriction, all traffic allowed.') +@description('CIDR range allowed to reach ACA directly. Empty = unrestricted. Must be empty when Front Door is enabled because ACA sees Front Door backend IPs, not client IPs.') param allowedCidr string = '' @description('Human-readable description for the IP restriction rule') @@ -99,18 +101,17 @@ param maxReplicas int = 1 @description('Azure Container Registry name (for managed identity pull). Used if acrResourceId is not provided.') param acrName string = '' -@description('Enable Private Endpoint for the ACA environment. When false, uses public access with IP restrictions.') -param enablePrivateEndpoint bool = true - -@description('VNet address prefix (used only when creating a new VNet)') +@description('Virtual network address prefix') param vnetAddressPrefix string = '10.0.0.0/16' -@description('Subnet address prefix for the Private Endpoint (used only when creating a new subnet)') -param subnetAddressPrefix string = '10.0.0.0/24' +@description('Dedicated ACA infrastructure subnet prefix') +param infrastructureSubnetAddressPrefix string = '10.0.1.0/26' + +@description('Existing Azure Policy IP tags to preserve when adopting a reserved egress public IP') +param egressPublicIpIpTags array = [] -@description('Resource ID of an existing subnet for the Private Endpoint. If empty, a new VNet + subnet is created.') -@metadata({ example: '/subscriptions//resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/' }) -param infrastructureSubnetId string = '' +@description('Protect the static egress public IP from accidental deletion') +param protectEgressPublicIp bool = false @description('Log Analytics retention in days (used only when creating a new workspace)') param logRetentionDays int = 90 @@ -131,6 +132,9 @@ param keyVaultResourceId string @description('Resource ID of the Azure Container Registry (for AcrPull role assignment). Recommended over acrName for IaC-managed access.') param acrResourceId string = '' +@description('Optional existing user-assigned managed identity resource ID. Empty creates a new identity using the existing naming behavior.') +param existingManagedIdentityResourceId string = '' + @description('Resource tags applied to all resources (ownership + data classification)') param tags object = { Service: 'pyrit-gui' @@ -141,49 +145,41 @@ param tags object = { @description('Enable OpenTelemetry managed agent for audit logging. Creates Application Insights and wires the ACA managed OTel collector.') param enableOtel bool = false -// Soft guardrail: detect :latest usage (enforced via output warning) -var imageUsesLatest = endsWith(containerImage, ':latest') +@description('Create Azure Front Door Premium as the public application endpoint') +param enableFrontDoor bool = false // Determine whether to create or reference existing resources +var effectiveAllowedCidr = enableFrontDoor && !empty(allowedCidr) + ? fail('allowedCidr must be empty when enableFrontDoor is true') + : allowedCidr var createLogAnalytics = logAnalyticsWorkspaceId == '' -var createVnet = enablePrivateEndpoint && infrastructureSubnetId == '' var createAcr = acrResourceId == '' && acrName == '' var useInlineEnvFile = !empty(envFileContents) - -// ============================================================================ -// VNet + Subnet (created only if infrastructureSubnetId is not provided) -// The subnet hosts the Private Endpoint for the ACA environment — no ACA -// delegation needed (that's only for VNet-integrated internal environments). -// ============================================================================ -resource vnet 'Microsoft.Network/virtualNetworks@2023-11-01' = if (createVnet) { - name: '${appName}-vnet' - location: location - tags: tags - properties: { - addressSpace: { - addressPrefixes: [ - vnetAddressPrefix - ] - } - subnets: [ - { - name: '${appName}-pe-subnet' - properties: { - addressPrefix: subnetAddressPrefix - privateEndpointNetworkPolicies: 'Disabled' - } - } - ] +var createManagedIdentity = empty(existingManagedIdentityResourceId) +var generatedAcrName = '${padLeft(replace(appName, '-', ''), 2, 'p')}acr' +var existingManagedIdentitySegments = split(existingManagedIdentityResourceId, '/') +var existingManagedIdentitySubscriptionId = createManagedIdentity ? subscription().subscriptionId : existingManagedIdentitySegments[2] +var existingManagedIdentityResourceGroupName = createManagedIdentity ? resourceGroup().name : existingManagedIdentitySegments[4] +var existingManagedIdentityName = createManagedIdentity ? '' : last(existingManagedIdentitySegments) + +module acaNatNetwork './modules/aca_nat_network.bicep' = { + name: '${appName}-aca-nat-network' + params: { + namePrefix: appName + location: location + tags: tags + vnetAddressPrefix: vnetAddressPrefix + infrastructureSubnetAddressPrefix: infrastructureSubnetAddressPrefix + egressPublicIpIpTags: egressPublicIpIpTags + protectEgressPublicIp: protectEgressPublicIp } } -var effectiveSubnetId = createVnet ? vnet.properties.subnets[0].id : infrastructureSubnetId - // ============================================================================ // Azure Container Registry (created only if neither acrResourceId nor acrName is provided) // ============================================================================ resource newAcr 'Microsoft.ContainerRegistry/registries@2023-08-01-preview' = if (createAcr) { - name: '${replace(appName, '-', '')}acr' + name: generatedAcrName location: location tags: tags sku: { @@ -218,8 +214,8 @@ resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2023-09-01' = if } } -var effectiveLogAnalyticsCustomerIdValue = createLogAnalytics ? logAnalytics.properties.customerId : logAnalyticsCustomerId -var effectiveLogAnalyticsKeyValue = createLogAnalytics ? logAnalytics.listKeys().primarySharedKey : logAnalyticsSharedKey +var effectiveLogAnalyticsCustomerIdValue = createLogAnalytics ? logAnalytics!.properties.customerId : logAnalyticsCustomerId +var effectiveLogAnalyticsKeyValue = createLogAnalytics ? logAnalytics!.listKeys().primarySharedKey : logAnalyticsSharedKey // ============================================================================ // Application Insights (created when OTel is enabled — destination for traces/logs) @@ -241,12 +237,25 @@ resource appInsights 'Microsoft.Insights/components@2020-02-02' = if (enableOtel // revision starts. This avoids the chicken-and-egg problem with system-assigned // MI where the revision tries to pull images / access KV before RBAC propagates. // ============================================================================ -resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = if (createManagedIdentity) { name: '${appName}-identity' location: location tags: tags } +resource referencedManagedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' existing = if (!createManagedIdentity) { + name: existingManagedIdentityName + scope: resourceGroup(existingManagedIdentitySubscriptionId, existingManagedIdentityResourceGroupName) +} + +var effectiveManagedIdentityId = createManagedIdentity ? managedIdentity!.id : referencedManagedIdentity!.id +var effectiveManagedIdentityClientId = createManagedIdentity + ? managedIdentity!.properties.clientId + : referencedManagedIdentity!.properties.clientId +var effectiveManagedIdentityPrincipalId = createManagedIdentity + ? managedIdentity!.properties.principalId + : referencedManagedIdentity!.properties.principalId + // ============================================================================ // Key Vault (existing — avoids soft-delete/purge-protection redeployment issues) // All auth uses managed identity (Azure SQL, ACR, AOAI). The vault is for @@ -266,12 +275,9 @@ var keyVaultName = last(split(keyVaultResourceId, '/')) // ============================================================================ // ============================================================================ -// Azure Container Apps Environment (workload profiles, public network disabled) -// Uses Private Endpoint pattern instead of VNet-integrated internal mode: -// - Environment is NOT VNet-integrated (no internal ILB) -// - Public network access is disabled -// - A Private Endpoint provides corp-reachable connectivity via Private Link -// - Private DNS zone resolves the FQDN to the private endpoint IP +// Azure Container Apps Environment (workload profiles) +// Public ACA-managed HTTPS ingress with optional app-level IP restrictions and +// VNet-integrated fixed NAT egress. // // OTel: When enableOtel=true, configure the managed OTel agent // as a post-deploy CLI step (2024-03-01 schema does not support it natively). @@ -288,13 +294,28 @@ resource acaEnvironment 'Microsoft.App/managedEnvironments@2024-10-02-preview' = sharedKey: effectiveLogAnalyticsKeyValue } } - publicNetworkAccess: enablePrivateEndpoint ? 'Disabled' : 'Enabled' + publicNetworkAccess: 'Enabled' workloadProfiles: [ { name: 'Consumption' workloadProfileType: 'Consumption' } ] + vnetConfiguration: { + infrastructureSubnetId: acaNatNetwork!.outputs.infrastructureSubnetId + internal: false + } + } +} + +var acaOriginHostName = '${appName}.${acaEnvironment.properties.defaultDomain}' + +module acaFrontDoor './modules/aca_front_door.bicep' = if (enableFrontDoor) { + name: '${appName}-aca-front-door' + params: { + namePrefix: appName + originHostName: acaOriginHostName + tags: tags } } @@ -305,69 +326,6 @@ resource acaEnvironment 'Microsoft.App/managedEnvironments@2024-10-02-preview' = // --connection-string // The Bicep API (2024-03-01) does not support openTelemetryConfiguration natively. -// ============================================================================ -// Private Endpoint for ACA Environment (corp-reachable via Private Link) -// The PE must be in a VNet that corp VPN/ExpressRoute can reach. -// ============================================================================ -resource privateEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = if (enablePrivateEndpoint) { - name: '${appName}-pe' - location: location - tags: tags - properties: { - subnet: { - id: effectiveSubnetId - } - privateLinkServiceConnections: [ - { - name: '${appName}-pe-connection' - properties: { - privateLinkServiceId: acaEnvironment.id - groupIds: [ - 'managedEnvironments' - ] - } - } - ] - } -} - -// Private DNS Zone for ACA Private Endpoint resolution -resource privateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = if (enablePrivateEndpoint) { - name: 'privatelink.${location}.azurecontainerapps.io' - location: 'global' - tags: tags -} - -// Link DNS zone to the VNet so clients in the VNet can resolve -resource dnsZoneVnetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = if (enablePrivateEndpoint) { - name: '${appName}-dns-link' - parent: privateDnsZone - location: 'global' - tags: tags - properties: { - virtualNetwork: { - id: createVnet ? vnet.id : join(take(split(infrastructureSubnetId, '/'), 9), '/') - } - registrationEnabled: false - } -} - -// DNS record group for the private endpoint -resource privateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-11-01' = if (enablePrivateEndpoint) { - name: 'default' - parent: privateEndpoint - properties: { - privateDnsZoneConfigs: [ - { - name: 'aca-dns-config' - properties: { - privateDnsZoneId: privateDnsZone.id - } - } - ] - } -} - // ============================================================================ // Container App — PyRIT GUI // ============================================================================ @@ -378,7 +336,7 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { identity: { type: 'UserAssigned' userAssignedIdentities: { - '${managedIdentity.id}': {} + '${effectiveManagedIdentityId}': {} } } // RBAC roles (AcrPull, KV Secrets User) must be granted manually before @@ -390,18 +348,17 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { // Single revision mode — only one revision serves traffic (appropriate for GUI) activeRevisionsMode: 'Single' - // Ingress — external at the app level (access is controlled by - // Private Endpoint + disabled public network access, not ingress type) + // ACA-managed public HTTPS ingress, optionally restricted by source CIDR. ingress: { external: true targetPort: 8000 transport: 'http' allowInsecure: false - ipSecurityRestrictions: allowedCidr != '' ? [ + ipSecurityRestrictions: effectiveAllowedCidr != '' ? [ { name: 'allowed-cidr' description: allowedCidrDescription - ipAddressRange: allowedCidr + ipAddressRange: effectiveAllowedCidr action: 'Allow' } ] : [] @@ -411,7 +368,7 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { registries: [ { server: effectiveAcrServer - identity: managedIdentity.id + identity: effectiveManagedIdentityId } ] @@ -425,7 +382,7 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { : { name: 'env-file' keyVaultUrl: 'https://${keyVaultName}${environment().suffixes.keyvaultDns}/secrets/${envSecretName}' - identity: managedIdentity.id + identity: effectiveManagedIdentityId } ] } @@ -457,7 +414,7 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { name: 'PYRIT_INITIALIZER' value: pyritInitializer } - // .env file contents from Key Vault — PyRIT parses this at startup + // .env contents from the inline or Key Vault-backed ACA secret { name: 'PYRIT_ENV_CONTENTS' secretRef: 'env-file' @@ -489,14 +446,14 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { // DefaultAzureCredential needs the UAMI client ID to pick the correct identity { name: 'AZURE_CLIENT_ID' - value: managedIdentity.properties.clientId + value: effectiveManagedIdentityClientId } - // CORS origin for the SPA. The ACA-generated FQDN is deterministic - // (.), so we compute it from upstream - // resources rather than self-referencing containerApp. + // Permit both the rollback ACA URL and the Front Door cutover URL. { name: 'PYRIT_CORS_ORIGINS' - value: 'https://${appName}.${acaEnvironment.properties.defaultDomain}' + value: enableFrontDoor + ? 'https://${acaOriginHostName},https://${acaFrontDoor!.outputs.endpointHostName}' + : 'https://${acaOriginHostName}' } ] } @@ -527,20 +484,37 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { @description('The FQDN of the deployed Container App') output appFqdn string = containerApp.properties.configuration.ingress.fqdn +@description('The Azure Front Door managed HTTPS hostname') +output frontDoorFqdn string = enableFrontDoor ? acaFrontDoor!.outputs.endpointHostName : '' + +@description('The Azure Front Door public URL') +output frontDoorUrl string = enableFrontDoor ? 'https://${acaFrontDoor!.outputs.endpointHostName}' : '' + +@description('The public application FQDN selected for this deployment') +output publicFqdn string = enableFrontDoor ? acaFrontDoor!.outputs.endpointHostName : containerApp.properties.configuration.ingress.fqdn + @description('The default domain of the ACA environment') output environmentDefaultDomain string = acaEnvironment.properties.defaultDomain -@description('Private Endpoint resource ID (empty when PE is disabled)') -output privateEndpointId string = enablePrivateEndpoint ? privateEndpoint.id : '' +@description('Static outbound IPv4 address') +output egressPublicIpAddress string = acaNatNetwork!.outputs.egressPublicIpAddress + +@description('NAT Gateway resource ID') +output natGatewayId string = acaNatNetwork!.outputs.natGatewayId + +@description('ACA infrastructure subnet resource ID') +output acaInfrastructureSubnetId string = acaNatNetwork!.outputs.infrastructureSubnetId @description('The principal ID of the user-assigned managed identity — grant this Cognitive Services OpenAI User on your AOAI instances and db_datareader/db_datawriter on Azure SQL') -output managedIdentityPrincipalId string = managedIdentity.properties.principalId +output managedIdentityPrincipalId string = effectiveManagedIdentityPrincipalId @description('The resource ID of the user-assigned managed identity') -output managedIdentityResourceId string = managedIdentity.id +output managedIdentityResourceId string = effectiveManagedIdentityId @description('IMPORTANT: Create an Azure AD contained user in the target database for this managed identity. See README post-deployment steps.') -output sqlAadSetupRequired string = 'Run CREATE USER [${appName}-identity] FROM EXTERNAL PROVIDER on database ${sqlDatabaseName}' +output sqlAadSetupRequired string = createManagedIdentity + ? 'Run CREATE USER [${appName}-identity] FROM EXTERNAL PROVIDER on database ${sqlDatabaseName}' + : 'Verify the existing managed identity has the required contained user and database roles on ${sqlDatabaseName}' @description('Key Vault name (existing)') output keyVaultName string = keyVaultName @@ -548,8 +522,8 @@ output keyVaultName string = keyVaultName @description('ACR login server') output acrLoginServer string = effectiveAcrServer -@description('VNet name (if created by this template)') -output vnetName string = createVnet ? vnet.name : 'N/A (existing VNet used)' +@description('Virtual network name') +output vnetName string = acaNatNetwork!.outputs.vnetName @description('Application Insights connection string (if OTel enabled)') -output appInsightsConnectionString string = enableOtel ? appInsights.properties.ConnectionString : 'N/A (OTel disabled)' +output appInsightsConnectionString string = enableOtel ? appInsights!.properties.ConnectionString : 'N/A (OTel disabled)' diff --git a/infra/modules/aca_front_door.bicep b/infra/modules/aca_front_door.bicep new file mode 100644 index 0000000000..335dce8ea9 --- /dev/null +++ b/infra/modules/aca_front_door.bicep @@ -0,0 +1,93 @@ +@description('Prefix used to name Front Door resources') +param namePrefix string + +@description('ACA-generated origin hostname without a scheme') +param originHostName string + +@description('Resource tags applied to the Front Door profile') +param tags object + +var endpointSuffix = take(uniqueString(subscription().id, resourceGroup().id, namePrefix), 8) + +resource profile 'Microsoft.Cdn/profiles@2024-09-01' = { + name: '${namePrefix}-afd' + location: 'global' + tags: tags + sku: { + name: 'Premium_AzureFrontDoor' + } + properties: { + originResponseTimeoutSeconds: 60 + } +} + +resource endpoint 'Microsoft.Cdn/profiles/afdEndpoints@2024-09-01' = { + parent: profile + name: '${namePrefix}-${endpointSuffix}' + location: 'global' + properties: { + enabledState: 'Enabled' + } +} + +resource originGroup 'Microsoft.Cdn/profiles/originGroups@2024-09-01' = { + parent: profile + name: '${namePrefix}-origin-group' + properties: { + healthProbeSettings: { + probeIntervalInSeconds: 30 + probePath: '/api/health' + probeProtocol: 'Https' + probeRequestType: 'GET' + } + loadBalancingSettings: { + additionalLatencyInMilliseconds: 50 + sampleSize: 4 + successfulSamplesRequired: 3 + } + sessionAffinityState: 'Disabled' + } +} + +resource origin 'Microsoft.Cdn/profiles/originGroups/origins@2024-09-01' = { + parent: originGroup + name: '${namePrefix}-aca-origin' + properties: { + enabledState: 'Enabled' + enforceCertificateNameCheck: true + hostName: originHostName + httpPort: 80 + httpsPort: 443 + originHostHeader: originHostName + priority: 1 + weight: 1000 + } +} + +resource route 'Microsoft.Cdn/profiles/afdEndpoints/routes@2024-09-01' = { + parent: endpoint + name: '${namePrefix}-route' + properties: { + enabledState: 'Enabled' + forwardingProtocol: 'HttpsOnly' + httpsRedirect: 'Enabled' + linkToDefaultDomain: 'Enabled' + originGroup: { + id: originGroup.id + } + patternsToMatch: [ + '/*' + ] + supportedProtocols: [ + 'Http' + 'Https' + ] + } + dependsOn: [ + origin + ] +} + +output endpointHostName string = endpoint.properties.hostName +output endpointId string = endpoint.id +output profileId string = profile.id \ No newline at end of file diff --git a/infra/modules/aca_nat_network.bicep b/infra/modules/aca_nat_network.bicep new file mode 100644 index 0000000000..fa4f29955c --- /dev/null +++ b/infra/modules/aca_nat_network.bicep @@ -0,0 +1,108 @@ +@description('Prefix used to name the Container Apps network resources') +param namePrefix string + +@description('Azure region for the network resources') +param location string + +@description('Resource tags applied to the network resources') +param tags object + +@description('Virtual network address prefix') +param vnetAddressPrefix string + +@description('Dedicated Container Apps infrastructure subnet address prefix') +param infrastructureSubnetAddressPrefix string + +@description('Existing Azure Policy IP tags to preserve when adopting a reserved egress public IP') +param egressPublicIpIpTags array = [] + +@description('Protect the static egress public IP from accidental deletion') +param protectEgressPublicIp bool = false + +var infrastructureSubnetName = '${namePrefix}-aca-subnet' + +resource egressPublicIp 'Microsoft.Network/publicIPAddresses@2024-05-01' = { + name: '${namePrefix}-egress-pip' + location: location + tags: tags + sku: { + name: 'Standard' + tier: 'Regional' + } + properties: { + ddosSettings: { + protectionMode: 'VirtualNetworkInherited' + } + ipTags: egressPublicIpIpTags + publicIPAllocationMethod: 'Static' + publicIPAddressVersion: 'IPv4' + idleTimeoutInMinutes: 4 + } +} + +resource egressPublicIpLock 'Microsoft.Authorization/locks@2020-05-01' = if (protectEgressPublicIp) { + name: '${namePrefix}-egress-pip-lock' + scope: egressPublicIp + properties: { + level: 'CanNotDelete' + notes: 'Protects the allow-listed static egress IP. Remove only through an approved egress migration.' + } +} + +resource natGateway 'Microsoft.Network/natGateways@2024-05-01' = { + name: '${namePrefix}-nat' + location: location + tags: tags + sku: { + name: 'Standard' + } + properties: { + idleTimeoutInMinutes: 4 + publicIpAddresses: [ + { + id: egressPublicIp.id + } + ] + } +} + +resource vnet 'Microsoft.Network/virtualNetworks@2024-05-01' = { + name: '${namePrefix}-vnet' + location: location + tags: tags + properties: { + privateEndpointVNetPolicies: 'Disabled' + addressSpace: { + addressPrefixes: [ + vnetAddressPrefix + ] + } + subnets: [ + { + name: infrastructureSubnetName + properties: { + addressPrefix: infrastructureSubnetAddressPrefix + defaultOutboundAccess: false + delegations: [ + { + name: 'aca-environment-delegation' + properties: { + serviceName: 'Microsoft.App/environments' + } + } + ] + natGateway: { + id: natGateway.id + } + } + } + ] + } +} + +output vnetId string = vnet.id +output vnetName string = vnet.name +output infrastructureSubnetId string = resourceId('Microsoft.Network/virtualNetworks/subnets', vnet.name, infrastructureSubnetName) +output natGatewayId string = natGateway.id +output egressPublicIpId string = egressPublicIp.id +output egressPublicIpAddress string = egressPublicIp.properties.ipAddress diff --git a/infra/parameters.demo.json b/infra/parameters.demo.json index fd09671419..f7f7867b25 100644 --- a/infra/parameters.demo.json +++ b/infra/parameters.demo.json @@ -40,7 +40,16 @@ "enableOtel": { "value": false }, - "enablePrivateEndpoint": { + "enableFrontDoor": { + "value": false + }, + "vnetAddressPrefix": { + "value": "10.0.0.0/16" + }, + "infrastructureSubnetAddressPrefix": { + "value": "10.0.1.0/26" + }, + "protectEgressPublicIp": { "value": false }, "tags": { @@ -54,12 +63,12 @@ "acrName": { "value": "REPLACE_ACR_NAME" }, + "existingManagedIdentityResourceId": { + "value": "REPLACE_MANAGED_IDENTITY_RESOURCE_ID" + }, "keyVaultResourceId": { "value": "REPLACE_KEY_VAULT_RESOURCE_ID" }, - "infrastructureSubnetId": { - "value": "" - }, "logAnalyticsWorkspaceId": { "value": "" }, diff --git a/infra/parameters.example.json b/infra/parameters.example.json index 1bf9ed92d7..c6869dd97e 100644 --- a/infra/parameters.example.json +++ b/infra/parameters.example.json @@ -1,6 +1,7 @@ { "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", "contentVersion": "1.0.0.0", + "_comment_resources": "For a healthy one-pass deployment, point acrName and existingManagedIdentityResourceId at pre-created resources. Bicep can create them when omitted, but image push and RBAC bootstrap must then be completed separately.", "parameters": { "appName": { "value": "pyrit-gui" @@ -41,6 +42,18 @@ "enableOtel": { "value": false }, + "enableFrontDoor": { + "value": false + }, + "vnetAddressPrefix": { + "value": "10.0.0.0/16" + }, + "infrastructureSubnetAddressPrefix": { + "value": "10.0.1.0/26" + }, + "protectEgressPublicIp": { + "value": false + }, "tags": { "value": { "Service": "pyrit-gui", @@ -49,17 +62,15 @@ } }, - "_comment_optional": "--- Below are optional: omit to let the template create resources ---", - - "infrastructureSubnetId": { - "value": "" - }, "acrName": { - "value": "" + "value": "YOUR_ACR_NAME" }, "acrResourceId": { "value": "" }, + "existingManagedIdentityResourceId": { + "value": "/subscriptions/YOUR_SUB/resourceGroups/YOUR_IDENTITY_RG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/YOUR_IDENTITY" + }, "keyVaultResourceId": { "value": "/subscriptions/YOUR_SUB/resourceGroups/YOUR_RG/providers/Microsoft.KeyVault/vaults/YOUR_VAULT" }, diff --git a/infra/pipelines/deploy_public_nat.sh b/infra/pipelines/deploy_public_nat.sh new file mode 100644 index 0000000000..e8ae5ed8af --- /dev/null +++ b/infra/pipelines/deploy_public_nat.sh @@ -0,0 +1,327 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +set -euo pipefail + +required_variables=( + PYRIT_SLOT + PYRIT_BUILD_ID + PYRIT_SOURCE_DIRECTORY + PYRIT_AGENT_TEMP_DIRECTORY + PYRIT_DEPLOYMENT_RESOURCE_GROUP + PYRIT_APP_NAME + PYRIT_CONTAINER_IMAGE + PYRIT_VNET_ADDRESS_PREFIX + PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX + PYRIT_MANAGED_IDENTITY_RESOURCE_ID + PYRIT_ENTRA_TENANT_ID + PYRIT_ENTRA_CLIENT_ID + PYRIT_ALLOWED_GROUP_OBJECT_IDS + PYRIT_SQL_SERVER_FQDN + PYRIT_SQL_DATABASE_NAME + PYRIT_KEY_VAULT_RESOURCE_ID + PYRIT_ACR_RESOURCE_ID + PYRIT_ENABLE_OTEL + PYRIT_ENV_SECRET_NAME +) + +for variable_name in "${required_variables[@]}"; do + if [[ -z "${!variable_name:-}" || "${!variable_name}" == '$('* ]]; then + echo "##vso[task.logissue type=error]Required deployment value is missing: $variable_name" + exit 1 + fi +done + +if [[ "${PYRIT_ALLOWED_CLIENT_CIDR:-}" == '$('* ]]; then + echo "##vso[task.logissue type=error]Optional deployment value is unresolved: PYRIT_ALLOWED_CLIENT_CIDR" + exit 1 +fi +if [[ -n "${PYRIT_ALLOWED_CLIENT_CIDR:-}" ]]; then + echo "##vso[task.logissue type=error]Front Door cannot use an ACA client CIDR restriction because ACA sees Front Door backend IPs, not client IPs; leave PYRIT_ALLOWED_CLIENT_CIDR empty" + exit 1 +fi + +if [[ ! "$PYRIT_SLOT" =~ ^(test|prod)$ || ! "$PYRIT_BUILD_ID" =~ ^[0-9]+$ ]]; then + echo "##vso[task.logissue type=error]Invalid slot or build ID" + exit 1 +fi + +validate_resource_group_name() { + local value=$1 + [[ "$value" =~ ^[[:alnum:]_.()-]{1,90}$ && "$value" != *. ]] +} + +validate_container_app_name() { + local value=$1 + [[ "$value" =~ ^[a-z][a-z0-9-]{0,30}[a-z0-9]$ ]] +} + +if ! validate_resource_group_name "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + || ! validate_container_app_name "$PYRIT_APP_NAME"; then + echo "##vso[task.logissue type=error]Invalid deployment resource group or app name" + exit 1 +fi + +if ! python3 - \ + "$PYRIT_VNET_ADDRESS_PREFIX" \ + "$PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX" \ + "${PYRIT_ALLOWED_CLIENT_CIDR:-}" \ + "$PYRIT_ENTRA_TENANT_ID" \ + "$PYRIT_ENTRA_CLIENT_ID" \ + "$PYRIT_ALLOWED_GROUP_OBJECT_IDS" <<'PY' +import ipaddress +import sys +import uuid + +try: + vnet = ipaddress.ip_network(sys.argv[1], strict=True) + subnet = ipaddress.ip_network(sys.argv[2], strict=True) + allowed = ipaddress.ip_network(sys.argv[3], strict=True) if sys.argv[3] else None + if vnet.version != 4 or subnet.version != 4 or (allowed is not None and allowed.version != 4): + raise ValueError + if not subnet.subnet_of(vnet) or subnet.prefixlen > 27: + raise ValueError + uuid.UUID(sys.argv[4]) + uuid.UUID(sys.argv[5]) + groups = [value.strip() for value in sys.argv[6].split(",") if value.strip()] + if not groups: + raise ValueError + for group in groups: + uuid.UUID(group) +except (ValueError, IndexError): + raise SystemExit(1) +PY +then + echo "##vso[task.logissue type=error]Invalid network prefix, subnet sizing, Entra ID, or allowed group ID" + exit 1 +fi + +guid_pattern='[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}' +if [[ ! "$PYRIT_ACR_RESOURCE_ID" =~ ^/subscriptions/($guid_pattern)/resourceGroups/[^/]+/providers/Microsoft\.ContainerRegistry/registries/([a-z0-9]{5,50})$ ]]; then + echo "##vso[task.logissue type=error]PYRIT_ACR_RESOURCE_ID is not canonical" + exit 1 +fi +expected_subscription=${BASH_REMATCH[1],,} +acr_name=${BASH_REMATCH[2]} +if [[ "$(az account show --query id -o tsv | tr '[:upper:]' '[:lower:]')" != "$expected_subscription" ]]; then + echo "##vso[task.logissue type=error]Azure subscription does not match ACR" + exit 1 +fi + +if [[ ! "$PYRIT_MANAGED_IDENTITY_RESOURCE_ID" =~ ^/subscriptions/($guid_pattern)/resourceGroups/[^/]+/providers/Microsoft\.ManagedIdentity/userAssignedIdentities/[a-zA-Z0-9_-]{3,128}$ ]] \ + || [[ "${BASH_REMATCH[1],,}" != "$expected_subscription" ]]; then + echo "##vso[task.logissue type=error]Managed identity resource ID is not canonical or is in another subscription" + exit 1 +fi + +if [[ ! "$PYRIT_KEY_VAULT_RESOURCE_ID" =~ ^/subscriptions/($guid_pattern)/resourceGroups/[^/]+/providers/Microsoft\.KeyVault/vaults/[a-zA-Z0-9-]{3,24}$ ]] \ + || [[ "${BASH_REMATCH[1],,}" != "$expected_subscription" ]]; then + echo "##vso[task.logissue type=error]Key Vault resource ID is not canonical or is in another subscription" + exit 1 +fi +if [[ ! "$PYRIT_SQL_SERVER_FQDN" =~ ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\.database\.windows\.net$ \ + || ! "$PYRIT_ENV_SECRET_NAME" =~ ^[a-zA-Z0-9-]{1,127}$ \ + || ! "$PYRIT_ENABLE_OTEL" =~ ^(true|false)$ ]]; then + echo "##vso[task.logissue type=error]Invalid SQL FQDN, Key Vault secret name, or enableOtel value" + exit 1 +fi +if ! az resource show --ids "$PYRIT_MANAGED_IDENTITY_RESOURCE_ID" --api-version 2023-01-31 -o none 2>/dev/null; then + echo "##vso[task.logissue type=error]Managed identity does not exist or is not readable" + exit 1 +fi + +deployment_resource_group_id=$(az group show \ + --name "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" --query id -o tsv 2>/dev/null || true) +if [[ -z "$deployment_resource_group_id" ]]; then + echo "##vso[task.logissue type=error]Deployment resource group must already exist" + exit 1 +fi +if [[ "${deployment_resource_group_id,,}" != "/subscriptions/$expected_subscription/resourcegroups/"* ]]; then + echo "##vso[task.logissue type=error]Deployment resource group is in another subscription" + exit 1 +fi + +expected_app_id="$deployment_resource_group_id/providers/Microsoft.App/containerApps/$PYRIT_APP_NAME" +expected_environment_id="$deployment_resource_group_id/providers/Microsoft.App/managedEnvironments/$PYRIT_APP_NAME-env" +expected_vnet_id="$deployment_resource_group_id/providers/Microsoft.Network/virtualNetworks/$PYRIT_APP_NAME-vnet" +expected_subnet_id="$expected_vnet_id/subnets/$PYRIT_APP_NAME-aca-subnet" +expected_nat_id="$deployment_resource_group_id/providers/Microsoft.Network/natGateways/$PYRIT_APP_NAME-nat" +expected_pip_id="$deployment_resource_group_id/providers/Microsoft.Network/publicIPAddresses/$PYRIT_APP_NAME-egress-pip" + +existing_app=$(az containerapp show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME" \ + --query '{id:id,environmentId:properties.managedEnvironmentId,tags:tags}' -o json 2>/dev/null || true) +existing_vnet=$(az network vnet show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-vnet" \ + --query '{id:id,prefix:addressSpace.addressPrefixes[0],tags:tags}' -o json 2>/dev/null || true) +existing_subnet=$(az network vnet subnet show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --vnet-name "$PYRIT_APP_NAME-vnet" \ + --name "$PYRIT_APP_NAME-aca-subnet" \ + --query '{id:id,prefix:addressPrefix,natId:natGateway.id}' -o json 2>/dev/null || true) +existing_nat=$(az network nat gateway show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-nat" \ + --query '{id:id,pipId:publicIpAddresses[0].id,tags:tags}' -o json 2>/dev/null || true) +existing_pip=$(az network public-ip show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-egress-pip" \ + --query '{id:id,ip:ipAddress,allocation:publicIPAllocationMethod,sku:sku.name,tags:tags}' -o json 2>/dev/null || true) + +if [[ -z "$existing_app" || -z "$existing_vnet" || -z "$existing_subnet" \ + || -z "$existing_nat" || -z "$existing_pip" ]]; then + echo "##vso[task.logissue type=error]Internal deployments must adopt an existing app, environment, VNet, subnet, NAT, and egress PIP" + exit 1 +fi + +deployment_tags=$(jq -cS '.tags' <<< "$existing_app") +pip_tags=$(jq -cS '.tags' <<< "$existing_pip") +nat_tags=$(jq -cS '.tags' <<< "$existing_nat") +vnet_tags=$(jq -cS '.tags' <<< "$existing_vnet") +existing_pip_ip_tags=$(az network public-ip show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-egress-pip" --query 'ipTags || `[]`' -o json | jq -c .) +expected_egress_ip=$(jq -r '.ip // empty' <<< "$existing_pip") + +if [[ "$(jq -r '.id | ascii_downcase' <<< "$existing_app")" != "${expected_app_id,,}" \ + || "$(jq -r '.environmentId | ascii_downcase' <<< "$existing_app")" != "${expected_environment_id,,}" \ + || "$(jq -r '.id | ascii_downcase' <<< "$existing_vnet")" != "${expected_vnet_id,,}" \ + || "$(jq -r '.id | ascii_downcase' <<< "$existing_subnet")" != "${expected_subnet_id,,}" \ + || "$(jq -r '.id | ascii_downcase' <<< "$existing_nat")" != "${expected_nat_id,,}" \ + || "$(jq -r '.id | ascii_downcase' <<< "$existing_pip")" != "${expected_pip_id,,}" \ + || "$(jq -r '.natId | ascii_downcase' <<< "$existing_subnet")" != "${expected_nat_id,,}" \ + || "$(jq -r '.pipId | ascii_downcase' <<< "$existing_nat")" != "${expected_pip_id,,}" \ + || "$(jq -r '.prefix' <<< "$existing_vnet")" != "$PYRIT_VNET_ADDRESS_PREFIX" \ + || "$(jq -r '.prefix' <<< "$existing_subnet")" != "$PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX" \ + || "$(jq -r '.allocation' <<< "$existing_pip")" != "Static" \ + || "$(jq -r '.sku' <<< "$existing_pip")" != "Standard" \ + || -z "$expected_egress_ip" ]]; then + echo "##vso[task.logissue type=error]Deployment variables do not match the existing protected topology" + exit 1 +fi + +if [[ "$deployment_tags" == *'<'* || "$deployment_tags" == "null" \ + || "$deployment_tags" != "$pip_tags" || "$deployment_tags" != "$nat_tags" \ + || "$deployment_tags" != "$vnet_tags" ]]; then + echo "##vso[task.logissue type=error]Protected resource tags are missing, placeholders, or inconsistent" + exit 1 +fi + +if [[ ! "$PYRIT_CONTAINER_IMAGE" =~ ^([^/]+)/(.+)@(sha256:[0-9a-fA-F]{64})$ ]]; then + echo "##vso[task.logissue type=error]Built image must be an immutable registry digest" + exit 1 +fi +registry_server=${BASH_REMATCH[1]} +repository=${BASH_REMATCH[2]} +digest=${BASH_REMATCH[3]} +if [[ "$registry_server" != "$acr_name.azurecr.io" ]]; then + echo "##vso[task.logissue type=error]Built image registry does not match ACR resource ID" + exit 1 +fi +repository_pattern='^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*$' +if [[ ! "$repository" =~ $repository_pattern ]]; then + echo "##vso[task.logissue type=error]Built image repository is invalid" + exit 1 +fi +immutable_image="$registry_server/$repository@$digest" + +parameters=( + "appName=$PYRIT_APP_NAME" + "containerImage=$immutable_image" + "entraTenantId=$PYRIT_ENTRA_TENANT_ID" + "entraClientId=$PYRIT_ENTRA_CLIENT_ID" + "allowedGroupObjectIds=$PYRIT_ALLOWED_GROUP_OBJECT_IDS" + "allowedCidr=${PYRIT_ALLOWED_CLIENT_CIDR:-}" + "sqlServerFqdn=$PYRIT_SQL_SERVER_FQDN" + "sqlDatabaseName=$PYRIT_SQL_DATABASE_NAME" + "keyVaultResourceId=$PYRIT_KEY_VAULT_RESOURCE_ID" + "acrResourceId=$PYRIT_ACR_RESOURCE_ID" + "existingManagedIdentityResourceId=$PYRIT_MANAGED_IDENTITY_RESOURCE_ID" + "enableOtel=$PYRIT_ENABLE_OTEL" + "envSecretName=$PYRIT_ENV_SECRET_NAME" + "enableFrontDoor=true" + "vnetAddressPrefix=$PYRIT_VNET_ADDRESS_PREFIX" + "infrastructureSubnetAddressPrefix=$PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX" + "egressPublicIpIpTags=$existing_pip_ip_tags" + "protectEgressPublicIp=true" + "tags=$deployment_tags" +) + +deployment_name="pyrit-$PYRIT_SLOT-$PYRIT_BUILD_ID" +what_if_file="$PYRIT_AGENT_TEMP_DIRECTORY/$deployment_name-what-if.json" +az deployment group what-if \ + --name "$deployment_name-preview" \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --template-file "$PYRIT_SOURCE_DIRECTORY/infra/main.bicep" \ + --parameters "${parameters[@]}" \ + --result-format FullResourcePayloads --no-pretty-print -o json > "$what_if_file" + +# ARM what-if reports these read-only server defaults as deletes when adopting +# existing Standard NAT/PIP resources. Every other protected-resource delta fails. +if ! python3 "$PYRIT_SOURCE_DIRECTORY/infra/pipelines/validate_what_if.py" \ + --what-if-file "$what_if_file" \ + --deployment-resource-group-id "$deployment_resource_group_id" \ + --expected-pip-id "$expected_pip_id" \ + --expected-nat-id "$expected_nat_id" \ + --expected-vnet-id "$expected_vnet_id" \ + --expected-subnet-id "$expected_subnet_id"; then + echo "##vso[task.logissue type=error]What-if contains a delete, cross-resource-group write, protected-network change, or core resource create" + exit 1 +fi + +az deployment group create \ + --name "$deployment_name" \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --template-file "$PYRIT_SOURCE_DIRECTORY/infra/main.bicep" \ + --parameters "${parameters[@]}" + +health="" +for attempt in {1..5}; do + health=$(az containerapp revision list \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME" \ + --query "[?properties.template.containers[0].image=='$immutable_image'] | sort_by(@,&properties.createdTime)[-1].properties.healthState" \ + -o tsv || true) + echo "Revision health attempt $attempt/5: ${health:-}" + [[ "$health" == "Healthy" ]] && break + [[ "$attempt" -lt 5 ]] && sleep 120 +done +if [[ "$health" != "Healthy" ]]; then + echo "##vso[task.logissue type=error]Deployed revision did not become healthy" + exit 1 +fi + +app_fqdn=$(az deployment group show \ + --name "$deployment_name" --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --query properties.outputs.appFqdn.value -o tsv) +front_door_fqdn=$(az deployment group show \ + --name "$deployment_name" --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --query properties.outputs.frontDoorFqdn.value -o tsv) +egress_ip=$(az deployment group show \ + --name "$deployment_name" --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --query properties.outputs.egressPublicIpAddress.value -o tsv) +actual_pip_id=$(az network public-ip show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-egress-pip" --query id -o tsv) +if [[ "$egress_ip" != "$expected_egress_ip" \ + || "${actual_pip_id,,}" != "${expected_pip_id,,}" ]]; then + echo "##vso[task.logissue type=error]Reserved egress PIP identity or address changed" + exit 1 +fi +front_door_health="" +for attempt in {1..20}; do + front_door_health=$(curl \ + --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --max-time 30 "https://$front_door_fqdn/api/health" || true) + echo "Front Door health attempt $attempt/20: ${front_door_health:-}" + [[ "$front_door_health" == "200" ]] && break + [[ "$attempt" -lt 20 ]] && sleep 30 +done +if [[ "$front_door_health" != "200" ]]; then + echo "##vso[task.logissue type=error]Front Door did not route a healthy response" + exit 1 +fi +echo "Deployment healthy; public URL: https://$front_door_fqdn; ACA origin: https://$app_fqdn; egress IPv4: $egress_ip" \ No newline at end of file diff --git a/infra/pipelines/validate_what_if.py b/infra/pipelines/validate_what_if.py new file mode 100644 index 0000000000..d8b65a3927 --- /dev/null +++ b/infra/pipelines/validate_what_if.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +"""Reject ARM what-if results that can replace protected deployment topology.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import cast + + +class WhatIfFormatError(ValueError): + """Raised when Azure returns a what-if payload that cannot be validated safely.""" + + +_CORE_RESOURCE_ID_PATTERN = re.compile( + r"/providers/microsoft\." + r"(?:network/(?:publicipaddresses/[^/]+|natgateways/[^/]+|virtualnetworks/[^/]+(?:/subnets/[^/]+)?)" + r"|app/(?:managedenvironments|containerapps)/[^/]+" + r"|operationalinsights/workspaces/[^/]+)$", + re.IGNORECASE, +) + + +def _expect_object(value: object, *, context: str) -> dict[str, object]: + if not isinstance(value, dict): + raise WhatIfFormatError(f"{context} must be a JSON object") + return cast(dict[str, object], value) + + +def _expect_array(value: object, *, context: str) -> list[object]: + if not isinstance(value, list): + raise WhatIfFormatError(f"{context} must be a JSON array") + return cast(list[object], value) + + +def _expect_string(value: object, *, context: str) -> str: + if not isinstance(value, str) or not value: + raise WhatIfFormatError(f"{context} must be a non-empty string") + return value + + +def validate_what_if( + payload: object, + *, + deployment_resource_group_id: str, + expected_pip_id: str, + expected_nat_id: str, + expected_vnet_id: str, + expected_subnet_id: str, +) -> list[str]: + """Return every destructive, cross-scope, protected, or core-create violation.""" + document = _expect_object(payload, context="what-if result") + changes = _expect_array(document.get("changes"), context="what-if changes") + resource_group_prefix = f"{deployment_resource_group_id.rstrip('/').casefold()}/" + protected_paths = { + expected_pip_id.rstrip("/").casefold(): {"sku.tier"}, + expected_nat_id.rstrip("/").casefold(): {"properties.scope", "sku.tier"}, + expected_vnet_id.rstrip("/").casefold(): set(), + expected_subnet_id.rstrip("/").casefold(): set(), + } + violations: list[str] = [] + + for index, value in enumerate(changes): + change = _expect_object(value, context=f"what-if change {index}") + change_type = _expect_string(change.get("changeType"), context=f"what-if change {index} type") + resource_id = _expect_string(change.get("resourceId"), context=f"what-if change {index} resource ID") + normalized_resource_id = resource_id.casefold().rstrip("/") + + if change_type == "Delete": + violations.append(f"delete: {resource_id}") + + if change_type != "Ignore" and not normalized_resource_id.startswith(resource_group_prefix): + violations.append(f"cross-resource-group write: {resource_id}") + + if change_type == "Create" and _CORE_RESOURCE_ID_PATTERN.search(normalized_resource_id): + violations.append(f"core resource create: {resource_id}") + + if normalized_resource_id not in protected_paths or change_type in {"NoChange", "Ignore"}: + continue + + delta_value = change.get("delta") + if delta_value is None: + violations.append(f"opaque protected-resource change: {resource_id}") + continue + + deltas = _expect_array(delta_value, context=f"protected change delta for {resource_id}") + if not deltas: + violations.append(f"opaque protected-resource change: {resource_id}") + continue + + allowed_paths = protected_paths[normalized_resource_id] + for delta_index, delta_value in enumerate(deltas): + delta = _expect_object(delta_value, context=f"delta {delta_index} for {resource_id}") + path = _expect_string(delta.get("path"), context=f"delta {delta_index} path for {resource_id}") + if path not in allowed_paths: + violations.append(f"protected-resource delta {path}: {resource_id}") + + return violations + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--what-if-file", required=True) + parser.add_argument("--deployment-resource-group-id", required=True) + parser.add_argument("--expected-pip-id", required=True) + parser.add_argument("--expected-nat-id", required=True) + parser.add_argument("--expected-vnet-id", required=True) + parser.add_argument("--expected-subnet-id", required=True) + return parser.parse_args() + + +def main() -> int: + """Validate a FullResourcePayloads what-if file for the internal update path.""" + parsed = _parse_args() + what_if_file = Path(cast(str, parsed.what_if_file)) + + try: + payload: object = json.loads(what_if_file.read_text(encoding="utf-8")) + violations = validate_what_if( + payload, + deployment_resource_group_id=cast(str, parsed.deployment_resource_group_id), + expected_pip_id=cast(str, parsed.expected_pip_id), + expected_nat_id=cast(str, parsed.expected_nat_id), + expected_vnet_id=cast(str, parsed.expected_vnet_id), + expected_subnet_id=cast(str, parsed.expected_subnet_id), + ) + except (OSError, json.JSONDecodeError, WhatIfFormatError) as error: + print(f"What-if validation failed closed: {error}", file=sys.stderr) + return 2 + + for violation in violations: + print(f"What-if rejected: {violation}", file=sys.stderr) + return 1 if violations else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/infra/teardown_instance.py b/infra/teardown_instance.py index 754b919d3a..71dd4fead4 100644 --- a/infra/teardown_instance.py +++ b/infra/teardown_instance.py @@ -10,11 +10,16 @@ Usage: python infra/teardown_instance.py --instance-name partners-demo \\ - --subscription "AI Red Team Tooling" + --subscription "" \ + --resource-group-id "/subscriptions//resourceGroups/copyrit-partners-demo" \ + --acknowledge-egress-ip-release # Include Entra cleanup: python infra/teardown_instance.py --instance-name partners-demo \\ - --subscription "AI Red Team Tooling" --delete-entra-app + --subscription "" \ + --resource-group-id "/subscriptions//resourceGroups/copyrit-partners-demo" \ + --acknowledge-egress-ip-release \ + --delete-entra-app --entra-app-id "" """ @@ -22,14 +27,25 @@ import json import logging import platform +import re import subprocess import sys +from typing import cast logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") logger = logging.getLogger(__name__) # On Windows, az CLI is a .cmd script that requires shell=True for subprocess to find it. _SHELL = platform.system() == "Windows" +_INSTANCE_NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,11}[a-z0-9])?$") +_RESOURCE_GROUP_ID_RE = re.compile( + r"^/subscriptions/([^/]+)/resourceGroups/([^/]+)$", + re.IGNORECASE, +) +_OWNERSHIP_TAGS: dict[str, str] = { + "Service": "pyrit-gui", + "ManagedBy": "infra/deploy_instance.py", +} def run_az( @@ -63,7 +79,28 @@ def run_az( ) -def run_az_json(*, args: list[str]) -> dict | list | str | None: +def _expect_json_object(value: object, *, context: str) -> dict[str, object]: + """Require a JSON object with string keys at an Azure CLI response boundary.""" + if not isinstance(value, dict): + raise RuntimeError(f"Azure CLI returned invalid {context} data") + return cast(dict[str, object], value) + + +def _expect_json_array(value: object, *, context: str) -> list[object]: + """Require a JSON array at an Azure CLI response boundary.""" + if not isinstance(value, list): + raise RuntimeError(f"Azure CLI returned invalid {context} data") + return cast(list[object], value) + + +def _expect_string(value: object, *, context: str) -> str: + """Require a nonempty string at an Azure CLI response boundary.""" + if not isinstance(value, str) or not value: + raise RuntimeError(f"Azure CLI returned invalid {context} data") + return value + + +def run_az_json(*, args: list[str]) -> object | None: """ Run an Azure CLI command and parse JSON output. @@ -71,12 +108,13 @@ def run_az_json(*, args: list[str]) -> dict | list | str | None: args (list[str]): The az CLI arguments (without the leading 'az'). Returns: - dict | list | str | None: The parsed JSON output, or None on failure. + object | None: The parsed JSON output, or None on command failure. """ result = run_az(args=args + ["-o", "json"], check=False) if result.returncode != 0: return None - return json.loads(result.stdout) + parsed: object = json.loads(result.stdout) + return parsed def parse_args(args: list[str] | None = None) -> argparse.Namespace: @@ -103,15 +141,30 @@ def parse_args(args: list[str] | None = None) -> argparse.Namespace: required=True, help="Azure subscription name or ID", ) + parser.add_argument( + "--resource-group-id", + required=True, + help="Exact resource ID of the instance resource group", + ) parser.add_argument( "--delete-entra-app", action="store_true", help="Also delete the Entra app registration and service principal", ) + parser.add_argument( + "--entra-app-id", + default="", + help="Exact Application (client) ID; required with --delete-entra-app", + ) + parser.add_argument( + "--acknowledge-egress-ip-release", + action="store_true", + help="Confirm that external allowlists have been updated before the static egress IP is released", + ) parser.add_argument( "--yes", action="store_true", - help="Skip confirmation prompt", + help="Skip the interactive prompt; ownership and egress-release checks still apply", ) return parser.parse_args(args) @@ -128,59 +181,171 @@ def main(args: list[str] | None = None) -> int: """ parsed = parse_args(args) - instance = parsed.instance_name + instance = _expect_string(parsed.instance_name, context="instance name") rg_name = f"copyrit-{instance}" entra_app_name = f"CoPyRIT GUI ({instance})" - logger.info("Instance: %s", instance) - logger.info("Resource group: %s", rg_name) - if parsed.delete_entra_app: - logger.info("Entra app: %s (will be deleted)", entra_app_name) - - if not parsed.yes: - confirm = input(f"\nDelete resource group '{rg_name}' and all its resources? [y/N] ") - if confirm.lower() != "y": - logger.info("Aborted.") - return 0 + if not _INSTANCE_NAME_RE.fullmatch(instance): + logger.error( + "--instance-name must be 1-13 lowercase letters, numbers, or internal hyphens, " + "and must start and end with a letter or number" + ) + return 1 + if not parsed.acknowledge_egress_ip_release: + logger.error( + "--acknowledge-egress-ip-release is required after removing the instance IP from external allowlists" + ) + return 1 + if parsed.delete_entra_app != bool(parsed.entra_app_id): + logger.error("--delete-entra-app and --entra-app-id must be provided together") + return 1 try: - # Set subscription logger.info("Setting subscription to: %s", parsed.subscription) run_az(args=["account", "set", "--subscription", parsed.subscription]) - # Delete resource group (and all Azure resources in it) - logger.info("Deleting resource group: %s (this may take several minutes)", rg_name) - run_az(args=["group", "delete", "--name", rg_name, "--yes", "--no-wait"]) - logger.info("Resource group deletion initiated (running in background)") + account = _expect_json_object( + run_az_json(args=["account", "show", "--query", "{id:id,name:name}"]), + context="active subscription", + ) + account_id = _expect_string(account.get("id"), context="active subscription ID") + account_name = _expect_string(account.get("name"), context="active subscription name") + + resource_group_match = _RESOURCE_GROUP_ID_RE.fullmatch(parsed.resource_group_id) + if resource_group_match is None: + raise RuntimeError("--resource-group-id is not a canonical Azure resource group ID") + resource_group_subscription_id, resource_group_name = resource_group_match.groups() + if resource_group_subscription_id.casefold() != account_id.casefold() or resource_group_name != rg_name: + raise RuntimeError("--resource-group-id does not match the active subscription and derived instance name") + + group_info = _expect_json_object( + run_az_json( + args=["group", "show", "--name", rg_name, "--query", "{id:id,name:name,tags:tags}"] + ), + context="resource group", + ) + group_id = _expect_string(group_info.get("id"), context="resource group ID") + if group_id.casefold() != parsed.resource_group_id.casefold(): + raise RuntimeError("Azure returned a resource group ID different from --resource-group-id") + tags = _expect_json_object(group_info.get("tags"), context="resource group tags") + expected_tags = {**_OWNERSHIP_TAGS, "Instance": instance} + if any(tags.get(key) != value for key, value in expected_tags.items()): + raise RuntimeError( + "Resource group ownership tags do not match deploy_instance.py; refuse automatic deletion" + ) - # Delete Entra app registration if requested + entra_app: dict[str, object] | None = None if parsed.delete_entra_app: - logger.info("Looking up Entra app: %s", entra_app_name) - app_info = run_az_json( + entra_app = _expect_json_object( + run_az_json( + args=[ + "ad", + "app", + "show", + "--id", + parsed.entra_app_id, + "--query", + "{appId:appId,displayName:displayName}", + ] + ), + context="Entra application", + ) + if ( + entra_app.get("appId") != parsed.entra_app_id + or entra_app.get("displayName") != entra_app_name + ): + raise RuntimeError("--entra-app-id does not identify the expected instance application") + + egress_ip_value = run_az_json( + args=[ + "network", + "public-ip", + "show", + "--resource-group", + rg_name, + "--name", + f"{rg_name}-egress-pip", + "--query", + "ipAddress", + ] + ) + egress_ip = egress_ip_value if isinstance(egress_ip_value, str) and egress_ip_value else None + principal_id = _expect_string( + run_az_json( + args=[ + "identity", + "show", + "--resource-group", + rg_name, + "--name", + f"{rg_name}-identity", + "--query", + "principalId", + ] + ), + context="managed identity principal ID", + ) + assignment_values = _expect_json_array( + run_az_json( args=[ - "ad", - "app", + "role", + "assignment", "list", - "--display-name", - entra_app_name, + "--assignee-object-id", + principal_id, + "--all", + "--fill-principal-name", + "false", "--query", - "[0].appId", + "[].{id:id,scope:scope}", ] - ) + ), + context="managed identity role assignments", + ) + assignments: list[dict[str, str]] = [] + for value in assignment_values: + assignment = _expect_json_object(value, context="role assignment") + assignment_id = _expect_string(assignment.get("id"), context="role assignment ID") + scope_value = assignment.get("scope") + scope = scope_value if isinstance(scope_value, str) and scope_value else "" + assignments.append({"id": assignment_id, "scope": scope}) + + logger.info("Instance: %s", instance) + logger.info("Subscription: %s (%s)", account_name, account_id) + logger.info("Resource group ID: %s", group_id) + logger.info("Static egress IP: %s (will be released)", egress_ip or "") + logger.info("Role assignments: %d (will be removed)", len(assignments)) + if entra_app is not None: + logger.info("Entra app ID: %s (will be deleted)", parsed.entra_app_id) + + if not parsed.yes: + confirm = input(f"\nDelete verified instance resource group '{rg_name}' and all its resources? [y/N] ") + if confirm.lower() != "y": + logger.info("Aborted.") + return 0 + + for assignment in assignments: + logger.info("Deleting role assignment on %s", assignment["scope"]) + run_az(args=["role", "assignment", "delete", "--ids", assignment["id"]]) - if app_info: - logger.info("Deleting Entra app registration: %s", app_info) - run_az(args=["ad", "app", "delete", "--id", app_info]) - logger.info("Entra app deleted") - else: - logger.warning("Entra app '%s' not found — skipping", entra_app_name) + logger.info("Deleting resource group: %s (this may take several minutes)", rg_name) + run_az(args=["group", "delete", "--name", rg_name, "--yes"]) + group_exists = run_az_json(args=["group", "exists", "--name", rg_name]) + if group_exists is not False: + raise RuntimeError(f"Resource group '{rg_name}' still exists after deletion returned") + + if parsed.delete_entra_app: + logger.info("Deleting Entra app registration: %s", parsed.entra_app_id) + run_az(args=["ad", "app", "delete", "--id", parsed.entra_app_id]) + logger.info("Entra app deleted") logger.info("") logger.info("=" * 60) logger.info("TEARDOWN COMPLETE") logger.info("=" * 60) - logger.info("Resource group '%s' is being deleted.", rg_name) + logger.info("Resource group '%s' was deleted.", rg_name) logger.info("This includes: Container App, SQL server, Key Vault, MI, networking, logs.") + logger.info("Static egress IP '%s' was released.", egress_ip or "") logger.info("") logger.info("Note: Key Vault uses purge protection. The vault name '%s'", f"copyrit-{instance}-kv") logger.info("will be reserved for ~90 days after deletion.") @@ -188,6 +353,9 @@ def main(args: list[str] | None = None) -> int: return 0 + except RuntimeError as error: + logger.error("%s", error) + return 1 except subprocess.CalledProcessError as e: logger.error("Command failed (exit code %d): %s", e.returncode, " ".join(e.cmd)) if e.stderr: diff --git a/tests/unit/infra/__init__.py b/tests/unit/infra/__init__.py new file mode 100644 index 0000000000..0eca6426d9 --- /dev/null +++ b/tests/unit/infra/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. \ No newline at end of file diff --git a/tests/unit/infra/test_bicep_topology.py b/tests/unit/infra/test_bicep_topology.py new file mode 100644 index 0000000000..824ccf6edc --- /dev/null +++ b/tests/unit/infra/test_bicep_topology.py @@ -0,0 +1,180 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +"""Compile the deployment Bicep and verify its public-NAT contract.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[3] +MAIN_BICEP = REPO_ROOT / "infra" / "main.bicep" +NETWORK_BICEP = REPO_ROOT / "infra" / "modules" / "aca_nat_network.bicep" +FRONT_DOOR_BICEP = REPO_ROOT / "infra" / "modules" / "aca_front_door.bicep" +AZ_CLI = shutil.which("az") + + +def _compile_bicep(source: Path, output: Path) -> dict[str, Any]: + """Compile one Bicep file and return its generated ARM template.""" + assert AZ_CLI is not None + command = [AZ_CLI, "bicep", "build", "--file", str(source), "--outfile", str(output)] + command_input: str | list[str] = subprocess.list2cmdline(command) if os.name == "nt" else command + result = subprocess.run(command_input, capture_output=True, text=True, check=False, shell=os.name == "nt") + assert result.returncode == 0, result.stderr + return json.loads(output.read_text(encoding="utf-8")) + + +def _resources(template: dict[str, Any], resource_type: str) -> list[dict[str, Any]]: + """Return resources of one ARM type from a compiled template.""" + return [resource for resource in template["resources"] if resource["type"] == resource_type] + + +@unittest.skipIf(AZ_CLI is None, "Azure CLI is not installed") +class BicepTopologyTests(unittest.TestCase): + """Verify the only supported public ACA topology with fixed NAT egress.""" + + def setUp(self): + self._temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self._temporary_directory.cleanup) + self.output_directory = Path(self._temporary_directory.name) + + def test_main_has_one_public_nat_topology(self): + template = _compile_bicep(MAIN_BICEP, self.output_directory / "main.json") + + unsupported_parameters = { + "networkMode", + "enablePrivateEndpoint", + "infrastructureSubnetId", + "infrastructureNsgName", + "applicationGatewayNsgName", + "enableFrontDoorPrivateLink", + } + assert unsupported_parameters.isdisjoint(template["parameters"]) + + existing_identity = template["parameters"]["existingManagedIdentityResourceId"] + assert existing_identity["defaultValue"] == "" + created_identity = _resources(template, "Microsoft.ManagedIdentity/userAssignedIdentities")[0] + assert created_identity["condition"] == "[variables('createManagedIdentity')]" + + modules = _resources(template, "Microsoft.Resources/deployments") + assert len(modules) == 2 + network_module = next(module for module in modules if "aca-nat-network" in module["name"]) + front_door_module = next(module for module in modules if "aca-front-door" in module["name"]) + assert "condition" not in network_module + assert "parameters('enableFrontDoor')" in front_door_module["condition"] + + nested_types = { + resource_type + for module in modules + for resource_type in (resource["type"] for resource in module["properties"]["template"]["resources"]) + } + assert "Microsoft.Network/natGateways" in nested_types + assert "Microsoft.Network/virtualNetworks" in nested_types + assert "Microsoft.Network/publicIPAddresses" in nested_types + assert "Microsoft.Authorization/locks" in nested_types + assert not any("privateDnsZones" in resource_type for resource_type in nested_types) + assert "Microsoft.Network/applicationGateways" not in nested_types + assert "Microsoft.Network/networkSecurityGroups" not in nested_types + assert "Microsoft.Cdn/profiles" in nested_types + assert "Microsoft.Cdn/profiles/afdEndpoints" in nested_types + assert "Microsoft.Cdn/profiles/originGroups" in nested_types + assert "Microsoft.Cdn/profiles/originGroups/origins" in nested_types + assert "Microsoft.Cdn/profiles/afdEndpoints/routes" in nested_types + + environment = _resources(template, "Microsoft.App/managedEnvironments")[0] + environment_properties = environment["properties"] + assert environment_properties["publicNetworkAccess"] == "Enabled" + assert environment_properties["vnetConfiguration"]["internal"] is False + assert "outputs.infrastructureSubnetId.value" in environment_properties["vnetConfiguration"][ + "infrastructureSubnetId" + ] + + container_app = _resources(template, "Microsoft.App/containerApps")[0] + assert container_app["properties"]["configuration"]["registries"][0]["identity"] == ( + "[variables('effectiveManagedIdentityId')]" + ) + ingress_restrictions = container_app["properties"]["configuration"]["ingress"]["ipSecurityRestrictions"] + assert "variables('effectiveAllowedCidr')" in ingress_restrictions + effective_allowed_cidr = template["variables"]["effectiveAllowedCidr"] + assert "parameters('allowedCidr')" in effective_allowed_cidr + assert "parameters('enableFrontDoor')" in effective_allowed_cidr + assert "fail(" in effective_allowed_cidr + assert not _resources(template, "Microsoft.Network/privateEndpoints") + assert not _resources(template, "Microsoft.Network/applicationGateways") + cors_value = next( + value["value"] + for value in container_app["properties"]["template"]["containers"][0]["env"] + if value["name"] == "PYRIT_CORS_ORIGINS" + ) + assert "aca-front-door" in cors_value + assert "outputs.endpointHostName.value" in cors_value + + def test_aca_nat_network_is_static_and_delegated(self): + template = _compile_bicep(NETWORK_BICEP, self.output_directory / "network.json") + + public_ips = _resources(template, "Microsoft.Network/publicIPAddresses") + assert len(public_ips) == 1 + public_ip = public_ips[0] + assert public_ip["sku"]["name"] == "Standard" + assert public_ip["sku"]["tier"] == "Regional" + assert public_ip["properties"]["publicIPAllocationMethod"] == "Static" + assert public_ip["properties"]["publicIPAddressVersion"] == "IPv4" + assert public_ip["properties"]["ddosSettings"]["protectionMode"] == "VirtualNetworkInherited" + assert public_ip["properties"]["ipTags"] == "[parameters('egressPublicIpIpTags')]" + + locks = _resources(template, "Microsoft.Authorization/locks") + assert len(locks) == 1 + assert "parameters('protectEgressPublicIp')" in locks[0]["condition"] + assert locks[0]["properties"]["level"] == "CanNotDelete" + assert "publicIPAddresses" in locks[0]["scope"] + + nat_gateway = _resources(template, "Microsoft.Network/natGateways")[0] + assert nat_gateway["sku"]["name"] == "Standard" + assert len(nat_gateway["properties"]["publicIpAddresses"]) == 1 + assert not _resources(template, "Microsoft.Network/routeTables") + + assert not _resources(template, "Microsoft.Network/virtualNetworks/subnets") + vnet = _resources(template, "Microsoft.Network/virtualNetworks")[0] + assert vnet["properties"]["privateEndpointVNetPolicies"] == "Disabled" + assert len(vnet["properties"]["subnets"]) == 1 + subnet = vnet["properties"]["subnets"][0] + assert subnet["properties"]["addressPrefix"] == "[parameters('infrastructureSubnetAddressPrefix')]" + assert subnet["properties"]["defaultOutboundAccess"] is False + assert subnet["properties"]["delegations"][0]["properties"]["serviceName"] == "Microsoft.App/environments" + assert "natGateway" in subnet["properties"] + assert "networkSecurityGroup" not in subnet["properties"] + + assert not _resources(template, "Microsoft.Network/networkSecurityGroups") + assert not _resources(template, "Microsoft.Network/networkSecurityGroups/securityRules") + + def test_front_door_uses_https_health_probe_without_caching(self): + template = _compile_bicep(FRONT_DOOR_BICEP, self.output_directory / "front-door.json") + + profile = _resources(template, "Microsoft.Cdn/profiles")[0] + assert profile["sku"]["name"] == "Premium_AzureFrontDoor" + + origin_group = _resources(template, "Microsoft.Cdn/profiles/originGroups")[0] + probe = origin_group["properties"]["healthProbeSettings"] + assert probe["probePath"] == "/api/health" + assert probe["probeProtocol"] == "Https" + assert probe["probeRequestType"] == "GET" + + origin = _resources(template, "Microsoft.Cdn/profiles/originGroups/origins")[0] + assert origin["properties"]["originHostHeader"] == "[parameters('originHostName')]" + assert origin["properties"]["enforceCertificateNameCheck"] is True + assert "sharedPrivateLinkResource" not in origin["properties"] + + route = _resources(template, "Microsoft.Cdn/profiles/afdEndpoints/routes")[0] + assert route["properties"]["forwardingProtocol"] == "HttpsOnly" + assert route["properties"]["httpsRedirect"] == "Enabled" + assert "cacheConfiguration" not in route["properties"] + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/unit/infra/test_instance_lifecycle.py b/tests/unit/infra/test_instance_lifecycle.py new file mode 100644 index 0000000000..6e74e549dd --- /dev/null +++ b/tests/unit/infra/test_instance_lifecycle.py @@ -0,0 +1,216 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +"""Guard deployment ownership and destructive teardown preconditions.""" + +from __future__ import annotations + +import importlib.util +import tempfile +import unittest +from pathlib import Path +from types import ModuleType +from unittest.mock import patch + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def _load_module(*, name: str, path: Path) -> ModuleType: + """Load an infrastructure script as a module without requiring a package.""" + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +DEPLOY = _load_module(name="copyrit_deploy_instance", path=REPO_ROOT / "infra" / "deploy_instance.py") +TEARDOWN = _load_module(name="copyrit_teardown_instance", path=REPO_ROOT / "infra" / "teardown_instance.py") + +INSTANCE = "audit-demo" +SUBSCRIPTION_ID = "11111111-1111-1111-1111-111111111111" +RESOURCE_GROUP = f"copyrit-{INSTANCE}" +RESOURCE_GROUP_ID = f"/subscriptions/{SUBSCRIPTION_ID}/resourceGroups/{RESOURCE_GROUP}" +GROUP_ID = "22222222-2222-2222-2222-222222222222" + + +class DeployInstanceContractTests(unittest.TestCase): + """Verify fail-fast inputs and ownership metadata for new instances.""" + + def test_deployment_tags_identify_script_owned_instance(self): + tags = DEPLOY._deployment_tags(instance=INSTANCE, owner="owner@example.com") + + self.assertEqual(tags["Service"], "pyrit-gui") + self.assertEqual(tags["Instance"], INSTANCE) + self.assertEqual(tags["ManagedBy"], "infra/deploy_instance.py") + self.assertEqual(tags["DataClass"], "Confidential") + self.assertEqual(tags["Owner"], "owner@example.com") + + def test_invalid_instance_name_fails_before_azure_calls(self): + with patch.object(DEPLOY, "run_az") as run_az: + result = DEPLOY.main( + [ + "--instance-name", + "Bad_Name", + "--env-file", + "missing.env", + "--subscription", + SUBSCRIPTION_ID, + "--acr-name", + "sharedacr", + "--container-image", + "sharedacr.azurecr.io/pyrit:abc123", + "--allowed-groups", + GROUP_ID, + ] + ) + + self.assertEqual(result, 1) + run_az.assert_not_called() + + def test_dry_run_accepts_matching_immutable_image(self): + with tempfile.TemporaryDirectory() as temporary_directory: + env_file = Path(temporary_directory) / "instance.env" + env_file.write_text("OPENAI_KEY=placeholder\n", encoding="utf-8") + result = DEPLOY.main( + [ + "--instance-name", + INSTANCE, + "--env-file", + str(env_file), + "--subscription", + SUBSCRIPTION_ID, + "--acr-name", + "sharedacr", + "--container-image", + "sharedacr.azurecr.io/pyrit:abc123", + "--allowed-groups", + GROUP_ID, + "--dry-run", + ] + ) + + self.assertEqual(result, 0) + + def test_invalid_ingress_cidr_fails_before_azure_calls(self): + with patch.object(DEPLOY, "run_az") as run_az: + result = DEPLOY.main( + [ + "--instance-name", + INSTANCE, + "--env-file", + "missing.env", + "--subscription", + SUBSCRIPTION_ID, + "--acr-name", + "sharedacr", + "--container-image", + "sharedacr.azurecr.io/pyrit:abc123", + "--allowed-groups", + GROUP_ID, + "--allowed-cidr", + "999.1.1.1/24", + ] + ) + + self.assertEqual(result, 1) + run_az.assert_not_called() + + def test_group_assignment_uses_resource_service_principal_relationship(self): + with patch.object(DEPLOY, "run_az") as run_az: + DEPLOY.assign_groups_to_app(sp_id=GROUP_ID, group_ids=[GROUP_ID]) + + command = run_az.call_args.kwargs["args"] + url = command[command.index("--url") + 1] + self.assertTrue(url.endswith(f"servicePrincipals/{GROUP_ID}/appRoleAssignedTo")) + self.assertNotIn("/appRoleAssignments", url) + + def test_data_plane_firewalls_use_only_static_egress_ip(self): + with patch.object(DEPLOY, "run_az") as run_az: + DEPLOY.configure_data_plane_network_access( + resource_group=RESOURCE_GROUP, + sql_server_name=f"{RESOURCE_GROUP}-sql", + storage_account_name="copyritauditdemosa", + egress_ip="20.30.40.50", + ) + + commands = [call.kwargs["args"] for call in run_az.call_args_list] + sql_command = commands[0] + self.assertEqual(sql_command[sql_command.index("--start-ip-address") + 1], "20.30.40.50") + self.assertEqual(sql_command[sql_command.index("--end-ip-address") + 1], "20.30.40.50") + self.assertNotIn("0.0.0.0", sql_command) + self.assertIn("--default-action", commands[2]) + self.assertEqual(commands[2][commands[2].index("--default-action") + 1], "Deny") + self.assertEqual(commands[2][commands[2].index("--bypass") + 1], "None") + + +class TeardownInstanceContractTests(unittest.TestCase): + """Verify teardown cannot bypass ownership and egress-release checks.""" + + def _arguments(self) -> list[str]: + return [ + "--instance-name", + INSTANCE, + "--subscription", + SUBSCRIPTION_ID, + "--resource-group-id", + RESOURCE_GROUP_ID, + "--acknowledge-egress-ip-release", + "--yes", + ] + + def test_missing_egress_acknowledgement_fails_before_azure_calls(self): + arguments = self._arguments() + arguments.remove("--acknowledge-egress-ip-release") + with patch.object(TEARDOWN, "run_az") as run_az: + result = TEARDOWN.main(arguments) + + self.assertEqual(result, 1) + run_az.assert_not_called() + + def test_mismatched_ownership_tags_refuse_deletion(self): + responses = [ + {"id": SUBSCRIPTION_ID, "name": "Audit Subscription"}, + {"id": RESOURCE_GROUP_ID, "name": RESOURCE_GROUP, "tags": {"Service": "unrelated"}}, + ] + with ( + patch.object(TEARDOWN, "run_az") as run_az, + patch.object(TEARDOWN, "run_az_json", side_effect=responses), + ): + result = TEARDOWN.main(self._arguments()) + + self.assertEqual(result, 1) + self.assertFalse(any(call.kwargs["args"][:2] == ["group", "delete"] for call in run_az.call_args_list)) + + def test_verified_instance_removes_roles_and_waits_for_group_deletion(self): + assignment_id = f"{RESOURCE_GROUP_ID}/providers/Microsoft.Authorization/roleAssignments/role-id" + responses = [ + {"id": SUBSCRIPTION_ID, "name": "Audit Subscription"}, + { + "id": RESOURCE_GROUP_ID, + "name": RESOURCE_GROUP, + "tags": { + "Service": "pyrit-gui", + "Instance": INSTANCE, + "ManagedBy": "infra/deploy_instance.py", + }, + }, + "20.30.40.50", + "33333333-3333-3333-3333-333333333333", + [{"id": assignment_id, "scope": RESOURCE_GROUP_ID}], + False, + ] + with ( + patch.object(TEARDOWN, "run_az") as run_az, + patch.object(TEARDOWN, "run_az_json", side_effect=responses), + ): + result = TEARDOWN.main(self._arguments()) + + self.assertEqual(result, 0) + commands = [call.kwargs["args"] for call in run_az.call_args_list] + self.assertIn(["role", "assignment", "delete", "--ids", assignment_id], commands) + self.assertIn(["group", "delete", "--name", RESOURCE_GROUP, "--yes"], commands) + self.assertFalse(any("--no-wait" in command for command in commands)) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/unit/infra/test_pipeline_guardrails.py b/tests/unit/infra/test_pipeline_guardrails.py new file mode 100644 index 0000000000..4fb9a18158 --- /dev/null +++ b/tests/unit/infra/test_pipeline_guardrails.py @@ -0,0 +1,238 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +"""Guard the single-topology Azure DevOps deployment contract.""" + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +PIPELINE = REPO_ROOT / "gui-deploy.yml" +DEPLOY_SCRIPT = REPO_ROOT / "infra" / "pipelines" / "deploy_public_nat.sh" +WHAT_IF_VALIDATOR = REPO_ROOT / "infra" / "pipelines" / "validate_what_if.py" +EXAMPLE_PARAMETERS = REPO_ROOT / "infra" / "parameters.example.json" +DEMO_PARAMETERS = REPO_ROOT / "infra" / "parameters.demo.json" + +SUBSCRIPTION_ID = "11111111-1111-1111-1111-111111111111" +RESOURCE_GROUP_ID = f"/subscriptions/{SUBSCRIPTION_ID}/resourceGroups/copyrit-prod-v2" +PIP_ID = f"{RESOURCE_GROUP_ID}/providers/Microsoft.Network/publicIPAddresses/copyrit-prod-v2-egress-pip" +NAT_ID = f"{RESOURCE_GROUP_ID}/providers/Microsoft.Network/natGateways/copyrit-prod-v2-nat" +VNET_ID = f"{RESOURCE_GROUP_ID}/providers/Microsoft.Network/virtualNetworks/copyrit-prod-v2-vnet" +SUBNET_ID = f"{VNET_ID}/subnets/copyrit-prod-v2-aca-subnet" + + +class PipelineGuardrailTests(unittest.TestCase): + """Verify one preview-first test/prod deployment workflow.""" + + @classmethod + def setUpClass(cls): + cls.pipeline = PIPELINE.read_text(encoding="utf-8") + cls.deploy_script = DEPLOY_SCRIPT.read_text(encoding="utf-8") + + def test_pipeline_has_one_test_and_prod_workflow(self): + assert "deploymentTarget" not in self.pipeline + assert "applyReplacement" not in self.pipeline + assert "stage: Build" in self.pipeline + assert "stage: DeployTest" in self.pipeline + assert "stage: ApproveProd" in self.pipeline + assert "stage: DeployProd" in self.pipeline + assert "DeployReplacement" not in self.pipeline + assert self.pipeline.count("scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh'") == 2 + + def test_production_remains_opt_in_and_independently_approved(self): + assert "task: ManualValidation@1" in self.pipeline + assert "onTimeout: reject" in self.pipeline + assert "approvers: '$(prodApprovers)'" in self.pipeline + assert "allowApproversToApproveTheirOwnRuns: false" in self.pipeline + approval_stage = self.pipeline[self.pipeline.index("stage: ApproveProd") : self.pipeline.index("stage: DeployProd")] + assert "- group: copyrit-gui-prod" in approval_stage + assert '"$BUILD_SOURCEBRANCH" != refs/heads/main' in self.pipeline + assert "eq(variables['Build.SourceBranch'], 'refs/heads/main')" in self.pipeline + assert "refs/heads/releases/" not in self.pipeline + assert "condition: succeeded('ApproveProd')" in self.pipeline + + def test_deploy_resolves_digest_and_previews_before_apply(self): + assert "name: BuildImage" in self.pipeline + assert "variable=immutableImage;isOutput=true" in self.pipeline + assert "stageDependencies.Build.BuildAndPush.outputs['BuildImage.immutableImage']" in self.pipeline + assert "PYRIT_CONTAINER_IMAGE: $(immutableImage)" in self.pipeline + assert '@(sha256:[0-9a-fA-F]{64})' in self.deploy_script + assert 'immutable_image="$registry_server/$repository@$digest"' in self.deploy_script + assert '"containerImage=$immutable_image"' in self.deploy_script + assert "az acr repository show" not in self.deploy_script + assert self.deploy_script.index("az deployment group what-if") < self.deploy_script.index( + "az deployment group create" + ) + assert "validate_what_if.py" in self.deploy_script + assert "cross-resource-group write" in self.deploy_script + assert "networkMode=" not in self.deploy_script + assert "enablePrivateEndpoint=" not in self.deploy_script + assert '"enableFrontDoor=true"' in self.deploy_script + assert "enableFrontDoorPrivateLink=" not in self.deploy_script + + def test_pipeline_passes_values_via_environment(self): + deploy_yaml = self.pipeline[self.pipeline.index("stage: DeployTest") :] + assert "PYRIT_DEPLOYMENT_RESOURCE_GROUP: $(deploymentResourceGroup)" in deploy_yaml + assert "PYRIT_CONTAINER_IMAGE: $(immutableImage)" in deploy_yaml + assert "PYRIT_ALLOWED_CLIENT_CIDR: $(deploymentAllowedClientCidr)" in deploy_yaml + assert "PYRIT_MANAGED_IDENTITY_RESOURCE_ID: $(managedIdentityResourceId)" in deploy_yaml + assert '="$(replacement' not in deploy_yaml + assert "PYRIT_FALLBACK" not in deploy_yaml + + def test_deploy_validates_structured_inputs_before_arm(self): + assert "ipaddress.ip_network" in self.deploy_script + assert "subnet.subnet_of(vnet)" in self.deploy_script + assert "subnet.prefixlen > 27" in self.deploy_script + assert "uuid.UUID" in self.deploy_script + assert "Microsoft\\.KeyVault/vaults" in self.deploy_script + assert "database\\.windows\\.net" in self.deploy_script + assert self.deploy_script.index("ipaddress.ip_network") < self.deploy_script.index("az deployment group what-if") + + def test_deploy_preserves_existing_network_and_tags(self): + assert "Front Door cannot use an ACA client CIDR restriction" in self.deploy_script + assert "Internal deployments must adopt an existing app" in self.deploy_script + assert '"tags=$deployment_tags"' in self.deploy_script + assert '"egressPublicIpIpTags=$existing_pip_ip_tags"' in self.deploy_script + assert '"protectEgressPublicIp=true"' in self.deploy_script + assert "--result-format FullResourcePayloads" in self.deploy_script + assert "--expected-pip-id" in self.deploy_script + assert "--expected-subnet-id" in self.deploy_script + assert "Reserved egress PIP identity or address changed" in self.deploy_script + assert self.deploy_script.index("expected_egress_ip=") < self.deploy_script.index("az deployment group what-if") + assert self.deploy_script.index("actual_pip_id=") > self.deploy_script.index("az deployment group create") + + def test_data_plane_health_probe_respects_ingress_restrictions(self): + assert "properties.outputs.frontDoorFqdn.value" in self.deploy_script + assert '"https://$front_door_fqdn/api/health"' in self.deploy_script + assert '"https://$app_fqdn/api/health"' not in self.deploy_script + assert "Front Door did not route a healthy response" in self.deploy_script + assert "ACA origin: https://$app_fqdn" in self.deploy_script + + def test_manual_parameter_files_use_the_single_topology(self): + example = json.loads(EXAMPLE_PARAMETERS.read_text(encoding="utf-8")) + demo = json.loads(DEMO_PARAMETERS.read_text(encoding="utf-8")) + + assert "_comment_resources" in example + assert "_comment_resources" not in example["parameters"] + unsupported = { + "enablePrivateEndpoint", + "networkMode", + "infrastructureNsgName", + "applicationGatewayNsgName", + "enableFrontDoorPrivateLink", + } + assert unsupported.isdisjoint(example["parameters"]) + assert unsupported.isdisjoint(demo["parameters"]) + assert "vnetAddressPrefix" in example["parameters"] + assert "infrastructureSubnetAddressPrefix" in example["parameters"] + assert example["parameters"]["acrName"]["value"] + assert example["parameters"]["existingManagedIdentityResourceId"]["value"] + assert demo["parameters"]["existingManagedIdentityResourceId"]["value"] + + def _run_what_if_validator( + self, + changes: list[dict[str, object]], + *, + expected_subnet_id: str = SUBNET_ID, + ) -> subprocess.CompletedProcess[str]: + with tempfile.TemporaryDirectory() as directory: + what_if_file = Path(directory) / "what-if.json" + what_if_file.write_text(json.dumps({"changes": changes}), encoding="utf-8") + return subprocess.run( + [ + sys.executable, + str(WHAT_IF_VALIDATOR), + "--what-if-file", + str(what_if_file), + "--deployment-resource-group-id", + RESOURCE_GROUP_ID, + "--expected-pip-id", + PIP_ID, + "--expected-nat-id", + NAT_ID, + "--expected-vnet-id", + VNET_ID, + "--expected-subnet-id", + expected_subnet_id, + ], + capture_output=True, + text=True, + check=False, + ) + + def test_what_if_validator_accepts_read_only_normalization_and_lock_create(self): + lock_id = f"{PIP_ID}/providers/Microsoft.Authorization/locks/copyrit-prod-v2-egress-pip-lock" + changes: list[dict[str, object]] = [ + { + "changeType": "Modify", + "resourceId": NAT_ID, + "delta": [{"path": "properties.scope"}, {"path": "sku.tier"}], + }, + {"changeType": "Modify", "resourceId": PIP_ID, "delta": [{"path": "sku.tier"}]}, + {"changeType": "Create", "resourceId": lock_id}, + ] + + result = self._run_what_if_validator(changes) + + assert result.returncode == 0, result.stderr + + def test_what_if_validator_rejects_each_protected_topology_violation(self): + fixtures: dict[str, tuple[dict[str, object], str]] = { + "delete": ({"changeType": "Delete", "resourceId": PIP_ID}, "delete"), + "cross-resource-group": ( + { + "changeType": "Modify", + "resourceId": f"/subscriptions/{SUBSCRIPTION_ID}/resourceGroups/other/providers/Microsoft.App/containerApps/app", + "delta": [{"path": "properties.configuration"}], + }, + "cross-resource-group write", + ), + "protected-subnet": ( + { + "changeType": "Modify", + "resourceId": SUBNET_ID, + "delta": [{"path": "properties.addressPrefix"}], + }, + "protected-resource delta", + ), + "opaque-protected-change": ({"changeType": "Modify", "resourceId": VNET_ID}, "opaque"), + "container-app-create": ( + { + "changeType": "Create", + "resourceId": f"{RESOURCE_GROUP_ID}/providers/Microsoft.App/containerApps/replacement", + }, + "core resource create", + ), + "workspace-create": ( + { + "changeType": "Create", + "resourceId": f"{RESOURCE_GROUP_ID}/providers/Microsoft.OperationalInsights/workspaces/copyrit-prod-v2-logs", + }, + "core resource create", + ), + } + + for name, (change, expected_error) in fixtures.items(): + with self.subTest(name=name): + result = self._run_what_if_validator([change]) + assert result.returncode == 1 + assert expected_error in result.stderr + + def test_what_if_validator_normalizes_expected_protected_resource_ids(self): + change: dict[str, object] = { + "changeType": "Modify", + "resourceId": SUBNET_ID, + "delta": [{"path": "properties.addressPrefix"}], + } + + result = self._run_what_if_validator([change], expected_subnet_id=f"{SUBNET_ID}/") + + assert result.returncode == 1 + assert "protected-resource delta" in result.stderr + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From 1e1ec05d52138eb4f6a0d234609bbc2f572badc0 Mon Sep 17 00:00:00 2001 From: Bashir Partovi Date: Wed, 19 Aug 2026 23:57:31 -0400 Subject: [PATCH 2/7] FIX: Satisfy infrastructure pre-commit checks Apply the repository-pinned Ruff formatter, resolve typing-only imports and long fixture lines, and normalize file endings for the new infrastructure Python files. --- infra/deploy_instance.py | 8 +++----- infra/pipelines/validate_what_if.py | 19 +++++++++---------- infra/teardown_instance.py | 13 ++++--------- tests/unit/infra/__init__.py | 2 +- tests/unit/infra/test_bicep_topology.py | 9 +++++---- tests/unit/infra/test_instance_lifecycle.py | 7 +++++-- tests/unit/infra/test_pipeline_guardrails.py | 18 ++++++++++++------ 7 files changed, 39 insertions(+), 37 deletions(-) diff --git a/infra/deploy_instance.py b/infra/deploy_instance.py index bc69d8d760..0c74e7b8b7 100644 --- a/infra/deploy_instance.py +++ b/infra/deploy_instance.py @@ -52,9 +52,7 @@ _GRAPH_USER_READ_SCOPE_ID = "e1fe6dd8-ba31-4d61-89e7-88639da4683d" _INSTANCE_NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,11}[a-z0-9])?$") _ACR_NAME_RE = re.compile(r"^[a-z0-9]{5,50}$") -_GROUP_ID_RE = re.compile( - r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" -) +_GROUP_ID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") _IMAGE_REPOSITORY_RE = r"[a-z0-9]+(?:[._-][a-z0-9]+)*(?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*" _IMAGE_VERSION_RE = r"(?:[A-Za-z0-9_][A-Za-z0-9_.-]*|sha256:[0-9a-fA-F]{64})" @@ -110,14 +108,14 @@ def _expect_json_object(value: object, *, context: str) -> dict[str, object]: """Require a JSON object with string keys at an Azure CLI response boundary.""" if not isinstance(value, dict): raise RuntimeError(f"Azure CLI returned invalid {context} data") - return cast(dict[str, object], value) + return cast("dict[str, object]", value) def _expect_json_array(value: object, *, context: str) -> list[object]: """Require a JSON array at an Azure CLI response boundary.""" if not isinstance(value, list): raise RuntimeError(f"Azure CLI returned invalid {context} data") - return cast(list[object], value) + return cast("list[object]", value) def _expect_string(value: object, *, context: str) -> str: diff --git a/infra/pipelines/validate_what_if.py b/infra/pipelines/validate_what_if.py index d8b65a3927..410aba83b2 100644 --- a/infra/pipelines/validate_what_if.py +++ b/infra/pipelines/validate_what_if.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. """Reject ARM what-if results that can replace protected deployment topology.""" @@ -29,13 +28,13 @@ class WhatIfFormatError(ValueError): def _expect_object(value: object, *, context: str) -> dict[str, object]: if not isinstance(value, dict): raise WhatIfFormatError(f"{context} must be a JSON object") - return cast(dict[str, object], value) + return cast("dict[str, object]", value) def _expect_array(value: object, *, context: str) -> list[object]: if not isinstance(value, list): raise WhatIfFormatError(f"{context} must be a JSON array") - return cast(list[object], value) + return cast("list[object]", value) def _expect_string(value: object, *, context: str) -> str: @@ -117,17 +116,17 @@ def _parse_args() -> argparse.Namespace: def main() -> int: """Validate a FullResourcePayloads what-if file for the internal update path.""" parsed = _parse_args() - what_if_file = Path(cast(str, parsed.what_if_file)) + what_if_file = Path(cast("str", parsed.what_if_file)) try: payload: object = json.loads(what_if_file.read_text(encoding="utf-8")) violations = validate_what_if( payload, - deployment_resource_group_id=cast(str, parsed.deployment_resource_group_id), - expected_pip_id=cast(str, parsed.expected_pip_id), - expected_nat_id=cast(str, parsed.expected_nat_id), - expected_vnet_id=cast(str, parsed.expected_vnet_id), - expected_subnet_id=cast(str, parsed.expected_subnet_id), + deployment_resource_group_id=cast("str", parsed.deployment_resource_group_id), + expected_pip_id=cast("str", parsed.expected_pip_id), + expected_nat_id=cast("str", parsed.expected_nat_id), + expected_vnet_id=cast("str", parsed.expected_vnet_id), + expected_subnet_id=cast("str", parsed.expected_subnet_id), ) except (OSError, json.JSONDecodeError, WhatIfFormatError) as error: print(f"What-if validation failed closed: {error}", file=sys.stderr) @@ -139,4 +138,4 @@ def main() -> int: if __name__ == "__main__": - raise SystemExit(main()) \ No newline at end of file + raise SystemExit(main()) diff --git a/infra/teardown_instance.py b/infra/teardown_instance.py index 71dd4fead4..a6a9558815 100644 --- a/infra/teardown_instance.py +++ b/infra/teardown_instance.py @@ -83,14 +83,14 @@ def _expect_json_object(value: object, *, context: str) -> dict[str, object]: """Require a JSON object with string keys at an Azure CLI response boundary.""" if not isinstance(value, dict): raise RuntimeError(f"Azure CLI returned invalid {context} data") - return cast(dict[str, object], value) + return cast("dict[str, object]", value) def _expect_json_array(value: object, *, context: str) -> list[object]: """Require a JSON array at an Azure CLI response boundary.""" if not isinstance(value, list): raise RuntimeError(f"Azure CLI returned invalid {context} data") - return cast(list[object], value) + return cast("list[object]", value) def _expect_string(value: object, *, context: str) -> str: @@ -219,9 +219,7 @@ def main(args: list[str] | None = None) -> int: raise RuntimeError("--resource-group-id does not match the active subscription and derived instance name") group_info = _expect_json_object( - run_az_json( - args=["group", "show", "--name", rg_name, "--query", "{id:id,name:name,tags:tags}"] - ), + run_az_json(args=["group", "show", "--name", rg_name, "--query", "{id:id,name:name,tags:tags}"]), context="resource group", ) group_id = _expect_string(group_info.get("id"), context="resource group ID") @@ -250,10 +248,7 @@ def main(args: list[str] | None = None) -> int: ), context="Entra application", ) - if ( - entra_app.get("appId") != parsed.entra_app_id - or entra_app.get("displayName") != entra_app_name - ): + if entra_app.get("appId") != parsed.entra_app_id or entra_app.get("displayName") != entra_app_name: raise RuntimeError("--entra-app-id does not identify the expected instance application") egress_ip_value = run_az_json( diff --git a/tests/unit/infra/__init__.py b/tests/unit/infra/__init__.py index 0eca6426d9..9a0454564d 100644 --- a/tests/unit/infra/__init__.py +++ b/tests/unit/infra/__init__.py @@ -1,2 +1,2 @@ # Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. \ No newline at end of file +# Licensed under the MIT license. diff --git a/tests/unit/infra/test_bicep_topology.py b/tests/unit/infra/test_bicep_topology.py index 824ccf6edc..324ba38b42 100644 --- a/tests/unit/infra/test_bicep_topology.py +++ b/tests/unit/infra/test_bicep_topology.py @@ -91,9 +91,10 @@ def test_main_has_one_public_nat_topology(self): environment_properties = environment["properties"] assert environment_properties["publicNetworkAccess"] == "Enabled" assert environment_properties["vnetConfiguration"]["internal"] is False - assert "outputs.infrastructureSubnetId.value" in environment_properties["vnetConfiguration"][ - "infrastructureSubnetId" - ] + assert ( + "outputs.infrastructureSubnetId.value" + in environment_properties["vnetConfiguration"]["infrastructureSubnetId"] + ) container_app = _resources(template, "Microsoft.App/containerApps")[0] assert container_app["properties"]["configuration"]["registries"][0]["identity"] == ( @@ -177,4 +178,4 @@ def test_front_door_uses_https_health_probe_without_caching(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/unit/infra/test_instance_lifecycle.py b/tests/unit/infra/test_instance_lifecycle.py index 6e74e549dd..dd34a56469 100644 --- a/tests/unit/infra/test_instance_lifecycle.py +++ b/tests/unit/infra/test_instance_lifecycle.py @@ -8,9 +8,12 @@ import tempfile import unittest from pathlib import Path -from types import ModuleType +from typing import TYPE_CHECKING from unittest.mock import patch +if TYPE_CHECKING: + from types import ModuleType + REPO_ROOT = Path(__file__).resolve().parents[3] @@ -213,4 +216,4 @@ def test_verified_instance_removes_roles_and_waits_for_group_deletion(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/unit/infra/test_pipeline_guardrails.py b/tests/unit/infra/test_pipeline_guardrails.py index 4fb9a18158..22be7553df 100644 --- a/tests/unit/infra/test_pipeline_guardrails.py +++ b/tests/unit/infra/test_pipeline_guardrails.py @@ -47,7 +47,9 @@ def test_production_remains_opt_in_and_independently_approved(self): assert "onTimeout: reject" in self.pipeline assert "approvers: '$(prodApprovers)'" in self.pipeline assert "allowApproversToApproveTheirOwnRuns: false" in self.pipeline - approval_stage = self.pipeline[self.pipeline.index("stage: ApproveProd") : self.pipeline.index("stage: DeployProd")] + approval_stage = self.pipeline[ + self.pipeline.index("stage: ApproveProd") : self.pipeline.index("stage: DeployProd") + ] assert "- group: copyrit-gui-prod" in approval_stage assert '"$BUILD_SOURCEBRANCH" != refs/heads/main' in self.pipeline assert "eq(variables['Build.SourceBranch'], 'refs/heads/main')" in self.pipeline @@ -59,7 +61,7 @@ def test_deploy_resolves_digest_and_previews_before_apply(self): assert "variable=immutableImage;isOutput=true" in self.pipeline assert "stageDependencies.Build.BuildAndPush.outputs['BuildImage.immutableImage']" in self.pipeline assert "PYRIT_CONTAINER_IMAGE: $(immutableImage)" in self.pipeline - assert '@(sha256:[0-9a-fA-F]{64})' in self.deploy_script + assert "@(sha256:[0-9a-fA-F]{64})" in self.deploy_script assert 'immutable_image="$registry_server/$repository@$digest"' in self.deploy_script assert '"containerImage=$immutable_image"' in self.deploy_script assert "az acr repository show" not in self.deploy_script @@ -89,7 +91,9 @@ def test_deploy_validates_structured_inputs_before_arm(self): assert "uuid.UUID" in self.deploy_script assert "Microsoft\\.KeyVault/vaults" in self.deploy_script assert "database\\.windows\\.net" in self.deploy_script - assert self.deploy_script.index("ipaddress.ip_network") < self.deploy_script.index("az deployment group what-if") + assert self.deploy_script.index("ipaddress.ip_network") < self.deploy_script.index( + "az deployment group what-if" + ) def test_deploy_preserves_existing_network_and_tags(self): assert "Front Door cannot use an ACA client CIDR restriction" in self.deploy_script @@ -180,12 +184,14 @@ def test_what_if_validator_accepts_read_only_normalization_and_lock_create(self) assert result.returncode == 0, result.stderr def test_what_if_validator_rejects_each_protected_topology_violation(self): + other_resource_group_id = f"/subscriptions/{SUBSCRIPTION_ID}/resourceGroups/other" + workspace_provider_id = f"{RESOURCE_GROUP_ID}/providers/Microsoft.OperationalInsights" fixtures: dict[str, tuple[dict[str, object], str]] = { "delete": ({"changeType": "Delete", "resourceId": PIP_ID}, "delete"), "cross-resource-group": ( { "changeType": "Modify", - "resourceId": f"/subscriptions/{SUBSCRIPTION_ID}/resourceGroups/other/providers/Microsoft.App/containerApps/app", + "resourceId": f"{other_resource_group_id}/providers/Microsoft.App/containerApps/app", "delta": [{"path": "properties.configuration"}], }, "cross-resource-group write", @@ -209,7 +215,7 @@ def test_what_if_validator_rejects_each_protected_topology_violation(self): "workspace-create": ( { "changeType": "Create", - "resourceId": f"{RESOURCE_GROUP_ID}/providers/Microsoft.OperationalInsights/workspaces/copyrit-prod-v2-logs", + "resourceId": f"{workspace_provider_id}/workspaces/copyrit-prod-v2-logs", }, "core resource create", ), @@ -235,4 +241,4 @@ def test_what_if_validator_normalizes_expected_protected_resource_ids(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() From 6816d793de364ea166c950fc0598cc487dd81cbc Mon Sep 17 00:00:00 2001 From: Bashir Partovi Date: Thu, 20 Aug 2026 01:07:55 -0400 Subject: [PATCH 3/7] FIX: Normalize infrastructure file endings Add the trailing newlines required by the cross-platform end-of-file pre-commit hook. --- gui-deploy.yml | 2 +- infra/modules/aca_front_door.bicep | 2 +- infra/pipelines/deploy_public_nat.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gui-deploy.yml b/gui-deploy.yml index fabe54c455..03aafde102 100644 --- a/gui-deploy.yml +++ b/gui-deploy.yml @@ -246,4 +246,4 @@ stages: azureSubscription: '$(azureServiceConnection)' scriptType: 'bash' scriptLocation: 'scriptPath' - scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh' \ No newline at end of file + scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh' diff --git a/infra/modules/aca_front_door.bicep b/infra/modules/aca_front_door.bicep index 335dce8ea9..c11b16904f 100644 --- a/infra/modules/aca_front_door.bicep +++ b/infra/modules/aca_front_door.bicep @@ -90,4 +90,4 @@ resource route 'Microsoft.Cdn/profiles/afdEndpoints/routes@2024-09-01' = { output endpointHostName string = endpoint.properties.hostName output endpointId string = endpoint.id -output profileId string = profile.id \ No newline at end of file +output profileId string = profile.id diff --git a/infra/pipelines/deploy_public_nat.sh b/infra/pipelines/deploy_public_nat.sh index e8ae5ed8af..23c320d822 100644 --- a/infra/pipelines/deploy_public_nat.sh +++ b/infra/pipelines/deploy_public_nat.sh @@ -324,4 +324,4 @@ if [[ "$front_door_health" != "200" ]]; then echo "##vso[task.logissue type=error]Front Door did not route a healthy response" exit 1 fi -echo "Deployment healthy; public URL: https://$front_door_fqdn; ACA origin: https://$app_fqdn; egress IPv4: $egress_ip" \ No newline at end of file +echo "Deployment healthy; public URL: https://$front_door_fqdn; ACA origin: https://$app_fqdn; egress IPv4: $egress_ip" From 6340cd665bf1aa553b0d7f5ae290a25279494cb3 Mon Sep 17 00:00:00 2001 From: Bashir Partovi Date: Thu, 20 Aug 2026 17:10:24 -0400 Subject: [PATCH 4/7] FEAT: Isolate CoPyRIT AFD origins with Private Link Add opt-in AFD Premium Private Link for ACA, disable direct ACA public access in the team deployment path, and automate connection approval, validation, and ordered rollback. Preserve community defaults, protect the managed environment in what-if validation, and document the cutover lifecycle. --- infra/README.md | 48 +++-- infra/main.bicep | 44 ++++- infra/modules/aca_front_door.bicep | 31 ++- infra/parameters.demo.json | 6 + infra/parameters.example.json | 6 + infra/pipelines/deploy_public_nat.sh | 192 ++++++++++++++++++- infra/pipelines/validate_what_if.py | 7 + tests/unit/infra/test_bicep_topology.py | 34 +++- tests/unit/infra/test_pipeline_guardrails.py | 50 ++++- 9 files changed, 375 insertions(+), 43 deletions(-) diff --git a/infra/README.md b/infra/README.md index cd4ee19fe0..528747542e 100644 --- a/infra/README.md +++ b/infra/README.md @@ -13,7 +13,7 @@ flowchart TB subgraph azure["Azure subscription"] frontDoor["Azure Front Door Premium
Optional managed HTTPS entry point"] - ingress["ACA-managed public HTTPS ingress
Optional allowedCidr restriction"] + ingress["ACA-managed HTTPS ingress
Public endpoint optionally disabled"] subgraph vnet["Virtual network"] subgraph subnet["Delegated ACA infrastructure subnet"] @@ -36,8 +36,8 @@ flowchart TB end user -->|"HTTPS when Front Door enabled"| frontDoor - frontDoor -->|"HTTPS origin"| ingress - user -->|"Direct ACA public URL"| ingress + frontDoor -->|"HTTPS public origin or Private Link"| ingress + user -.->|"Direct ACA URL when public access is enabled"| ingress ingress --> environment user -->|"MSAL PKCE sign-in"| entra entra -->|"Delegated Graph token"| user @@ -64,7 +64,7 @@ flowchart TB app -.->|"Traces after agent setup"| appInsights ``` -The base topology is public ACA-managed HTTPS ingress plus VNet-integrated fixed NAT egress. `enableFrontDoor=true` adds Front Door Premium as the preferred managed HTTPS URL while the ACA origin remains concurrently public. Requests can therefore bypass Front Door through the ACA hostname; Front Door is a routing and reliability layer, not the exclusive ingress security boundary. The Microsoft team ADO workflow enables Front Door; community deployments leave it disabled by default. Front Door mode requires `allowedCidr` to be empty; Bicep rejects that combination instead of silently dropping a requested client restriction. Front Door changes inbound routing only: outbound connections from ACA continue to use the NAT Gateway's static IPv4. +The base topology is public ACA-managed HTTPS ingress plus VNet-integrated fixed NAT egress. `enableFrontDoor=true` adds Front Door Premium as the preferred managed HTTPS URL. By default, the ACA origin remains concurrently public and can bypass Front Door. `enableFrontDoorPrivateLink=true` instead connects Premium Front Door to the ACA environment through Private Link; setting `disableContainerAppsPublicAccess=true` then removes the direct public ACA path. Bicep rejects public-access shutdown unless both Front Door and its Private Link origin are enabled. The team ADO workflow uses this isolated-origin mode; community examples leave all three Front Door settings disabled. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door rather than the original client. Front Door changes inbound routing only: outbound connections from ACA continue to use the NAT Gateway's static IPv4. ## Development Workflow @@ -97,9 +97,9 @@ Community users can deploy `main.bicep` directly using the instructions below. F - **Authentication**: [MSAL](https://learn.microsoft.com/en-us/entra/msal/) [PKCE](https://oauth.net/2/pkce/) on the frontend (`@azure/msal-browser`) + Microsoft Graph-backed middleware on the backend. The frontend sends a delegated Graph token, and the backend authenticates it through Graph `/me`. PKCE (public client) requires no client secrets or certificates. - **Authorization**: Entra group check via `allowedGroupObjectIds` param. Requires delegated Graph `User.Read`; the backend calls `/me/checkMemberGroups` and compares the returned transitive memberships with the configured group IDs. Each security group must also be assigned to the enterprise app (see Prerequisites §3). Authenticated deployments require at least one allowed group and fail to start without one. `/api/health`, `/api/auth/config`, and `/api/media` are intentional public exceptions; other `/api` routes require authentication when auth is enabled. Successful identity and membership results are cached in-process for 60 seconds, keyed by a SHA-256 token digest, to reduce Graph latency and throttling. Bearer tokens themselves are not stored in the cache. - **Identity**: `deploy_instance.py` creates its user-assigned managed identity (UAMI) and grants AcrPull and Storage Blob Data Contributor before deploying Bicep. A direct Bicep deployment can create `-identity`, but the template creates no role assignments, so its first revision can remain unhealthy until required roles are granted and the revision is restarted. A healthy one-pass direct deployment uses an existing, pre-authorized UAMI. `AZURE_CLIENT_ID` is set to the UAMI's client ID so `DefaultAzureCredential` selects the correct identity. -- **Network**: The template always creates a VNet-integrated public Container Apps environment, one delegated ACA infrastructure subnet, a Standard NAT Gateway, and a static outbound IPv4. ACA supplies the generated HTTPS hostname and trusted certificate. In direct-ACA mode, `allowedCidr` optionally restricts public ingress to one IPv4 CIDR; an empty value permits public ingress. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door backend addresses, not the original client; Bicep and the internal pipeline reject the invalid combination. Entra sign-in, enterprise-app assignment, and backend group checks remain mandatory application access controls. -- **Front Door**: `enableFrontDoor=true` creates a Premium profile, managed `azurefd.net` endpoint, HTTPS ACA origin, `/api/health` probe, and uncached catch-all route. The module does not create a WAF policy or isolate the public ACA origin, so application authentication and authorization remain mandatory on both hostnames. -- **Routing**: Public inbound requests reach ACA either directly or through Front Door and do not traverse the NAT Gateway. Outbound connections from the ACA environment that leave the virtual network use the NAT Gateway's static public IPv4. +- **Network**: The template always creates a VNet-integrated external Container Apps environment, one delegated ACA infrastructure subnet, a Standard NAT Gateway, and a static outbound IPv4. ACA supplies the generated HTTPS hostname and trusted certificate. In direct-ACA mode, `allowedCidr` optionally restricts public ingress to one IPv4 CIDR; an empty value permits public ingress. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door backend addresses, not the original client; Bicep and the team pipeline reject the invalid combination. Entra sign-in, enterprise-app assignment, and backend group checks remain mandatory application access controls. +- **Front Door**: `enableFrontDoor=true` creates a Premium profile, managed `azurefd.net` endpoint, HTTPS ACA origin, `/api/health` probe, and uncached catch-all route. `enableFrontDoorPrivateLink=true` targets the ACA managed environment with group ID `managedEnvironments`. The resulting private endpoint connection must be approved before AFD can route privately. `disableContainerAppsPublicAccess=true` disables the ACA environment public endpoint and CORS then permits only the AFD origin. The module does not create a WAF policy; application authentication and authorization remain mandatory. +- **Routing**: Inbound requests through Front Door do not traverse the NAT Gateway. When ACA public access remains enabled, users can also reach ACA directly. When Private Link is enabled and public access is disabled, all public application traffic enters through Front Door. Outbound connections from the ACA environment that leave the virtual network use the NAT Gateway's static public IPv4. - **Response headers**: `SecurityHeadersMiddleware` adds [CSP](https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP), HTTP Strict Transport Security (HSTS, production only), X-Frame-Options, X-Content-Type-Options, Referrer-Policy, Permissions-Policy, and Cache-Control (`no-store` on API routes). Swagger/OpenAPI disabled in production. - **Data**: Azure SQL with managed identity authentication (no passwords) - **Secrets**: When `envFileContents` is nonempty, Bicep stores it as an inline ACA secret. Otherwise, Bicep creates a versionless Key Vault reference to `envSecretName` using the app UAMI; that path requires `Key Vault Secrets User` and network access to the vault. `deploy_instance.py` uses the inline path. @@ -114,7 +114,7 @@ Community users can deploy `main.bicep` directly using the instructions below. F The Bicep template creates the Container Apps resources, dedicated network, NAT Gateway, static egress IP, and (unless supplied) Log Analytics workspace. It can also declare an ACR and UAMI, but it does not push an image or create RBAC role assignments. The supported one-pass workflows therefore use an existing ACR; a healthy one-pass direct Bicep deployment also uses an existing, pre-authorized UAMI. Entra resources must be created separately through Microsoft Graph. Bicep requires an existing Key Vault. See [Post-Deployment §2](#post-deployment) for direct-deployment RBAC. -Front Door is optional. When enabled, the subscription must have the `Microsoft.Cdn` resource provider registered. +Front Door is optional. When enabled, the subscription must have the `Microsoft.Cdn` resource provider registered. Private Link requires Front Door Premium and a workload-profiles ACA environment in a [supported Private Link region](https://learn.microsoft.com/azure/frontdoor/private-link#region-availability). The deployment principal also needs permission to read AFD origins and approve `Microsoft.App/managedEnvironments/privateEndpointConnections`. Azure requires ACA public network access to be disabled before private endpoints can be enabled, so converting an existing public origin has an unavoidable interval while the request is pending and AFD propagates the approval. The team workflow performs this cutover in a maintenance window and redeploys the prior public-origin configuration if post-cutover validation fails. > **Migration boundary:** This template owns a dedicated VNet and creates a VNet-integrated workload-profiles environment. ACA environment network type is creation-time configuration. Do not apply this template in place to a legacy environment created without this VNet or with the former private-endpoint parameters. Deploy a parallel resource group/app/environment, validate it, and then migrate users and redirect URIs. @@ -162,18 +162,21 @@ az ad sp create --id "$APP_ID" --output none az account show --query tenantId -o tsv ``` -> **Fresh app registrations only**: The ACA and optional Front Door hostnames are known after deployment. For a newly created app with no existing SPA redirects, register both the direct ACA URL and selected public URL (they are identical when Front Door is disabled): +> **Fresh app registrations only**: The ACA and optional Front Door hostnames are known after deployment. For a newly created app with no existing SPA redirects, register the selected public URL and add the ACA URL only while ACA public access is enabled: > > ```bash > ACA_FQDN=$(az deployment group show -g -n \ > --query properties.outputs.appFqdn.value -o tsv) > PUBLIC_FQDN=$(az deployment group show -g -n \ > --query properties.outputs.publicFqdn.value -o tsv) +> ACA_PUBLIC_ACCESS=$(az deployment group show -g -n \ +> --query properties.outputs.containerAppsPublicNetworkAccess.value -o tsv) > APP_OBJECT_ID=$(az ad app show --id "$APP_ID" --query id -o tsv) > REDIRECT_URIS=$(jq -cn \ > --arg aca "https://$ACA_FQDN" \ > --arg public "https://$PUBLIC_FQDN" \ -> '[$aca, $public] | unique') +> --arg publicAccess "$ACA_PUBLIC_ACCESS" \ +> '(if $publicAccess == "Disabled" then [$public] else [$aca, $public] end) | unique') > PATCH_BODY=$(jq -cn --argjson uris "$REDIRECT_URIS" \ > '{spa:{redirectUris:$uris}}') > az rest --method PATCH \ @@ -354,7 +357,9 @@ Use deployment outputs rather than reconstructing public hostnames: | --- | --- | | `publicFqdn` | User-facing hostname: Front Door when enabled, otherwise ACA | | `frontDoorFqdn`, `frontDoorUrl` | Managed Front Door hostname/URL; empty when disabled | -| `appFqdn` | Direct public ACA origin for diagnostics and rollback | +| `appFqdn` | Generated ACA hostname; inaccessible when ACA public access is disabled | +| `containerAppsPublicNetworkAccess` | Effective ACA environment public-access state | +| `frontDoorPrivateLinkRequestMessage` | Deterministic Private Link approval request message; empty when disabled | | `egressPublicIpAddress` | Static outbound NAT IPv4 for provider allowlists | | `natGatewayId`, `acaInfrastructureSubnetId`, `vnetName` | Created network resources | | `managedIdentityPrincipalId`, `managedIdentityResourceId` | UAMI identifiers for RBAC and SQL setup | @@ -376,9 +381,10 @@ az deployment group show -g -n \ 2. Capture the exact pushed digest and pass it across stages. 3. Require the existing app, environment, VNet, subnet, NAT, and reserved PIP; validate their IDs, prefixes, tags, SKU, allocation, and attachments. 4. Run a full ARM `what-if` through a fail-closed validator; reject malformed results, deletions, cross-resource-group writes, protected-network deltas other than the documented read-only NAT/PIP normalization, and core network, app, or Log Analytics workspace creates. The expected PIP protection lock may be created. -5. Preserve policy-managed PIP tags and deploy with `enableFrontDoor=true` and `protectEgressPublicIp=true`. -6. Verify the digest-pinned ACA revision, Front Door `/api/health`, and the same PIP resource ID/address after deployment. -7. Print the Front Door URL, direct ACA origin, and static egress IPv4. +5. Preserve policy-managed PIP tags and deploy with Front Door Private Link, ACA public access disabled, and PIP protection enabled. +6. Validate the AFD origin targets the expected ACA environment, approve only active requests with the deterministic message, and require the ACA-side connection to report `Approved`. AFD can continue to display `Pending` after approval, so successful AFD health is the data-plane readiness signal. +7. Verify ACA public access is disabled, the digest-pinned revision and Front Door `/api/health` are healthy, direct ACA access is unavailable, and the PIP resource ID/address is unchanged. +8. If cutover validation fails, redeploy the prior public AFD origin and re-enable ACA public access; otherwise print the Front Door URL and static egress IPv4. Qualifying merges to `main` automatically deploy test. Production deployment is independent of PyRIT package releases: manually queue a commit merged to `main` with `deployToProd=true`. The workflow deploys test first, then requires a timeout-rejecting manual approval whose requester cannot self-approve. @@ -415,7 +421,7 @@ The resource group, registry, image-pull authorization, managed identity, Key Va The internal workflow is update-only for networking: its app name and prefixes must resolve to the existing app/environment/VNet/subnet/NAT/PIP. It records the current PIP resource ID and address before preview, requires protected resources to remain unchanged except Azure read-only normalization, and verifies the same PIP/address after deployment. -The workflow also creates a `CanNotDelete` lock scoped to the reserved PIP. The validated Front Door origin is the public ACA hostname; direct ACA access remains available and can bypass Front Door. +The workflow also creates a `CanNotDelete` lock scoped to the reserved PIP. Its validated Front Door origin uses Private Link to the ACA environment, and the ACA public endpoint is disabled after deployment. ## Post-Deployment @@ -426,6 +432,8 @@ The workflow also creates a `CanNotDelete` lock scoped to the reserved PIP. The --query properties.outputs.appFqdn.value -o tsv) PUBLIC_FQDN=$(az deployment group show -g -n \ --query properties.outputs.publicFqdn.value -o tsv) + ACA_PUBLIC_ACCESS=$(az deployment group show -g -n \ + --query properties.outputs.containerAppsPublicNetworkAccess.value -o tsv) APP_OBJECT_ID=$(az ad app show --id --query id -o tsv) CURRENT_URIS=$(az rest --method GET \ --uri "https://graph.microsoft.com/v1.0/applications/$APP_OBJECT_ID?\$select=spa" \ @@ -434,7 +442,8 @@ The workflow also creates a `CanNotDelete` lock scoped to the reserved PIP. The --argjson existing "$CURRENT_URIS" \ --arg aca "https://$ACA_FQDN" \ --arg public "https://$PUBLIC_FQDN" \ - '($existing // []) + [$aca, $public] | unique') + --arg publicAccess "$ACA_PUBLIC_ACCESS" \ + '($existing // []) + (if $publicAccess == "Disabled" then [$public] else [$aca, $public] end) | unique') PATCH_BODY=$(jq -cn --argjson uris "$UPDATED_URIS" '{spa:{redirectUris:$uris}}') az rest --method PATCH \ --uri "https://graph.microsoft.com/v1.0/applications/$APP_OBJECT_ID" \ @@ -498,13 +507,10 @@ The workflow also creates a `CanNotDelete` lock scoped to the reserved PIP. The ```bash PUBLIC_FQDN=$(az deployment group show -g -n \ --query properties.outputs.publicFqdn.value -o tsv) -ACA_FQDN=$(az deployment group show -g -n \ - --query properties.outputs.appFqdn.value -o tsv) echo "Public URL: https://$PUBLIC_FQDN" -echo "Direct ACA origin: https://$ACA_FQDN" ``` -Open the public URL and verify unauthenticated users are redirected to Entra and only assigned users in an allowed backend group can complete access. When Front Door is enabled, the direct ACA origin remains publicly reachable and bypasses Front Door; retain it only as an intentional diagnostic/rollback path. +Open the public URL and verify unauthenticated users are redirected to Entra and only assigned users in an allowed backend group can complete access. When `containerAppsPublicNetworkAccess` is `Disabled`, verify the generated ACA hostname is no longer publicly reachable. ## Configuration: .pyrit_conf and .env @@ -568,7 +574,7 @@ Supported Azure integrations, including OpenAI, Content Safety, and Speech, can ## Notes -- **Network topology**: Public ACA-managed HTTPS ingress with optional `allowedCidr` plus VNet-integrated fixed NAT egress is the base topology. Front Door Premium is an optional inbound layer and is enabled by the internal ADO workflow. `allowedCidr` must be empty when Front Door is enabled; Bicep rejects the combination. +- **Network topology**: Public ACA-managed HTTPS ingress with optional `allowedCidr` plus VNet-integrated fixed NAT egress is the base topology. Front Door Premium is an optional inbound layer. Private Link plus disabled ACA public access makes Front Door the only public application path. The team ADO workflow enables this isolated-origin mode. `allowedCidr` must be empty when Front Door is enabled; Bicep rejects the combination. - **Ingress vs. egress**: Front Door affects inbound requests only. The reserved NAT public IP remains the source for ACA-originated outbound connections. - **NAT routing**: NAT Gateway supplies the outbound source IP only while the subnet's effective default route remains `Internet`. A UDR or propagated BGP `0.0.0.0/0` route to a firewall or gateway takes precedence; in that topology, allow-list the egress device's public IP instead. - **Network outputs**: `egressPublicIpAddress`, `natGatewayId`, `acaInfrastructureSubnetId`, and `vnetName` describe the created network. diff --git a/infra/main.bicep b/infra/main.bicep index fa22bf559d..1434b371ba 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -148,10 +148,22 @@ param enableOtel bool = false @description('Create Azure Front Door Premium as the public application endpoint') param enableFrontDoor bool = false +@description('Connect Azure Front Door Premium to the ACA environment through Private Link') +param enableFrontDoorPrivateLink bool = false + +@description('Disable the ACA environment public endpoint after Front Door Private Link is configured') +param disableContainerAppsPublicAccess bool = false + // Determine whether to create or reference existing resources var effectiveAllowedCidr = enableFrontDoor && !empty(allowedCidr) ? fail('allowedCidr must be empty when enableFrontDoor is true') : allowedCidr +var effectiveFrontDoorPrivateLink = enableFrontDoorPrivateLink && !enableFrontDoor + ? fail('enableFrontDoor must be true when enableFrontDoorPrivateLink is true') + : enableFrontDoorPrivateLink +var effectiveContainerAppsPublicAccess = disableContainerAppsPublicAccess + ? (effectiveFrontDoorPrivateLink ? 'Disabled' : fail('Front Door Private Link is required before ACA public access can be disabled')) + : 'Enabled' var createLogAnalytics = logAnalyticsWorkspaceId == '' var createAcr = acrResourceId == '' && acrName == '' var useInlineEnvFile = !empty(envFileContents) @@ -291,10 +303,21 @@ resource acaEnvironment 'Microsoft.App/managedEnvironments@2024-10-02-preview' = destination: 'log-analytics' logAnalyticsConfiguration: { customerId: effectiveLogAnalyticsCustomerIdValue + dynamicJsonColumns: false sharedKey: effectiveLogAnalyticsKeyValue } } - publicNetworkAccess: 'Enabled' + peerAuthentication: { + mtls: { + enabled: false + } + } + peerTrafficConfiguration: { + encryption: { + enabled: false + } + } + publicNetworkAccess: effectiveContainerAppsPublicAccess workloadProfiles: [ { name: 'Consumption' @@ -316,6 +339,9 @@ module acaFrontDoor './modules/aca_front_door.bicep' = if (enableFrontDoor) { namePrefix: appName originHostName: acaOriginHostName tags: tags + enablePrivateLink: effectiveFrontDoorPrivateLink + originResourceId: acaEnvironment.id + originLocation: location } } @@ -448,11 +474,13 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { name: 'AZURE_CLIENT_ID' value: effectiveManagedIdentityClientId } - // Permit both the rollback ACA URL and the Front Door cutover URL. + // The ACA URL is usable only while environment public access remains enabled. { name: 'PYRIT_CORS_ORIGINS' value: enableFrontDoor - ? 'https://${acaOriginHostName},https://${acaFrontDoor!.outputs.endpointHostName}' + ? (effectiveContainerAppsPublicAccess == 'Disabled' + ? 'https://${acaFrontDoor!.outputs.endpointHostName}' + : 'https://${acaOriginHostName},https://${acaFrontDoor!.outputs.endpointHostName}') : 'https://${acaOriginHostName}' } ] @@ -481,7 +509,7 @@ resource containerApp 'Microsoft.App/containerApps@2024-03-01' = { // Outputs // ============================================================================ -@description('The FQDN of the deployed Container App') +@description('The generated ACA FQDN; inaccessible when ACA public network access is disabled') output appFqdn string = containerApp.properties.configuration.ingress.fqdn @description('The Azure Front Door managed HTTPS hostname') @@ -490,6 +518,14 @@ output frontDoorFqdn string = enableFrontDoor ? acaFrontDoor!.outputs.endpointHo @description('The Azure Front Door public URL') output frontDoorUrl string = enableFrontDoor ? 'https://${acaFrontDoor!.outputs.endpointHostName}' : '' +@description('The deterministic ACA Private Link approval request message; empty when Private Link is disabled') +output frontDoorPrivateLinkRequestMessage string = effectiveFrontDoorPrivateLink + ? acaFrontDoor!.outputs.privateLinkRequestMessage + : '' + +@description('ACA environment public network access state') +output containerAppsPublicNetworkAccess string = effectiveContainerAppsPublicAccess + @description('The public application FQDN selected for this deployment') output publicFqdn string = enableFrontDoor ? acaFrontDoor!.outputs.endpointHostName : containerApp.properties.configuration.ingress.fqdn diff --git a/infra/modules/aca_front_door.bicep b/infra/modules/aca_front_door.bicep index c11b16904f..418557a9d1 100644 --- a/infra/modules/aca_front_door.bicep +++ b/infra/modules/aca_front_door.bicep @@ -7,7 +7,23 @@ param originHostName string @description('Resource tags applied to the Front Door profile') param tags object +@description('Connect Front Door to the ACA environment through Private Link') +param enablePrivateLink bool = false + +@description('ACA managed environment resource ID used by Front Door Private Link') +param originResourceId string = '' + +@description('ACA managed environment location used by Front Door Private Link') +param originLocation string = '' + var endpointSuffix = take(uniqueString(subscription().id, resourceGroup().id, namePrefix), 8) +var privateLinkRequestMessage = 'Azure Front Door private access to ${namePrefix}' +var effectiveOriginResourceId = enablePrivateLink && empty(originResourceId) + ? fail('originResourceId is required when enablePrivateLink is true') + : originResourceId +var effectiveOriginLocation = enablePrivateLink && empty(originLocation) + ? fail('originLocation is required when enablePrivateLink is true') + : originLocation resource profile 'Microsoft.Cdn/profiles@2024-09-01' = { name: '${namePrefix}-afd' @@ -52,7 +68,7 @@ resource originGroup 'Microsoft.Cdn/profiles/originGroups@2024-09-01' = { resource origin 'Microsoft.Cdn/profiles/originGroups/origins@2024-09-01' = { parent: originGroup name: '${namePrefix}-aca-origin' - properties: { + properties: union({ enabledState: 'Enabled' enforceCertificateNameCheck: true hostName: originHostName @@ -61,7 +77,17 @@ resource origin 'Microsoft.Cdn/profiles/originGroups/origins@2024-09-01' = { originHostHeader: originHostName priority: 1 weight: 1000 - } + }, enablePrivateLink ? { + sharedPrivateLinkResource: { + groupId: 'managedEnvironments' + privateLink: { + id: effectiveOriginResourceId + } + privateLinkLocation: effectiveOriginLocation + requestMessage: privateLinkRequestMessage + status: 'Pending' + } + } : {}) } resource route 'Microsoft.Cdn/profiles/afdEndpoints/routes@2024-09-01' = { @@ -91,3 +117,4 @@ resource route 'Microsoft.Cdn/profiles/afdEndpoints/routes@2024-09-01' = { output endpointHostName string = endpoint.properties.hostName output endpointId string = endpoint.id output profileId string = profile.id +output privateLinkRequestMessage string = enablePrivateLink ? privateLinkRequestMessage : '' diff --git a/infra/parameters.demo.json b/infra/parameters.demo.json index f7f7867b25..43439063b5 100644 --- a/infra/parameters.demo.json +++ b/infra/parameters.demo.json @@ -43,6 +43,12 @@ "enableFrontDoor": { "value": false }, + "enableFrontDoorPrivateLink": { + "value": false + }, + "disableContainerAppsPublicAccess": { + "value": false + }, "vnetAddressPrefix": { "value": "10.0.0.0/16" }, diff --git a/infra/parameters.example.json b/infra/parameters.example.json index c6869dd97e..53ee9af036 100644 --- a/infra/parameters.example.json +++ b/infra/parameters.example.json @@ -45,6 +45,12 @@ "enableFrontDoor": { "value": false }, + "enableFrontDoorPrivateLink": { + "value": false + }, + "disableContainerAppsPublicAccess": { + "value": false + }, "vnetAddressPrefix": { "value": "10.0.0.0/16" }, diff --git a/infra/pipelines/deploy_public_nat.sh b/infra/pipelines/deploy_public_nat.sh index 23c320d822..bd0cf9baab 100644 --- a/infra/pipelines/deploy_public_nat.sh +++ b/infra/pipelines/deploy_public_nat.sh @@ -98,7 +98,8 @@ then fi guid_pattern='[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}' -if [[ ! "$PYRIT_ACR_RESOURCE_ID" =~ ^/subscriptions/($guid_pattern)/resourceGroups/[^/]+/providers/Microsoft\.ContainerRegistry/registries/([a-z0-9]{5,50})$ ]]; then +normalized_acr_resource_id=${PYRIT_ACR_RESOURCE_ID,,} +if [[ ! "$normalized_acr_resource_id" =~ ^/subscriptions/($guid_pattern)/resourcegroups/[^/]+/providers/microsoft\.containerregistry/registries/([a-z0-9]{5,50})$ ]]; then echo "##vso[task.logissue type=error]PYRIT_ACR_RESOURCE_ID is not canonical" exit 1 fi @@ -109,13 +110,15 @@ if [[ "$(az account show --query id -o tsv | tr '[:upper:]' '[:lower:]')" != "$e exit 1 fi -if [[ ! "$PYRIT_MANAGED_IDENTITY_RESOURCE_ID" =~ ^/subscriptions/($guid_pattern)/resourceGroups/[^/]+/providers/Microsoft\.ManagedIdentity/userAssignedIdentities/[a-zA-Z0-9_-]{3,128}$ ]] \ +normalized_managed_identity_resource_id=${PYRIT_MANAGED_IDENTITY_RESOURCE_ID,,} +if [[ ! "$normalized_managed_identity_resource_id" =~ ^/subscriptions/($guid_pattern)/resourcegroups/[^/]+/providers/microsoft\.managedidentity/userassignedidentities/[a-z0-9_-]{3,128}$ ]] \ || [[ "${BASH_REMATCH[1],,}" != "$expected_subscription" ]]; then echo "##vso[task.logissue type=error]Managed identity resource ID is not canonical or is in another subscription" exit 1 fi -if [[ ! "$PYRIT_KEY_VAULT_RESOURCE_ID" =~ ^/subscriptions/($guid_pattern)/resourceGroups/[^/]+/providers/Microsoft\.KeyVault/vaults/[a-zA-Z0-9-]{3,24}$ ]] \ +normalized_key_vault_resource_id=${PYRIT_KEY_VAULT_RESOURCE_ID,,} +if [[ ! "$normalized_key_vault_resource_id" =~ ^/subscriptions/($guid_pattern)/resourcegroups/[^/]+/providers/microsoft\.keyvault/vaults/[a-z0-9-]{3,24}$ ]] \ || [[ "${BASH_REMATCH[1],,}" != "$expected_subscription" ]]; then echo "##vso[task.logissue type=error]Key Vault resource ID is not canonical or is in another subscription" exit 1 @@ -153,6 +156,10 @@ existing_app=$(az containerapp show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME" \ --query '{id:id,environmentId:properties.managedEnvironmentId,tags:tags}' -o json 2>/dev/null || true) +existing_environment=$(az containerapp env show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-env" \ + --query '{id:id,publicNetworkAccess:properties.publicNetworkAccess}' -o json 2>/dev/null || true) existing_vnet=$(az network vnet show \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --name "$PYRIT_APP_NAME-vnet" \ @@ -171,7 +178,7 @@ existing_pip=$(az network public-ip show \ --name "$PYRIT_APP_NAME-egress-pip" \ --query '{id:id,ip:ipAddress,allocation:publicIPAllocationMethod,sku:sku.name,tags:tags}' -o json 2>/dev/null || true) -if [[ -z "$existing_app" || -z "$existing_vnet" || -z "$existing_subnet" \ +if [[ -z "$existing_app" || -z "$existing_environment" || -z "$existing_vnet" || -z "$existing_subnet" \ || -z "$existing_nat" || -z "$existing_pip" ]]; then echo "##vso[task.logissue type=error]Internal deployments must adopt an existing app, environment, VNet, subnet, NAT, and egress PIP" exit 1 @@ -188,6 +195,8 @@ expected_egress_ip=$(jq -r '.ip // empty' <<< "$existing_pip") if [[ "$(jq -r '.id | ascii_downcase' <<< "$existing_app")" != "${expected_app_id,,}" \ || "$(jq -r '.environmentId | ascii_downcase' <<< "$existing_app")" != "${expected_environment_id,,}" \ + || "$(jq -r '.id | ascii_downcase' <<< "$existing_environment")" != "${expected_environment_id,,}" \ + || ! "$(jq -r '.publicNetworkAccess' <<< "$existing_environment")" =~ ^(Enabled|Disabled)$ \ || "$(jq -r '.id | ascii_downcase' <<< "$existing_vnet")" != "${expected_vnet_id,,}" \ || "$(jq -r '.id | ascii_downcase' <<< "$existing_subnet")" != "${expected_subnet_id,,}" \ || "$(jq -r '.id | ascii_downcase' <<< "$existing_nat")" != "${expected_nat_id,,}" \ @@ -243,6 +252,8 @@ parameters=( "enableOtel=$PYRIT_ENABLE_OTEL" "envSecretName=$PYRIT_ENV_SECRET_NAME" "enableFrontDoor=true" + "enableFrontDoorPrivateLink=true" + "disableContainerAppsPublicAccess=true" "vnetAddressPrefix=$PYRIT_VNET_ADDRESS_PREFIX" "infrastructureSubnetAddressPrefix=$PYRIT_INFRASTRUCTURE_SUBNET_ADDRESS_PREFIX" "egressPublicIpIpTags=$existing_pip_ip_tags" @@ -250,6 +261,15 @@ parameters=( "tags=$deployment_tags" ) +rollback_parameters=() +for parameter in "${parameters[@]}"; do + case "$parameter" in + enableFrontDoorPrivateLink=*) rollback_parameters+=("enableFrontDoorPrivateLink=false") ;; + disableContainerAppsPublicAccess=*) rollback_parameters+=("disableContainerAppsPublicAccess=false") ;; + *) rollback_parameters+=("$parameter") ;; + esac +done + deployment_name="pyrit-$PYRIT_SLOT-$PYRIT_BUILD_ID" what_if_file="$PYRIT_AGENT_TEMP_DIRECTORY/$deployment_name-what-if.json" az deployment group what-if \ @@ -267,17 +287,168 @@ if ! python3 "$PYRIT_SOURCE_DIRECTORY/infra/pipelines/validate_what_if.py" \ --expected-pip-id "$expected_pip_id" \ --expected-nat-id "$expected_nat_id" \ --expected-vnet-id "$expected_vnet_id" \ - --expected-subnet-id "$expected_subnet_id"; then + --expected-subnet-id "$expected_subnet_id" \ + --expected-environment-id "$expected_environment_id"; then echo "##vso[task.logissue type=error]What-if contains a delete, cross-resource-group write, protected-network change, or core resource create" exit 1 fi +cutover_in_progress=false +rollback_public_origin() { + local exit_code=$? + trap - EXIT + if [[ "$cutover_in_progress" == "true" && "$exit_code" != "0" ]]; then + echo "##vso[task.logissue type=warning]Private Link cutover failed; restoring the public ACA origin" + local rollback_origin_host + local rollback_request_message + rollback_origin_host=$(az containerapp show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME" --query properties.configuration.ingress.fqdn -o tsv || true) + rollback_request_message=${private_link_request_message:-"Azure Front Door private access to $PYRIT_APP_NAME"} + + if [[ -n "$rollback_origin_host" ]]; then + az deployment group create \ + --name "$deployment_name-rollback-origin" \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --template-file "$PYRIT_SOURCE_DIRECTORY/infra/modules/aca_front_door.bicep" \ + --parameters \ + "namePrefix=$PYRIT_APP_NAME" \ + "originHostName=$rollback_origin_host" \ + "tags=$deployment_tags" \ + "enablePrivateLink=false" || true + fi + + local rollback_connections + rollback_connections=$(az network private-endpoint-connection list \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-env" \ + --type Microsoft.App/managedEnvironments -o json || echo '[]') + while IFS= read -r connection_id; do + [[ -z "$connection_id" ]] && continue + if [[ "${connection_id,,}" == "${expected_environment_id,,}/privateendpointconnections/"* ]]; then + az network private-endpoint-connection delete --id "$connection_id" --yes || true + fi + done < <(jq -r --arg message "$rollback_request_message" \ + '.[] | select(.properties.privateLinkServiceConnectionState.description == $message) | .id' \ + <<< "$rollback_connections") + + for attempt in {1..20}; do + rollback_connections=$(az network private-endpoint-connection list \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-env" \ + --type Microsoft.App/managedEnvironments -o json || echo '[]') + rollback_connection_count=$(jq --arg message "$rollback_request_message" \ + '[.[] | select(.properties.privateLinkServiceConnectionState.description == $message)] | length' \ + <<< "$rollback_connections") + [[ "$rollback_connection_count" == "0" ]] && break + [[ "$attempt" -lt 20 ]] && sleep 15 + done + + if az deployment group create \ + --name "$deployment_name-rollback" \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --template-file "$PYRIT_SOURCE_DIRECTORY/infra/main.bicep" \ + --parameters "${rollback_parameters[@]}"; then + echo "##vso[task.logissue type=warning]Public ACA origin rollback completed" + else + echo "##vso[task.logissue type=error]Public ACA origin rollback failed; manual recovery is required" + fi + fi + exit "$exit_code" +} +trap rollback_public_origin EXIT +cutover_in_progress=true + az deployment group create \ --name "$deployment_name" \ --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ --template-file "$PYRIT_SOURCE_DIRECTORY/infra/main.bicep" \ --parameters "${parameters[@]}" +private_link_request_message=$(az deployment group show \ + --name "$deployment_name" --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --query properties.outputs.frontDoorPrivateLinkRequestMessage.value -o tsv) +if [[ -z "$private_link_request_message" ]]; then + echo "##vso[task.logissue type=error]Deployment did not return a Private Link approval request message" + exit 1 +fi +origin_private_link=$(az afd origin show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --profile-name "$PYRIT_APP_NAME-afd" \ + --origin-group-name "$PYRIT_APP_NAME-origin-group" \ + --origin-name "$PYRIT_APP_NAME-aca-origin" \ + --query '{status:sharedPrivateLinkResource.status,resourceId:sharedPrivateLinkResource.privateLink.id}' -o json) +private_link_status=$(jq -r '.status // empty' <<< "$origin_private_link") +private_link_resource_id=$(jq -r '.resourceId // empty | ascii_downcase' <<< "$origin_private_link") +if [[ "$private_link_resource_id" != "${expected_environment_id,,}" \ + || ! "$private_link_status" =~ ^(Pending|Approved)$ ]]; then + echo "##vso[task.logissue type=error]Front Door Private Link does not target the expected ACA environment" + exit 1 +fi + +matching_connections='' +for attempt in {1..20}; do + connections=$(az network private-endpoint-connection list \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-env" \ + --type Microsoft.App/managedEnvironments -o json || echo '[]') + matching_connections=$(jq -c --arg message "$private_link_request_message" \ + '[.[] | select( + .properties.privateLinkServiceConnectionState.description == $message + and (.properties.privateLinkServiceConnectionState.status == "Pending" + or .properties.privateLinkServiceConnectionState.status == "Approved"))]' <<< "$connections") + connection_count=$(jq 'length' <<< "$matching_connections") + echo "Private Link request discovery attempt $attempt/20: $connection_count active connection(s)" + [[ "$connection_count" -gt 0 ]] && break + [[ "$attempt" -lt 20 ]] && sleep 15 +done +if [[ "$(jq 'length' <<< "$matching_connections")" == "0" ]]; then + echo "##vso[task.logissue type=error]Front Door did not create the expected ACA Private Link request" + exit 1 +fi + +while IFS=$'\t' read -r connection_id connection_status; do + if [[ "${connection_id,,}" != "${expected_environment_id,,}/privateendpointconnections/"* ]]; then + echo "##vso[task.logissue type=error]Private Link request is outside the expected ACA environment" + exit 1 + fi + if [[ "$connection_status" == "Pending" ]]; then + az network private-endpoint-connection approve \ + --id "$connection_id" \ + --description "$private_link_request_message" -o none + fi +done < <(jq -r '.[] | [.id, .properties.privateLinkServiceConnectionState.status] | @tsv' \ + <<< "$matching_connections") + +approved_connection_count=0 +for attempt in {1..20}; do + connections=$(az network private-endpoint-connection list \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-env" \ + --type Microsoft.App/managedEnvironments -o json || echo '[]') + approved_connection_count=$(jq --arg message "$private_link_request_message" \ + '[.[] | select( + .properties.privateLinkServiceConnectionState.description == $message + and .properties.privateLinkServiceConnectionState.status == "Approved")] | length' <<< "$connections") + echo "ACA Private Link approval attempt $attempt/20: $approved_connection_count approved connection(s)" + [[ "$approved_connection_count" -gt 0 ]] && break + [[ "$attempt" -lt 20 ]] && sleep 15 +done +if [[ "$approved_connection_count" == "0" ]]; then + echo "##vso[task.logissue type=error]ACA Private Link connection did not become approved" + exit 1 +fi +echo "AFD origin status is ${private_link_status}; ACA approval and AFD health determine readiness" + +public_network_access=$(az containerapp env show \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --name "$PYRIT_APP_NAME-env" \ + --query properties.publicNetworkAccess -o tsv) +if [[ "$public_network_access" != "Disabled" ]]; then + echo "##vso[task.logissue type=error]ACA environment public network access remains enabled" + exit 1 +fi + health="" for attempt in {1..5}; do health=$(az containerapp revision list \ @@ -324,4 +495,13 @@ if [[ "$front_door_health" != "200" ]]; then echo "##vso[task.logissue type=error]Front Door did not route a healthy response" exit 1 fi -echo "Deployment healthy; public URL: https://$front_door_fqdn; ACA origin: https://$app_fqdn; egress IPv4: $egress_ip" +direct_aca_health=$(curl \ + --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --max-time 15 "https://$app_fqdn/api/health" || true) +if [[ "$direct_aca_health" == "200" ]]; then + echo "##vso[task.logissue type=error]Direct ACA public access remains reachable" + exit 1 +fi +cutover_in_progress=false +trap - EXIT +echo "Deployment healthy; public URL: https://$front_door_fqdn; ACA public access: disabled; egress IPv4: $egress_ip" diff --git a/infra/pipelines/validate_what_if.py b/infra/pipelines/validate_what_if.py index 410aba83b2..7df820abc7 100644 --- a/infra/pipelines/validate_what_if.py +++ b/infra/pipelines/validate_what_if.py @@ -51,6 +51,7 @@ def validate_what_if( expected_nat_id: str, expected_vnet_id: str, expected_subnet_id: str, + expected_environment_id: str, ) -> list[str]: """Return every destructive, cross-scope, protected, or core-create violation.""" document = _expect_object(payload, context="what-if result") @@ -61,6 +62,10 @@ def validate_what_if( expected_nat_id.rstrip("/").casefold(): {"properties.scope", "sku.tier"}, expected_vnet_id.rstrip("/").casefold(): set(), expected_subnet_id.rstrip("/").casefold(): set(), + expected_environment_id.rstrip("/").casefold(): { + "properties.appLogsConfiguration.logAnalyticsConfiguration.customerId", + "properties.publicNetworkAccess", + }, } violations: list[str] = [] @@ -110,6 +115,7 @@ def _parse_args() -> argparse.Namespace: parser.add_argument("--expected-nat-id", required=True) parser.add_argument("--expected-vnet-id", required=True) parser.add_argument("--expected-subnet-id", required=True) + parser.add_argument("--expected-environment-id", required=True) return parser.parse_args() @@ -127,6 +133,7 @@ def main() -> int: expected_nat_id=cast("str", parsed.expected_nat_id), expected_vnet_id=cast("str", parsed.expected_vnet_id), expected_subnet_id=cast("str", parsed.expected_subnet_id), + expected_environment_id=cast("str", parsed.expected_environment_id), ) except (OSError, json.JSONDecodeError, WhatIfFormatError) as error: print(f"What-if validation failed closed: {error}", file=sys.stderr) diff --git a/tests/unit/infra/test_bicep_topology.py b/tests/unit/infra/test_bicep_topology.py index 324ba38b42..f23087c89c 100644 --- a/tests/unit/infra/test_bicep_topology.py +++ b/tests/unit/infra/test_bicep_topology.py @@ -53,9 +53,10 @@ def test_main_has_one_public_nat_topology(self): "infrastructureSubnetId", "infrastructureNsgName", "applicationGatewayNsgName", - "enableFrontDoorPrivateLink", } assert unsupported_parameters.isdisjoint(template["parameters"]) + assert template["parameters"]["enableFrontDoorPrivateLink"]["defaultValue"] is False + assert template["parameters"]["disableContainerAppsPublicAccess"]["defaultValue"] is False existing_identity = template["parameters"]["existingManagedIdentityResourceId"] assert existing_identity["defaultValue"] == "" @@ -68,6 +69,14 @@ def test_main_has_one_public_nat_topology(self): front_door_module = next(module for module in modules if "aca-front-door" in module["name"]) assert "condition" not in network_module assert "parameters('enableFrontDoor')" in front_door_module["condition"] + assert ( + "effectiveFrontDoorPrivateLink" + in front_door_module["properties"]["parameters"]["enablePrivateLink"]["value"] + ) + assert ( + "Microsoft.App/managedEnvironments" + in front_door_module["properties"]["parameters"]["originResourceId"]["value"] + ) nested_types = { resource_type @@ -89,8 +98,17 @@ def test_main_has_one_public_nat_topology(self): environment = _resources(template, "Microsoft.App/managedEnvironments")[0] environment_properties = environment["properties"] - assert environment_properties["publicNetworkAccess"] == "Enabled" + assert environment_properties["publicNetworkAccess"] == "[variables('effectiveContainerAppsPublicAccess')]" + effective_public_access = template["variables"]["effectiveContainerAppsPublicAccess"] + assert "disableContainerAppsPublicAccess" in effective_public_access + assert "effectiveFrontDoorPrivateLink" in effective_public_access + assert "fail(" in effective_public_access assert environment_properties["vnetConfiguration"]["internal"] is False + assert ( + environment_properties["appLogsConfiguration"]["logAnalyticsConfiguration"]["dynamicJsonColumns"] is False + ) + assert environment_properties["peerAuthentication"]["mtls"]["enabled"] is False + assert environment_properties["peerTrafficConfiguration"]["encryption"]["enabled"] is False assert ( "outputs.infrastructureSubnetId.value" in environment_properties["vnetConfiguration"]["infrastructureSubnetId"] @@ -167,9 +185,15 @@ def test_front_door_uses_https_health_probe_without_caching(self): assert probe["probeRequestType"] == "GET" origin = _resources(template, "Microsoft.Cdn/profiles/originGroups/origins")[0] - assert origin["properties"]["originHostHeader"] == "[parameters('originHostName')]" - assert origin["properties"]["enforceCertificateNameCheck"] is True - assert "sharedPrivateLinkResource" not in origin["properties"] + origin_properties = origin["properties"] + assert "originHostHeader" in origin_properties + assert "enforceCertificateNameCheck" in origin_properties + assert "parameters('enablePrivateLink')" in origin_properties + assert "sharedPrivateLinkResource" in origin_properties + assert "managedEnvironments" in origin_properties + assert "effectiveOriginResourceId" in origin_properties + assert "effectiveOriginLocation" in origin_properties + assert "Pending" in origin_properties route = _resources(template, "Microsoft.Cdn/profiles/afdEndpoints/routes")[0] assert route["properties"]["forwardingProtocol"] == "HttpsOnly" diff --git a/tests/unit/infra/test_pipeline_guardrails.py b/tests/unit/infra/test_pipeline_guardrails.py index 22be7553df..9da291a067 100644 --- a/tests/unit/infra/test_pipeline_guardrails.py +++ b/tests/unit/infra/test_pipeline_guardrails.py @@ -22,6 +22,7 @@ NAT_ID = f"{RESOURCE_GROUP_ID}/providers/Microsoft.Network/natGateways/copyrit-prod-v2-nat" VNET_ID = f"{RESOURCE_GROUP_ID}/providers/Microsoft.Network/virtualNetworks/copyrit-prod-v2-vnet" SUBNET_ID = f"{VNET_ID}/subnets/copyrit-prod-v2-aca-subnet" +ENVIRONMENT_ID = f"{RESOURCE_GROUP_ID}/providers/Microsoft.App/managedEnvironments/copyrit-prod-v2-env" class PipelineGuardrailTests(unittest.TestCase): @@ -73,7 +74,8 @@ def test_deploy_resolves_digest_and_previews_before_apply(self): assert "networkMode=" not in self.deploy_script assert "enablePrivateEndpoint=" not in self.deploy_script assert '"enableFrontDoor=true"' in self.deploy_script - assert "enableFrontDoorPrivateLink=" not in self.deploy_script + assert '"enableFrontDoorPrivateLink=true"' in self.deploy_script + assert '"disableContainerAppsPublicAccess=true"' in self.deploy_script def test_pipeline_passes_values_via_environment(self): deploy_yaml = self.pipeline[self.pipeline.index("stage: DeployTest") :] @@ -89,7 +91,9 @@ def test_deploy_validates_structured_inputs_before_arm(self): assert "subnet.subnet_of(vnet)" in self.deploy_script assert "subnet.prefixlen > 27" in self.deploy_script assert "uuid.UUID" in self.deploy_script - assert "Microsoft\\.KeyVault/vaults" in self.deploy_script + assert "normalized_managed_identity_resource_id=" in self.deploy_script + assert "normalized_key_vault_resource_id=" in self.deploy_script + assert "microsoft\\.keyvault/vaults" in self.deploy_script assert "database\\.windows\\.net" in self.deploy_script assert self.deploy_script.index("ipaddress.ip_network") < self.deploy_script.index( "az deployment group what-if" @@ -104,6 +108,7 @@ def test_deploy_preserves_existing_network_and_tags(self): assert "--result-format FullResourcePayloads" in self.deploy_script assert "--expected-pip-id" in self.deploy_script assert "--expected-subnet-id" in self.deploy_script + assert "--expected-environment-id" in self.deploy_script assert "Reserved egress PIP identity or address changed" in self.deploy_script assert self.deploy_script.index("expected_egress_ip=") < self.deploy_script.index("az deployment group what-if") assert self.deploy_script.index("actual_pip_id=") > self.deploy_script.index("az deployment group create") @@ -111,9 +116,23 @@ def test_deploy_preserves_existing_network_and_tags(self): def test_data_plane_health_probe_respects_ingress_restrictions(self): assert "properties.outputs.frontDoorFqdn.value" in self.deploy_script assert '"https://$front_door_fqdn/api/health"' in self.deploy_script - assert '"https://$app_fqdn/api/health"' not in self.deploy_script + assert "direct_aca_health=$(curl" in self.deploy_script + assert '"https://$app_fqdn/api/health"' in self.deploy_script + assert '[[ "$direct_aca_health" == "200" ]]' in self.deploy_script assert "Front Door did not route a healthy response" in self.deploy_script - assert "ACA origin: https://$app_fqdn" in self.deploy_script + assert "private-endpoint-connection approve" in self.deploy_script + assert '--description "$private_link_request_message"' in self.deploy_script + assert "sharedPrivateLinkResource.status" in self.deploy_script + assert "sharedPrivateLinkResource.privateLink.id" in self.deploy_script + assert "approved_connection_count=" in self.deploy_script + assert "ACA approval and AFD health determine readiness" in self.deploy_script + assert "cutover_in_progress=true" in self.deploy_script + assert '"$deployment_name-rollback-origin"' in self.deploy_script + assert "private-endpoint-connection delete" in self.deploy_script + assert '"$deployment_name-rollback"' in self.deploy_script + assert '"${rollback_parameters[@]}"' in self.deploy_script + assert "Direct ACA public access remains reachable" in self.deploy_script + assert "ACA public access: disabled" in self.deploy_script def test_manual_parameter_files_use_the_single_topology(self): example = json.loads(EXAMPLE_PARAMETERS.read_text(encoding="utf-8")) @@ -126,7 +145,6 @@ def test_manual_parameter_files_use_the_single_topology(self): "networkMode", "infrastructureNsgName", "applicationGatewayNsgName", - "enableFrontDoorPrivateLink", } assert unsupported.isdisjoint(example["parameters"]) assert unsupported.isdisjoint(demo["parameters"]) @@ -134,6 +152,10 @@ def test_manual_parameter_files_use_the_single_topology(self): assert "infrastructureSubnetAddressPrefix" in example["parameters"] assert example["parameters"]["acrName"]["value"] assert example["parameters"]["existingManagedIdentityResourceId"]["value"] + assert example["parameters"]["enableFrontDoorPrivateLink"]["value"] is False + assert example["parameters"]["disableContainerAppsPublicAccess"]["value"] is False + assert demo["parameters"]["enableFrontDoorPrivateLink"]["value"] is False + assert demo["parameters"]["disableContainerAppsPublicAccess"]["value"] is False assert demo["parameters"]["existingManagedIdentityResourceId"]["value"] def _run_what_if_validator( @@ -161,6 +183,8 @@ def _run_what_if_validator( VNET_ID, "--expected-subnet-id", expected_subnet_id, + "--expected-environment-id", + ENVIRONMENT_ID, ], capture_output=True, text=True, @@ -176,6 +200,14 @@ def test_what_if_validator_accepts_read_only_normalization_and_lock_create(self) "delta": [{"path": "properties.scope"}, {"path": "sku.tier"}], }, {"changeType": "Modify", "resourceId": PIP_ID, "delta": [{"path": "sku.tier"}]}, + { + "changeType": "Modify", + "resourceId": ENVIRONMENT_ID, + "delta": [ + {"path": "properties.appLogsConfiguration.logAnalyticsConfiguration.customerId"}, + {"path": "properties.publicNetworkAccess"}, + ], + }, {"changeType": "Create", "resourceId": lock_id}, ] @@ -205,6 +237,14 @@ def test_what_if_validator_rejects_each_protected_topology_violation(self): "protected-resource delta", ), "opaque-protected-change": ({"changeType": "Modify", "resourceId": VNET_ID}, "opaque"), + "protected-environment": ( + { + "changeType": "Modify", + "resourceId": ENVIRONMENT_ID, + "delta": [{"path": "properties.vnetConfiguration.internal"}], + }, + "protected-resource delta", + ), "container-app-create": ( { "changeType": "Create", From 9731995574913d146081e380c93d95b2a7874f71 Mon Sep 17 00:00:00 2001 From: Bashir Partovi Date: Thu, 20 Aug 2026 19:38:04 -0400 Subject: [PATCH 5/7] FIX: Harden AFD Private Link cutovers Approve ACA private endpoint connections through ARM instead of MSYS-sensitive resource-ID CLI calls, and use REST cleanup during rollback. Allow the full observed Front Door propagation window, set explicit deployment-job timeouts, and cover the approval contract in tests. --- gui-deploy.yml | 2 ++ infra/README.md | 2 +- .../aca_private_endpoint_approval.bicep | 25 +++++++++++++++++++ infra/pipelines/deploy_public_nat.sh | 22 ++++++++++------ tests/unit/infra/test_bicep_topology.py | 10 ++++++++ tests/unit/infra/test_pipeline_guardrails.py | 9 ++++--- 6 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 infra/modules/aca_private_endpoint_approval.bicep diff --git a/gui-deploy.yml b/gui-deploy.yml index 03aafde102..59fa50ee75 100644 --- a/gui-deploy.yml +++ b/gui-deploy.yml @@ -134,6 +134,7 @@ stages: jobs: - deployment: DeployToTest displayName: 'Deploy test environment' + timeoutInMinutes: 120 environment: 'copyrit-test' strategy: runOnce: @@ -211,6 +212,7 @@ stages: jobs: - deployment: DeployToProd displayName: 'Deploy production environment' + timeoutInMinutes: 120 environment: 'copyrit-prod' strategy: runOnce: diff --git a/infra/README.md b/infra/README.md index 528747542e..9422545b04 100644 --- a/infra/README.md +++ b/infra/README.md @@ -383,7 +383,7 @@ az deployment group show -g -n \ 4. Run a full ARM `what-if` through a fail-closed validator; reject malformed results, deletions, cross-resource-group writes, protected-network deltas other than the documented read-only NAT/PIP normalization, and core network, app, or Log Analytics workspace creates. The expected PIP protection lock may be created. 5. Preserve policy-managed PIP tags and deploy with Front Door Private Link, ACA public access disabled, and PIP protection enabled. 6. Validate the AFD origin targets the expected ACA environment, approve only active requests with the deterministic message, and require the ACA-side connection to report `Approved`. AFD can continue to display `Pending` after approval, so successful AFD health is the data-plane readiness signal. -7. Verify ACA public access is disabled, the digest-pinned revision and Front Door `/api/health` are healthy, direct ACA access is unavailable, and the PIP resource ID/address is unchanged. +7. Allow up to 30 minutes for Front Door propagation, then verify ACA public access is disabled, the digest-pinned revision and Front Door `/api/health` are healthy, direct ACA access is unavailable, and the PIP resource ID/address is unchanged. 8. If cutover validation fails, redeploy the prior public AFD origin and re-enable ACA public access; otherwise print the Front Door URL and static egress IPv4. Qualifying merges to `main` automatically deploy test. Production deployment is independent of PyRIT package releases: manually queue a commit merged to `main` with `deployToProd=true`. The workflow deploys test first, then requires a timeout-rejecting manual approval whose requester cannot self-approve. diff --git a/infra/modules/aca_private_endpoint_approval.bicep b/infra/modules/aca_private_endpoint_approval.bicep new file mode 100644 index 0000000000..20a3662869 --- /dev/null +++ b/infra/modules/aca_private_endpoint_approval.bicep @@ -0,0 +1,25 @@ +@description('Name of the existing Azure Container Apps managed environment') +param environmentName string + +@description('Name of the existing private endpoint connection to approve') +param connectionName string + +@description('Approval description preserved for deterministic discovery') +param approvalDescription string + +resource environment 'Microsoft.App/managedEnvironments@2024-10-02-preview' existing = { + name: environmentName +} + +resource connection 'Microsoft.App/managedEnvironments/privateEndpointConnections@2024-10-02-preview' = { + parent: environment + name: connectionName + properties: { + privateLinkServiceConnectionState: { + description: approvalDescription + status: 'Approved' + } + } +} + +output connectionId string = connection.id diff --git a/infra/pipelines/deploy_public_nat.sh b/infra/pipelines/deploy_public_nat.sh index bd0cf9baab..31c77a8734 100644 --- a/infra/pipelines/deploy_public_nat.sh +++ b/infra/pipelines/deploy_public_nat.sh @@ -326,7 +326,8 @@ rollback_public_origin() { while IFS= read -r connection_id; do [[ -z "$connection_id" ]] && continue if [[ "${connection_id,,}" == "${expected_environment_id,,}/privateendpointconnections/"* ]]; then - az network private-endpoint-connection delete --id "$connection_id" --yes || true + az rest --method delete \ + --url "https://management.azure.com${connection_id}?api-version=2024-10-02-preview" || true fi done < <(jq -r --arg message "$rollback_request_message" \ '.[] | select(.properties.privateLinkServiceConnectionState.description == $message) | .id' \ @@ -413,9 +414,16 @@ while IFS=$'\t' read -r connection_id connection_status; do exit 1 fi if [[ "$connection_status" == "Pending" ]]; then - az network private-endpoint-connection approve \ - --id "$connection_id" \ - --description "$private_link_request_message" -o none + connection_name=${connection_id##*/} + connection_suffix=${connection_name:0:8} + az deployment group create \ + --name "$deployment_name-private-link-approval-$connection_suffix" \ + --resource-group "$PYRIT_DEPLOYMENT_RESOURCE_GROUP" \ + --template-file "$PYRIT_SOURCE_DIRECTORY/infra/modules/aca_private_endpoint_approval.bicep" \ + --parameters \ + "environmentName=$PYRIT_APP_NAME-env" \ + "connectionName=$connection_name" \ + "approvalDescription=$private_link_request_message" -o none fi done < <(jq -r '.[] | [.id, .properties.privateLinkServiceConnectionState.status] | @tsv' \ <<< "$matching_connections") @@ -483,13 +491,13 @@ if [[ "$egress_ip" != "$expected_egress_ip" \ exit 1 fi front_door_health="" -for attempt in {1..20}; do +for attempt in {1..60}; do front_door_health=$(curl \ --silent --show-error --output /dev/null --write-out '%{http_code}' \ --max-time 30 "https://$front_door_fqdn/api/health" || true) - echo "Front Door health attempt $attempt/20: ${front_door_health:-}" + echo "Front Door health attempt $attempt/60: ${front_door_health:-}" [[ "$front_door_health" == "200" ]] && break - [[ "$attempt" -lt 20 ]] && sleep 30 + [[ "$attempt" -lt 60 ]] && sleep 30 done if [[ "$front_door_health" != "200" ]]; then echo "##vso[task.logissue type=error]Front Door did not route a healthy response" diff --git a/tests/unit/infra/test_bicep_topology.py b/tests/unit/infra/test_bicep_topology.py index f23087c89c..5cc8ca2924 100644 --- a/tests/unit/infra/test_bicep_topology.py +++ b/tests/unit/infra/test_bicep_topology.py @@ -17,6 +17,7 @@ MAIN_BICEP = REPO_ROOT / "infra" / "main.bicep" NETWORK_BICEP = REPO_ROOT / "infra" / "modules" / "aca_nat_network.bicep" FRONT_DOOR_BICEP = REPO_ROOT / "infra" / "modules" / "aca_front_door.bicep" +PRIVATE_ENDPOINT_APPROVAL_BICEP = REPO_ROOT / "infra" / "modules" / "aca_private_endpoint_approval.bicep" AZ_CLI = shutil.which("az") @@ -200,6 +201,15 @@ def test_front_door_uses_https_health_probe_without_caching(self): assert route["properties"]["httpsRedirect"] == "Enabled" assert "cacheConfiguration" not in route["properties"] + def test_private_endpoint_approval_preserves_discovery_description(self): + template = _compile_bicep(PRIVATE_ENDPOINT_APPROVAL_BICEP, self.output_directory / "approval.json") + + connections = _resources(template, "Microsoft.App/managedEnvironments/privateEndpointConnections") + assert len(connections) == 1 + state = connections[0]["properties"]["privateLinkServiceConnectionState"] + assert state["status"] == "Approved" + assert state["description"] == "[parameters('approvalDescription')]" + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/infra/test_pipeline_guardrails.py b/tests/unit/infra/test_pipeline_guardrails.py index 9da291a067..456cf11014 100644 --- a/tests/unit/infra/test_pipeline_guardrails.py +++ b/tests/unit/infra/test_pipeline_guardrails.py @@ -41,6 +41,7 @@ def test_pipeline_has_one_test_and_prod_workflow(self): assert "stage: ApproveProd" in self.pipeline assert "stage: DeployProd" in self.pipeline assert "DeployReplacement" not in self.pipeline + assert self.pipeline.count("timeoutInMinutes: 120") == 2 assert self.pipeline.count("scriptPath: '$(Build.SourcesDirectory)/infra/pipelines/deploy_public_nat.sh'") == 2 def test_production_remains_opt_in_and_independently_approved(self): @@ -120,15 +121,17 @@ def test_data_plane_health_probe_respects_ingress_restrictions(self): assert '"https://$app_fqdn/api/health"' in self.deploy_script assert '[[ "$direct_aca_health" == "200" ]]' in self.deploy_script assert "Front Door did not route a healthy response" in self.deploy_script - assert "private-endpoint-connection approve" in self.deploy_script - assert '--description "$private_link_request_message"' in self.deploy_script + assert "aca_private_endpoint_approval.bicep" in self.deploy_script + assert "connection_suffix=${connection_name:0:8}" in self.deploy_script + assert '"$deployment_name-private-link-approval-$connection_suffix"' in self.deploy_script + assert '"approvalDescription=$private_link_request_message"' in self.deploy_script assert "sharedPrivateLinkResource.status" in self.deploy_script assert "sharedPrivateLinkResource.privateLink.id" in self.deploy_script assert "approved_connection_count=" in self.deploy_script assert "ACA approval and AFD health determine readiness" in self.deploy_script assert "cutover_in_progress=true" in self.deploy_script assert '"$deployment_name-rollback-origin"' in self.deploy_script - assert "private-endpoint-connection delete" in self.deploy_script + assert "az rest --method delete" in self.deploy_script assert '"$deployment_name-rollback"' in self.deploy_script assert '"${rollback_parameters[@]}"' in self.deploy_script assert "Direct ACA public access remains reachable" in self.deploy_script From c7487eda70f1198bd522eeb0d7ce55fb62ecccbd Mon Sep 17 00:00:00 2001 From: Bashir Partovi Date: Thu, 20 Aug 2026 19:45:27 -0400 Subject: [PATCH 6/7] DOC: Clarify AFD Private Link topology --- infra/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/infra/README.md b/infra/README.md index 9422545b04..7356a6fed6 100644 --- a/infra/README.md +++ b/infra/README.md @@ -13,11 +13,12 @@ flowchart TB subgraph azure["Azure subscription"] frontDoor["Azure Front Door Premium
Optional managed HTTPS entry point"] - ingress["ACA-managed HTTPS ingress
Public endpoint optionally disabled"] + privateLink["AFD-managed Private Link
Optional origin isolation"] + ingress["ACA-managed HTTPS ingress
Public network access enabled or disabled"] subgraph vnet["Virtual network"] subgraph subnet["Delegated ACA infrastructure subnet"] - environment["Public ACA workload-profiles environment"] + environment["External ACA workload-profiles environment"] app["Container App
React SPA + FastAPI API"] environment --> app end @@ -36,7 +37,9 @@ flowchart TB end user -->|"HTTPS when Front Door enabled"| frontDoor - frontDoor -->|"HTTPS public origin or Private Link"| ingress + frontDoor -.->|"Public HTTPS origin when Private Link is disabled"| ingress + frontDoor -->|"Private origin when enabled"| privateLink + privateLink --> ingress user -.->|"Direct ACA URL when public access is enabled"| ingress ingress --> environment user -->|"MSAL PKCE sign-in"| entra From bf39017242d5478b841f21f0ae182dde46a960d7 Mon Sep 17 00:00:00 2001 From: Bashir Partovi Date: Fri, 21 Aug 2026 15:23:59 -0400 Subject: [PATCH 7/7] FIX: Raise Front Door origin timeout --- infra/README.md | 2 +- infra/modules/aca_front_door.bicep | 2 +- tests/unit/infra/test_bicep_topology.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/infra/README.md b/infra/README.md index 7356a6fed6..ad28305c33 100644 --- a/infra/README.md +++ b/infra/README.md @@ -101,7 +101,7 @@ Community users can deploy `main.bicep` directly using the instructions below. F - **Authorization**: Entra group check via `allowedGroupObjectIds` param. Requires delegated Graph `User.Read`; the backend calls `/me/checkMemberGroups` and compares the returned transitive memberships with the configured group IDs. Each security group must also be assigned to the enterprise app (see Prerequisites §3). Authenticated deployments require at least one allowed group and fail to start without one. `/api/health`, `/api/auth/config`, and `/api/media` are intentional public exceptions; other `/api` routes require authentication when auth is enabled. Successful identity and membership results are cached in-process for 60 seconds, keyed by a SHA-256 token digest, to reduce Graph latency and throttling. Bearer tokens themselves are not stored in the cache. - **Identity**: `deploy_instance.py` creates its user-assigned managed identity (UAMI) and grants AcrPull and Storage Blob Data Contributor before deploying Bicep. A direct Bicep deployment can create `-identity`, but the template creates no role assignments, so its first revision can remain unhealthy until required roles are granted and the revision is restarted. A healthy one-pass direct deployment uses an existing, pre-authorized UAMI. `AZURE_CLIENT_ID` is set to the UAMI's client ID so `DefaultAzureCredential` selects the correct identity. - **Network**: The template always creates a VNet-integrated external Container Apps environment, one delegated ACA infrastructure subnet, a Standard NAT Gateway, and a static outbound IPv4. ACA supplies the generated HTTPS hostname and trusted certificate. In direct-ACA mode, `allowedCidr` optionally restricts public ingress to one IPv4 CIDR; an empty value permits public ingress. Front Door mode requires `allowedCidr` to be empty because ACA sees Front Door backend addresses, not the original client; Bicep and the team pipeline reject the invalid combination. Entra sign-in, enterprise-app assignment, and backend group checks remain mandatory application access controls. -- **Front Door**: `enableFrontDoor=true` creates a Premium profile, managed `azurefd.net` endpoint, HTTPS ACA origin, `/api/health` probe, and uncached catch-all route. `enableFrontDoorPrivateLink=true` targets the ACA managed environment with group ID `managedEnvironments`. The resulting private endpoint connection must be approved before AFD can route privately. `disableContainerAppsPublicAccess=true` disables the ACA environment public endpoint and CORS then permits only the AFD origin. The module does not create a WAF policy; application authentication and authorization remain mandatory. +- **Front Door**: `enableFrontDoor=true` creates a Premium profile, managed `azurefd.net` endpoint, HTTPS ACA origin, `/api/health` probe, uncached catch-all route, and 240-second origin response timeout matching the ACA HTTP ingress limit. `enableFrontDoorPrivateLink=true` targets the ACA managed environment with group ID `managedEnvironments`. The resulting private endpoint connection must be approved before AFD can route privately. `disableContainerAppsPublicAccess=true` disables the ACA environment public endpoint and CORS then permits only the AFD origin. The module does not create a WAF policy; application authentication and authorization remain mandatory. - **Routing**: Inbound requests through Front Door do not traverse the NAT Gateway. When ACA public access remains enabled, users can also reach ACA directly. When Private Link is enabled and public access is disabled, all public application traffic enters through Front Door. Outbound connections from the ACA environment that leave the virtual network use the NAT Gateway's static public IPv4. - **Response headers**: `SecurityHeadersMiddleware` adds [CSP](https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP), HTTP Strict Transport Security (HSTS, production only), X-Frame-Options, X-Content-Type-Options, Referrer-Policy, Permissions-Policy, and Cache-Control (`no-store` on API routes). Swagger/OpenAPI disabled in production. - **Data**: Azure SQL with managed identity authentication (no passwords) diff --git a/infra/modules/aca_front_door.bicep b/infra/modules/aca_front_door.bicep index 418557a9d1..6f1b1fd438 100644 --- a/infra/modules/aca_front_door.bicep +++ b/infra/modules/aca_front_door.bicep @@ -33,7 +33,7 @@ resource profile 'Microsoft.Cdn/profiles@2024-09-01' = { name: 'Premium_AzureFrontDoor' } properties: { - originResponseTimeoutSeconds: 60 + originResponseTimeoutSeconds: 240 } } diff --git a/tests/unit/infra/test_bicep_topology.py b/tests/unit/infra/test_bicep_topology.py index 5cc8ca2924..e9e20e7635 100644 --- a/tests/unit/infra/test_bicep_topology.py +++ b/tests/unit/infra/test_bicep_topology.py @@ -178,6 +178,7 @@ def test_front_door_uses_https_health_probe_without_caching(self): profile = _resources(template, "Microsoft.Cdn/profiles")[0] assert profile["sku"]["name"] == "Premium_AzureFrontDoor" + assert profile["properties"]["originResponseTimeoutSeconds"] == 240 origin_group = _resources(template, "Microsoft.Cdn/profiles/originGroups")[0] probe = origin_group["properties"]["healthProbeSettings"]