gh-115582 and gh-115545: Windows release build mixes up free-threaded files (#98)

Author: zooba

windows-release/README.md

@@ -40,10 +40,27 @@ Without this, the build records will be lost after 30 days.
 The code signing certificate is stored in Azure Key Vault, and is authenticated using the
 variables in a Variable group called CPythonSign. The variable group is what triggers approvals.
 The group is at https://dev.azure.com/Python/cpython/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=1&path=CPythonSign
+A second group called CPythonTestSign exists without approvals, but only has access to a test signing certificate.
+
+The five variables in the Variable Group identify the Entra ID
+[App registration](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app) with access,
+and the name of the certificate to use.
+
+* `KeyVaultApplication` - the "Application (client) ID" of the App registration
+* `KeyVaultDirectory` - the "Directory (tenant) ID" of the App registration
+* `KeyVaultSecret` - the current "Client secret" of the App registration
+* `KeyVaultUri` - the base `https://*.vault.azure.net/` URI of the Key Vault
+* `KeyVaultCertificateName` - the name of the certificate. This is not a secret
+
+The Key Vault should be configured to use Azure role-based access control (soon to be the only option),
+and the App registration should have the "Key Vault Certificate User" and "Key Vault Crypto User" roles.
+The trusted owner of the Key Vault should have the "Owner" role, but the App registration should not.
 
 To upload a new code signing certificate (which will be provided by the PSF),
-or to change to a new Azure Keyvault instance,
-see the documentation at https://github.com/vcsjones/AzureSignTool/blob/main/WALKTHROUGH.md.
+you need the certificate in encrypted .pfx format.
+This can then be uploaded directly through the Azure Portal into the Key Vault along with the passphrase.
+If reusing an existing Key Vault, upload it as a new version of the existing certificate.
+If it is uploaded as a new certificate, the Variable Group must be updated.
 
 GPG signature generation uses a GPG key stored in the Secure Files library.
 This can be found at https://dev.azure.com/Python/cpython/_library?itemType=SecureFiles

windows-release/azure-pipelines.yml

@@ -145,6 +145,7 @@ stages:
       parameters:
         ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}:
           SigningCertificate: ${{ parameters.SigningCertificate }}
+        DoFreethreaded: ${{ parameters.DoFreethreaded }}
 
   - stage: Layout
     displayName: Generate layouts
@@ -204,6 +205,7 @@ stages:
           ARM64TclTk: ${{ parameters.ARM64TclTk }}
           ${{ if and(parameters.SigningCertificate, ne(parameters.SigningCertificate, 'Unsigned')) }}:
             SigningCertificate: ${{ parameters.SigningCertificate }}
+          DoARM64: ${{ parameters.DoARM64}}
           DoFreethreaded: ${{ parameters.DoFreethreaded }}
 
     - stage: Test_MSI

windows-release/build-steps-pgo.yml

@@ -2,7 +2,6 @@ parameters:
   PGInstrument: false
   PGRun: false
   PGUpdate: false
-  DoFreethreaded: false
 
 steps:
 - template: ./checkout.yml
@@ -23,21 +22,12 @@ steps:
 - ${{ if eq(parameters.PGInstrument, 'true') }}:
   - powershell: |
       $env:SigningCertificate = $null
-      .\PCbuild\build.bat -v -p $(Platform) -c PGInstrument
+      .\PCbuild\build.bat -v -p $(Platform) -c PGInstrument $(ExtraOptions)
     displayName: 'Run build'
     env:
       IncludeUwp: true
       Py_OutDir: '$(Build.BinariesDirectory)\bin'
 
-  - ${{ if eq(parameters.DoFreethreaded, 'true') }}:
-    - powershell: |
-        $env:SigningCertificate = $null
-        .\PCbuild\build.bat -v -p $(Platform) -c PGInstrument --disable-gil
-      displayName: 'Run free-threaded build'
-      env:
-        IncludeUwp: true
-        Py_OutDir: '$(Build.BinariesDirectory)\bin'
-
   - ${{ if ne(parameters.PGRun, 'true') }}:
     # Not running in this job, so we publish our entire build and object files
 
@@ -70,21 +60,13 @@ steps:
         targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)\instrumented'
 
   - powershell: |
-      & "$(Build.BinariesDirectory)\bin\$(Arch)\instrumented\python.exe" -m test --pgo
+      $exe = (gci "$(Build.BinariesDirectory)\bin\$(Arch)\instrumented\$(PythonExePattern)" | select -First 1)
+      & $exe -m test --pgo
     ignoreLASTEXITCODE: true
     displayName: 'Collect profile'
     env:
       PYTHONHOME: '$(Build.SourcesDirectory)'
 
-  - ${{ if eq(parameters.DoFreethreaded, 'true') }}:
-    - powershell: |
-        $exe = (gci "$(Build.BinariesDirectory)\bin\$(Arch)\instrumented\python3*t.exe" | select -First 1)
-        & $exe -m test --pgo
-      ignoreLASTEXITCODE: true
-      displayName: 'Collect free-threaded profile'
-      env:
-        PYTHONHOME: '$(Build.SourcesDirectory)'
-
 
   - ${{ if ne(parameters.PGUpdate, 'true') }}:
     # Not finishing in this job, so publish the binaries
@@ -121,21 +103,12 @@ steps:
 
   - powershell: |
       $env:SigningCertificate = $null
-      .\PCbuild\build.bat -v -p $(Platform) -c PGUpdate
+      .\PCbuild\build.bat -v -p $(Platform) -c PGUpdate $(ExtraOptions)
     displayName: 'Run build with PGO'
     env:
       IncludeUwp: true
       Py_OutDir: '$(Build.BinariesDirectory)\bin'
 
-  - ${{ if eq(parameters.DoFreethreaded, 'true') }}:
-    - powershell: |
-        $env:SigningCertificate = $null
-        .\PCbuild\build.bat -v -p $(Platform) -c PGUpdate --disable-gil
-      displayName: 'Run free-threaded build with PGO'
-      env:
-        IncludeUwp: true
-        Py_OutDir: '$(Build.BinariesDirectory)\bin'
-
   - powershell: |
       $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10
       $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1)

windows-release/build-steps.yml

@@ -1,6 +1,5 @@
 parameters:
   ShouldPGO: false
-  DoFreethreaded: false
 
 steps:
 - template: ./checkout.yml
@@ -16,48 +15,24 @@ steps:
     Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)-$(Name)"
   displayName: 'Extract version numbers'
 
-- ${{ if eq(parameters.DoFreethreaded, 'true') }}:
-  - powershell: |
-      Write-Host "##vso[build.addbuildtag]freethreaded"
-      Write-Host "##vso[build.addbuildtag]$(VersionText)-freethreaded"
-    displayName: 'Add freethreaded tags'
-
 - ${{ if eq(parameters.ShouldPGO, 'false') }}:
   - powershell: |
       $env:SigningCertificate = $null
-      .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration)
+      .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration) $(ExtraOptions)
     displayName: 'Run build'
     env:
       IncludeUwp: true
       Py_OutDir: '$(Build.BinariesDirectory)\bin'
 
-  - ${{ if eq(parameters.DoFreethreaded, 'true') }}:
-    - powershell: |
-        $env:SigningCertificate = $null
-        .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration) --disable-gil
-      displayName: 'Run free-threaded build'
-      env:
-        IncludeUwp: true
-        Py_OutDir: '$(Build.BinariesDirectory)\bin'
-
 - ${{ if eq(parameters.ShouldPGO, 'true') }}:
   - powershell: |
       $env:SigningCertificate = $null
-      .\PCbuild\build.bat -v -p $(Platform) --pgo
+      .\PCbuild\build.bat -v -p $(Platform) --pgo $(ExtraOptions)
     displayName: 'Run build with PGO'
     env:
       IncludeUwp: true
       Py_OutDir: '$(Build.BinariesDirectory)\bin'
 
-  - ${{ if eq(parameters.DoFreethreaded, 'true') }}:
-    - powershell: |
-        $env:SigningCertificate = $null
-        .\PCbuild\build.bat -v -p $(Platform) --pgo --disable-gil
-      displayName: 'Run free-threaded build'
-      env:
-        IncludeUwp: true
-        Py_OutDir: '$(Build.BinariesDirectory)\bin'
-
 - powershell: |
     $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10
     $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1)

windows-release/layout-command.yml

@@ -1,23 +1,37 @@
+parameters:
+  Binaries: $(Pipeline.Workspace)\bin_$(Name)
+  Sources: $(Build.SourcesDirectory)
+  Temp: $(Build.BinariesDirectory)\layout-temp
+  Docs: $(Build.BinariesDirectory)\doc
+
 steps:
-- task: DownloadPipelineArtifact@1
+- download: current
+  artifact: bin_$(HostArch)
   displayName: 'Download artifact: bin_$(HostArch)'
   condition: and(succeeded(), variables['HostArch'])
-  inputs:
-    artifactName: bin_$(HostArch)
-    targetPath: $(Build.BinariesDirectory)\bin_$(HostArch)
+
+- powershell: |
+    Write-Host "##vso[task.setvariable variable=Python]${{ parameters.Binaries }}\python.exe"
+  condition: and(succeeded(), not(variables['HostArch']))
+  displayName: 'Set Python command'
+
+- powershell: |
+    Write-Host "##vso[task.setvariable variable=Python]$(Pipeline.Workspace)\bin_$(HostArch)\python.exe"
+  condition: and(succeeded(), variables['HostArch'])
+  displayName: 'Set Python command'
 
 - powershell: >
-    Write-Host (
-    '##vso[task.setvariable variable=LayoutCmd]&
+    $layout_cmd = '&
     "$(Python)"
     "{1}\PC\layout"
     -vv
     --source "{1}"
-    --build "{0}\bin"
-    --arch "$(Name)"
-    --temp "{0}\layout-temp"
-    --include-cat "{0}\bin\python.cat"
-    --doc-build "{0}\doc"'
-    -f ("$(Build.BinariesDirectory)", "$(Build.SourcesDirectory)")
-    )
+    --build "{0}"
+    --arch "$(Arch)"
+    --temp "{2}"
+    --include-cat "{0}\python.cat"
+    --doc-build "{3}"'
+    -f ("${{ parameters.Binaries }}", "${{ parameters.Sources }}", "${{ parameters.Temp }}", "${{ parameters.Docs}}");
+    Write-Host "##vso[task.setvariable variable=LayoutCmd]$layout_cmd";
+    Write-Host "Setting LayoutCmd=$layout_cmd"
   displayName: 'Set LayoutCmd'

windows-release/libffi-build.yml

@@ -24,7 +24,10 @@ name: ${{ parameters.SourceTag }}_$(Date:yyyyMMdd)$(Rev:.rr)
 
 
 variables:
-- group: CPythonSign
+- ${{ if eq(parameters.SigningCertificate, 'PythonSoftwareFoundation') }}:
+  - group: CPythonSign
+- ${{ if eq(parameters.SigningCertificate, 'TestSign') }}:
+  - group: CPythonTestSign
 - name: IntDir
   value: '$(Build.BinariesDirectory)'
 - name: OutDir