Merge branch 'main' into regex-syntax-anchors

Author: JelleZijlstra

.azure-pipelines/ci.yml

@@ -1,18 +1,14 @@
 variables:
-  manylinux: false
   coverage: false
 
-resources:
-  containers:
-  - container: manylinux1
-    image: pyca/cryptography-manylinux1:x86_64
+trigger: ['main', '3.11', '3.10', '3.9', '3.8', '3.7']
 
 jobs:
 - job: Prebuild
   displayName: Pre-build checks
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-20.04
 
   steps:
   - template: ./prebuild-checks.yml
@@ -24,7 +20,7 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-20.04
 
   steps:
   - template: ./docs-steps.yml
@@ -35,14 +31,16 @@ jobs:
 - job: macOS_CI_Tests
   displayName: macOS CI Tests
   dependsOn: Prebuild
-  condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+  #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
+  # bpo-39837: macOS tests on Azure Pipelines are disabled
+  condition: false
 
   variables:
     testRunTitle: '$(build.sourceBranchName)-macos'
     testRunPlatform: macos
 
   pool:
-    vmImage: macos-10.14
+    vmImage: macos-10.15
 
   steps:
   - template: ./macos-steps.yml
@@ -54,50 +52,19 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-20.04
 
   variables:
     testRunTitle: '$(build.sourceBranchName)-linux'
     testRunPlatform: linux
-    openssl_version: 1.1.1d
+    openssl_version: 1.1.1n
 
   steps:
   - template: ./posix-steps.yml
     parameters:
       dependencies: apt
 
 
-- job: ManyLinux1_CI_Tests
-  displayName: ManyLinux1 CI Tests
-  dependsOn: Prebuild
-  condition: |
-    and(
-        and(
-            succeeded(),
-            eq(variables['manylinux'], 'true')
-        ),
-        eq(dependencies.Prebuild.outputs['tests.run'], 'true')
-    )
-
-  pool:
-    vmImage: ubuntu-16.04
-
-  container: manylinux1
-
-  variables:
-    testRunTitle: '$(build.sourceBranchName)-manylinux1'
-    testRunPlatform: manylinux1
-    openssl_version: ''
-
-  steps:
-  - template: ./posix-steps.yml
-    parameters:
-      dependencies: yum
-      sudo_dependencies: ''
-      xvfb: false
-      patchcheck: false
-
-
 - job: Ubuntu_Coverage_CI_Tests
   displayName: Ubuntu CI Tests (coverage)
   dependsOn: Prebuild
@@ -111,12 +78,12 @@ jobs:
     )
 
   pool:
-    vmImage: ubuntu-16.04
+    vmImage: ubuntu-20.04
 
   variables:
     testRunTitle: '$(Build.SourceBranchName)-linux-coverage'
     testRunPlatform: linux-coverage
-    openssl_version: 1.1.1d
+    openssl_version: 1.1.1n
 
   steps:
   - template: ./posix-steps.yml
@@ -131,13 +98,13 @@ jobs:
   condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true'))
 
   pool:
-    vmImage: windows-2019
+    vmImage: windows-2022
 
   strategy:
     matrix:
       win32:
         arch: win32
-        buildOpt:
+        buildOpt: '-p Win32'
         testRunTitle: '$(Build.SourceBranchName)-win32'
         testRunPlatform: win32
       win64:

.azure-pipelines/docs-steps.yml

@@ -12,11 +12,12 @@ steps:
   inputs:
     versionSpec: '>=3.6'
 
-- script: python -m pip install sphinx==2.2.0 blurb python-docs-theme
+- script: python -m pip install -r requirements.txt
+  workingDirectory: '$(build.sourcesDirectory)/Doc'
   displayName: 'Install build dependencies'
 
 - ${{ if ne(parameters.latex, 'true') }}:
-  - script: make check suspicious html PYTHON=python
+  - script: make check html PYTHON=python
     workingDirectory: '$(build.sourcesDirectory)/Doc'
     displayName: 'Build documentation'
 
@@ -31,7 +32,7 @@ steps:
 - ${{ if eq(parameters.upload, 'true') }}:
   - task: PublishBuildArtifacts@1
     displayName: 'Publish docs'
-  
+
     inputs:
       PathToPublish: '$(build.sourcesDirectory)/Doc/build'
       ArtifactName: docs

.azure-pipelines/find-tools.yml

@@ -0,0 +1,26 @@
+# Locate a set of the tools used for builds
+
+steps:
+  - template: windows-release/find-sdk.yml
+    parameters:
+      toolname: 'signtool.exe'
+
+  - powershell: |
+      $vcvarsall = (& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" `
+          -prerelease `
+          -latest `
+          -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 `
+          -find VC\Auxiliary\Build\vcvarsall.bat)
+      Write-Host "Found vcvarsall at $vcvarsall"
+      Write-Host "##vso[task.setVariable variable=vcvarsall]$vcvarsall"
+    displayName: 'Find vcvarsall.bat'
+
+  - powershell: |
+      $msbuild = (& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" `
+          -prerelease `
+          -latest `
+          -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 `
+          -find MSBuild\Current\Bin\msbuild.exe)
+      Write-Host "Found MSBuild at $msbuild"
+      Write-Host "##vso[task.setVariable variable=msbuild]$msbuild"
+    displayName: 'Find MSBuild'

.azure-pipelines/libffi-build.yml

@@ -0,0 +1,86 @@
+name: $(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+variables:
+  IntDir: '$(Build.BinariesDirectory)'
+  OutDir: '$(Build.ArtifactStagingDirectory)'
+
+  # MUST BE SET AT QUEUE TIME
+  # SigningCertificate: 'Python Software Foundation'
+  # SourcesRepo: 'https://github.com/python/cpython-source-deps'
+  # SourceTag: 'libffi-3.4.2'
+
+jobs:
+- job: Build_LibFFI
+  displayName: LibFFI
+  pool:
+    vmImage: windows-latest
+
+  workspace:
+    clean: all
+
+  steps:
+    - checkout: none
+
+    - template: ./find-tools.yml
+
+    - powershell: |
+       mkdir -Force "$(IntDir)\script"
+       iwr "https://github.com/python/cpython/raw/main/PCbuild/prepare_libffi.bat" `
+           -outfile "$(IntDir)\script\prepare_libffi.bat"
+      displayName: 'Download build script'
+
+    - powershell: |
+        git clone $(SourcesRepo) -b $(SourceTag) --depth 1 -c core.autocrlf=false -c core.eol=lf .
+      displayName: 'Check out LibFFI sources'
+
+    - script: 'prepare_libffi.bat --install-cygwin'
+      workingDirectory: '$(IntDir)\script'
+      displayName: 'Install Cygwin and build'
+      env:
+        VCVARSALL: '$(vcvarsall)'
+        LIBFFI_SOURCE: '$(Build.SourcesDirectory)'
+        LIBFFI_OUT: '$(OutDir)'
+
+    - powershell: |
+       if ((gci *\*.dll).Count -lt 4) {
+           Write-Error "Did not generate enough DLL files"
+       }
+       if ((gci *\Include\ffi.h).Count -lt 4) {
+           Write-Error "Did not generate enough include files"
+       }
+      failOnStderr: true
+      workingDirectory: '$(OutDir)'
+      displayName: 'Verify files were created'
+
+    - publish: '$(OutDir)'
+      artifact: 'unsigned'
+      displayName: 'Publish unsigned build'
+
+- job: Sign_LibFFI
+  displayName: Sign LibFFI
+  dependsOn: Build_LibFFI
+  pool:
+    name: 'Windows Release'
+
+  workspace:
+    clean: all
+
+  steps:
+    - checkout: none
+    - download: current
+      artifact: unsigned
+
+    - template: ./find-tools.yml
+
+    - powershell: |
+        signtool sign /q /a `
+            /n "Python Software Foundation" `
+            /fd sha256 `
+            /tr http://timestamp.digicert.com/ /td sha256 `
+            /d "LibFFI for Python" `
+            (gci "$(Pipeline.Workspace)\unsigned\*.dll" -r)
+      displayName: 'Sign files'
+
+    - publish: '$(Pipeline.Workspace)\unsigned'
+      artifact: 'libffi'
+      displayName: 'Publish libffi'

.azure-pipelines/macos-steps.yml

@@ -6,7 +6,7 @@ steps:
 - script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev
   displayName: 'Configure CPython (debug)'
 
-- script: make -s -j4
+- script: make -j4
   displayName: 'Build CPython'
 
 - script: make pythoninfo

.azure-pipelines/openssl-build.yml

@@ -0,0 +1,110 @@
+name: $(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr)
+
+variables:
+  IntDir: '$(Build.BinariesDirectory)'
+  OutDir: '$(Build.ArtifactStagingDirectory)'
+
+  # MUST BE SET AT QUEUE TIME
+  # SigningCertificate: 'Python Software Foundation'
+  # SourcesRepo: 'https://github.com/python/cpython-source-deps'
+  # SourceTag: 'openssl-1.1.1k'
+
+jobs:
+- job: Build_SSL
+  displayName: OpenSSL
+  pool:
+    name: 'Windows Release'
+    #vmImage: windows-latest
+
+  strategy:
+    matrix:
+      win32:
+        Platform: 'win32'
+        VCPlatform: 'amd64_x86'
+        OpenSSLPlatform: 'VC-WIN32 no-asm'
+      amd64:
+        Platform: 'amd64'
+        VCPlatform: 'amd64'
+        OpenSSLPlatform: 'VC-WIN64A-masm'
+      arm32:
+        Platform: 'arm32'
+        VCPlatform: 'amd64_arm'
+        OpenSSLPlatform: 'VC-WIN32-ARM'
+      arm64:
+        Platform: 'arm64'
+        VCPlatform: 'amd64_arm64'
+        OpenSSLPlatform: 'VC-WIN64-ARM'
+
+  workspace:
+    clean: all
+
+  steps:
+    - checkout: none
+
+    - template: ./find-tools.yml
+
+    - powershell: |
+        git clone $(SourcesRepo) -b $(SourceTag) --depth 1 .
+      displayName: 'Check out OpenSSL sources'
+
+    - powershell: |
+        $f = gi ms\uplink.c
+        $c1 = gc $f
+        $c2 = $c1 -replace '\(\(h = GetModuleHandle\(NULL\)\) == NULL\)', '((h = GetModuleHandleA("_ssl.pyd")) == NULL) if ((h = GetModuleHandleA("_ssl_d.pyd")) == NULL) if ((h = GetModuleHandle(NULL)) == NULL /*patched*/)'
+        if ($c2 -ne $c1) {
+            $c2 | Out-File $f -Encoding ASCII
+        } else {
+            Write-Host '##warning Failed to patch uplink.c'
+        }
+      displayName: 'Apply uplink.c patch'
+
+    - script: |
+        call "$(vcvarsall)" $(VCPlatform)
+        perl "$(Build.SourcesDirectory)\Configure" $(OpenSSLPlatform)
+        nmake
+      workingDirectory: '$(IntDir)'
+      displayName: 'Build OpenSSL'
+
+    - script: |
+        call "$(vcvarsall)" $(VCPlatform)
+        signtool sign /q /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "OpenSSL for Python" *.dll
+      workingDirectory: '$(IntDir)'
+      displayName: 'Sign OpenSSL Build'
+      condition: and(succeeded(), variables['SigningCertificate'])
+
+    - task: CopyFiles@2
+      displayName: 'Copy built libraries for upload'
+      inputs:
+        SourceFolder: '$(IntDir)'
+        Contents: |
+          lib*.dll
+          lib*.pdb
+          lib*.lib
+          include\openssl\*.h
+        TargetFolder: '$(OutDir)'
+
+    - task: CopyFiles@2
+      displayName: 'Copy header files for upload'
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)'
+        Contents: |
+          include\openssl\*
+        TargetFolder: '$(OutDir)'
+
+    - task: CopyFiles@2
+      displayName: 'Copy applink files for upload'
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)\ms'
+        Contents: applink.c
+        TargetFolder: '$(OutDir)\include'
+
+    - task: CopyFiles@2
+      displayName: 'Copy LICENSE for upload'
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)'
+        Contents: LICENSE
+        TargetFolder: '$(OutDir)'
+
+    - publish: '$(OutDir)'
+      artifact: '$(Platform)'
+      displayName: 'Publishing $(Platform)'