azure-pipelines.yml

################################################################################
# TorchSharp's PR validation build
################################################################################

name: $(TeamProject)_$(Build.DefinitionName)_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r)

parameters:
  # Set this to 'true' to build the libtorch-* packages as part of main branch CI and
  # push them to the artifacts feed of the Azure CI project
  - name: BuildLibTorchPackages
    type: boolean
    default: true
  - name: PushPackagesToFeed
    type: boolean
    default: false

  # Set which source branch to build libtorch-* packages.
  # The build-libtorch jobs are only run if the source branch is the same as this value.
  # The default is 'main' branch.
  - name: SourceBranchName
    type: string
    default: 'main'

variables:
  MyRunNumber:  $[counter('MyRunNumber', 52201)]
  LinuxPrepScript: |
    ldd --version && (/sbin/ldconfig -p | grep stdc++) && (strings /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | grep LIBCXX)
    sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys DE19EB17684BA42D
    sudo apt-get -y update
    sudo apt-get -y install clang git libunwind8 curl libomp-dev libomp5 wget gpg
    wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
    echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ resolute main' | sudo tee /etc/apt/sources.list.d/kitware.list
    sudo apt-get -y update
    sudo apt-get -y install cmake
    cmake --version
    wget https://packages.microsoft.com/config/ubuntu/26.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
    sudo dpkg --purge packages-microsoft-prod && sudo dpkg -i packages-microsoft-prod.deb
    sudo apt-get update; sudo apt-get install -y apt-transport-https && sudo apt-get update
    ldd --version && (/sbin/ldconfig -p | grep stdc++) && (strings /usr/lib/x86_64-linux-gnu/libstdc++.so.6 | grep LIBCXX)

resources:
  containers:
#   - container: CentosContainer
#     image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033
# dotnet-buildtools/prereqs containers can be browsed at
#   https://github.com/dotnet/versions/blob/main/build-info/docker/image-info.dotnet-dotnet-buildtools-prereqs-docker-main.json

   - container: UbuntuContainer
     image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-26.04-amd64

jobs:
- template: /build/ci/job-template.yml
  parameters:
    name: Ubuntu_x64
    # This reports the GLIB_ and GLIBCXX_ dependencies on the system which ends up getting baked into
    # the generated libTorchSharp.so (and partly determines which Linux systems that binary will
    # be usable on), then installs clang-7.0 (LibTorch likes this for building C++ 14), then installs .NET 8.0
    prepScript: ${{ variables.LinuxPrepScript }}
    buildScript: dotnet build /p:SkipCuda=true -c
    testScript: dotnet test /p:SkipCuda=true --blame test/TorchSharpTest/TorchSharpTest.csproj -c
    pool:
      vmImage: 'ubuntu-latest'
    container: UbuntuContainer

- template: /build/ci/job-template.yml
  parameters:
    prepScript: echo "no prep needed"
    name: Windows_x64_NetCore
    buildScript: dotnet build /p:SkipCuda=true /p:SkipNetFxBuild=true -c
    testScript: dotnet test /p:SkipCuda=true /p:SkipNetFxBuild=true --blame test\TorchSharpTest\TorchSharpTest.csproj -c
    pool:
      vmImage: 'windows-latest'

- template: /build/ci/job-template.yml
  parameters:
    prepScript: echo "no prep needed"
    name: Windows_x64_NetFX
    buildScript: dotnet build /p:SkipCuda=true /p:SkipNetCoreBuild=true -c
    testScript: dotnet test /p:SkipCuda=true /p:SkipNetCoreBuild=true --blame test\TorchSharpTest\TorchSharpTest.csproj -c
    pool:
      vmImage: 'windows-latest'

- template: /build/ci/job-template.yml
  parameters:
    prepScript: echo "no prep needed"
    name: MacOS_arm64
    buildScript: dotnet build /p:SkipCuda=true /p:TargetArchitecture=arm64 -c
    testScript: echo "Azure Pipelines does not support arm64 yet, can't run tests"
    pool:
      vmImage: 'macos-latest'

################################################################################
# {Build} --> combine --> package to build native bits on multiple OS's
################################################################################


################################################################################
- job: Linux_Native_Build_For_Packages
################################################################################
  condition: and(ne(variables['system.pullrequest.isfork'], true), eq(variables['build.sourcebranchname'], '${{ parameters.SourceBranchName }}'))
  variables:
    BuildConfig: Release
    OfficialBuildId: $(BUILD.BUILDNUMBER)
    DOTNET_CLI_TELEMETRY_OPTOUT: 1
    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
    DOTNET_MULTILEVEL_LOOKUP: 0
  pool:
    vmImage: 'ubuntu-latest'
  container: UbuntuContainer
  steps:
  # Initial cleanup
  - script: |
      rm -rf bin/obj
      find . -name "*.pdb" -type f -delete
      find . -name "*.xml" -type f -delete
      df -h
    displayName: Initial cleanup and check disk space
    continueOnError: true

  - task: UseDotNet@2
    displayName: 'Use .NET Core sdk'
    inputs:
      packageType: sdk
      version: 8.0.x
      installationPath: $(Agent.ToolsDirectory)/dotnet

  - script: ${{ variables.LinuxPrepScript }}
    displayName: Install dependencies

  # Build libtorch CPU and clean immediately
  - script: |
      dotnet build -c $(BuildConfig) src/Redist/libtorch-cpu/libtorch-cpu.proj /p:UpdateSHA=true /p:SkipTests=true /p:TargetOS=linux /t:Build /p:IncludeLibTorchCpuPackages=true
      # Clean up downloaded archives after extraction
      find . -name "*.zip" -type f -delete
      find . -name "*.tar.gz" -type f -delete
    condition: eq('${{ parameters.BuildLibTorchPackages }}', true)
    displayName: Download libtorch native binaries and cleanup archives

  # Build libtorch CUDA and clean immediately
  - script: |
      dotnet build -c $(BuildConfig) src/Redist/libtorch-cuda-12.8/libtorch-cuda-12.8.proj /p:UpdateSHA=true /p:SkipTests=true /p:TargetOS=linux /t:Build /p:IncludeLibTorchCudaPackages=true
      # Clean up downloaded archives after extraction
      find . -name "*.zip" -type f -delete
      find . -name "*.tar.gz" -type f -delete
    condition: eq('${{ parameters.BuildLibTorchPackages }}', true)
    displayName: Download libtorch CUDA binaries and cleanup archives

  - script: dotnet build -c $(BuildConfig) src/TorchSharp/TorchSharp.csproj /p:SkipCuda=true /p:SkipTests=true
    displayName: Build TorchSharp

  - script: dotnet build -c $(BuildConfig) src/TorchVision/TorchVision.csproj /p:SkipCuda=true /p:SkipTests=true
    displayName: Build TorchVision

  - script: dotnet build -c $(BuildConfig) src/TorchAudio/TorchAudio.csproj /p:SkipCuda=true /p:SkipTests=true
    displayName: Build TorchAudio

  # Clean up unnecessary files before publishing
  - script: |
      find $(Build.SourcesDirectory)/bin -name "*.pdb" -type f -delete
      find $(Build.SourcesDirectory)/bin -name "*.xml" -type f -delete
      rm -rf $(Build.SourcesDirectory)/bin/obj/packprep/$(BuildConfig)/**/*.a
      df -h
    displayName: Clean up unnecessary files before publish
    continueOnError: true

  - publish: $(Build.SourcesDirectory)/bin/obj/packprep/$(BuildConfig)
    artifact: LinuxAssets

################################################################################
- job: Windows_Native_Build_For_Packages
################################################################################
  condition: and(ne(variables['system.pullrequest.isfork'], true), eq(variables['build.sourcebranchname'], '${{ parameters.SourceBranchName }}'))
  variables:
    BuildConfig: Release
    OfficialBuildId: $(BUILD.BUILDNUMBER)
    DOTNET_CLI_TELEMETRY_OPTOUT: 1
    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
    DOTNET_MULTILEVEL_LOOKUP: 0
  pool:
    vmImage: 'windows-latest'
  steps:
  # Initial cleanup
  - script: |
      rmdir /s /q .git 2>nul
      dotnet nuget locals all --clear
      dir
    displayName: Initial cleanup
    continueOnError: true

  - task: UseDotNet@2
    displayName: 'Use .NET Core sdk'
    inputs:
      packageType: sdk
      version: 8.0.x
      installationPath: $(Agent.ToolsDirectory)/dotnet

  # Build libtorch CPU and clean immediately
  - script: |
      dotnet build -c $(BuildConfig) src/Redist/libtorch-cpu/libtorch-cpu.proj /p:UpdateSHA=true /p:SkipTests=true /p:TargetOS=windows /t:Build /p:IncludeLibTorchCpuPackages=true
      del /s /q *.zip 2>nul
      del /s /q *.tar.gz 2>nul
    displayName: Download libtorch native binaries and cleanup
    condition: eq('${{ parameters.BuildLibTorchPackages }}', true)

  # Build libtorch CUDA and clean immediately  
  - script: |
      dotnet build -c $(BuildConfig) src/Redist/libtorch-cuda-12.8/libtorch-cuda-12.8.proj /p:UpdateSHA=true /p:SkipTests=true /p:TargetOS=windows /t:Build /p:IncludeLibTorchCudaPackages=true
      del /s /q *.zip 2>nul
      del /s /q *.tar.gz 2>nul
    condition: eq('${{ parameters.BuildLibTorchPackages }}', true)
    displayName: Download libtorch CUDA binaries and cleanup

  - script: dotnet build -c $(BuildConfig) src/TorchSharp/TorchSharp.csproj /p:SkipCuda=true /p:SkipTests=true
    condition: eq('${{ parameters.BuildLibTorchPackages }}', true)
    displayName: Build TorchSharp

  - script: dotnet build -c $(BuildConfig) src/TorchVision/TorchVision.csproj /p:SkipCuda=true /p:SkipTests=true
    displayName: Build TorchVision

  - script: dotnet build -c $(BuildConfig) src/TorchAudio/TorchAudio.csproj /p:SkipCuda=true /p:SkipTests=true
    displayName: Build TorchAudio

  # Clean up unnecessary files before publishing
  - script: |
      del /s /q $(Build.SourcesDirectory)\bin\*.pdb 2>nul
      del /s /q $(Build.SourcesDirectory)\bin\*.xml 2>nul
      del /s /q $(Build.SourcesDirectory)\bin\obj\packprep\$(BuildConfig)\*.lib 2>nul
    displayName: Clean up unnecessary files
    continueOnError: true

  - publish: $(Build.SourcesDirectory)/bin/obj/packprep/$(BuildConfig)
    artifact: WindowsAssets

################################################################################
- job: MacOS_arm64_Native_Build_For_Packages
################################################################################
  condition: and(ne(variables['system.pullrequest.isfork'], true), eq(variables['build.sourcebranchname'], '${{ parameters.SourceBranchName }}'))
  variables:
    BuildConfig: Release
    OfficialBuildId: $(BUILD.BUILDNUMBER)
    DOTNET_CLI_TELEMETRY_OPTOUT: 1
    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
    DOTNET_MULTILEVEL_LOOKUP: 0
    CMAKE_POLICY_VERSION_MINIMUM: 3.5
  pool:
    vmImage: 'macos-latest'
  steps:
  # Initial cleanup
  - script: |
      rm -rf .git
      dotnet nuget locals all --clear
      # Check CMake version and update if needed
      cmake --version || brew install cmake
      brew upgrade cmake || true
      cmake --version
      df -h
    displayName: Initial cleanup and setup
    continueOnError: true

  # Build and clean archives immediately
  - script: |
      dotnet build -c $(BuildConfig) src/Redist/libtorch-cpu/libtorch-cpu.proj /p:UpdateSHA=true /p:TargetOS=mac /p:TargetArchitecture=arm64 /t:Build /p:IncludeLibTorchCpuPackages=true /p:CmakePolicyVersionMinimum=3.5
      find . -name "*.zip" -type f -delete
      find . -name "*.tar.gz" -type f -delete
    displayName: Download libtorch native binaries and cleanup
    condition: eq('${{ parameters.BuildLibTorchPackages }}', true)

  - script: dotnet build -c $(BuildConfig) src/TorchSharp/TorchSharp.csproj /p:SkipCuda=true /p:SkipTests=true /p:TargetArchitecture=arm64 /p:CmakePolicyVersionMinimum=3.5
    displayName: Build TorchSharp mac-arm64

  - script: dotnet build -c $(BuildConfig) src/TorchVision/TorchVision.csproj /p:SkipCuda=true /p:SkipTests=true /p:TargetArchitecture=arm64 /p:CmakePolicyVersionMinimum=3.5
    displayName: Build TorchVision

  - script: dotnet build -c $(BuildConfig) src/TorchAudio/TorchAudio.csproj /p:SkipCuda=true /p:SkipTests=true /p:TargetArchitecture=arm64 /p:CmakePolicyVersionMinimum=3.5
    displayName: Build TorchAudio

  # Clean up before publishing
  - script: |
      find $(Build.SourcesDirectory)/bin -name "*.pdb" -type f -delete
      find $(Build.SourcesDirectory)/bin -name "*.xml" -type f -delete
      df -h
    displayName: Clean up unnecessary files
    continueOnError: true

  - publish: $(Build.SourcesDirectory)/bin/obj/packprep/$(BuildConfig)
    artifact: MacAssets_arm64

################################################################################
- job: Build_TorchSharp_And_libtorch_cpu_Packages
################################################################################
  condition: and(ne(variables['system.pullrequest.isfork'], true), eq(variables['build.sourcebranchname'], '${{ parameters.SourceBranchName }}'))
  dependsOn:
  - Linux_Native_Build_For_Packages
  - Windows_Native_Build_For_Packages
  - MacOS_arm64_Native_Build_For_Packages
  timeoutInMinutes: 90
  variables:
    BuildConfig: Release
    OfficialBuildId: $(BUILD.BUILDNUMBER)
    DOTNET_CLI_TELEMETRY_OPTOUT: 1
    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
    DOTNET_MULTILEVEL_LOOKUP: 0
  pool:
    vmImage: 'windows-latest'
  steps:

  # We are 10GB space-constrained on the Azure Pipelines CI system so clean up what we can
  # yup we even nuke the .git
  - script: rmdir /q /s .git
    displayName: Clean up space (.git)
    continueOnError: true

  # Download Linux assets in chunks
  - task: DownloadPipelineArtifact@2
    displayName: Download Linux TorchSharp assets
    inputs:
      artifact: LinuxAssets
      patterns: |
        TorchSharp/**
      path: $(Pipeline.Workspace)/LinuxAssets
    retryCountOnTaskFailure: 3

  - task: DownloadPipelineArtifact@2
    displayName: Download Linux TorchAudio assets
    inputs:
      artifact: LinuxAssets
      patterns: |
        TorchAudio/**
      path: $(Pipeline.Workspace)/LinuxAssets
    retryCountOnTaskFailure: 3

  - task: DownloadPipelineArtifact@2
    displayName: Download Linux TorchVision assets
    inputs:
      artifact: LinuxAssets
      patterns: |
        TorchVision/**
      path: $(Pipeline.Workspace)/LinuxAssets
    retryCountOnTaskFailure: 3

  # Download libtorch assets if they exist
  - task: DownloadPipelineArtifact@2
    displayName: Download Linux libtorch-cpu assets
    condition: eq('${{ parameters.BuildLibTorchPackages }}', true)
    inputs:
      artifact: LinuxAssets
      patterns: |
        libtorch-cpu-linux-x64/**
      path: $(Pipeline.Workspace)/LinuxAssets
    retryCountOnTaskFailure: 3
    continueOnError: true

  # Download CUDA assets if they exist (shouldn't be in this artifact but just in case)
  - task: DownloadPipelineArtifact@2
    displayName: Download any Linux CUDA assets (if present)
    condition: eq('${{ parameters.BuildLibTorchPackages }}', true)
    inputs:
      artifact: LinuxAssets
      patterns: |
        libtorch-cuda-*-linux-x64/**
      path: $(Pipeline.Workspace)/LinuxAssets
    retryCountOnTaskFailure: 2
    continueOnError: true

  # Copy Linux assets to build directory
  - task: CopyFiles@2
    displayName: Copy linux native assets (TorchSharp) to correct folder
    inputs:
      sourceFolder: $(Pipeline.Workspace)/LinuxAssets/TorchSharp
      targetFolder: $(Build.SourcesDirectory)/bin/obj/packprep/$(BuildConfig)/TorchSharp
      cleanTargetFolder: false

  - task: CopyFiles@2
    displayName: Copy linux native assets (TorchAudio) to correct folder
    inputs:
      sourceFolder: $(Pipeline.Workspace)/LinuxAssets/TorchAudio
      targetFolder: $(Build.SourcesDirectory)/bin/obj/packprep/$(BuildConfig)/TorchAudio
      cleanTargetFolder: false

  - task: CopyFiles@2
    displayName: Copy linux native assets (TorchVision) to correct folder
    inputs:
      sourceFolder: $(Pipeline.Workspace)/LinuxAssets/TorchVision
      targetFolder: $(Build.SourcesDirectory)/bin/obj/packprep/$(BuildConfig)/TorchVision
      cleanTargetFolder: false

  - task: CopyFiles@2
    displayName: Copy linux native assets to correct folder
    inputs:
      sourceFolder: $(Pipeline.Workspace)/LinuxAssets
      targetFolder: $(Build.SourcesDirectory)/bin/obj/packprep/$(BuildConfig)
      cleanTargetFolder: false

  - script: rmdir /s /q $(Pipeline.Workspace)\LinuxAssets
    displayName: Clean LinuxAssets immediately
    continueOnError: true

  # Process Mac assets - download in chunks
  - task: DownloadPipelineArtifact@2
    displayName: Download Mac TorchSharp assets
    inputs:
      artifact: MacAssets_arm64
      patterns: |
        TorchSharp/**
      path: $(Pipeline.Workspace)/MacAssets_arm64
    retryCountOnTaskFailure: 3

  - task: DownloadPipelineArtifact@2
    displayName: Download Mac TorchAudio assets
    inputs:
      artifact: MacAssets_arm64
      patterns: |
        TorchAudio/**
      path: $(Pipeline.Workspace)/MacAssets_arm64
    retryCountOnTaskFailure: 3

  - task: DownloadPipelineArtifact@2
    displayName: Download Mac TorchVision assets
    inputs:
      artifact: MacAssets_arm64
      patterns: |
        TorchVision/**
      path: $(Pipeline.Workspace)/MacAssets_arm64
    retryCountOnTaskFailure: 3

  # Download Mac libtorch assets if they exist
  - task: DownloadPipelineArtifact@2
    displayName: Download Mac libtorch-cpu assets (if present)
    condition: eq('${{ parameters.BuildLibTorchPackages }}', true)
    inputs:
      artifact: MacAssets_arm64
      patterns: |
        libtorch-cpu-osx-arm64/**
      path: $(Pipeline.Workspace)/MacAssets_arm64
    retryCountOnTaskFailure: 2
    continueOnError: true

  - task: CopyFiles@2
    displayName: Copy mac-arm64 native assets (batch)
    inputs:
      sourceFolder: $(Pipeline.Workspace)/MacAssets_arm64
      targetFolder: $(Build.SourcesDirectory)/bin/obj/packprep/$(BuildConfig)
      cleanTargetFolder: false

  - script: rmdir /s /q $(Pipeline.Workspace)\MacAssets_arm64
    displayName: Clean MacAssets immediately
    continueOnError: true

  # Process Windows assets - download in chunks
  - task: DownloadPipelineArtifact@2
    displayName: Download Windows TorchSharp assets
    inputs:
      artifact: WindowsAssets
      patterns: |
        TorchSharp/**
      path: $(Pipeline.Workspace)/WindowsAssets
    retryCountOnTaskFailure: 3

  - task: DownloadPipelineArtifact@2
    displayName: Download Windows TorchAudio assets
    inputs:
      artifact: WindowsAssets
      patterns: |
        TorchAudio/**
      path: $(Pipeline.Workspace)/WindowsAssets
    retryCountOnTaskFailure: 3

  - task: DownloadPipelineArtifact@2
    displayName: Download Windows TorchVision assets
    inputs:
      artifact: WindowsAssets
      patterns: |
        TorchVision/**
      path: $(Pipeline.Workspace)/WindowsAssets
    retryCountOnTaskFailure: 3

  - task: DownloadPipelineArtifact@2
    displayName: Download Windows libtorch-cpu assets
    condition: eq('${{ parameters.BuildLibTorchPackages }}', true)
    inputs:
      artifact: WindowsAssets
      patterns: |
        libtorch-cpu-win-x64/**
      path: $(Pipeline.Workspace)/WindowsAssets
    retryCountOnTaskFailure: 3
    continueOnError: true

  - task: CopyFiles@2
    displayName: Copy windows native assets (batch)
    inputs:
      sourceFolder: $(Pipeline.Workspace)/WindowsAssets
      targetFolder: $(Build.SourcesDirectory)/bin/obj/packprep/$(BuildConfig)
      cleanTargetFolder: false

  - script: rmdir /s /q $(Pipeline.Workspace)\WindowsAssets
    displayName: Clean WindowsAssets immediately
    continueOnError: true

  # Restore and pack
  - script: dotnet restore pkg/pack.proj /p:Configuration=Release --nologo
    displayName: Restore package projects

  - script: dotnet pack -c $(BuildConfig) --no-build -v:n /p:SkipNative=true /p:SkipTests=true /p:IncludeTorchSharpPackage=true /p:IncludeLibTorchCpuPackages=${{ parameters.BuildLibTorchPackages }} /p:GenerateCompatibilitySuppressionFile=true pkg/pack.proj
    displayName: Create Packages

  # Final cleanup before staging
  - script: |
      rmdir /q /s bin\obj
      for /d %%i in ($(Build.SourcesDirectory)\bin\packages\$(BuildConfig)\*.symbols.nupkg) do del "%%i"
    displayName: Clean up before staging
    continueOnError: true

  - task: CopyFiles@2
    displayName: Copy packaged assets to staging folder
    inputs:
      sourceFolder: $(Build.SourcesDirectory)/bin/packages/$(BuildConfig)
      targetFolder: $(Build.ArtifactStagingDirectory)
      contents: |
        **/*.nupkg
        !**/*.symbols.nupkg

  - publish: $(Build.ArtifactStagingDirectory)
    displayName: Publish build packages
    artifact: BuildTorchSharpPackages

  - publish: $(Build.SourcesDirectory)/config
    displayName: Publish signing config
    artifact: config

################################################################################
# Only run if BuildLibTorchPackages is true
- job: Build_libtorch_cuda_win_Packages
################################################################################
  condition: and(not(canceled()), ne(variables['system.pullrequest.isfork'], true), eq(variables['build.sourcebranchname'], '${{ parameters.SourceBranchName }}'), eq('${{ parameters.BuildLibTorchPackages }}', true))
  dependsOn:
  - Windows_Native_Build_For_Packages
  - Build_TorchSharp_And_libtorch_cpu_Packages
  timeoutInMinutes: 90
  variables:
    BuildConfig: Release
    OfficialBuildId: $(BUILD.BUILDNUMBER)
    DOTNET_CLI_TELEMETRY_OPTOUT: 1
    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
    DOTNET_MULTILEVEL_LOOKUP: 0
  pool:
    vmImage: 'windows-latest'
  steps:
  # Aggressive cleanup
  - script: |
      rmdir /q /s bin\obj 2>nul
      del /s /q *.pdb 2>nul
      del /s /q *.xml 2>nul
    displayName: Final cleanup
    continueOnError: true

  # Download Windows CUDA assets in smaller chunks
  - task: DownloadPipelineArtifact@2
    displayName: Download Windows CUDA 12.8 assets
    inputs:
      artifact: WindowsAssets
      patterns: |
        libtorch-cuda-12.8-win-x64/**
      path: $(Pipeline.Workspace)/WindowsAssets
    retryCountOnTaskFailure: 3

  - task: DownloadPipelineArtifact@2
    displayName: Download remaining Windows CUDA assets
    inputs:
      artifact: WindowsAssets
      patterns: |
        libtorch-cuda*/**
        !libtorch-cuda-12.8-win-x64/**
      path: $(Pipeline.Workspace)/WindowsAssets
    retryCountOnTaskFailure: 3
    continueOnError: true

  - task: CopyFiles@2
    displayName: Copy windows CUDA assets
    inputs:
      sourceFolder: $(Pipeline.Workspace)/WindowsAssets
      targetFolder: $(Build.SourcesDirectory)/bin/obj/packprep/$(BuildConfig)
      contents: |
        libtorch-cuda*/**

  - script: rmdir /s /q $(Pipeline.Workspace)\WindowsAssets
    displayName: Clean workspace immediately
    continueOnError: true

  - script: dotnet restore pkg/pack.proj /p:Configuration=Release --nologo
    displayName: Restore package projects

  - script: dotnet pack -c $(BuildConfig) --no-build -v:n /p:SkipNative=true /p:SkipTests=true /p:IncludeTorchSharpPackage=false /p:IncludeLibTorchCpuPackages=false /p:IncludeLibTorchCudaPackages=true /p:TargetOS=windows pkg/pack.proj
    displayName: Create CUDA Packages

  # Aggressive cleanup before staging
  - script: |
      rmdir /q /s bin\obj
      del /s /q *.pdb 2>nul
      del /s /q *.xml 2>nul
    displayName: Final cleanup
    continueOnError: true

  - task: CopyFiles@2
    displayName: Copy packaged assets to staging
    inputs:
      sourceFolder: $(Build.SourcesDirectory)/bin/packages/$(BuildConfig)
      targetFolder: $(Build.ArtifactStagingDirectory)
      contents: |
        *cuda*.nupkg
        !*.symbols.nupkg

  - publish: $(Build.ArtifactStagingDirectory)
    displayName: Publish Windows CUDA packages
    artifact: BuildWinCUDAPackages

################################################################################
- job: Build_libtorch_cuda_linux_Packages
################################################################################
  condition: and(not(canceled()), ne(variables['system.pullrequest.isfork'], true), eq(variables['build.sourcebranchname'], '${{ parameters.SourceBranchName }}'), eq('${{ parameters.BuildLibTorchPackages }}', true))
  dependsOn:
  - Linux_Native_Build_For_Packages
  - Build_libtorch_cuda_win_Packages
  timeoutInMinutes: 90
  variables:
    BuildConfig: Release
    OfficialBuildId: $(BUILD.BUILDNUMBER)
    DOTNET_CLI_TELEMETRY_OPTOUT: 1
    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
    DOTNET_MULTILEVEL_LOOKUP: 0
  pool:
    vmImage: 'ubuntu-latest'
  container: UbuntuContainer
  steps:
  - task: UseDotNet@2
    displayName: 'Use .NET Core sdk'
    inputs:
      packageType: sdk
      version: 8.0.x
      installationPath: $(Agent.ToolsDirectory)/dotnet

  - script: ${{ variables.LinuxPrepScript }}
    displayName: Install dependencies

  # Aggressive cleanup
  - script: |
      rm -rf .git
      dotnet nuget locals all --clear
      df -h
    displayName: Final cleanup
    continueOnError: true

  # Download Linux CUDA assets in smaller chunks
  - task: DownloadPipelineArtifact@2
    displayName: Download Linux CUDA 12.8 assets
    inputs:
      artifact: LinuxAssets
      patterns: |
        libtorch-cuda-12.8-linux-x64/**
      path: $(Pipeline.Workspace)/LinuxAssets
    retryCountOnTaskFailure: 3

  - task: DownloadPipelineArtifact@2
    displayName: Download remaining Linux CUDA assets
    inputs:
      artifact: LinuxAssets
      patterns: |
        libtorch-cuda*/**
        !libtorch-cuda-12.8-linux-x64/**
      path: $(Pipeline.Workspace)/LinuxAssets
    retryCountOnTaskFailure: 3
    continueOnError: true

  - task: CopyFiles@2
    displayName: Copy linux CUDA assets
    inputs:
      sourceFolder: $(Pipeline.Workspace)/LinuxAssets
      targetFolder: $(Build.SourcesDirectory)/bin/obj/packprep/$(BuildConfig)
      contents: |
        libtorch-cuda*/**

  - script: rm -rf $(Pipeline.Workspace)/LinuxAssets
    displayName: Clean workspace immediately
    continueOnError: true

  - script: dotnet restore pkg/pack.proj /p:Configuration=Release --nologo
    displayName: Restore package projects

  - script: dotnet pack -c $(BuildConfig) --no-build -v:n /p:SkipNative=true /p:SkipTests=true /p:ApiCompatGenerateSuppressionFile=true /p:IncludeTorchSharpPackage=false /p:IncludeLibTorchCpuPackages=false /p:IncludeLibTorchCudaPackages=true /p:TargetOS=linux pkg/pack.proj
    displayName: Create CUDA Packages

  # Final cleanup
  - script: |
      rm -rf bin/obj
      find . -name "*.pdb" -type f -delete
      find . -name "*.xml" -type f -delete
      df -h
    displayName: Final cleanup
    continueOnError: true

  - task: CopyFiles@2
    displayName: Copy packaged assets to staging
    inputs:
      sourceFolder: $(Build.SourcesDirectory)/bin/packages/$(BuildConfig)
      targetFolder: $(Build.ArtifactStagingDirectory)
      contents: |
        *cuda*.nupkg
        !*.symbols.nupkg

  - publish: $(Build.ArtifactStagingDirectory)
    displayName: Publish Linux CUDA packages
    artifact: BuildLinuxCUDAPackages

################################################################################
- job: CodeSign_Core
################################################################################
  condition: and(ne(variables['system.pullrequest.isfork'], true), eq(variables['build.sourcebranchname'], '${{ parameters.SourceBranchName }}'))
  dependsOn:
  - Build_TorchSharp_And_libtorch_cpu_Packages
  variables:
  - group: SignClient Credentials
  pool:
    vmImage: 'windows-latest'
  steps:

  - task: DotNetCoreCLI@2
    inputs:
      command: custom
      custom: tool
      arguments: install --tool-path . --prerelease sign
    displayName: Install SignTool tool

  - download: current
    displayName: Download configuration
    artifact: config

  - download: current
    displayName: Download TorchSharp Packages
    artifact: BuildTorchSharpPackages

  - task: PowerShell@2
    displayName: Sign packages
    inputs:
      targetType: 'inline'
      script: |
        & .\sign code azure-key-vault '**/*.nupkg' `
          --base-directory "$(Pipeline.Workspace)\BuildTorchSharpPackages" `
          --file-list "$(Pipeline.Workspace)\config\signedfiles.txt" `
          --azure-key-vault-url "$(SignVaultURL)" `
          --publisher-name "TorchSharp" `
          --description "TorchSharp" `
          --description-url "https://github.com/dotnet/TorchSharp" `
          --azure-key-vault-tenant-id "$(SignTenantId)" `
          --azure-key-vault-client-id "$(SignClientId)" `
          --azure-key-vault-client-secret "$(SignClientSecret)" `
          --azure-key-vault-certificate "$(SignCertFriendlyName)"

  - publish: $(Pipeline.Workspace)/BuildTorchSharpPackages
    displayName: Publish Signed TorchSharp Packages
    artifact: SignedTorchSharpPackages
    continueOnError: true

  - script: rmdir /s /q  $(Pipeline.Workspace)\BuildTorchSharpPackages
    displayName: Free up space (TorchSharp packages in workspace)

################################################################################
- job: CodeSign_Extras
################################################################################
  condition: and(ne(variables['system.pullrequest.isfork'], true), eq(variables['build.sourcebranchname'], '${{ parameters.SourceBranchName }}'), eq('${{ parameters.BuildLibTorchPackages }}', true))
  dependsOn:
  - Build_libtorch_cuda_win_Packages
  - Build_libtorch_cuda_linux_Packages
  variables:
  - group: SignClient Credentials
  pool:
    vmImage: 'windows-latest'
  steps:

  - task: DotNetCoreCLI@2
    inputs:
      command: custom
      custom: tool
      arguments: install --tool-path . --prerelease sign
    displayName: Install SignTool tool

  - download: current
    displayName: Download configuration
    artifact: config

  - download: current
    displayName: Download Windows CUDA Packages
    artifact: BuildWinCUDAPackages

  - task: PowerShell@2
    displayName: Sign packages
    inputs:
      targetType: 'inline'
      script: |
        & .\sign code azure-key-vault '**/*.nupkg' `
          --base-directory "$(Pipeline.Workspace)\BuildWinCUDAPackages" `
          --file-list "$(Pipeline.Workspace)\config\signedfiles.txt" `
          --azure-key-vault-url "$(SignVaultURL)" `
          --publisher-name "TorchSharp" `
          --description "TorchSharp" `
          --description-url "https://github.com/dotnet/TorchSharp" `
          --azure-key-vault-tenant-id "$(SignTenantId)"  `
          --azure-key-vault-client-id "$(SignClientId)" `
          --azure-key-vault-client-secret "$(SignClientSecret)" `
          --azure-key-vault-certificate "$(SignCertFriendlyName)"

  - publish: $(Pipeline.Workspace)/BuildWinCUDAPackages
    displayName: Publish Signed Windows CUDA Packages
    artifact: SignedWinCUDAPackages
    continueOnError: true

  - script: rmdir /s /q  $(Pipeline.Workspace)\BuildWinCUDAPackages
    displayName: Free up space (TorchSharp packages in workspace)

  - download: current
    displayName: Download Linux CUDA Packages
    artifact: BuildLinuxCUDAPackages

  - task: PowerShell@2
    displayName: Sign packages with Sign CLI
    inputs:
      targetType: 'inline'
      script: |
        & .\sign code azure-key-vault '**/*.nupkg' `
          --base-directory "$(Pipeline.Workspace)\BuildLinuxCUDAPackages" `
          --file-list "$(Pipeline.Workspace)\config\signedfiles.txt" `
          --azure-key-vault-url "$(SignVaultURL)" `
          --publisher-name "TorchSharp" `
          --description "TorchSharp" `
          --description-url "https://github.com/dotnet/TorchSharp" `
          --azure-key-vault-tenant-id "$(SignTenantId)" `
          --azure-key-vault-client-id "$(SignClientId)" `
          --azure-key-vault-client-secret "$(SignClientSecret)" `
          --azure-key-vault-certificate "$(SignCertFriendlyName)"

  - publish: $(Pipeline.Workspace)/BuildLinuxCUDAPackages
    displayName: Publish Signed Linux CUDA Packages
    artifact: SignedLinuxCUDAPackages
    continueOnError: true

  - script: rmdir /s /q  $(Pipeline.Workspace)\BuildLinuxCUDAPackages
    displayName: Free up space (TorchSharp packages in workspace)

################################################################################
- job: Push_TorchSharp_And_libtorch_cpu_Packages
################################################################################
  condition: and(eq('${{ parameters.PushPackagesToFeed }}', true), ne(variables['system.pullrequest.isfork'], true), eq(variables['build.sourcebranchname'], '${{ parameters.SourceBranchName }}'))
  dependsOn:
  - Build_TorchSharp_And_libtorch_cpu_Packages
  - CodeSign_Core
  variables:
    BuildConfig: Release
    OfficialBuildId: $(BUILD.BUILDNUMBER)
    DOTNET_CLI_TELEMETRY_OPTOUT: 1
    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
    DOTNET_MULTILEVEL_LOOKUP: 0
  pool:
    vmImage: 'windows-latest'
  steps:

  # Push packages to feed
  #
  # Doing these one by one as we're getting a lot of failures pushing them
  # Also replaying them multiple times for the same reason
  #
  - task: NuGetAuthenticate@0
    displayName: 'NuGet Authenticate'

  - download: current
    displayName: Download Signed TorchSharp Packages
    artifact: SignedTorchSharpPackages

  - task: NuGetCommand@2
    displayName: 'NuGet push (TorchSharp)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/TorchSharp.*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
    continueOnError: true

  - task: NuGetCommand@2
    displayName: 'NuGet push (TorchAudio)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/TorchAudio.*.nupkg'
      publishVstsFeed: 'TorchSharp/SignedPackages'
      allowPackageConflicts: true
    continueOnError: true

  - task: NuGetCommand@2
    displayName: 'NuGet push (TorchVision)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/TorchVision.*.nupkg'
      publishVstsFeed: 'TorchSharp/SignedPackages'
      allowPackageConflicts: true
    continueOnError: true

  - task: NuGetCommand@2
    displayName: 'NuGet push (TorchSharp-cpu)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/TorchSharp-*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
    continueOnError: true

  # push the CPU runtime packages
  - task: NuGetCommand@2
    displayName: 'NuGet push (libtorch-cpu)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
    continueOnError: true

  # push the CPU runtime packages (retry - we get so many failures!)
  - task: NuGetCommand@2
    displayName: 'NuGet push (libtorch-cpu - retry)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
    continueOnError: true

  # push the CPU runtime packages (retry - we get so many failures!)
  - task: NuGetCommand@2
    displayName: 'NuGet push (libtorch-cpu - retry)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
    continueOnError: true

  # Terminate all dotnet build processes.
  - script: dotnet build-server shutdown
    displayName: Dotnet Server Shutdown


################################################################################
# Only run if BuildLibTorchPackages is true
- job: Push_libtorch_cuda_win_Packages
################################################################################
  condition: and(eq('${{ parameters.BuildLibTorchPackages }}', true), eq('${{ parameters.PushPackagesToFeed }}', true), ne(variables['system.pullrequest.isfork'], true))
  dependsOn:
  - Build_libtorch_cuda_win_Packages
  - CodeSign_Extras
  variables:
    BuildConfig: Release
    OfficialBuildId: $(BUILD.BUILDNUMBER)
    DOTNET_CLI_TELEMETRY_OPTOUT: 1
    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
    DOTNET_MULTILEVEL_LOOKUP: 0
  pool:
    vmImage: 'windows-latest'
  steps:

  # Push packages to feed
  #
  # Doing these one by one as we're getting a lot of failures pushing them
  # Also replaying them multiple times for the same reason
  #
  - task: NuGetAuthenticate@0
    displayName: 'NuGet Authenticate'

  - download: current
    displayName: Download Signed Windows CUDA Packages
    artifact: SignedWinCUDAPackages

  # push the Windows Cuda packages
  - task: NuGetCommand@2
    displayName: 'NuGet push (cuda win-x64)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/*cuda*win*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
      # often fails - try but ignore the error until we sort it out
    continueOnError: true

  # push the Windows Cuda packages
  - task: NuGetCommand@2
    displayName: 'NuGet push (cuda win-x64 retry)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/*cuda*win*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
      # often fails - try but ignore the error until we sort it out
    continueOnError: true

  # push the Windows Cuda packages
  - task: NuGetCommand@2
    displayName: 'NuGet push (cuda win-x64 retry)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/*cuda*win*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
      # often fails - try but ignore the error until we sort it out
    continueOnError: true

  # push the Windows Cuda packages
  - task: NuGetCommand@2
    displayName: 'NuGet push (cuda win-x64 retry)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/*cuda*win*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
      # often fails - try but ignore the error until we sort it out
    continueOnError: true

  # push the Windows Cuda packages
  - task: NuGetCommand@2
    displayName: 'NuGet push (cuda win-x64 retry)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/*cuda*win*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
      # often fails - try but ignore the error until we sort it out
    continueOnError: true


################################################################################
- job: Push_libtorch_cuda_linux_Packages
################################################################################
  condition: and(eq('${{ parameters.BuildLibTorchPackages }}', 'true'), eq('${{ parameters.PushPackagesToFeed }}', true), ne(variables['system.pullrequest.isfork'], true))
  dependsOn:
  - Build_libtorch_cuda_linux_Packages
  - CodeSign_Extras
  variables:
    BuildConfig: Release
    OfficialBuildId: $(BUILD.BUILDNUMBER)
    DOTNET_CLI_TELEMETRY_OPTOUT: 1
    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1
    DOTNET_MULTILEVEL_LOOKUP: 0
  pool:
    vmImage: 'windows-latest'
  # container: UbuntuContainer
  # Exact copy of the dependency install above - TODO share this somewhere
  steps:

  # Push packages to feed
  #
  # Doing these one by one as we're getting a lot of failures pushing them
  # Also replaying them multiple times for the same reason
  #
  - task: NuGetAuthenticate@0
    displayName: 'NuGet Authenticate'

  - download: current
    displayName: Download Signed Linux CUDA Packages
    artifact: SignedLinuxCUDAPackages

  # push the Linux Cuda packages
  - task: NuGetCommand@2
    displayName: 'NuGet push (cuda linux-x64)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
      # often fails - try but ignore the error until we sort it out
    continueOnError: true

  # push the Linux Cuda packages
  - task: NuGetCommand@2
    displayName: 'NuGet push (cuda linux-x64)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
      # often fails - try but ignore the error until we sort it out
    continueOnError: true

  # push the Linux Cuda packages
  - task: NuGetCommand@2
    displayName: 'NuGet push (cuda linux-x64)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
      # often fails - try but ignore the error until we sort it out
    continueOnError: true

  # push the Linux Cuda packages
  - task: NuGetCommand@2
    displayName: 'NuGet push (cuda linux-x64)'
    inputs:
      command: push
      packagesToPush:  '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg'
      publishVstsFeed: 'TorchSharp/TestPackages'
      allowPackageConflicts: true
      # often fails - try but ignore the error until we sort it out
    continueOnError: true