From 107a2634f7f93313b594ce5ff1d63d34b7fdfcf5 Mon Sep 17 00:00:00 2001 From: Dylan Simon Date: Thu, 9 Jul 2026 17:12:39 -0400 Subject: [PATCH 1/5] jenkins: WIP migration to jenkins-new --- Jenkinsfile | 526 ++++++++++++++-------------------------------------- 1 file changed, 144 insertions(+), 382 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0330bf5a1d..e90219f5a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,397 +1,159 @@ -@Library('StanUtils') -import org.stan.Utils - -def utils = new org.stan.Utils() -def skipRemainingStages = false - -def setupCXX(CXX = env.CXX) { - unstash 'CmdStanSetup' - - stanc3_bin_url_str = params.stanc3_bin_url != "nightly" ? "\nSTANC3_TEST_BIN_URL=${params.stanc3_bin_url}\n" : "" - writeFile(file: "make/local", text: "CXX=${CXX} \n${stanc3_bin_url_str} \nCXXFLAGS+=-Wp,-D_GLIBCXX_ASSERTIONS\n") -} - -def runTests(String prefix = "") { - "${prefix}runCmdStanTests.py -j${env.PARALLEL} src/test/interface" -} - -def runWinTests(String prefix = "") { - withEnv(["PATH+TBB=${WORKSPACE}\\stan\\lib\\stan_math\\lib\\tbb"]) { - bat """ - SET \"PATH=${env.RTOOLS40_HOME};%PATH%\" - SET \"PATH=${env.RTOOLS40_HOME}\\usr\\bin;${LLVM7}\\bin;%PATH%\" // - SET \"PATH=${env.RTOOLS40_HOME}\\mingw64\\bin;%PATH%\" - SET \"PATH=C:\\PROGRA~1\\R\\R-4.1.2\\bin;%PATH%\" - SET \"PATH=C:\\PROGRA~1\\Microsoft^ MPI\\Bin;%PATH%\" - SET \"MPI_HOME=C:\\PROGRA~1\\Microsoft^ MPI\\Bin\" - SET \"PATH=C:\\Users\\jenkins\\Anaconda3;%PATH%\" - python ${prefix}runCmdStanTests.py -j${env.PARALLEL} src/test/interface +properties([ + disableConcurrentBuilds(abortPrevious: env.BRANCH_NAME != "downstream_tests" && env.BRANCH_NAME != "downstream_hotfix"), + buildDiscarder(logRotator(numToKeepStr: '20', daysToKeepStr: '30')), + parameters([ + string(defaultValue: '', name: 'stan_pr', + description: "Stan PR to test against. Will check out this PR in the downstream Stan repo."), + string(defaultValue: '', name: 'math_pr', + description: "Math PR to test against. Will check out this PR in the downstream Math repo."), + string(defaultValue: 'nightly', name: 'stanc3_bin_url', + description: 'Custom stanc3 binary url') + ]) +]) + +def runRemainingStages = false +def WIN_CXX = 'g++' +def MPI_CXX = 'mpicxx.openmpi' +def MAC_CXX = 'clang++' + +catchError { + withEnv([ + 'GIT_AUTHOR_NAME=Stan Jenkins', + 'GIT_AUTHOR_EMAIL=mc.stanislaw@gmail.com', + 'GIT_COMMITTER_NAME=Stan Jenkins', + 'GIT_COMMITTER_EMAIL=mc.stanislaw@gmail.com' + ]) { + runPod(image: "stanorg/ci:gpu", cpus: 2) { + runRemainingStages = filesChanged('src/cmdstan', 'src/test', 'lib', 'examples', 'make', 'stan', 'install-tbb.bat', 'makefile', 'runCmdStanTests.py', 'test-all.sh', 'Jenkinsfile') + + stage('clang-format') { + def dirty = sh returnStatus: true, script: """ + clang-format --version + git ls-files 'src/*.hpp' 'src/*.cpp' | xargs -n20 -P\$PARALLEL clang-format -i + git diff --exit-code """ + if (dirty) { + def branch = env.CHANGE_BRANCH ?: env.BRANCH_NAME + def repo = env.CHANGE_FORK ?: "stan-dev" + if (!("/" in repo)) + repo += "/cmdstan.git" + echo "Exiting build because clang-format found changes." + emailext ( + subject: "[StanJenkins] Autoformattted: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", + body: """ +Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' has been autoformatted and the +changes committed to your branch, if permissions allowed. Please pull these +changes before continuing. + +See https://github.com/stan-dev/stan/wiki/Coding-Style-and-Idioms for setting +up the autoformatter locally. (Check console output at ${env.BUILD_URL}) +""", + recipientProviders: [[$class: 'RequesterRecipientProvider']], + to: env.CHANGE_AUTHOR_EMAIL) + sh ''' + git add -u src + git commit -m "[Jenkins] auto-formatting by `clang-format --version`" + ''' + gitPush(gitScm: scmGit( + userRemoteConfigs: [[credentialsId: "stan-github", name: 'dest', url: "https://github.com/$repo"]], + branches: [[name: "refs/heads/$branch"]]), + targetBranch: branch, + targetRepo: 'dest') + echo "Those changes are now found on stan-dev/cmdstan under $repo branch $branch" + echo "Please 'git pull' before continuing to develop." + error "clang-format changes" + } + } } -} - -def deleteDirWin() { - bat "attrib -r -s /s /d" - deleteDir() -} - -Boolean isPR() { env.CHANGE_URL != null } -String fork() { env.CHANGE_FORK ?: "stan-dev" } -String branchName() { isPR() ? env.CHANGE_BRANCH :env.BRANCH_NAME } - -pipeline { - agent none - options { - skipDefaultCheckout() - disableConcurrentBuilds(abortPrevious: env.BRANCH_NAME != "downstream_tests" && env.BRANCH_NAME != "downstream_hotfix") - } - parameters { - string(defaultValue: '', name: 'stan_pr', - description: "Stan PR to test against. Will check out this PR in the downstream Stan repo.") - string(defaultValue: '', name: 'math_pr', - description: "Math PR to test against. Will check out this PR in the downstream Math repo.") - string(defaultValue: 'nightly', name: 'stanc3_bin_url', - description: 'Custom stanc3 binary url') - } - environment { - MAC_CXX = 'clang++' - LINUX_CXX = 'clang++-6.0' - WIN_CXX = 'g++' - PARALLEL = 4 - MPICXX = 'mpicxx.openmpi' - GIT_AUTHOR_NAME = 'Stan Jenkins' - GIT_AUTHOR_EMAIL = 'mc.stanislaw@gmail.com' - GIT_COMMITTER_NAME = 'Stan Jenkins' - GIT_COMMITTER_EMAIL = 'mc.stanislaw@gmail.com' + def scmFull = scmGit( + branches: scm.branches, + userRemoteConfigs: scm.userRemoteConfigs, + extensions: scm.extensions + [cleanBeforeCheckout(), + submodule(recursiveSubmodules: true, shallow: true, depth: 2)]) + + def prepTests = { extra -> + checkout scmFull + checkoutPR("stan", params.stan_pr) + checkoutPR("stan/lib/stan_math", params.math_pr) + + def local = "CXXFLAGS+=-Wp,-D_GLIBCXX_ASSERTIONS\n" + if (params.stanc3_bin_url != "nightly") { + local += "STANC3_TEST_BIN_URL=${params.stanc3_bin_url}\n" + } + writeFile(file: "make/local", text: local + extra) } - stages { - stage('Clean & Setup') { - agent { - docker { - image 'stanorg/ci:gpu' - label 'linux' - } - } - steps { - retry(3) { checkout scm } - sh 'git clean -xffd' - sh 'make stan-revert' - script { - utils.checkout_pr("stan", "stan", params.stan_pr) - utils.checkout_pr("math", "stan/lib/stan_math", params.math_pr) - } - stash 'CmdStanSetup' + if (runRemainingStages) { + parallel windows: { + node('windows') { + stage('Windows interface tests') { + prepTests("CXX=${WIN_CXX}") + withEnv(["PATH+TBB=${WORKSPACE}\\stan\\lib\\stan_math\\lib\\tbb"]) { + bat ''' + SET "PATH=%RTOOLS%;%RTOOLS%\\usr\\bin;%PATH%" + SET "PATH=%RTOOLS%\\x86_64-w64-mingw32.static.posix\\bin;%PATH%" + SET "PATH=%CONDA%;%PATH%" + python runCmdStanTests.py -j%PARALLEL% src/test/interface + ''' } - post { always { deleteDir() }} + } } - stage('Verify changes') { - agent { - docker { - image 'stanorg/ci:gpu' - label 'linux' - } - } - steps { - script { - retry(3) { checkout scm } - sh 'git clean -xffd' - - def paths = ['src/cmdstan', 'src/test', 'lib', 'examples', 'make', 'stan', 'install-tbb.bat', 'makefile', 'runCmdStanTests.py', 'test-all.sh', 'Jenkinsfile'].join(" ") - skipRemainingStages = utils.verifyChanges(paths) - } - } - post { - always { - deleteDir() - } - } + }, linux: { + runPod(image: "stanorg/ci:gpu", checkout: false) { + stage('Linux interface tests with MPI') { + prepTests("""CXX=${MPI_CXX} +STAN_MPI=true +CXX_TYPE=gcc""") + sh "make build-mpi > build-mpi.log 2>&1" + sh './runCmdStanTests.py -j$PARALLEL src/test/interface' + } } - stage("Clang-format") { - agent { - docker { - image 'stanorg/ci:gpu' - label 'linux' - } - } - steps { - retry(3) { checkout scm } - withCredentials([usernamePassword(credentialsId: 'a630aebc-6861-4e69-b497-fd7f496ec46b', - usernameVariable: 'GIT_USERNAME', passwordVariable: 'GIT_PASSWORD')]) { - sh """#!/bin/bash - set -x - git checkout -b ${branchName()} - clang-format --version - find src -name '*.hpp' -o -name '*.cpp' | xargs -n20 -P${env.PARALLEL} clang-format -i - if [[ `git diff` != "" ]]; then - git add src - git commit -m "[Jenkins] auto-formatting by `clang-format --version`" - git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${fork()}/cmdstan.git ${branchName()} - echo "Exiting build because clang-format found changes." - echo "Those changes are now found on stan-dev/cmdstan under branch ${branchName()}" - echo "Please 'git pull' before continuing to develop." - exit 1 - fi - """ - } - } - post { - always { deleteDir() } - failure { - script { - emailext ( - subject: "[StanJenkins] Autoformattted: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'", - body: "Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' " + - "has been autoformatted and the changes committed " + - "to your branch, if permissions allowed." + - "Please pull these changes before continuing." + - "\n\n" + - "See https://github.com/stan-dev/stan/wiki/Coding-Style-and-Idioms" + - " for setting up the autoformatter locally.\n"+ - "(Check console output at ${env.BUILD_URL})", - recipientProviders: [[$class: 'RequesterRecipientProvider']], - to: "${env.CHANGE_AUTHOR_EMAIL}" - ) - } - } - } + }, mac: { + node('macos') { + stage('Mac interface tests') { + prepTests("CXX=${MAC_CXX}") + sh 'python3 ./runCmdStanTests.py -j$PARALLEL src/test/interface' + } } - stage('Parallel tests') { - when { - expression { - !skipRemainingStages - } - } - parallel { - stage('Windows interface tests') { - agent { label 'windows' } - steps { - setupCXX(WIN_CXX) - runWinTests() - } - post { - always { - - recordIssues( - id: "Windows", - name: "Windows interface tests", - enabledForFailure: true, - aggregatingResults : false, - filters: [ - excludeFile('/lib/.*'), - excludeFile('tbb/*'), - excludeFile('stan/lib/stan_math/lib/*'), - excludeMessage(".*'sprintf' is deprecated.*") - ], - tools: [ - gcc4(id: "Windows_gcc4", name: "Windows interface tests@GCC4"), - clang(id: "Windows_clang", name: "Windows interface tests@CLANG") - ], - qualityGates: [[threshold: 1, type: 'TOTAL', unstable: true]], - healthy: 10, unhealthy: 100, minimumSeverity: 'HIGH' - ) - - deleteDirWin() - } - } - } - - stage('Linux interface tests with MPI') { - agent { - docker { - image 'stanorg/ci:gpu' - label 'linux' - } - } - steps { - setupCXX(MPICXX) - sh "echo STAN_MPI=true >> make/local" - sh "echo CXX_TYPE=gcc >> make/local" - sh "make build-mpi > build-mpi.log 2>&1" - sh runTests("./") - } - post { - always { - - recordIssues( - id: "Linux_mpi", - name: "Linux interface tests with MPI", - enabledForFailure: true, - aggregatingResults : false, - filters: [ - excludeFile('/lib/.*'), - excludeFile('tbb/*'), - excludeFile('stan/lib/stan_math/lib/*'), - excludeMessage(".*'sprintf' is deprecated.*") - ], - tools: [ - gcc4(id: "Linux_mpi_gcc4", name: "Linux interface tests with MPI@GCC4"), - clang(id: "Linux_mpi_clang", name: "Linux interface tests with MPI@CLANG") - ], - qualityGates: [[threshold: 1, type: 'TOTAL', unstable: true]], - healthy: 10, unhealthy: 100, minimumSeverity: 'HIGH' - ) - - deleteDir() - } - } - } - - stage('Mac interface tests') { - agent { label 'osx' } - steps { - setupCXX(MAC_CXX) - sh runTests("python3 ./") - } - post { - always { - - recordIssues( - id: "Mac", - name: "Mac interface tests", - enabledForFailure: true, - aggregatingResults : false, - filters: [ - excludeFile('/lib/.*'), - excludeFile('tbb/*'), - excludeFile('stan/lib/stan_math/lib/*'), - excludeMessage(".*'sprintf' is deprecated.*") - ], - tools: [ - gcc4(id: "Mac_gcc4", name: "Mac interface tests@GCC4"), - clang(id: "Mac_clang", name: "Mac interface tests@CLANG") - ], - qualityGates: [[threshold: 1, type: 'TOTAL', unstable: true]], - healthy: 10, unhealthy: 100, minimumSeverity: 'HIGH' - ) - - deleteDir() - } - } - } - - stage('Upstream CmdStan Performance tests') { - when { - expression { - env.BRANCH_NAME ==~ /PR-\d+/ || - env.BRANCH_NAME == "downstream_tests" || - env.BRANCH_NAME == "downstream_hotfix" - } - } - steps { - script{ - build( - job: "Stan/CmdStan Performance Tests/downstream_tests", - parameters: [ - string(name: 'cmdstan_pr', value: env.BRANCH_NAME), - string(name: 'stan_pr', value: params.stan_pr), - string(name: 'math_pr', value: params.math_pr), - string(name: 'stanc3_bin_url', value: params.stanc3_bin_url) - ], - wait:true - ) - } - } - } - - } + }, upstream: { + if (env.BRANCH_NAME == "downstream_tests" + || env.BRANCH_NAME == "downstream_hotfix" + || env.CHANGE_TARGET) { + stage('Upstream CmdStan Performance tests') { + build( + job: "CCM/Stan/CmdStan Performance Tests/downstream_tests", + parameters: [ + string(name: 'cmdstan_pr', value: env.BRANCH_NAME), + string(name: 'stan_pr', value: params.stan_pr), + string(name: 'math_pr', value: params.math_pr), + string(name: 'stanc3_bin_url', value: params.stanc3_bin_url) + ], + wait:true) + } } - stage('Update downstream branches') { - parallel { - stage('Update downstream_hotfix - master') { - agent { - docker { - image 'ellerbrock/alpine-bash-git' - label 'linux' - args '--entrypoint=' - } - } - when { - beforeAgent true - branch 'master' - } - steps { - script { - retry(3) { - checkout([ - $class: 'GitSCM', - branches: [[name: '*/master'], [name: '*/downstream_hotfix']], - userRemoteConfigs: scm.userRemoteConfigs - ]) - } - withCredentials([usernamePassword(credentialsId: 'a630aebc-6861-4e69-b497-fd7f496ec46b', - usernameVariable: 'GIT_USERNAME', passwordVariable: 'GIT_PASSWORD')]) { - sh """#!/bin/bash - set -x - - git checkout downstream_hotfix - git reset --hard origin/master - git status - git push -f https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/stan-dev/cmdstan.git downstream_hotfix - """ - } - } - } - post { - always { - deleteDir() - } - } - } - stage('Update downstream_tests - develop') { - agent { - docker { - image 'ellerbrock/alpine-bash-git' - label 'linux' - args '--entrypoint=' - } - } - when { - beforeAgent true - branch 'develop' - } - steps { - script { - retry(3) { - checkout([ - $class: 'GitSCM', - branches: [[name: '*/develop'], [name: '*/downstream_tests']], - userRemoteConfigs: scm.userRemoteConfigs - ]) - } - withCredentials([usernamePassword(credentialsId: 'a630aebc-6861-4e69-b497-fd7f496ec46b', - usernameVariable: 'GIT_USERNAME', passwordVariable: 'GIT_PASSWORD')]) { - sh """#!/bin/bash - set -x + } + /* TODO recordIssues? */ + } - git checkout downstream_tests - git reset --hard origin/develop - git status - git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/stan-dev/cmdstan.git downstream_tests - """ - } - } - } - post { - always { - deleteDir() - } - } - } + def downstreams = [ + master: 'downstream_hotfix', + develop: 'downstream_tests' + ] + def downstream = downstreams[env.BRANCH_NAME] + if (downstream) { + podTemplate(inheritFrom: 'jnlp') { + node(POD_LABEL) { + stage('Update downstream branch') { + checkout scm + withCredentials([gitUsernamePassword(credentialsId: 'stan-github', gitToolName: 'git-tool')]) { + sh "git push --force origin HEAD:refs/heads/$downstream" } + } } + } } - post { - success { - script { - if (env.BRANCH_NAME == "develop") { - build job: "Stan/CmdStan Performance Tests/master", wait:false - } - utils.mailBuildResults("SUCCESSFUL") - } - } - unstable { script { utils.mailBuildResults("UNSTABLE", "stan-buildbot@googlegroups.com") } } - failure { script { utils.mailBuildResults("FAILURE", "stan-buildbot@googlegroups.com") } } - } + } } + +emailFailure() From 3d7191fe9334ec531497e55b814f3dfa4eeab8a4 Mon Sep 17 00:00:00 2001 From: Dylan Simon Date: Sat, 11 Jul 2026 19:32:29 -0400 Subject: [PATCH 2/5] jenkins: don't assume globbing works The old windows setup must have been using a different shell somehow --- src/test/interface/diagnose_test.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test/interface/diagnose_test.cpp b/src/test/interface/diagnose_test.cpp index 455d2ba594..a365ed30af 100644 --- a/src/test/interface/diagnose_test.cpp +++ b/src/test/interface/diagnose_test.cpp @@ -74,9 +74,10 @@ TEST(CommandDiagnose, mix) { std::string command = "bin" + path_separator + "diagnose"; std::string csv_file = "src" + path_separator + "test" + path_separator + "interface" + path_separator + "example_output" - + path_separator + "mix_output.*"; + + path_separator + "mix_output."; - auto out = run_command(command + " " + csv_file); + auto out + = run_command(command + " " + csv_file + "1.csv " + csv_file + "2.csv"); ASSERT_FALSE(out.hasError) << "\"" << out.command << "\" quit with an error"; compare_to_stored_output(out.output, @@ -89,9 +90,10 @@ TEST(CommandDiagnose, divergences) { std::string command = "bin" + path_separator + "diagnose"; std::string csv_file = "src" + path_separator + "test" + path_separator + "interface" + path_separator + "example_output" - + path_separator + "div_output*.csv"; + + path_separator + "div_output_"; - auto out = run_command(command + " " + csv_file); + auto out + = run_command(command + " " + csv_file + "1.csv " + csv_file + "2.csv"); ASSERT_FALSE(out.hasError) << "\"" << out.command << "\" quit with an error"; compare_to_stored_output(out.output, From 5842f148c8f9888f2f4a3a94372c9073726772e3 Mon Sep 17 00:00:00 2001 From: Dylan Simon Date: Mon, 13 Jul 2026 22:19:05 -0400 Subject: [PATCH 3/5] jenkins: try with -Werror --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index e90219f5a6..ba5181db3a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,7 +77,7 @@ up the autoformatter locally. (Check console output at ${env.BUILD_URL}) checkoutPR("stan", params.stan_pr) checkoutPR("stan/lib/stan_math", params.math_pr) - def local = "CXXFLAGS+=-Wp,-D_GLIBCXX_ASSERTIONS\n" + def local = "CXXFLAGS+=-Wp,-D_GLIBCXX_ASSERTIONS -Werror\n" if (params.stanc3_bin_url != "nightly") { local += "STANC3_TEST_BIN_URL=${params.stanc3_bin_url}\n" } From 0c72767647c52eaefbe4e9ad4d201dd8df3927ce Mon Sep 17 00:00:00 2001 From: Dylan Simon Date: Mon, 13 Jul 2026 23:22:59 -0400 Subject: [PATCH 4/5] jenkins: mac doesn't like a -c on the dep line --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index ba5181db3a..6c7e3cb6e0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -77,7 +77,7 @@ up the autoformatter locally. (Check console output at ${env.BUILD_URL}) checkoutPR("stan", params.stan_pr) checkoutPR("stan/lib/stan_math", params.math_pr) - def local = "CXXFLAGS+=-Wp,-D_GLIBCXX_ASSERTIONS -Werror\n" + def local = "CXXFLAGS+=-Wp,-D_GLIBCXX_ASSERTIONS -Werror -Wno-unused-command-line-argument\n" if (params.stanc3_bin_url != "nightly") { local += "STANC3_TEST_BIN_URL=${params.stanc3_bin_url}\n" } From d2209f5d66fbaa718b97883a38bd183f2d4fd1f2 Mon Sep 17 00:00:00 2001 From: Dylan Simon Date: Fri, 17 Jul 2026 21:54:23 -0400 Subject: [PATCH 5/5] jenkins: update performance tests target --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6c7e3cb6e0..010d2f85ff 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -122,7 +122,7 @@ CXX_TYPE=gcc""") || env.CHANGE_TARGET) { stage('Upstream CmdStan Performance tests') { build( - job: "CCM/Stan/CmdStan Performance Tests/downstream_tests", + job: "CCM/Stan/cmdstan performance tests/master", parameters: [ string(name: 'cmdstan_pr', value: env.BRANCH_NAME), string(name: 'stan_pr', value: params.stan_pr),