diff --git a/.github/workflows/dsBaseClient_test_suite.yaml b/.github/workflows/dsBaseClient_test_suite.yaml index b8a6f3ccf..573d73d1b 100644 --- a/.github/workflows/dsBaseClient_test_suite.yaml +++ b/.github/workflows/dsBaseClient_test_suite.yaml @@ -1,13 +1,35 @@ ################################################################################ # DataSHIELD GHA test suite - dsBaseClient -# Adapted from `armadillo_azure-pipelines.yml` by Roberto Villegas-Diaz +# Replaces azure-pipelines.yml / opal_azure-pipelines.yml / armadillo_azure-pipelines.yml. # -# Inside the root directory $(Pipeline.Workspace) will be a file tree like: -# /dsBaseClient <- Checked out version of datashield/dsBaseClient -# /dsBaseClient/logs <- Where results of tests and logs are collated -# /testStatus <- Checked out version of datashield/testStatus +# Structure (all jobs below run in parallel except where "needs" says otherwise): +# r-checks - devtools::document()/check() sync checks. No backend needed. +# opal-dsbase (matrix x4 shards) - dsBase suite against Opal, one shard each. +# opal-dsdanger - dsDanger suite against Opal (small, not sharded). +# opal-report - needs the two Opal jobs above; merges + publishes report. +# armadillo-dsbase (matrix x4 shards) - dsBase suite against Armadillo. +# armadillo-dsdanger - dsDanger suite against Armadillo. +# armadillo-report - needs the two Armadillo jobs above; merges + publishes report. # -# As of Sept. 2025 this takes ~ 95 mins to run. +# Each dsbase/dsdanger job spins up its OWN backend instance (isolated - no +# shared server state / concurrency risk between shards). This is what makes +# splitting the ~260-file dsBase suite into shards safe: shard 4x lets that +# phase run in roughly a quarter of the wall-clock instead of one long +# sequential testthat::test_package() call. +# +# Opal runs via docker-compose (docker-compose_opal.yml). +# Armadillo runs as a plain `java -jar` process (not docker-compose): Armadillo +# self-manages its Rock container over the host Docker socket +# (docker-management-enabled: true / docker-run-in-container: false), which skips +# building/pulling the old custom armadillo_citest image. See +# molgenis-service-armadillo's application.template.yml for that flag pairing. +# +# Every job starts its backend as the very first step so it boots in the +# background while R dependencies install, instead of paying for both serially. +# +# As of Sept. 2025 the single-backend, dsBase-only, unsharded version of this +# took ~ 95 mins; the full (Opal+Armadillo, dsBase+dsDanger) unsharded run is +# well over an hour. ################################################################################ name: dsBaseClient tests' suite @@ -17,27 +39,94 @@ on: - cron: '0 0 * * 0' # Weekly - cron: '0 1 * * *' # Nightly +permissions: + contents: read + +env: + _r_check_system_clock_: 0 + WORKFLOW_ID: ${{ github.run_id }}-${{ github.run_attempt }} + PROJECT_NAME: dsBaseClient + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + REPO_OWNER: ${{ github.repository_owner }} + R_KEEP_PKG_SOURCE: yes + GITHUB_TOKEN: ${{ github.token || 'placeholder-token' }} + # These should all be constant, except the two TEST_FILTER_* values. These can + # be used to test subsets of test files in the testthat directory. Options are + # like: '*' <- run all tests, '*_smk_*' <- run all the smoke tests. + TEST_FILTER_DSBASE: '_-|datachk-|smk-|arg-|disc-|perf-|smk_expt-|expt-|math-' + TEST_FILTER_DSDANGER: '__dgr-|datachk_dgr-|smk_dgr-|arg_dgr-|disc_dgr-|smk_expt_dgr-|expt_dgr-|math_dgr-' + jobs: - dsBaseClient_test_suite: + + ################################################################################ + # Doc-sync / R CMD check - no backend needed, so this runs once, not per + # backend or per shard. + ################################################################################ + r-checks: + name: Package checks (doc sync, R CMD check) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + http-user-agent: release + use-public-rspm: true + + - name: Install system libraries + run: | + sudo apt-get update -qq + sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev -y + sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev libuv1-dev -y + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + dependencies: 'c("Imports")' + extra-packages: | + any::rcmdcheck + cran::devtools + needs: check + + - name: Check manual updated + run: | + orig_sum=$(find man -type f | sort -u | xargs cat | md5sum) + R -q -e "devtools::document()" + new_sum=$(find man -type f | sort -u | xargs cat | md5sum) + if [ "$orig_sum" != "$new_sum" ]; then + echo "Your committed man/*.Rd files are out of sync with the R headers." + exit 1 + fi + continue-on-error: true + + - name: Devtools checks + run: | + R -q -e "devtools::check(args = c('--no-examples', '--no-tests'))" | tee azure-pipelines_check.Rout + grep --quiet "^0 errors" azure-pipelines_check.Rout && grep --quiet " 0 warnings" azure-pipelines_check.Rout && grep --quiet " 0 notes" azure-pipelines_check.Rout + continue-on-error: true + + + ################################################################################ + # Opal - dsBase suite, sharded 4 ways. Each shard is a fully isolated job + # with its own Opal instance. + ################################################################################ + opal-dsbase: + name: Opal dsBase tests (shard ${{ matrix.shard }}) runs-on: ubuntu-latest - timeout-minutes: 120 - permissions: - contents: read - - # These should all be constant, except TEST_FILTER. This can be used to test - # subsets of test files in the testthat directory. Options are like: - # '*' <- Run all tests. - # 'asNumericDS*' <- Run all asNumericDS tests, i.e. all the arg, etc. tests. - # '*_smk_*' <- Run all the smoke tests for all functions. + timeout-minutes: 60 + strategy: + fail-fast: false + # Shard count is hardcoded here and in "Compute shard filter" below (both + # must match) - there's no clean way to share a single value between a + # matrix definition and a step's env in plain workflow YAML. + matrix: + shard: [0, 1, 2, 3] env: - TEST_FILTER: '_-|datachk-|smk-|arg-|disc-|perf-|smk_expt-|expt-|math-' - _r_check_system_clock_: 0 - WORKFLOW_ID: ${{ github.run_id }}-${{ github.run_attempt }} - PROJECT_NAME: dsBaseClient - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - REPO_OWNER: ${{ github.repository_owner }} - R_KEEP_PKG_SOURCE: yes - GITHUB_TOKEN: ${{ github.token || 'placeholder-token' }} + DS_DRIVER: OpalDriver + DSBASE_REF: v7.0-dev steps: - name: Checkout dsBaseClient @@ -45,15 +134,148 @@ jobs: with: path: dsBaseClient - - name: Checkout testStatus - if: ${{ github.actor != 'nektos/act' }} # for local deployment only + - name: Start Opal docker-compose + run: docker compose -f docker-compose_opal.yml up -d --build + working-directory: dsBaseClient + + - name: Uninstall default MySQL + run: | + curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - + sudo service mysql stop || true + sudo apt-get update + sudo apt-get remove --purge mysql-client mysql-server mysql-common -y + sudo apt-get autoremove -y + sudo apt-get autoclean -y + sudo rm -rf /var/lib/mysql/ + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + http-user-agent: release + use-public-rspm: true + + - name: Install system libraries + run: | + sudo apt-get update -qq + sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev -y + sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev libuv1-dev -y + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + dependencies: 'c("Imports")' + extra-packages: | + cran::devtools + cran::covr + cran::fields + cran::meta + cran::metafor + cran::ggplot2 + cran::gridExtra + cran::data.table + cran::DSI + cran::DSOpal + cran::DSLite + cran::MolgenisAuth + cran::MolgenisArmadillo + cran::DSMolgenisArmadillo + cran::DescTools + cran::e1071 + needs: check + + - name: Install test datasets to Opal + run: | + sleep 60 + R -q -f "obiba_opal-upload_testing_datasets.R" + working-directory: dsBaseClient/tests/testthat/data_files + + - name: Install dsBase to Opal, set disclosure test options + run: | + R -q -e "library(opalr); opal <- opal.login(username = 'administrator', password = 'datashield_test&', url = 'http://localhost:8080/'); opal.put(opal, 'system', 'conf', 'general', '_rPackage'); opal.logout(opal)" + R -q -e "library(opalr); opal <- opal.login('administrator','datashield_test&', url='http://localhost:8080/'); dsadmin.install_github_package(opal, 'dsBase', username = 'datashield', ref = '${{ env.DSBASE_REF }}'); opal.logout(opal)" + + sleep 60 + + expected_version=$(curl -sf "https://raw.githubusercontent.com/datashield/dsBase/${{ env.DSBASE_REF }}/DESCRIPTION" | sed -n 's/^Version: //p' | tr -d '\r') + if [ -z "$expected_version" ]; then + echo "Could not determine expected dsBase version from GitHub" + exit 1 + fi + echo "Expected dsBase version: $expected_version" + + R -q -e "library(opalr); opal <- opal.login('administrator', 'datashield_test&', url='http://localhost:8080/'); desc <- dsadmin.package_description(opal, 'dsBase'); opal.logout(opal); installed_version <- desc[['Version']]; cat('Installed dsBase version:', installed_version, '\n'); if (is.null(installed_version) || installed_version != '$expected_version') stop('dsBase version mismatch: expected $expected_version, found ', installed_version)" + R -q -e "library(opalr); opal <- opal.login('administrator', 'datashield_test&', url='http://localhost:8080/'); dsadmin.profile_init(opal, name = 'default', packages = c('dsBase', 'dsTidyverse', 'resourcer')); opal.logout(opal)" + R -q -e "library(opalr); opal <- opal.login('administrator', 'datashield_test&', url='http://localhost:8080/'); dsadmin.set_option(opal, 'default.datashield.privacyControlLevel', 'permissive'); opal.logout(opal)" + working-directory: dsBaseClient/tests/testthat/data_files + + - name: Compute shard filter + run: | + all_files=$(cd tests/testthat && ls test-*.R | grep -E "${{ env.TEST_FILTER_DSBASE }}" | sort) + shard_files=$(echo "$all_files" | awk -v n=4 -v s=${{ matrix.shard }} 'NR % n == s') + echo "Shard ${{ matrix.shard }}/4 running $(echo "$shard_files" | grep -c .) files:" + echo "$shard_files" + shard_filter=$(echo "$shard_files" | sed -E 's/^test-//; s/\.R$//' | sed 's/\./\\./g' | paste -sd'|' -) + echo "SHARD_FILTER=$shard_filter" >> "$GITHUB_ENV" + working-directory: dsBaseClient + + - name: Run dsBase tests with coverage & JUnit report + run: | + R -q -e "devtools::reload();" + R -q -e ' + write.csv( + covr::coverage_to_list( + covr::package_coverage( + type = c("none"), + code = c('"'"' + output_file <- file("test_console_output_dsbase.txt"); + sink(output_file); + sink(output_file, type = "message"); + junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results_dsbase.xml")); + progress_rep <- testthat::ProgressReporter$new(max_failures = 999999); + multi_rep <- testthat::MultiReporter$new(reporters = list(progress_rep, junit_rep)); + options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}"); + testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ env.SHARD_FILTER }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"' + ) + ) + ), + "coveragelist.csv" + )' + cat test_console_output_dsbase.txt + working-directory: dsBaseClient + + - name: Upload shard results + uses: actions/upload-artifact@v4 + with: + name: opal-dsbase-shard-${{ matrix.shard }} + path: | + dsBaseClient/test_results_dsbase.xml + dsBaseClient/test_console_output_dsbase.txt + dsBaseClient/coveragelist.csv + + + ################################################################################ + # Opal - dsDanger suite. Small (~15 files), so it stays a single job rather + # than sharding further. + ################################################################################ + opal-dsdanger: + name: Opal dsDanger tests + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + DS_DRIVER: OpalDriver + DSBASE_REF: v7.0-dev + DSDANGER_REF: '6.3.4' + + steps: + - name: Checkout dsBaseClient uses: actions/checkout@v4 with: - repository: ${{ env.REPO_OWNER }}/testStatus - ref: master - path: testStatus - persist-credentials: false - token: ${{ env.GITHUB_TOKEN }} + path: dsBaseClient + + - name: Start Opal docker-compose + run: docker compose -f docker-compose_opal.yml up -d --build + working-directory: dsBaseClient - name: Uninstall default MySQL run: | @@ -73,82 +295,414 @@ jobs: http-user-agent: release use-public-rspm: true - - name: Install R and dependencies + - name: Install system libraries run: | - sudo apt-get install --no-install-recommends software-properties-common dirmngr -y - wget -qO- https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc - sudo add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" sudo apt-get update -qq - sudo apt-get upgrade -y - sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev r-base -y - sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev xml-twig-tools -y - sudo R -q -e "install.packages(c('devtools','covr','fields','meta','metafor','ggplot2','gridExtra','data.table','DSI','DSOpal','DSLite','MolgenisAuth','MolgenisArmadillo','DSMolgenisArmadillo','DescTools','e1071'), repos='https://cloud.r-project.org')" - sudo R -q -e "devtools::install_github(repo='datashield/dsDangerClient', ref=Sys.getenv('BRANCH_NAME'))" + sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev -y + sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev libuv1-dev -y - uses: r-lib/actions/setup-r-dependencies@v2 with: dependencies: 'c("Imports")' extra-packages: | - any::rcmdcheck cran::devtools cran::git2r - cran::RCurl - cran::readr - cran::magrittr - cran::xml2 - cran::purrr - cran::dplyr - cran::stringr - cran::tidyr + cran::covr + cran::fields + cran::meta + cran::metafor + cran::ggplot2 + cran::gridExtra + cran::data.table + cran::DSI + cran::DSOpal + cran::DSLite + cran::MolgenisAuth + cran::MolgenisArmadillo + cran::DSMolgenisArmadillo + cran::DescTools + cran::e1071 + needs: check + + - name: Install dsDangerClient + run: R -q -e "devtools::install_github(repo='datashield/dsDangerClient', ref=Sys.getenv('BRANCH_NAME'))" + + - name: Install test datasets to Opal + run: | + sleep 60 + R -q -f "obiba_opal-upload_testing_datasets.R" + working-directory: dsBaseClient/tests/testthat/data_files + + - name: Install dsBase to Opal, set disclosure test options + run: | + R -q -e "library(opalr); opal <- opal.login(username = 'administrator', password = 'datashield_test&', url = 'http://localhost:8080/'); opal.put(opal, 'system', 'conf', 'general', '_rPackage'); opal.logout(opal)" + R -q -e "library(opalr); opal <- opal.login('administrator','datashield_test&', url='http://localhost:8080/'); dsadmin.install_github_package(opal, 'dsBase', username = 'datashield', ref = '${{ env.DSBASE_REF }}'); opal.logout(opal)" + + sleep 60 + + expected_version=$(curl -sf "https://raw.githubusercontent.com/datashield/dsBase/${{ env.DSBASE_REF }}/DESCRIPTION" | sed -n 's/^Version: //p' | tr -d '\r') + if [ -z "$expected_version" ]; then + echo "Could not determine expected dsBase version from GitHub" + exit 1 + fi + echo "Expected dsBase version: $expected_version" + + R -q -e "library(opalr); opal <- opal.login('administrator', 'datashield_test&', url='http://localhost:8080/'); desc <- dsadmin.package_description(opal, 'dsBase'); opal.logout(opal); installed_version <- desc[['Version']]; cat('Installed dsBase version:', installed_version, '\n'); if (is.null(installed_version) || installed_version != '$expected_version') stop('dsBase version mismatch: expected $expected_version, found ', installed_version)" + R -q -e "library(opalr); opal <- opal.login('administrator', 'datashield_test&', url='http://localhost:8080/'); dsadmin.profile_init(opal, name = 'default', packages = c('dsBase', 'dsTidyverse', 'resourcer')); opal.logout(opal)" + R -q -e "library(opalr); opal <- opal.login('administrator', 'datashield_test&', url='http://localhost:8080/'); dsadmin.set_option(opal, 'default.datashield.privacyControlLevel', 'permissive'); opal.logout(opal)" + working-directory: dsBaseClient/tests/testthat/data_files + + - name: Install dsDanger package on Opal server + run: | + R -q -e "library(opalr); opal <- opal.login(username = 'administrator', password = 'datashield_test&', url = 'http://localhost:8080'); opal.put(opal, 'system', 'conf', 'general', '_rPackage'); opal.logout(opal)" + R -q -e "library(opalr); opal <- opal.login('administrator','datashield_test&', url='http://localhost:8080/'); dsadmin.install_github_package(opal, 'dsDanger', username = 'datashield', ref = '${{ env.DSDANGER_REF }}'); opal.logout(opal)" + working-directory: dsBaseClient + + - name: Run dsDanger tests with JUnit report + run: | + R -q -e ' + library(testthat); + output_file <- file("test_console_output_dsdanger.txt"); + sink(output_file); + sink(output_file, type = "message"); + junit_rep <- JunitReporter$new(file = "test_results_dsdanger.xml"); + progress_rep <- ProgressReporter$new(max_failures = 999999); + multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep)); + options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}"); + testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ env.TEST_FILTER_DSDANGER }}", reporter = multi_rep, stop_on_failure = FALSE)' + cat test_console_output_dsdanger.txt + working-directory: dsBaseClient + + - name: Upload dsDanger results + uses: actions/upload-artifact@v4 + with: + name: opal-dsdanger + path: | + dsBaseClient/test_results_dsdanger.xml + dsBaseClient/test_console_output_dsdanger.txt + + + ################################################################################ + # Opal - merge all shard + dsDanger results and publish the report. + ################################################################################ + opal-report: + name: Opal report + needs: [opal-dsbase, opal-dsdanger] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout dsBaseClient + uses: actions/checkout@v4 + with: + path: dsBaseClient + + - name: Checkout testStatus + if: ${{ github.actor != 'nektos/act' }} # for local deployment only + uses: actions/checkout@v4 + with: + repository: ${{ env.REPO_OWNER }}/testStatus + ref: master + path: testStatus + persist-credentials: false + token: ${{ env.GITHUB_TOKEN }} + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + use-public-rspm: true + + - name: Install xml-twig-tools + run: | + sudo apt-get update -qq + sudo apt-get install -qq xml-twig-tools -y + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + dependencies: 'NA' + extra-packages: | cran::quarto cran::knitr cran::kableExtra cran::rmarkdown cran::downlit - needs: check + cran::xml2 + cran::purrr + cran::dplyr + cran::stringr + cran::tidyr + cran::readr + cran::magrittr - - name: Check manual updated + - name: Download shard/dsdanger results + uses: actions/download-artifact@v4 + with: + pattern: 'opal-*' + path: dsBaseClient/artifacts + + - name: Merge results run: | - orig_sum=$(find man -type f | sort -u | xargs cat | md5sum) - R -q -e "devtools::document()" - new_sum=$(find man -type f | sort -u | xargs cat | md5sum) - if [ "$orig_sum" != "$new_sum" ]; then - echo "Your committed man/*.Rd files are out of sync with the R headers." - exit 1 - fi + mkdir -p logs + xml_grep --pretty_print indented --wrap "testsuites" --descr "" --cond "testsuite" artifacts/*/test_results_*.xml > logs/test_results.xml + cat artifacts/*/test_console_output_*.txt > logs/test_console_output.txt + + first=1 + for f in artifacts/opal-dsbase-shard-*/coveragelist.csv; do + if [ $first -eq 1 ]; then + cat "$f" > logs/coveragelist.csv + first=0 + else + tail -n +2 "$f" >> logs/coveragelist.csv + fi + done working-directory: dsBaseClient - continue-on-error: true - - name: Devtools checks + - name: Check for JUnit errors run: | - R -q -e "devtools::check(args = c('--no-examples', '--no-tests'))" | tee azure-pipelines_check.Rout - grep --quiet "^0 errors" azure-pipelines_check.Rout && grep --quiet " 0 warnings" azure-pipelines_check.Rout && grep --quiet " 0 notes" azure-pipelines_check.Rout + issue_count=$(sed 's/failures="0" errors="0"//' test_results.xml | grep -c errors= || true) + echo "Number of testsuites with issues: $issue_count" + sed 's/failures="0" errors="0"//' test_results.xml | grep errors= > issues.log || true + cat issues.log || true + # continue with workflow even when some tests fail + exit 0 + working-directory: dsBaseClient/logs + + - name: Write versions to file + run: | + echo "branch:${{ env.BRANCH_NAME }}" > ${{ env.WORKFLOW_ID }}.txt + echo "os:$(lsb_release -ds)" >> ${{ env.WORKFLOW_ID }}.txt + echo "R:$(R --version | head -n1)" >> ${{ env.WORKFLOW_ID }}.txt + working-directory: dsBaseClient/logs + + - name: Parse results from testthat and covr + run: Rscript --verbose --vanilla ../testStatus/source/parse_test_report.R logs/ working-directory: dsBaseClient - continue-on-error: true - - name: Start Armadillo docker-compose - run: docker compose -f docker-compose_armadillo.yml up -d --build + - name: Render report + run: | + cd testStatus + + mkdir -p new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest/ + + cp -rv ../dsBaseClient/logs/* new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + cp -rv ../dsBaseClient/logs/${{ env.WORKFLOW_ID }}.txt new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ + + R -e 'input_dir <- file.path("../new/logs", Sys.getenv("PROJECT_NAME"), Sys.getenv("BRANCH_NAME"), Sys.getenv("WORKFLOW_ID")); quarto::quarto_render("source/test_report.qmd", execute_params = list(input_dir = input_dir))' + mv source/test_report.html new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/index.html + cp -r new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/* new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest + env: + PROJECT_NAME: ${{ env.PROJECT_NAME }} + BRANCH_NAME: ${{ env.BRANCH_NAME }} + WORKFLOW_ID: ${{ env.WORKFLOW_ID }} + + - name: Upload test logs + uses: actions/upload-artifact@v4 + with: + name: dsbaseclient-logs-opal + path: testStatus/new + + + ################################################################################ + # Armadillo - dsBase suite, sharded 4 ways. Each shard downloads and runs its + # own Armadillo jar instance. + # + # Runs as a plain `java -jar` process instead of docker-compose: Armadillo + # self-manages its own Rock container over the host Docker socket + # (docker-management-enabled: true / docker-run-in-container: false), which + # avoids building/pulling the old custom armadillo_citest image and its + # dockerised Armadillo layer. The latest GitHub release jar is downloaded at + # run time. No process is restarted after installing dsBase - install then + # whitelist directly. + ################################################################################ + armadillo-dsbase: + name: Armadillo dsBase tests (shard ${{ matrix.shard }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + shard: [0, 1, 2, 3] + env: + DS_DRIVER: ArmadilloDriver + DSBASE_TARBALL: dsBase_7.0.0-permissive.tar.gz + + steps: + - name: Checkout dsBaseClient + uses: actions/checkout@v4 + with: + path: dsBaseClient + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + + - name: Download and start Armadillo (jar) + run: | + mkdir -p armadillo_home/config armadillo_home/storage armadillo_home/logs + + cat > armadillo_home/config/application.yml < armadillo_home/logs/stdout.log 2>&1 & + echo $! > armadillo_home/armadillo.pid working-directory: dsBaseClient - - name: Install test datasets + - name: Uninstall default MySQL run: | - sleep 60 - R -q -f "molgenis_armadillo-upload_testing_datasets.R" + curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - + sudo service mysql stop || true + sudo apt-get update + sudo apt-get remove --purge mysql-client mysql-server mysql-common -y + sudo apt-get autoremove -y + sudo apt-get autoclean -y + sudo rm -rf /var/lib/mysql/ + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + http-user-agent: release + use-public-rspm: true + + - name: Install system libraries + run: | + sudo apt-get update -qq + sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev -y + sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev libuv1-dev -y + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + dependencies: 'c("Imports")' + extra-packages: | + cran::devtools + cran::covr + cran::fields + cran::meta + cran::metafor + cran::ggplot2 + cran::gridExtra + cran::data.table + cran::DSI + cran::DSOpal + cran::DSLite + cran::MolgenisAuth + cran::MolgenisArmadillo + cran::DSMolgenisArmadillo + cran::DescTools + cran::e1071 + needs: check + + - name: Wait for Armadillo to be ready + run: | + for i in $(seq 1 60); do + if curl -fsS http://localhost:8080/actuator/health 2>/dev/null | grep -q '"status":"UP"'; then + echo "Armadillo is ready" + exit 0 + fi + sleep 5 + done + echo "Armadillo did not become ready in time" + cat armadillo_home/logs/stdout.log || true + exit 1 + working-directory: dsBaseClient + + - name: Install test datasets to Armadillo + run: R -q -f "molgenis_armadillo-upload_testing_datasets.R" working-directory: dsBaseClient/tests/testthat/data_files - name: Install dsBase to Armadillo run: | curl -u admin:admin -X GET http://localhost:8080/packages - curl -u admin:admin -H 'Content-Type: multipart/form-data' -F "file=@dsBase_7.0.0-permissive.tar.gz" -X POST http://localhost:8080/install-package - sleep 60 - docker restart dsbaseclient-armadillo-1 - sleep 30 + install_status=$(curl -u admin:admin -H 'Content-Type: multipart/form-data' -F "file=@${{ env.DSBASE_TARBALL }}" -o /dev/null -w '%{http_code}' -X POST http://localhost:8080/install-package) + if [ "$install_status" != "200" ]; then + echo "dsBase install request failed with HTTP status $install_status" + exit 1 + fi + + expected_version=$(tar -xOzf ${{ env.DSBASE_TARBALL }} dsBase/DESCRIPTION | sed -n 's/^Version: //p' | tr -d '\r') + if [ -z "$expected_version" ]; then + echo "Could not determine expected dsBase version from ${{ env.DSBASE_TARBALL }}" + exit 1 + fi + echo "Expected dsBase version: $expected_version" + + actual_version="" + for i in $(seq 1 30); do + packages_json=$(curl -sf -u admin:admin -X GET http://localhost:8080/packages || true) + actual_version=$(echo "$packages_json" | jq -r '.[] | select(.name == "dsBase") | .version' 2>/dev/null || true) + if [ "$actual_version" == "$expected_version" ]; then + break + fi + sleep 10 + done + echo "Installed dsBase version: $actual_version" + if [ "$actual_version" != "$expected_version" ]; then + echo "dsBase version mismatch: expected $expected_version, found '$actual_version'" + exit 1 + fi + curl -u admin:admin -X POST http://localhost:8080/whitelist/dsBase working-directory: dsBaseClient - - name: Run tests with coverage & JUnit report + - name: Compute shard filter + run: | + all_files=$(cd tests/testthat && ls test-*.R | grep -E "${{ env.TEST_FILTER_DSBASE }}" | sort) + shard_files=$(echo "$all_files" | awk -v n=4 -v s=${{ matrix.shard }} 'NR % n == s') + echo "Shard ${{ matrix.shard }}/4 running $(echo "$shard_files" | grep -c .) files:" + echo "$shard_files" + shard_filter=$(echo "$shard_files" | sed -E 's/^test-//; s/\.R$//' | sed 's/\./\\./g' | paste -sd'|' -) + echo "SHARD_FILTER=$shard_filter" >> "$GITHUB_ENV" + working-directory: dsBaseClient + + - name: Run dsBase tests with coverage & JUnit report run: | - mkdir -p logs R -q -e "devtools::reload();" R -q -e ' write.csv( @@ -156,22 +710,329 @@ jobs: covr::package_coverage( type = c("none"), code = c('"'"' - output_file <- file("test_console_output.txt"); + output_file <- file("test_console_output_dsbase.txt"); sink(output_file); sink(output_file, type = "message"); - junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results.xml")); + junit_rep <- testthat::JunitReporter$new(file = file.path(getwd(), "test_results_dsbase.xml")); progress_rep <- testthat::ProgressReporter$new(max_failures = 999999); multi_rep <- testthat::MultiReporter$new(reporters = list(progress_rep, junit_rep)); - options("datashield.return_errors" = FALSE, "default_driver" = "ArmadilloDriver"); - testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ env.TEST_FILTER }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"' + options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}"); + testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ env.SHARD_FILTER }}", reporter = multi_rep, stop_on_failure = FALSE)'"'"' ) ) ), "coveragelist.csv" )' + cat test_console_output_dsbase.txt + working-directory: dsBaseClient + + - name: Upload shard results + uses: actions/upload-artifact@v4 + with: + name: armadillo-dsbase-shard-${{ matrix.shard }} + path: | + dsBaseClient/test_results_dsbase.xml + dsBaseClient/test_console_output_dsbase.txt + dsBaseClient/coveragelist.csv + + + ################################################################################ + # Armadillo - dsDanger suite. Small, so it stays a single job. + ################################################################################ + armadillo-dsdanger: + name: Armadillo dsDanger tests + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + DS_DRIVER: ArmadilloDriver + DSBASE_TARBALL: dsBase_7.0.0-permissive.tar.gz + DSDANGER_TARBALL: dsDanger_6.3.4.tar.gz + + steps: + - name: Checkout dsBaseClient + uses: actions/checkout@v4 + with: + path: dsBaseClient + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + + - name: Download and start Armadillo (jar) + run: | + mkdir -p armadillo_home/config armadillo_home/storage armadillo_home/logs + + cat > armadillo_home/config/application.yml < armadillo_home/logs/stdout.log 2>&1 & + echo $! > armadillo_home/armadillo.pid + working-directory: dsBaseClient + + - name: Uninstall default MySQL + run: | + curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - + sudo service mysql stop || true + sudo apt-get update + sudo apt-get remove --purge mysql-client mysql-server mysql-common -y + sudo apt-get autoremove -y + sudo apt-get autoclean -y + sudo rm -rf /var/lib/mysql/ + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + http-user-agent: release + use-public-rspm: true + + - name: Install system libraries + run: | + sudo apt-get update -qq + sudo apt-get install -qq libxml2-dev libcurl4-openssl-dev libssl-dev libgsl-dev libgit2-dev -y + sudo apt-get install -qq libharfbuzz-dev libfribidi-dev libmagick++-dev libudunits2-dev libuv1-dev -y + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + dependencies: 'c("Imports")' + extra-packages: | + cran::devtools + cran::git2r + cran::covr + cran::fields + cran::meta + cran::metafor + cran::ggplot2 + cran::gridExtra + cran::data.table + cran::DSI + cran::DSOpal + cran::DSLite + cran::MolgenisAuth + cran::MolgenisArmadillo + cran::DSMolgenisArmadillo + cran::DescTools + cran::e1071 + needs: check + + - name: Install dsDangerClient + run: R -q -e "devtools::install_github(repo='datashield/dsDangerClient', ref=Sys.getenv('BRANCH_NAME'))" - mv coveragelist.csv logs/ - mv test_* logs/ + - name: Wait for Armadillo to be ready + run: | + for i in $(seq 1 60); do + if curl -fsS http://localhost:8080/actuator/health 2>/dev/null | grep -q '"status":"UP"'; then + echo "Armadillo is ready" + exit 0 + fi + sleep 5 + done + echo "Armadillo did not become ready in time" + cat armadillo_home/logs/stdout.log || true + exit 1 + working-directory: dsBaseClient + + - name: Install test datasets to Armadillo + run: R -q -f "molgenis_armadillo-upload_testing_datasets.R" + working-directory: dsBaseClient/tests/testthat/data_files + + - name: Install dsBase to Armadillo + run: | + curl -u admin:admin -X GET http://localhost:8080/packages + install_status=$(curl -u admin:admin -H 'Content-Type: multipart/form-data' -F "file=@${{ env.DSBASE_TARBALL }}" -o /dev/null -w '%{http_code}' -X POST http://localhost:8080/install-package) + if [ "$install_status" != "200" ]; then + echo "dsBase install request failed with HTTP status $install_status" + exit 1 + fi + + expected_version=$(tar -xOzf ${{ env.DSBASE_TARBALL }} dsBase/DESCRIPTION | sed -n 's/^Version: //p' | tr -d '\r') + if [ -z "$expected_version" ]; then + echo "Could not determine expected dsBase version from ${{ env.DSBASE_TARBALL }}" + exit 1 + fi + echo "Expected dsBase version: $expected_version" + + actual_version="" + for i in $(seq 1 30); do + packages_json=$(curl -sf -u admin:admin -X GET http://localhost:8080/packages || true) + actual_version=$(echo "$packages_json" | jq -r '.[] | select(.name == "dsBase") | .version' 2>/dev/null || true) + if [ "$actual_version" == "$expected_version" ]; then + break + fi + sleep 10 + done + echo "Installed dsBase version: $actual_version" + if [ "$actual_version" != "$expected_version" ]; then + echo "dsBase version mismatch: expected $expected_version, found '$actual_version'" + exit 1 + fi + + curl -u admin:admin -X POST http://localhost:8080/whitelist/dsBase + working-directory: dsBaseClient + + - name: Install dsDanger package on Armadillo server + run: | + curl -u admin:admin http://localhost:8080/whitelist + install_status=$(curl -u admin:admin -H 'Content-Type: multipart/form-data' -F "file=@${{ env.DSDANGER_TARBALL }}" -o /dev/null -w '%{http_code}' -X POST http://localhost:8080/install-package) + if [ "$install_status" != "200" ]; then + echo "dsDanger install request failed with HTTP status $install_status" + exit 1 + fi + + for i in $(seq 1 30); do + packages_json=$(curl -sf -u admin:admin -X GET http://localhost:8080/packages || true) + if echo "$packages_json" | jq -e '.[] | select(.name == "dsDanger")' >/dev/null 2>&1; then + break + fi + sleep 10 + done + + curl -u admin:admin -X POST http://localhost:8080/whitelist/dsDanger + curl -u admin:admin http://localhost:8080/whitelist + working-directory: dsBaseClient + + - name: Run dsDanger tests with JUnit report + run: | + R -q -e ' + library(testthat); + output_file <- file("test_console_output_dsdanger.txt"); + sink(output_file); + sink(output_file, type = "message"); + junit_rep <- JunitReporter$new(file = "test_results_dsdanger.xml"); + progress_rep <- ProgressReporter$new(max_failures = 999999); + multi_rep <- MultiReporter$new(reporters = list(progress_rep, junit_rep)); + options("datashield.return_errors" = FALSE, "default_driver" = "${{ env.DS_DRIVER }}"); + testthat::test_package("${{ env.PROJECT_NAME }}", filter = "${{ env.TEST_FILTER_DSDANGER }}", reporter = multi_rep, stop_on_failure = FALSE)' + cat test_console_output_dsdanger.txt + working-directory: dsBaseClient + + - name: Upload dsDanger results + uses: actions/upload-artifact@v4 + with: + name: armadillo-dsdanger + path: | + dsBaseClient/test_results_dsdanger.xml + dsBaseClient/test_console_output_dsdanger.txt + + + ################################################################################ + # Armadillo - merge all shard + dsDanger results and publish the report. + ################################################################################ + armadillo-report: + name: Armadillo report + needs: [armadillo-dsbase, armadillo-dsdanger] + if: always() + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout dsBaseClient + uses: actions/checkout@v4 + with: + path: dsBaseClient + + - name: Checkout testStatus + if: ${{ github.actor != 'nektos/act' }} # for local deployment only + uses: actions/checkout@v4 + with: + repository: ${{ env.REPO_OWNER }}/testStatus + ref: master + path: testStatus + persist-credentials: false + token: ${{ env.GITHUB_TOKEN }} + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: release + use-public-rspm: true + + - name: Install xml-twig-tools + run: | + sudo apt-get update -qq + sudo apt-get install -qq xml-twig-tools -y + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + dependencies: 'NA' + extra-packages: | + cran::quarto + cran::knitr + cran::kableExtra + cran::rmarkdown + cran::downlit + cran::xml2 + cran::purrr + cran::dplyr + cran::stringr + cran::tidyr + cran::readr + cran::magrittr + + - name: Download shard/dsdanger results + uses: actions/download-artifact@v4 + with: + pattern: 'armadillo-*' + path: dsBaseClient/artifacts + + - name: Merge results + run: | + mkdir -p logs + xml_grep --pretty_print indented --wrap "testsuites" --descr "" --cond "testsuite" artifacts/*/test_results_*.xml > logs/test_results.xml + cat artifacts/*/test_console_output_*.txt > logs/test_console_output.txt + + first=1 + for f in artifacts/armadillo-dsbase-shard-*/coveragelist.csv; do + if [ $first -eq 1 ]; then + cat "$f" > logs/coveragelist.csv + first=0 + else + tail -n +2 "$f" >> logs/coveragelist.csv + fi + done working-directory: dsBaseClient - name: Check for JUnit errors @@ -192,8 +1053,7 @@ jobs: working-directory: dsBaseClient/logs - name: Parse results from testthat and covr - run: | - Rscript --verbose --vanilla ../testStatus/source/parse_test_report.R logs/ + run: Rscript --verbose --vanilla ../testStatus/source/parse_test_report.R logs/ working-directory: dsBaseClient - name: Render report @@ -204,14 +1064,12 @@ jobs: mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ mkdir -p new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest/ - # Copy logs to new logs directory location cp -rv ../dsBaseClient/logs/* new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ cp -rv ../dsBaseClient/logs/${{ env.WORKFLOW_ID }}.txt new/logs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/ R -e 'input_dir <- file.path("../new/logs", Sys.getenv("PROJECT_NAME"), Sys.getenv("BRANCH_NAME"), Sys.getenv("WORKFLOW_ID")); quarto::quarto_render("source/test_report.qmd", execute_params = list(input_dir = input_dir))' mv source/test_report.html new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/index.html cp -r new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/${{ env.WORKFLOW_ID }}/* new/docs/${{ env.PROJECT_NAME }}/${{ env.BRANCH_NAME }}/latest - env: PROJECT_NAME: ${{ env.PROJECT_NAME }} BRANCH_NAME: ${{ env.BRANCH_NAME }} @@ -220,25 +1078,5 @@ jobs: - name: Upload test logs uses: actions/upload-artifact@v4 with: - name: dsbaseclient-logs + name: dsbaseclient-logs-armadillo path: testStatus/new - - - name: Dump environment info - run: | - echo -e "\n#############################" - echo -e "ls /: ######################" - ls -al . - echo -e "\n#############################" - echo -e "lscpu: ######################" - lscpu - echo -e "\n#############################" - echo -e "memory: #####################" - free -m - echo -e "\n#############################" - echo -e "env: ########################" - env - echo -e "\n#############################" - echo -e "R sessionInfo(): ############" - R -e 'sessionInfo()' - sudo apt install tree -y - tree .