diff --git a/.azuredevops/integration-tests.yml b/.azuredevops/integration-tests.yml index ceadc25178..76437fa34f 100644 --- a/.azuredevops/integration-tests.yml +++ b/.azuredevops/integration-tests.yml @@ -20,3 +20,17 @@ jobs: newDir: integration_test_directory testsFolder: integration makeTarget: integration-test + # Run each group in a fresh pytest process to release memory between groups. + testBatches: + - tests/integration/ai_recruiter + - tests/integration/cli + - tests/integration/converter + - tests/integration/datasets + - tests/integration/embeddings + - tests/integration/executor + - tests/integration/executors + - tests/integration/memory + - tests/integration/message_normalizer + - tests/integration/scenarios + - tests/integration/score + - tests/integration/targets diff --git a/.azuredevops/test-job-template.yml b/.azuredevops/test-job-template.yml index 65905af5cc..0b3a40f4dd 100644 --- a/.azuredevops/test-job-template.yml +++ b/.azuredevops/test-job-template.yml @@ -11,6 +11,9 @@ parameters: type: string - name: makeTarget type: string + - name: testBatches + type: object + default: [] jobs: - job: ${{ parameters.jobName }} @@ -114,7 +117,16 @@ jobs: echo "Azure SQL Database access token cached successfully." # Run tests - make ${{ parameters.makeTarget }} + test_batches="${{ join(' ', parameters.testBatches) }}" + if [ -n "$test_batches" ]; then + batch_number=0 + for test_batch in $test_batches; do + batch_number=$((batch_number + 1)) + make ${{ parameters.makeTarget }} INTEGRATION_TESTS="$test_batch" JUNIT_XML="junit/test-results-${batch_number}.xml" + done + else + make ${{ parameters.makeTarget }} + fi - bash: | rm -f ~/.pyrit/.env ~/.pyrit/.env.local name: clean_up_env_files @@ -123,4 +135,5 @@ jobs: condition: always() inputs: testResultsFormat: 'JUnit' - testResultsFiles: 'junit/test-results.xml' + testResultsFiles: 'junit/test-results*.xml' + mergeTestResults: true diff --git a/Makefile b/Makefile index fffa180d2f..f9d050d9b9 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ UNIT_TESTS:=tests/unit INTEGRATION_TESTS:=tests/integration PARTNER_INTEGRATION_TESTS:=tests/partner_integration END_TO_END_TESTS:=tests/end_to_end +JUNIT_XML?=junit/test-results.xml all: pre-commit @@ -67,7 +68,7 @@ unit-test-diff-cover: uv run python -m diff_cover.diff_cover_tool coverage.xml --compare-branch=origin/main --diff-range-notation=.. --fail-under=90 integration-test: - $(CMD) pytest $(INTEGRATION_TESTS) --cov=$(PYMODULE) $(INTEGRATION_TESTS) --cov-report xml --junitxml=junit/test-results.xml --doctest-modules + $(CMD) pytest $(INTEGRATION_TESTS) --cov=$(PYMODULE) --cov-report xml --junitxml=$(JUNIT_XML) --doctest-modules end-to-end-test: $(CMD) pytest $(END_TO_END_TESTS) -v --junitxml=junit/test-results.xml