From 79b49fbc903b64acc6a3b9eca542f0a16391a321 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 22 Oct 2026 00:29:36 +0300 Subject: [PATCH 1/5] CI: run tests --- .github/workflows/ci.yml | 74 ++++++++++++++++++++ Dockerfile--alpine.tmpl | 144 +++++++++++++++++++++++++++++++++++++++ run_tests_own.sh | 127 ++++++++++++++++++++++++++++++++++ tests/requirements.txt | 6 ++ 4 files changed, 351 insertions(+) create mode 100644 Dockerfile--alpine.tmpl create mode 100755 run_tests_own.sh create mode 100644 tests/requirements.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b10df9..43ad6cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,3 +66,77 @@ jobs: - name: Check package metadata run: | twine check dist/* + + test-linux-container: + runs-on: ubuntu-latest + needs: build-check + strategy: + fail-fast: false + matrix: + include: + - platform: "alpine" + python: "3.7" + postgres: "17" + case_suffix: "py3_07_xx-pg17_xx" + - platform: "alpine" + python: "3.8.0" + postgres: "17" + case_suffix: "py3_08_00-pg17_xx" + - platform: "alpine" + python: "3.8" + postgres: "17" + case_suffix: "py3_08_xx-pg17_xx" + - platform: "alpine" + python: "3.9" + postgres: "17" + case_suffix: "py3_09_xx-pg17_xx" + - platform: "alpine" + python: "3.10" + postgres: "17" + case_suffix: "py3_10_xx-pg17_xx" + - platform: "alpine" + python: "3.11" + postgres: "17" + case_suffix: "py3_11_xx-pg17_xx" + - platform: "alpine" + python: "3.12" + postgres: "17" + case_suffix: "py3_12_xx-pg17_xx" + - platform: "alpine" + python: "3.13" + postgres: "17" + case_suffix: "py3_13_xx-pg17_xx" + - platform: "alpine" + python: "3.14" + postgres: "17" + case_suffix: "py3_14_xx-pg17_xx" + + name: "test: ${{ matrix.platform }} | ${{ matrix.case_suffix }}" + + env: + BASE_SIGN: "${{ matrix.platform }}-${{ matrix.case_suffix }}" + + steps: + - name: Prepare variables + run: | + echo "RUN_CFG__NOW=$(date +'%Y%m%d_%H%M%S')" >> $GITHUB_ENV + echo "RUN_CFG__LOGS_DIR=logs-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "RUN_CFG__DOCKER_IMAGE_NAME=tests-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "---------- [$GITHUB_ENV]" + cat $GITHUB_ENV + - name: Checkout + uses: actions/checkout@v7 + - name: Prepare logs folder on the host + run: mkdir -p "${{ env.RUN_CFG__LOGS_DIR }}" + - name: Adjust logs folder permission + run: chmod -R 777 "${{ env.RUN_CFG__LOGS_DIR }}" + - name: Build local image ${{ matrix.alpine }} + run: docker build --build-arg PG_VERSION="${{ matrix.postgres }}" --build-arg PYTHON_VERSION="${{ matrix.python }}" -t "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" -f Dockerfile--${{ matrix.platform }}.tmpl . + - name: Run + run: docker run -t -v ${{ github.workspace }}/${{ env.RUN_CFG__LOGS_DIR }}:/home/test/work/logs "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" + - name: Upload Logs + uses: actions/upload-artifact@v7 + if: always() # IT IS IMPORTANT! + with: + name: testgres.pbk2--test_logs--${{ env.RUN_CFG__NOW }}-${{ env.BASE_SIGN }}-id${{ github.run_id }} + path: "${{ env.RUN_CFG__LOGS_DIR }}/" diff --git a/Dockerfile--alpine.tmpl b/Dockerfile--alpine.tmpl new file mode 100644 index 0000000..7bcbdb9 --- /dev/null +++ b/Dockerfile--alpine.tmpl @@ -0,0 +1,144 @@ +ARG PG_VERSION=17 +ARG PYTHON_VERSION=3.12 + +# --------------------------------------------- base1 +FROM python:${PYTHON_VERSION}-alpine AS base1 + +RUN apk add --no-cache \ + coreutils \ + bash \ + mc \ + procps \ + openssh \ + sshpass \ + sudo \ + git \ + curl + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config + +ENV PYTHON_BINARY=python3 + +# --------------------------------------------- postgres +FROM base1 AS base1_with_dev_tools + +RUN apk add --no-cache \ + gcc \ + make \ + meson \ + flex \ + bison \ + perl \ + pkgconfig \ + openssl-dev \ + icu-dev \ + zstd-dev \ + zlib-dev \ + lz4-dev \ + libxml2-dev \ + musl-dev \ + linux-headers + +# --------------------------------------------- postgres +FROM base1_with_dev_tools AS base1_with_pg-17 + +RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pgpro/postgres/source + +WORKDIR /pgpro/postgres/source + +RUN ./configure --prefix=/pgpro/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +ENV PG_CONFIG="/pgpro/postgres/install/bin/pg_config" + +# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir +RUN ln -s /pgpro/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +# libpq.so.5 is enough +# RUN ln -s /pgpro/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 + +# --------------------------------------------- base4-pbk +FROM base1_with_pg-${PG_VERSION} AS base4-pbk + +RUN git clone https://github.com/postgrespro/pg_probackup.git /pgpro/pbk2/source + +WORKDIR /pgpro/pbk2/source + +RUN top_srcdir=/pgpro/postgres/source/ CFLAGS='-g3 -O0' make USE_PGXS=1 + +ENV PGPROBACKUPSRC="/pgpro/pbk2/source" +ENV PGPROBACKUPBIN="/pgpro/pbk2/source/pg_probackup" + +# --------------------------------------------- final +FROM base4-pbk AS final + +EXPOSE 22 +RUN ssh-keygen -A + +RUN adduser -D test && addgroup -S sudo && adduser test sudo + +# It allows to use sudo without password +RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN echo "test:*" | chpasswd -e + +COPY --chown=test:test . /home/test/work +WORKDIR /home/test/work + +RUN chown -R test /pgpro/pbk2/source + +ENV LANG=C.UTF-8 + +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + echo 'Host *' > /home/test/.ssh/config && \ + echo ' ControlMaster auto' >> /home/test/.ssh/config && \ + echo ' ControlPath /home/test/.ssh/master-%r@%h:%p' >> /home/test/.ssh/config && \ + echo ' ControlPersist 30m' >> /home/test/.ssh/config && \ + echo ' StrictHostKeyChecking no' >> /home/test/.ssh/config && \ + echo ' UserKnownHostsFile /dev/null' >> /home/test/.ssh/config && \ + # echo ' GSSAPIAuthentication no' >> /home/test/.ssh/config && \ + chown -R test:test /home/test/.ssh && \ + chmod 700 /home/test/.ssh && \ + chmod 600 /home/test/.ssh/config + +# +# \"$@\" +# - quote is important! +# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ. +# +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + /usr/sbin/sshd; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Run tests by default (master machine role) +CMD ["bash", "-c", " \ +set -eux; \ +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su test -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\"; \ + su test -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\"; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +su test -c \"TEST_FILTER='' bash ./run_tests_own.sh\"; \ +"] diff --git a/run_tests_own.sh b/run_tests_own.sh new file mode 100755 index 0000000..781c87d --- /dev/null +++ b/run_tests_own.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +set -eux + +# Filter tests for local execution (without remote/ssh) +if [ -z ${TEST_FILTER+x} ]; then + export TEST_FILTER="" +fi + +echo NPROC: $(nproc) + +# Check for the presence of pg_config +echo check that pg_config is in PATH +command -v pg_config + +# Setting up the Python environment +VENV_PATH="/tmp/testgres_venv" +rm -rf $VENV_PATH +${PYTHON_BINARY} -m venv "${VENV_PATH}" +export VIRTUAL_ENV_DISABLE_PROMPT=1 +source "${VENV_PATH}/bin/activate" +pip install --upgrade pip setuptools wheel +pip install -r tests/requirements.txt + +# remove existing coverage file +export COVERAGE_FILE=.coverage +rm -f $COVERAGE_FILE + +pip install coverage + +if [ -n "${TEST_CFG__REMOTE_HOST:-}" ] && [ -n "${TEST_CFG__REMOTE_USERNAME:-}" ]; then + cmd_str="ssh" + + if [ -n "${TEST_CFG__REMOTE_PASSWORD:-}" ]; then + cmd_str="sshpass -p \"$TEST_CFG__REMOTE_PASSWORD\" $cmd_str" + fi + + [ -n "${TEST_CFG__REMOTE_SSH_KEY:-}" ] && cmd_str="$cmd_str -i \"$TEST_CFG__REMOTE_SSH_KEY\"" + [ -n "${TEST_CFG__REMOTE_PORT:-}" ] && cmd_str="$cmd_str -p \"$TEST_CFG__REMOTE_PORT\"" + + REMOTE_SSH_PREFIX="$cmd_str \"$TEST_CFG__REMOTE_USERNAME@$TEST_CFG__REMOTE_HOST\"" +else + REMOTE_SSH_PREFIX="" +fi + +exec_command() { + local cmd="$1" + local prefix="$2" + + eval "$prefix $cmd" +} + +show_fs_state__impl() { + local prefix="$1" + local host_label="$2" + + set +x + echo "------------- ${host_label} FS STATE" + set -x + exec_command "df -P" "$prefix" +} + +check_leftover_ports__impl() { + local prefix="$1" + local host_label="$2" + local ports_dir="/tmp/testgres/ports" + + set +x + echo "------------- Checking ${host_label} ports lock directory" + set -x + + # Check command: will print FOUND if the directory exists and is not empty + local check_cmd="if [ -d '${ports_dir}' ] && [ \"\$(ls -A '${ports_dir}' 2>/dev/null)\" ]; then echo 'FOUND'; fi" + + # Temporarily disable bash's instant drop (set +e) to safely intercept the result + set +e + local result + result=$(exec_command "$check_cmd" "$prefix") + set -e + + set +x + if [ "$result" = "FOUND" ]; then + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "ERROR: Leftover ports detected in $ports_dir on $host_label machine!" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + set -x + + # We display a list of frozen ports so that the culprits can be identified + exec_command "ls -la '$ports_dir'" "$prefix" + + # We hard-drop the entire control script + # sleep 3600 + exit 1 + else + echo "Clear. No leftover port locks." + fi + set -x +} + +fs_verification__impl() { + show_fs_state__impl "$1" "$2" + + check_leftover_ports__impl "$1" "$2" +} + +fs_verification() { + fs_verification__impl "" "LOCAL" + + if [ -n "$REMOTE_SSH_PREFIX" ]; then + fs_verification__impl "$REMOTE_SSH_PREFIX" "REMOTE" + fi +} + +# ---------------------------------------- PATH + +fs_verification + +# run tests (PATH) +time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" + +# ---------------------------------------- finish + +fs_verification + +# ---------------------------------------- coverage + +coverage report diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..f8634cf --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,6 @@ +pytest +pytest-xdist +testgres>=1.17 +pefile +lz4 +zstd From 8a7da2588a23347949e857f4fc1dedbe0deaa70a Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 22 Oct 2026 01:07:40 +0300 Subject: [PATCH 2/5] run_tests_own.sh is updated (logging) --- run_tests_own.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/run_tests_own.sh b/run_tests_own.sh index 781c87d..4a16b79 100755 --- a/run_tests_own.sh +++ b/run_tests_own.sh @@ -2,6 +2,9 @@ set -eux +echo "HOME DIR IS [`realpath ~/`]" +echo "WORK DIR IS [$(pwd)]" + # Filter tests for local execution (without remote/ssh) if [ -z ${TEST_FILTER+x} ]; then export TEST_FILTER="" @@ -116,6 +119,9 @@ fs_verification() { fs_verification # run tests (PATH) +echo "HOME DIR IS [`realpath ~/`]" +echo "WORK DIR IS [$(pwd)]" + time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" # ---------------------------------------- finish From c96a6fbc4b81a9350cb676bdf724958562a192eb Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 22 Oct 2026 02:34:54 +0300 Subject: [PATCH 3/5] CI: pg14-pg18 is tested --- .github/workflows/ci.yml | 16 ++++++++++++++++ Dockerfile--alpine.tmpl | 24 ++++++++++++++++++++---- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43ad6cd..30e59cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,6 +110,22 @@ jobs: python: "3.14" postgres: "17" case_suffix: "py3_14_xx-pg17_xx" + - platform: "alpine" + python: "3.12" + postgres: "14" + case_suffix: "py3_12_xx-pg14_xx" + - platform: "alpine" + python: "3.12" + postgres: "15" + case_suffix: "py3_12_xx-pg15_xx" + - platform: "alpine" + python: "3.12" + postgres: "16" + case_suffix: "py3_12_xx-pg16_xx" + - platform: "alpine" + python: "3.12" + postgres: "18" + case_suffix: "py3_12_xx-pg18_xx" name: "test: ${{ matrix.platform }} | ${{ matrix.case_suffix }}" diff --git a/Dockerfile--alpine.tmpl b/Dockerfile--alpine.tmpl index 7bcbdb9..5eca570 100644 --- a/Dockerfile--alpine.tmpl +++ b/Dockerfile--alpine.tmpl @@ -42,9 +42,10 @@ RUN apk add --no-cache \ linux-headers # --------------------------------------------- postgres -FROM base1_with_dev_tools AS base1_with_pg-17 +FROM base1_with_dev_tools AS base2a_with_pg +ARG PG_VERSION -RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pgpro/postgres/source +RUN git clone https://github.com/postgres/postgres.git -b REL_${PG_VERSION}_STABLE /pgpro/postgres/source WORKDIR /pgpro/postgres/source @@ -60,10 +61,25 @@ RUN ln -s /pgpro/postgres/install/bin/pg_config -t /usr/local/bin # SETUP PG CLIENT LIBRARY # libpq.so.5 is enough -# RUN ln -s /pgpro/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 +# RUN ln -s /pgpro/postgres/install/lib/libpq.so.5.${PG_VERSION} /usr/lib64/libpq.so.5 + +# --------------------------------------------- postgres 14 +FROM base2a_with_pg AS base2_with_pg-14 + +# --------------------------------------------- postgres 15 +FROM base2a_with_pg AS base2_with_pg-15 + +# --------------------------------------------- postgres 16 +FROM base2a_with_pg AS base2_with_pg-16 + +# --------------------------------------------- postgres 17 +FROM base2a_with_pg AS base2_with_pg-17 + +# --------------------------------------------- postgres 18 +FROM base2a_with_pg AS base2_with_pg-18 # --------------------------------------------- base4-pbk -FROM base1_with_pg-${PG_VERSION} AS base4-pbk +FROM base2_with_pg-${PG_VERSION} AS base4-pbk RUN git clone https://github.com/postgrespro/pg_probackup.git /pgpro/pbk2/source From dd4a47db690e1983fa7b7eedc1b058596e82cf61 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 22 Oct 2026 02:44:33 +0300 Subject: [PATCH 4/5] CI: pbk2 does not support pg18 --- .github/workflows/ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30e59cc..f5c0fdf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,10 +122,13 @@ jobs: python: "3.12" postgres: "16" case_suffix: "py3_12_xx-pg16_xx" - - platform: "alpine" - python: "3.12" - postgres: "18" - case_suffix: "py3_12_xx-pg18_xx" +# +# 11.06 /usr/lib/gcc/x86_64-alpine-linux-musl/15.2.0/../../../../x86_64-alpine-linux-musl/bin/ld: src/pg_crc.o: in function `crc32_bytea': +# 11.06 pg_crc.c:(.text+0x9): undefined reference to `pg_detoast_datum_packed' +# - platform: "alpine" +# python: "3.12" +# postgres: "18" +# case_suffix: "py3_12_xx-pg18_xx" name: "test: ${{ matrix.platform }} | ${{ matrix.case_suffix }}" From 0b518223fbff0d84782de6a82f0cb72c06a0ebbd Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 22 Oct 2026 03:02:02 +0300 Subject: [PATCH 5/5] testgres>=1.17,<1.18 --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index f8634cf..94359ed 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,6 @@ pytest pytest-xdist -testgres>=1.17 +testgres>=1.17,<1.18 pefile lz4 zstd