Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 93 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,96 @@ jobs:
- name: Check package metadata
run: |
twine check dist/*

test-linux-container:
runs-on: ubuntu-latest
needs: build-check
strategy:
fail-fast: false
matrix:
include:
- platform: "alpine"
python: "3.7"
postgres: "17"
case_suffix: "py3_07_xx-pg17_xx"
- platform: "alpine"
python: "3.8.0"
postgres: "17"
case_suffix: "py3_08_00-pg17_xx"
- platform: "alpine"
python: "3.8"
postgres: "17"
case_suffix: "py3_08_xx-pg17_xx"
- platform: "alpine"
python: "3.9"
postgres: "17"
case_suffix: "py3_09_xx-pg17_xx"
- platform: "alpine"
python: "3.10"
postgres: "17"
case_suffix: "py3_10_xx-pg17_xx"
- platform: "alpine"
python: "3.11"
postgres: "17"
case_suffix: "py3_11_xx-pg17_xx"
- platform: "alpine"
python: "3.12"
postgres: "17"
case_suffix: "py3_12_xx-pg17_xx"
- platform: "alpine"
python: "3.13"
postgres: "17"
case_suffix: "py3_13_xx-pg17_xx"
- platform: "alpine"
python: "3.14"
postgres: "17"
case_suffix: "py3_14_xx-pg17_xx"
- platform: "alpine"
python: "3.12"
postgres: "14"
case_suffix: "py3_12_xx-pg14_xx"
- platform: "alpine"
python: "3.12"
postgres: "15"
case_suffix: "py3_12_xx-pg15_xx"
- platform: "alpine"
python: "3.12"
postgres: "16"
case_suffix: "py3_12_xx-pg16_xx"
#
# 11.06 /usr/lib/gcc/x86_64-alpine-linux-musl/15.2.0/../../../../x86_64-alpine-linux-musl/bin/ld: src/pg_crc.o: in function `crc32_bytea':
# 11.06 pg_crc.c:(.text+0x9): undefined reference to `pg_detoast_datum_packed'
# - platform: "alpine"
# python: "3.12"
# postgres: "18"
# case_suffix: "py3_12_xx-pg18_xx"

name: "test: ${{ matrix.platform }} | ${{ matrix.case_suffix }}"

env:
BASE_SIGN: "${{ matrix.platform }}-${{ matrix.case_suffix }}"

steps:
- name: Prepare variables
run: |
echo "RUN_CFG__NOW=$(date +'%Y%m%d_%H%M%S')" >> $GITHUB_ENV
echo "RUN_CFG__LOGS_DIR=logs-${{ env.BASE_SIGN }}" >> $GITHUB_ENV
echo "RUN_CFG__DOCKER_IMAGE_NAME=tests-${{ env.BASE_SIGN }}" >> $GITHUB_ENV
echo "---------- [$GITHUB_ENV]"
cat $GITHUB_ENV
- name: Checkout
uses: actions/checkout@v7
- name: Prepare logs folder on the host
run: mkdir -p "${{ env.RUN_CFG__LOGS_DIR }}"
- name: Adjust logs folder permission
run: chmod -R 777 "${{ env.RUN_CFG__LOGS_DIR }}"
- name: Build local image ${{ matrix.alpine }}
run: docker build --build-arg PG_VERSION="${{ matrix.postgres }}" --build-arg PYTHON_VERSION="${{ matrix.python }}" -t "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" -f Dockerfile--${{ matrix.platform }}.tmpl .
- name: Run
run: docker run -t -v ${{ github.workspace }}/${{ env.RUN_CFG__LOGS_DIR }}:/home/test/work/logs "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}"
- name: Upload Logs
uses: actions/upload-artifact@v7
if: always() # IT IS IMPORTANT!
with:
name: testgres.pbk2--test_logs--${{ env.RUN_CFG__NOW }}-${{ env.BASE_SIGN }}-id${{ github.run_id }}
path: "${{ env.RUN_CFG__LOGS_DIR }}/"
160 changes: 160 additions & 0 deletions Dockerfile--alpine.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
ARG PG_VERSION=17
ARG PYTHON_VERSION=3.12

# --------------------------------------------- base1
FROM python:${PYTHON_VERSION}-alpine AS base1

RUN apk add --no-cache \
coreutils \
bash \
mc \
procps \
openssh \
sshpass \
sudo \
git \
curl

RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \
sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \
sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config

ENV PYTHON_BINARY=python3

# --------------------------------------------- postgres
FROM base1 AS base1_with_dev_tools

RUN apk add --no-cache \
gcc \
make \
meson \
flex \
bison \
perl \
pkgconfig \
openssl-dev \
icu-dev \
zstd-dev \
zlib-dev \
lz4-dev \
libxml2-dev \
musl-dev \
linux-headers

# --------------------------------------------- postgres
FROM base1_with_dev_tools AS base2a_with_pg
ARG PG_VERSION

RUN git clone https://github.com/postgres/postgres.git -b REL_${PG_VERSION}_STABLE /pgpro/postgres/source

WORKDIR /pgpro/postgres/source

RUN ./configure --prefix=/pgpro/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml
RUN make -j 4 install
RUN make -j 4 -C contrib install

# SETUP PG_CONFIG
ENV PG_CONFIG="/pgpro/postgres/install/bin/pg_config"

# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir
RUN ln -s /pgpro/postgres/install/bin/pg_config -t /usr/local/bin

# SETUP PG CLIENT LIBRARY
# libpq.so.5 is enough
# RUN ln -s /pgpro/postgres/install/lib/libpq.so.5.${PG_VERSION} /usr/lib64/libpq.so.5

# --------------------------------------------- postgres 14
FROM base2a_with_pg AS base2_with_pg-14

# --------------------------------------------- postgres 15
FROM base2a_with_pg AS base2_with_pg-15

# --------------------------------------------- postgres 16
FROM base2a_with_pg AS base2_with_pg-16

# --------------------------------------------- postgres 17
FROM base2a_with_pg AS base2_with_pg-17

# --------------------------------------------- postgres 18
FROM base2a_with_pg AS base2_with_pg-18

# --------------------------------------------- base4-pbk
FROM base2_with_pg-${PG_VERSION} AS base4-pbk

RUN git clone https://github.com/postgrespro/pg_probackup.git /pgpro/pbk2/source

WORKDIR /pgpro/pbk2/source

RUN top_srcdir=/pgpro/postgres/source/ CFLAGS='-g3 -O0' make USE_PGXS=1

ENV PGPROBACKUPSRC="/pgpro/pbk2/source"
ENV PGPROBACKUPBIN="/pgpro/pbk2/source/pg_probackup"

# --------------------------------------------- final
FROM base4-pbk AS final

EXPOSE 22
RUN ssh-keygen -A

RUN adduser -D test && addgroup -S sudo && adduser test sudo

# It allows to use sudo without password
RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers

# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD
RUN echo "test:*" | chpasswd -e

COPY --chown=test:test . /home/test/work
WORKDIR /home/test/work

RUN chown -R test /pgpro/pbk2/source

ENV LANG=C.UTF-8

RUN chmod 700 /home/test/ && \
mkdir -p /home/test/.ssh && \
echo 'Host *' > /home/test/.ssh/config && \
echo ' ControlMaster auto' >> /home/test/.ssh/config && \
echo ' ControlPath /home/test/.ssh/master-%r@%h:%p' >> /home/test/.ssh/config && \
echo ' ControlPersist 30m' >> /home/test/.ssh/config && \
echo ' StrictHostKeyChecking no' >> /home/test/.ssh/config && \
echo ' UserKnownHostsFile /dev/null' >> /home/test/.ssh/config && \
# echo ' GSSAPIAuthentication no' >> /home/test/.ssh/config && \
chown -R test:test /home/test/.ssh && \
chmod 700 /home/test/.ssh && \
chmod 600 /home/test/.ssh/config

#
# \"$@\"
# - quote is important!
# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ.
#
ENTRYPOINT ["sh", "-c", " \
set -eux; \
echo 'SYSTEM START: PREPARING SSH'; \
/usr/sbin/sshd; \
ls -la /home/test/.ssh/; \
\"$@\" \
", "DUMMY-DUMMY-DUMMY"]

# Run tests by default (master machine role)
CMD ["bash", "-c", " \
set -eux; \
echo \"HOME DIR IS [`realpath ~/`]\"; \
echo \"WORK DIR IS [$(pwd)]\"; \
if [ ! -f /home/test/.ssh/id_rsa ]; then \
su test -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\"; \
su test -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\"; \
chmod 600 /home/test/.ssh/authorized_keys; \
fi; \
ls -la /home/test/.ssh/; \
if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \
cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \
export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \
chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \
chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \
ls -la /home/test/.ssh/; \
fi; \
ls -la ./; \
su test -c \"TEST_FILTER='' bash ./run_tests_own.sh\"; \
"]
133 changes: 133 additions & 0 deletions run_tests_own.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
#!/usr/bin/env bash

set -eux

echo "HOME DIR IS [`realpath ~/`]"
echo "WORK DIR IS [$(pwd)]"

# Filter tests for local execution (without remote/ssh)
if [ -z ${TEST_FILTER+x} ]; then
export TEST_FILTER=""
fi

echo NPROC: $(nproc)

# Check for the presence of pg_config
echo check that pg_config is in PATH
command -v pg_config

# Setting up the Python environment
VENV_PATH="/tmp/testgres_venv"
rm -rf $VENV_PATH
${PYTHON_BINARY} -m venv "${VENV_PATH}"
export VIRTUAL_ENV_DISABLE_PROMPT=1
source "${VENV_PATH}/bin/activate"
pip install --upgrade pip setuptools wheel
pip install -r tests/requirements.txt

# remove existing coverage file
export COVERAGE_FILE=.coverage
rm -f $COVERAGE_FILE

pip install coverage

if [ -n "${TEST_CFG__REMOTE_HOST:-}" ] && [ -n "${TEST_CFG__REMOTE_USERNAME:-}" ]; then
cmd_str="ssh"

if [ -n "${TEST_CFG__REMOTE_PASSWORD:-}" ]; then
cmd_str="sshpass -p \"$TEST_CFG__REMOTE_PASSWORD\" $cmd_str"
fi

[ -n "${TEST_CFG__REMOTE_SSH_KEY:-}" ] && cmd_str="$cmd_str -i \"$TEST_CFG__REMOTE_SSH_KEY\""
[ -n "${TEST_CFG__REMOTE_PORT:-}" ] && cmd_str="$cmd_str -p \"$TEST_CFG__REMOTE_PORT\""

REMOTE_SSH_PREFIX="$cmd_str \"$TEST_CFG__REMOTE_USERNAME@$TEST_CFG__REMOTE_HOST\""
else
REMOTE_SSH_PREFIX=""
fi

exec_command() {
local cmd="$1"
local prefix="$2"

eval "$prefix $cmd"
}

show_fs_state__impl() {
local prefix="$1"
local host_label="$2"

set +x
echo "------------- ${host_label} FS STATE"
set -x
exec_command "df -P" "$prefix"
}

check_leftover_ports__impl() {
local prefix="$1"
local host_label="$2"
local ports_dir="/tmp/testgres/ports"

set +x
echo "------------- Checking ${host_label} ports lock directory"
set -x

# Check command: will print FOUND if the directory exists and is not empty
local check_cmd="if [ -d '${ports_dir}' ] && [ \"\$(ls -A '${ports_dir}' 2>/dev/null)\" ]; then echo 'FOUND'; fi"

# Temporarily disable bash's instant drop (set +e) to safely intercept the result
set +e
local result
result=$(exec_command "$check_cmd" "$prefix")
set -e

set +x
if [ "$result" = "FOUND" ]; then
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo "ERROR: Leftover ports detected in $ports_dir on $host_label machine!"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
set -x

# We display a list of frozen ports so that the culprits can be identified
exec_command "ls -la '$ports_dir'" "$prefix"

# We hard-drop the entire control script
# sleep 3600
exit 1
else
echo "Clear. No leftover port locks."
fi
set -x
}

fs_verification__impl() {
show_fs_state__impl "$1" "$2"

check_leftover_ports__impl "$1" "$2"
}

fs_verification() {
fs_verification__impl "" "LOCAL"

if [ -n "$REMOTE_SSH_PREFIX" ]; then
fs_verification__impl "$REMOTE_SSH_PREFIX" "REMOTE"
fi
}

# ---------------------------------------- PATH

fs_verification

# run tests (PATH)
echo "HOME DIR IS [`realpath ~/`]"
echo "WORK DIR IS [$(pwd)]"

time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"

# ---------------------------------------- finish

fs_verification

# ---------------------------------------- coverage

coverage report
6 changes: 6 additions & 0 deletions tests/requirements.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pytest
pytest-xdist
testgres>=1.17,<1.18
pefile
lz4
zstd
Loading