diff --git a/.github/actions/setup-dev-python-env/action.yml b/.github/actions/setup-dev-python-env/action.yml new file mode 100644 index 0000000..463ed31 --- /dev/null +++ b/.github/actions/setup-dev-python-env/action.yml @@ -0,0 +1,15 @@ +name: Setup Dev Python Environment +description: Set up Python 3.13 and install dev dependencies (requirements-dev.txt). + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + with: + python-version: '3.13' + cache: 'pip' + + - name: Install dev dependencies + shell: bash + run: pip install -r requirements-dev.txt diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 37223dc..2760062 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -53,5 +53,8 @@ Quality gates (run after changes, fix only if below threshold) - Run all quality gates at once: `make qa` - Once a quality gate passes, do not re-run it in different scenarios +Dependencies +- Two requirements files: `requirements.txt` (prod/Docker only) and `requirements-dev.txt` (extends prod with test and lint tools) + Git workflow - Do NOT create git commits; committing is the developer's responsibility diff --git a/.github/workflows/check_python.yml b/.github/workflows/check_python.yml index 9402dd2..0af585e 100644 --- a/.github/workflows/check_python.yml +++ b/.github/workflows/check_python.yml @@ -39,9 +39,9 @@ jobs: if [[ "${{ github.event_name }}" == "pull_request" ]]; then CHANGED_FILES=$(gh api \ "repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \ - --jq '.[].filename | select(endswith(".py") or . == "requirements.txt")') + --jq '.[].filename | select(endswith(".py") or (startswith("requirements") and endswith(".txt")))') else - CHANGED_FILES=$(git diff --name-only "${{ github.sha }}~1" "${{ github.sha }}" -- '*.py' 'requirements.txt') + CHANGED_FILES=$(git diff --name-only "${{ github.sha }}~1" "${{ github.sha }}" -- '*.py' 'requirements*.txt') fi if [[ -n "$CHANGED_FILES" ]]; then @@ -62,14 +62,8 @@ jobs: persist-credentials: false fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: '3.13' - cache: 'pip' - - - name: Install dependencies - run: pip install -r requirements.txt + - name: Set up dev Python environment + uses: ./.github/actions/setup-dev-python-env - name: Analyze code with Pylint id: analyze-code @@ -98,14 +92,8 @@ jobs: persist-credentials: false fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: '3.13' - cache: 'pip' - - - name: Install dependencies - run: pip install -r requirements.txt + - name: Set up dev Python environment + uses: ./.github/actions/setup-dev-python-env - name: Check code format with Black id: check-format @@ -123,14 +111,8 @@ jobs: persist-credentials: false fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: '3.13' - cache: 'pip' - - - name: Install dependencies - run: pip install -r requirements.txt + - name: Set up dev Python environment + uses: ./.github/actions/setup-dev-python-env - name: Check types with Mypy id: check-types @@ -148,14 +130,8 @@ jobs: persist-credentials: false fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: '3.13' - cache: 'pip' - - - name: Install Python dependencies - run: pip install -r requirements.txt + - name: Set up dev Python environment + uses: ./.github/actions/setup-dev-python-env - name: Check code coverage with Pytest run: pytest --cov=. -v tests/unit/ --cov-fail-under=80 @@ -173,14 +149,8 @@ jobs: persist-credentials: false fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 - with: - python-version: '3.13' - cache: 'pip' - - - name: Install dependencies - run: pip install -r requirements.txt + - name: Set up dev Python environment + uses: ./.github/actions/setup-dev-python-env - name: Run integration tests run: pytest tests/integration/ -v --tb=short --log-cli-level=INFO diff --git a/DEVELOPER.md b/DEVELOPER.md index d1399a3..0eeff70 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -26,14 +26,13 @@ EventGate ships two Lambda functions: ## Prerequisites - Python 3.13 (current required runtime) -- PostgreSQL client dev package - - For local development, you may temporarily switch to the commented `psycopg2-binary==2.9.10` in `requirements.txt`. +- Docker (for local integration tests using testcontainers) ## Set Up Python Environment ```shell python3 -m venv .venv source .venv/bin/activate -pip3 install -r requirements.txt +pip3 install -r requirements-dev.txt ``` ## Run Pylint Tool Locally diff --git a/Dockerfile b/Dockerfile index e11a990..a90fa0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,9 @@ ARG SASL_SSL_ARTIFACTS=./sasl_ssl_artifacts # Trusted certs COPY $TRUSTED_SSL_CERTS /opt/certs/ +# Production dependencies +COPY requirements.txt ${LAMBDA_TASK_ROOT}/requirements.txt + RUN \ echo "######################################################" && \ echo "### Import trusted certs before doing anything else ###" && \ @@ -36,33 +39,28 @@ RUN \ echo "### -> Basics ###" && \ echo "### -> GCC (some makefiles require cmd which)###" && \ echo "### -> dependencies for kerberos SASL_SSL ###" && \ + echo "### -> PostgreSQL dev headers (psycopg2) ###" && \ echo "##############################################" && \ dnf install -y \ wget tar xz bzip2-devel zlib-devel \ which make gcc gcc-c++ \ - libffi-devel cyrus-sasl-devel cyrus-sasl-gssapi openssl-devel krb5-workstation && \ + libffi-devel cyrus-sasl-devel cyrus-sasl-gssapi openssl-devel krb5-workstation postgresql-devel && \ echo "#################" && \ echo "### librdkafka ###" && \ echo "#################" && \ mkdir -p /tmp/env-install-workdir/librdkafka && \ cd /tmp/env-install-workdir/librdkafka && \ - wget --ca-certificate=/etc/pki/tls/certs/ca-bundle.crt https://github.com/edenhill/librdkafka/archive/v2.4.0.tar.gz && \ - tar -xf v2.4.0.tar.gz && \ - cd /tmp/env-install-workdir/librdkafka/librdkafka-2.4.0 && \ + wget --ca-certificate=/etc/pki/tls/certs/ca-bundle.crt https://github.com/confluentinc/librdkafka/archive/v2.14.0.tar.gz && \ + tar -xf v2.14.0.tar.gz && \ + cd /tmp/env-install-workdir/librdkafka/librdkafka-2.14.0 && \ ./configure && make && make install && \ echo "###################" && \ echo "### pip installs ###" && \ echo "###################" && \ - pip install requests==2.31.0 urllib3==1.26.18 setuptools cryptography jsonschema PyJWT psycopg2-binary && \ - echo "######################" && \ - echo "### confluent-kafka ###" && \ - echo "######################" && \ - mkdir -p /tmp/env-install-workdir/confluent-kafka && \ - cd /tmp/env-install-workdir/confluent-kafka && \ - wget --ca-certificate=/etc/pki/tls/certs/ca-bundle.crt https://github.com/confluentinc/confluent-kafka-python/archive/v2.4.0.tar.gz && \ - tar -xf v2.4.0.tar.gz && \ - cd /tmp/env-install-workdir/confluent-kafka/confluent-kafka-python-2.4.0 && \ - CPPFLAGS="-I/usr/local/include" LDFLAGS="-L/opt" python setup.py install && \ + # requirements.txt pins the version of confluent-kafka. + # --no-binary confluent-kafka forces source compilation against the system librdkafka + # built above, which includes Kerberos/GSSAPI support. The PyPI compiles without GSSAPI. + pip install -r ${LAMBDA_TASK_ROOT}/requirements.txt --no-binary confluent-kafka && \ echo "##############" && \ echo "### cleanup ###" && \ echo "##############" && \ diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..9fc1a00 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,12 @@ +-r requirements.txt +psycopg2-binary==2.9.12 +pytest==9.0.3 +pytest-cov==7.1.0 +pytest-mock==3.15.1 +pylint==4.0.5 +black==26.5.1 +mypy==2.1.0 +mypy-extensions==1.1.0 +moto[s3,secretsmanager,events]==5.2.1 +testcontainers==4.14.2 +docker==7.1.0 diff --git a/requirements.txt b/requirements.txt index 0b0404e..081f37a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,9 @@ -pytest==9.0.3 -pytest-cov==7.1.0 -pytest-mock==3.15.1 -pylint==4.0.5 -black==26.5.1 -mypy==2.1.0 -mypy-extensions==1.1.0 -urllib3==2.7.0 cryptography==48.0.0 jsonschema==4.26.0 PyJWT==2.13.0 requests==2.34.2 boto3==1.43.14 -aiosql==15.0 botocore==1.43.14 +aiosql==15.0 confluent-kafka==2.14.0 -moto[s3,secretsmanager,events]==5.2.1 -testcontainers==4.14.2 -docker==7.1.0 -# psycopg2-binary==2.9.10 # Ideal for local development, but not for long-term production use psycopg2==2.9.12