Repository navigation
fix(datasets): hydrate media under numeric object keys #7494
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - "main" | |
| tags: | |
| - "v*" | |
| merge_group: | |
| pull_request: | |
| branches: | |
| - "*" | |
| permissions: {} | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| linting: | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Install uv and set Python version | |
| uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 | |
| with: | |
| version: "0.11.2" | |
| python-version: "3.13" | |
| enable-cache: true # zizmor: ignore[cache-poisoning] CI-only, no artifacts published | |
| - name: Install dependencies | |
| run: uv sync --locked | |
| - name: Run Ruff | |
| run: uv run --frozen ruff check . | |
| type-checking: | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Install uv and set Python version | |
| uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 | |
| with: | |
| version: "0.11.2" | |
| python-version: "3.13" | |
| enable-cache: true # zizmor: ignore[cache-poisoning] CI-only, no artifacts published | |
| - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # zizmor: ignore[cache-poisoning] | |
| name: Cache mypy cache | |
| with: | |
| path: ./.mypy_cache | |
| key: mypy-${{ hashFiles('**/*.py', 'pyproject.toml') }} | |
| restore-keys: | | |
| mypy- | |
| - name: Install dependencies | |
| run: uv sync --locked | |
| - name: Run mypy type checking | |
| run: uv run --frozen mypy langfuse --no-error-summary | |
| unit-tests: | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| timeout-minutes: 30 | |
| env: | |
| LANGFUSE_BASE_URL: "http://localhost:3000" | |
| LANGFUSE_PUBLIC_KEY: "pk-lf-test" | |
| LANGFUSE_SECRET_KEY: "sk-lf-test" | |
| OPENAI_API_KEY: "test-openai-key" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: | |
| - "3.11" | |
| - "3.12" | |
| - "3.13" | |
| - "3.14" | |
| - "3.15" | |
| name: Unit tests on Python ${{ matrix.python-version }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| # The pinned uv only knows Python 3.15 pre-releases; install the final release for uv to pick up. | |
| - name: Set up Python 3.15 | |
| if: ${{ matrix.python-version == '3.15' }} | |
| uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 | |
| with: | |
| python-version: "3.15" | |
| - name: Install uv and set Python version | |
| uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 | |
| with: | |
| version: "0.11.2" | |
| python-version: ${{ matrix.python-version }} | |
| enable-cache: true # zizmor: ignore[cache-poisoning] CI-only, no artifacts published | |
| - name: Check Python version | |
| run: python --version | |
| - name: Install the project dependencies | |
| run: uv sync --locked | |
| - name: Run the automated tests | |
| run: | | |
| python --version | |
| uv run --frozen pytest -n auto --dist worksteal -s -v --log-cli-level=INFO tests/unit | |
| # pydantic < 2.8 matches undiscriminated unions left to right, so the | |
| # CreateScoresResponse member order matters there. | |
| - name: Run score response parsing on pydantic 2.7 | |
| if: ${{ matrix.python-version == '3.11' }} | |
| run: | | |
| uv run --frozen --with "pydantic==2.7.4" python -c "import pydantic; assert pydantic.VERSION == '2.7.4', pydantic.VERSION" | |
| uv run --frozen --with "pydantic==2.7.4" pytest -v tests/unit/test_scores_batch_response.py | |
| e2e-tests: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - suite: e2e | |
| job_name: E2E shard 1 tests on Python 3.13 | |
| shard_name: shard-1 | |
| shard_index: 0 | |
| shard_count: 2 | |
| - suite: e2e | |
| job_name: E2E shard 2 tests on Python 3.13 | |
| shard_name: shard-2 | |
| shard_index: 1 | |
| shard_count: 2 | |
| - suite: live_provider | |
| job_name: E2E live-provider tests on Python 3.13 | |
| shard_name: live-provider | |
| env: | |
| LANGFUSE_BASE_URL: "http://localhost:3000" | |
| LANGFUSE_PUBLIC_KEY: "pk-lf-1234567890" | |
| LANGFUSE_SECRET_KEY: "sk-lf-1234567890" | |
| LANGFUSE_INIT_ORG_ID: "0c6c96f4-0ca0-4f16-92a8-6dd7d7c6a501" | |
| LANGFUSE_INIT_ORG_NAME: "SDK Test Org" | |
| LANGFUSE_INIT_PROJECT_ID: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a" | |
| LANGFUSE_INIT_PROJECT_NAME: "SDK Test Project" | |
| LANGFUSE_INIT_PROJECT_PUBLIC_KEY: "pk-lf-1234567890" | |
| LANGFUSE_INIT_PROJECT_SECRET_KEY: "sk-lf-1234567890" | |
| LANGFUSE_INIT_USER_EMAIL: "sdk-tests@langfuse.local" | |
| LANGFUSE_INIT_USER_NAME: "SDK Tests" | |
| LANGFUSE_INIT_USER_PASSWORD: "langfuse-ci-password" | |
| LANGFUSE_E2E_READ_TIMEOUT_SECONDS: "60" | |
| LANGFUSE_E2E_READ_INTERVAL_SECONDS: "0.5" | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| # SERPAPI_API_KEY: ${{ secrets.SERPAPI_API_KEY }} | |
| HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| # GitHub withholds secrets from fork PRs, so provider calls there can only fail. | |
| # The suite still runs on same-repo PRs and on main, where a missing key fails loudly. | |
| SKIP_LIVE_PROVIDER: ${{ matrix.suite == 'live_provider' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} | |
| name: ${{ matrix.job_name }} | |
| steps: | |
| - name: Skip live-provider tests on fork PRs | |
| if: ${{ env.SKIP_LIVE_PROVIDER == 'true' }} | |
| run: echo "::notice::Live-provider tests are skipped on fork PRs because provider secrets are unavailable. They run after merge on main." | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| if: ${{ env.SKIP_LIVE_PROVIDER != 'true' }} | |
| with: | |
| persist-credentials: false | |
| - name: Install uv and set Python version | |
| if: ${{ env.SKIP_LIVE_PROVIDER != 'true' }} | |
| uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 | |
| with: | |
| version: "0.11.2" | |
| python-version: "3.13" | |
| enable-cache: true # zizmor: ignore[cache-poisoning] CI-only, no artifacts published | |
| - name: Install the project dependencies | |
| if: ${{ env.SKIP_LIVE_PROVIDER != 'true' }} | |
| run: uv sync --locked | |
| - name: Check uv Python version | |
| if: ${{ env.SKIP_LIVE_PROVIDER != 'true' }} | |
| run: uv run --frozen python --version | |
| - name: Prepare langfuse server compose | |
| if: ${{ env.SKIP_LIVE_PROVIDER != 'true' }} | |
| run: | | |
| mkdir -p ./langfuse-server | |
| LANGFUSE_SERVER_SHA="$(git ls-remote https://github.com/langfuse/langfuse.git HEAD | cut -f1)" | |
| curl -fsSL "https://raw.githubusercontent.com/langfuse/langfuse/${LANGFUSE_SERVER_SHA}/docker-compose.yml" \ | |
| -o ./langfuse-server/docker-compose.yml | |
| echo "${LANGFUSE_SERVER_SHA}" | |
| - name: Run langfuse server | |
| if: ${{ env.SKIP_LIVE_PROVIDER != 'true' }} | |
| run: | | |
| cd ./langfuse-server | |
| echo "::group::Start langfuse server" | |
| TELEMETRY_ENABLED=false \ | |
| NEXT_PUBLIC_LANGFUSE_RUN_NEXT_INIT=true \ | |
| LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT=http://localhost:9090 \ | |
| LANGFUSE_INGESTION_QUEUE_DELAY_MS=10 \ | |
| LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=10 \ | |
| LANGFUSE_EXPERIMENT_INSERT_INTO_EVENTS_TABLE=true \ | |
| QUEUE_CONSUMER_EVENT_PROPAGATION_QUEUE_IS_ENABLED=true \ | |
| LANGFUSE_ENABLE_EVENTS_TABLE_V2_APIS=true \ | |
| LANGFUSE_ENABLE_EVENTS_TABLE_OBSERVATIONS=true \ | |
| docker compose up -d | |
| echo "::endgroup::" | |
| - name: Health check for langfuse server | |
| if: ${{ env.SKIP_LIVE_PROVIDER != 'true' }} | |
| run: | | |
| echo "Checking if the langfuse server is up..." | |
| retry_count=0 | |
| max_retries=20 | |
| until curl --output /dev/null --silent --head --fail http://localhost:3000/api/public/health && \ | |
| uv run --frozen python -c "from langfuse import Langfuse; client = Langfuse(); project_id = client._get_project_id(); assert project_id == '7a88fb47-b4e2-43b8-a06c-a5ce950dc53a', project_id; print(project_id)" | |
| do | |
| retry_count=`expr $retry_count + 1` | |
| echo "Attempt $retry_count of $max_retries..." | |
| if [ $retry_count -ge $max_retries ]; then | |
| echo "Langfuse server did not respond in time. Printing logs..." | |
| (cd ./langfuse-server && docker compose ps) | |
| (cd ./langfuse-server && docker compose logs langfuse-web langfuse-worker) | |
| echo "Failing the step..." | |
| exit 1 | |
| fi | |
| sleep 5 | |
| done | |
| echo "Langfuse server is up and running!" | |
| - name: Select e2e shard files | |
| if: ${{ matrix.suite == 'e2e' }} | |
| run: | | |
| uv run --frozen python scripts/select_e2e_shard.py \ | |
| --shard-index ${{ matrix.shard_index }} \ | |
| --shard-count ${{ matrix.shard_count }} \ | |
| --json | |
| uv run --frozen python scripts/select_e2e_shard.py \ | |
| --shard-index ${{ matrix.shard_index }} \ | |
| --shard-count ${{ matrix.shard_count }} \ | |
| > "$RUNNER_TEMP/e2e-shard-files.txt" | |
| cat "$RUNNER_TEMP/e2e-shard-files.txt" | |
| - name: Run the parallel end-to-end tests | |
| if: ${{ matrix.suite == 'e2e' }} | |
| run: | | |
| uv run --frozen python --version | |
| mapfile -t e2e_files < "$RUNNER_TEMP/e2e-shard-files.txt" | |
| set +e | |
| uv run --frozen pytest \ | |
| -n 4 --dist worksteal \ | |
| -s -v --log-cli-level=INFO \ | |
| --timeout=180 --timeout-method=thread --durations=20 \ | |
| "${e2e_files[@]}" -m "not serial_e2e" | |
| status=$? | |
| set -e | |
| if [ "$status" -eq 5 ]; then | |
| echo "No parallel e2e tests selected for this shard." | |
| elif [ "$status" -ne 0 ]; then | |
| exit "$status" | |
| fi | |
| - name: Run serial end-to-end tests | |
| if: ${{ matrix.suite == 'e2e' }} | |
| run: | | |
| mapfile -t e2e_files < "$RUNNER_TEMP/e2e-shard-files.txt" | |
| set +e | |
| uv run --frozen pytest \ | |
| -s -v --log-cli-level=INFO \ | |
| --timeout=180 --timeout-method=thread --durations=20 \ | |
| "${e2e_files[@]}" -m "serial_e2e" | |
| status=$? | |
| set -e | |
| if [ "$status" -eq 5 ]; then | |
| echo "No serial e2e tests selected for this shard." | |
| elif [ "$status" -ne 0 ]; then | |
| exit "$status" | |
| fi | |
| - name: Run live-provider tests | |
| if: ${{ matrix.suite == 'live_provider' && env.SKIP_LIVE_PROVIDER != 'true' }} | |
| run: | | |
| uv run --frozen python --version | |
| uv run --frozen pytest \ | |
| -n 4 --dist worksteal \ | |
| -s -v --log-cli-level=INFO \ | |
| --timeout=180 --timeout-method=thread --durations=20 \ | |
| tests/live_provider -m "live_provider" | |
| all-tests-passed: | |
| # This allows us to have a branch protection rule for tests and deploys with matrix | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| needs: [unit-tests, e2e-tests, linting, type-checking] | |
| if: always() | |
| steps: | |
| - name: Successful deploy | |
| if: ${{ !(contains(needs.*.result, 'failure')) }} | |
| run: exit 0 | |
| - name: Failing deploy | |
| if: ${{ contains(needs.*.result, 'failure') }} | |
| run: exit 1 |