Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 164 additions & 0 deletions .github/scripts/build-slo-image.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
#!/usr/bin/env bash
#
# Builds the Docker image for the YDB Java SDK SLO workload.
#
# The script assembles a temporary build context containing two checkouts
# side by side — the SDK source tree and the ydb-java-examples checkout —
# and feeds that context to `docker build` using the Dockerfile shipped
# inside `ydb-java-examples/slo/`.
#
# The Dockerfile takes care of building the SDK from source, installing it
# into an in-image local Maven repository, and then building the workload
# against that exact SDK version. So the script does not need any Maven /
# JDK setup on the host; only `docker` and standard POSIX tools.
#
# If the initial build fails and `--fallback-image` is provided, the script
# tags the fallback image as `--tag` and exits successfully. This mirrors
# the behaviour of the equivalent script in `ydb-go-sdk` and is used by the
# baseline build, where we want to keep going even if the historical commit
# can't be built any more.

set -euo pipefail

usage() {
cat >&2 <<'EOF'
Usage:
build-slo-image.sh \
--sdk <path> \
--examples <path> \
--tag <docker-tag> \
[--fallback-image <docker-tag>]

Options:
--sdk Path to the ydb-java-sdk checkout to build against.
--examples Path to the ydb-java-examples checkout that owns the
SLO workload sources (must contain slo/Dockerfile).
--tag Docker tag to assign to the built image
(e.g. ydb-app-current).
--fallback-image If the initial Docker build fails, tag this image as
--tag and exit successfully. Useful for the baseline
build, which may be unable to compile a historical
SDK commit.
EOF
}

die() {
echo "ERROR: $*" >&2
exit 1
}

sdk_dir=""
examples_dir=""
tag=""
fallback_image=""

while [[ $# -gt 0 ]]; do
case "$1" in
--sdk)
sdk_dir="${2:-}"
shift 2
;;
--examples)
examples_dir="${2:-}"
shift 2
;;
--tag)
tag="${2:-}"
shift 2
;;
--fallback-image)
fallback_image="${2:-}"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
die "Unknown argument: $1 (use --help)"
;;
esac
done

if [[ -z "$sdk_dir" || -z "$examples_dir" || -z "$tag" ]]; then
usage
die "Incomplete argument set"
fi

[[ -d "$sdk_dir" ]] || die "--sdk does not exist: $sdk_dir"
[[ -d "$examples_dir" ]] || die "--examples does not exist: $examples_dir"

sdk_dir="$(cd "$sdk_dir" && pwd)"
examples_dir="$(cd "$examples_dir" && pwd)"

dockerfile="${examples_dir}/slo/Dockerfile"
[[ -f "$dockerfile" ]] || die "Dockerfile not found: $dockerfile"

# Assemble a build context that contains the two checkouts side by side.
# We use hard links where possible to avoid copying large amounts of data;
# `cp -al` falls back to a regular copy when hard links aren't supported
# (e.g. across filesystems on the GitHub runner cache).
context_dir="$(mktemp -d)"
trap 'rm -rf "$context_dir"' EXIT

echo "Assembling build context in $context_dir"
echo " ydb-java-sdk: $sdk_dir"
echo " ydb-java-examples: $examples_dir"
echo " tag: $tag"

copy_tree() {
local src="$1"
local dst="$2"
mkdir -p "$dst"
# The "/." suffix on src and "/" on dst asks cp to copy CONTENTS of src
# into dst, regardless of whether dst pre-existed. Without this, partial
# hardlink failures can leave dst partially populated and the fallback
# then nests src inside dst (dst/src/...) instead of overwriting.
if cp -al "$src"/. "$dst"/ 2>/dev/null; then
return 0
fi
cp -a "$src"/. "$dst"/
}

copy_tree "$sdk_dir" "$context_dir/ydb-java-sdk"
copy_tree "$examples_dir" "$context_dir/ydb-java-examples"

# Drop .git from each copied tree so it doesn't ship into image layers or
# confuse Maven plugins that probe for git metadata.
rm -rf "$context_dir/ydb-java-sdk/.git" "$context_dir/ydb-java-examples/.git"

# Fail fast with a clear message if the layout is wrong (e.g. because of a
# silent copy failure on the runner).
for required in \
"$context_dir/ydb-java-sdk/pom.xml" \
"$context_dir/ydb-java-examples/slo/Dockerfile"
do
[[ -f "$required" ]] || die "Build context missing required file: $required"
done

echo "Build context layout:"
ls -la "$context_dir"
echo " ydb-java-sdk: $(ls -1 "$context_dir/ydb-java-sdk" | wc -l) entries"

set +e
docker build \
--platform linux/amd64 \
-t "$tag" \
-f "$dockerfile" \
"$context_dir"
exit_code=$?
set -e

if [[ $exit_code -eq 0 ]]; then
echo "Docker image $tag built successfully"
exit 0
fi

echo "Docker build for $tag failed (exit code $exit_code)" >&2

if [[ -z "$fallback_image" ]]; then
die "Docker build failed and --fallback-image is not set"
fi

echo "Falling back to image $fallback_image, tagging as $tag"
docker tag "$fallback_image" "$tag"
40 changes: 40 additions & 0 deletions .github/workflows/slo-report.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
name: slo-report

on:
workflow_run:
workflows: ["SLO"]
types:
- completed

jobs:
publish-slo-report:
if: github.event.workflow_run.conclusion == 'success'
runs-on: ubuntu-latest
name: Publish YDB SLO Report
permissions:
checks: write
contents: read
pull-requests: write
steps:
- name: Publish YDB SLO Report
uses: ydb-platform/ydb-slo-action/report@v2
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
github_run_id: ${{ github.event.workflow_run.id }}

remove-slo-label:
if: github.event.workflow_run.event == 'pull_request'
runs-on: ubuntu-latest
name: Remove SLO Label
permissions:
pull-requests: write
steps:
- name: Remove SLO label from PR
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PRS: ${{ toJSON(github.event.workflow_run.pull_requests) }}
REPO: ${{ github.event.workflow_run.repository.full_name }}
run: |
set -euo pipefail
PR=$(jq -r '.[0].number' <<<"$PRS")
gh pr edit "$PR" --repo "$REPO" --remove-label SLO
129 changes: 129 additions & 0 deletions .github/workflows/slo.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
name: SLO

on:
pull_request:
types: [opened, reopened, synchronize, labeled]

jobs:
ydb-slo-action:
if: contains(github.event.pull_request.labels.*.name, 'SLO')

name: Run YDB SLO Tests
runs-on: large-runner-java-sdk

permissions:
contents: read

strategy:
fail-fast: false
matrix:
sdk:
- name: java-query-kv
command: ""

concurrency:
group: slo-${{ github.ref }}-${{ matrix.sdk.name }}
cancel-in-progress: true

steps:
- name: Install dependencies
run: |
set -euxo pipefail
YQ_VERSION=v4.48.2
BUILDX_VERSION=0.30.1
COMPOSE_VERSION=2.40.3

sudo curl -fLo /usr/local/bin/yq \
"https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64"
sudo chmod +x /usr/local/bin/yq

sudo mkdir -p /usr/local/lib/docker/cli-plugins

sudo curl -fLo /usr/local/lib/docker/cli-plugins/docker-buildx \
"https://github.com/docker/buildx/releases/download/v${BUILDX_VERSION}/buildx-v${BUILDX_VERSION}.linux-amd64"
sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx

sudo curl -fLo /usr/local/lib/docker/cli-plugins/docker-compose \
"https://github.com/docker/compose/releases/download/v${COMPOSE_VERSION}/docker-compose-linux-x86_64"
sudo chmod +x /usr/local/lib/docker/cli-plugins/docker-compose

yq --version
docker --version
docker buildx version
docker compose version

- name: Checkout current SDK version
uses: actions/checkout@v5
with:
path: sdk-current
fetch-depth: 0

- name: Determine baseline commit
id: baseline
working-directory: sdk-current
run: |
set -euo pipefail
BASELINE=$(git merge-base HEAD origin/master)
echo "sha=${BASELINE}" >> "$GITHUB_OUTPUT"

if git merge-base --is-ancestor "${BASELINE}" origin/master && \
[ "$(git rev-parse origin/master)" = "${BASELINE}" ]; then
BASELINE_REF="master"
else
BRANCH=$(git branch -r --contains "${BASELINE}" | grep -v HEAD | head -1 | sed 's|.*/||' || echo "")
if [ -n "${BRANCH}" ]; then
BASELINE_REF="${BRANCH}@${BASELINE:0:7}"
else
BASELINE_REF="${BASELINE:0:7}"
fi
fi
echo "ref=${BASELINE_REF}" >> "$GITHUB_OUTPUT"

- name: Checkout baseline SDK version
uses: actions/checkout@v5
with:
ref: ${{ steps.baseline.outputs.sha }}
path: sdk-baseline
fetch-depth: 1

- name: Checkout ydb-java-examples
uses: actions/checkout@v5
with:
repository: ydb-platform/ydb-java-examples
ref: master
path: examples

- name: Build current workload image
run: |
set -euxo pipefail
chmod +x sdk-current/.github/scripts/build-slo-image.sh
sdk-current/.github/scripts/build-slo-image.sh \
--sdk "${GITHUB_WORKSPACE}/sdk-current" \
--examples "${GITHUB_WORKSPACE}/examples" \
--tag ydb-app-current

- name: Build baseline workload image
run: |
set -euxo pipefail
# Reuse the build script from the current checkout — it doesn't
# depend on SDK contents, only on the layout it produces.
sdk-current/.github/scripts/build-slo-image.sh \
--sdk "${GITHUB_WORKSPACE}/sdk-baseline" \
--examples "${GITHUB_WORKSPACE}/examples" \
--tag ydb-app-baseline \
--fallback-image ydb-app-current

- name: Run SLO Tests
uses: ydb-platform/ydb-slo-action/init@v2
timeout-minutes: 30
with:
github_issue: ${{ github.event.pull_request.number }}
github_token: ${{ secrets.GITHUB_TOKEN }}
workload_name: ${{ matrix.sdk.name }}
workload_duration: "600"
workload_current_ref: ${{ github.head_ref || github.ref_name }}
workload_current_image: ydb-app-current
workload_current_command: ${{ matrix.sdk.command }} --read-rps 1000 --write-rps 100
workload_baseline_ref: ${{ steps.baseline.outputs.ref }}
workload_baseline_image: ydb-app-baseline
workload_baseline_command: ${{ matrix.sdk.command }} --read-rps 1000 --write-rps 100