diff --git a/.github/workflows/dev-instance.yml b/.github/workflows/dev-instance.yml new file mode 100644 index 0000000000..cf6bd5ac4c --- /dev/null +++ b/.github/workflows/dev-instance.yml @@ -0,0 +1,139 @@ +name: Dev instance +run-name: Dev instance (${{ inputs.ref || github.ref_name }}) + +# Builds the images that differ from upstream and deploys them to the dev +# server (deploy/dev-instance). A merge to the `dev` branch ships it. +# +# From the Actions tab, "Run workflow" can also deploy another branch, tag or +# commit (`ref`), for example a pull request that is not merged yet. There is +# only one server: the last deploy wins, and the next merge to `dev` replaces +# it. The ref must contain deploy/dev-instance. +on: + push: + branches: + - dev + workflow_dispatch: + inputs: + ref: + description: "Branch, tag or commit to deploy. The last deploy wins: the next merge to dev replaces it." + type: string + default: dev + +permissions: + contents: read + +# Never cancel a deploy halfway; queue the next one instead. +concurrency: + group: dev-instance + cancel-in-progress: false + +jobs: + resolve: + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.sha.outputs.sha }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.sha }} + - name: Commit to deploy + id: sha + run: | + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "Commit: $(git log -1 --format='%h %s')" >> "$GITHUB_STEP_SUMMARY" + + build: + needs: resolve + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: true + matrix: + include: + - name: frontend + file: src/frontend/Dockerfile + target: frontend-production + - name: yhub + file: src/yhub-server/Dockerfile + target: yhub + - name: y-provider + file: src/frontend/servers/y-provider/Dockerfile + target: y-provider + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.resolve.outputs.sha }} + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Image name + id: image + run: echo "name=ghcr.io/${GITHUB_REPOSITORY,,}/${{ matrix.name }}" >> "$GITHUB_OUTPUT" + - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7 + with: + context: . + file: ${{ matrix.file }} + target: ${{ matrix.target }} + platforms: linux/amd64 + build-args: | + DOCKER_USER=1001:127 + PUBLISH_AS_MIT=false + push: true + provenance: false + tags: ${{ steps.image.outputs.name }}:${{ needs.resolve.outputs.sha }} + cache-from: type=gha,scope=dev-${{ matrix.name }} + cache-to: type=gha,scope=dev-${{ matrix.name }},mode=max + + deploy: + needs: [resolve, build] + runs-on: ubuntu-latest + timeout-minutes: 150 + permissions: + contents: read + packages: read + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ needs.resolve.outputs.sha }} + - name: Set up SSH + env: + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + DEPLOY_USER: ${{ secrets.DEPLOY_USER }} + run: | + install -d -m 700 ~/.ssh + printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519 + printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts + chmod 600 ~/.ssh/id_ed25519 ~/.ssh/known_hosts + cat > ~/.ssh/config </dev/null \ + && trap 'docker logout ghcr.io >/dev/null' EXIT \ + && /opt/docs/scripts/deploy.sh '${{ needs.resolve.outputs.sha }}'" + - name: Check the instance answers + run: | + host=$(ssh dev-instance "grep '^DOCS_HOST=' /opt/docs/.env | cut -d= -f2") + curl -fsS --retry 10 --retry-delay 6 --retry-all-errors -o /dev/null "https://$host/" + curl -fsS --retry 10 --retry-delay 6 --retry-all-errors "https://$host/api/v1.0/config/" -o /dev/null + echo "Deployed to https://$host/" >> "$GITHUB_STEP_SUMMARY" diff --git a/deploy/dev-instance/Caddyfile b/deploy/dev-instance/Caddyfile new file mode 100644 index 0000000000..7885e9ab69 --- /dev/null +++ b/deploy/dev-instance/Caddyfile @@ -0,0 +1,17 @@ +# TLS for the three public host names, with certificates from Let's Encrypt. +# The host names come from the .env file on the server. + +{$DOCS_HOST} { + encode zstd gzip + reverse_proxy frontend:8083 +} + +{$KEYCLOAK_HOST} { + reverse_proxy keycloak:8080 +} + +# The media bucket. Access is signed by the backend (see /media-auth in the +# nginx template), the bucket itself is private. +{$S3_HOST} { + reverse_proxy minio:9000 +} diff --git a/deploy/dev-instance/README.md b/deploy/dev-instance/README.md new file mode 100644 index 0000000000..ca9a4b2423 --- /dev/null +++ b/deploy/dev-instance/README.md @@ -0,0 +1,56 @@ +# Dev instance + +A dev instance of Docs on one server, with the BlockNote version history work. +`.github/workflows/dev-instance.yml` deploys it. + +- Docs: https://167-233-225-101.sslip.io +- Keycloak: https://id.167-233-225-101.sslip.io + +## Demo users + +The password of each demo user is the user name. + +| User | Password | +| --- | --- | +| alice | alice | +| bob | bob | +| carol | carol | +| dave | dave | + +Every deploy makes sure that the sample document "07/10 meeting agenda (sample)" +exists. alice owns it, and bob, carol and dave can edit it. Its history has 15 +named versions by the four users (`scripts/seed.sh`). + +## Admin accounts + +The admin accounts have random passwords. They are only on the server, in +`/opt/docs/.env`: + +```bash +ssh deploy@167.233.225.101 'grep -E "^(SEED_PASSWORD_ADMIN|KEYCLOAK_ADMIN_PASSWORD)=" /opt/docs/.env' +``` + +| Account | User | Password variable | +| --- | --- | --- | +| Django admin (`/admin`) | admin@example.com | `SEED_PASSWORD_ADMIN` | +| "admin" user of the "docs" realm | admin | `SEED_PASSWORD_ADMIN` | +| Keycloak administration console | admin | `KEYCLOAK_ADMIN_PASSWORD` | + +## Deploy a change + +1. Open a pull request against `dev`. +2. Merge it. The workflow builds the frontend, yhub and y-provider images and + deploys them. + +To deploy a branch that is not merged, run the "Dev instance" workflow from the +Actions tab and set `ref`. There is only one server: the last deploy wins. + +## Server + +- The stack is in `/opt/docs` (`compose.yaml`). The `deploy` user owns it. +- The first deploy writes `/opt/docs/.env` with random secrets. It never + changes them after that. +- The data of the sample document is `/opt/docs/seed-data/meetingVersions.json`. + It is not in the repository. Without it, the deploy skips the sample + document. +- There are no backups. diff --git a/deploy/dev-instance/compose.yaml b/deploy/dev-instance/compose.yaml new file mode 100644 index 0000000000..6322164855 --- /dev/null +++ b/deploy/dev-instance/compose.yaml @@ -0,0 +1,203 @@ +# Dev instance of Docs on one server, deployed by +# .github/workflows/dev-instance.yml. It follows the production example in +# documentation/examples/compose/compose.yaml, with the identity provider, the +# object storage and the TLS proxy on the same machine. +# +# Every value that is specific to the server (host names, passwords, keys) is +# in the `.env` file next to this one on the server, which +# scripts/init-env.sh generates. Nothing secret is in the repository. +# +# Only caddy publishes ports (80 and 443). Docker bypasses the host firewall +# for published ports, so do not publish ports on other services. + +name: docs + +services: + caddy: + image: caddy:2 + restart: unless-stopped + ports: + - "80:80" + - "443:443" + - "443:443/udp" + environment: + - DOCS_HOST=${DOCS_HOST} + - KEYCLOAK_HOST=${KEYCLOAK_HOST} + - S3_HOST=${S3_HOST} + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy-data:/data + - caddy-config:/config + + postgresql: + image: postgres:16 + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] + interval: 1s + timeout: 2s + retries: 300 + env_file: + - env.d/postgresql + environment: + - PGDATA=/var/lib/postgresql/data/pgdata + volumes: + - postgresql:/var/lib/postgresql/data + + redis: + image: redis:8 + restart: unless-stopped + + # Holds the updates that no yhub worker has persisted yet: a store, not a + # cache. Hence the append-only file. + yhub-valkey: + image: valkey/valkey:8-alpine + restart: unless-stopped + command: ["valkey-server", "--maxmemory-policy", "volatile-lru", + "--appendonly", "yes", "--appendfsync", "everysec"] + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 1s + timeout: 2s + retries: 60 + volumes: + - yhub-valkey:/data + + minio: + image: pgsty/silo + restart: unless-stopped + command: server /data + environment: + - MINIO_ROOT_USER=${S3_ACCESS_KEY} + - MINIO_ROOT_PASSWORD=${S3_SECRET_KEY} + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 2s + timeout: 20s + retries: 150 + volumes: + - minio:/data + + # One-shot, run by deploy.sh + createbuckets: + image: pgsty/mc + profiles: ["setup"] + depends_on: + minio: + condition: service_healthy + environment: + - S3_ACCESS_KEY=${S3_ACCESS_KEY} + - S3_SECRET_KEY=${S3_SECRET_KEY} + - BUCKET_NAME=${BUCKET_NAME:-docs-media-storage} + entrypoint: > + sh -c " + mc alias set docs http://minio:9000 $${S3_ACCESS_KEY} $${S3_SECRET_KEY} && + mc mb --ignore-existing docs/$${BUCKET_NAME} && + mc version enable docs/$${BUCKET_NAME}" + + keycloak: + image: quay.io/keycloak/keycloak:26.7 + restart: unless-stopped + # Development mode with the embedded database: enough for a dev instance. + # The realm and its users are imported on the first start only. + command: ["start-dev", "--import-realm"] + environment: + - KC_HOSTNAME=https://${KEYCLOAK_HOST} + - KC_PROXY_HEADERS=xforwarded + - KC_HTTP_ENABLED=true + - KC_BOOTSTRAP_ADMIN_USERNAME=admin + - KC_BOOTSTRAP_ADMIN_PASSWORD=${KEYCLOAK_ADMIN_PASSWORD} + # Read by keycloak/realm.json + - DOCS_HOST=${DOCS_HOST} + - OIDC_CLIENT_SECRET=${OIDC_CLIENT_SECRET} + - SEED_PASSWORD_ADMIN=${SEED_PASSWORD_ADMIN} + volumes: + - keycloak:/opt/keycloak/data + - ./keycloak/realm.json:/opt/keycloak/data/import/realm.json:ro + + backend: + # Upstream image of the commit this branch is based on: Docs' backend is not + # changed here. Update it when merging a newer upstream main. + image: ${BACKEND_IMAGE:-lasuite/impress-backend:main@sha256:525fbb920e7ad8de69ccfde490030db608d2f11693cbcc6af81e681850700558} + user: "1000" + restart: unless-stopped + environment: + - DJANGO_CONFIGURATION=Production + env_file: + - env.d/common + - env.d/backend + - env.d/yprovider + - env.d/postgresql + volumes: + - ./keys/private.pem:/keys/private.pem:ro + healthcheck: + test: ["CMD", "python", "manage.py", "check"] + interval: 15s + timeout: 30s + retries: 20 + start_period: 10s + depends_on: + postgresql: + condition: service_healthy + restart: true + redis: + condition: service_started + + # The collaboration server. Run `yarn init-db` before it starts (deploy.sh + # does it on every deploy, it is idempotent). + yhub: + image: ${IMAGE_REGISTRY:-ghcr.io/typecellos/docs}/yhub:${IMAGE_TAG} + user: "1000" + restart: unless-stopped + env_file: + - env.d/common + - env.d/yhub + volumes: + - ./keys/yhub-private.pem:/keys/yhub-private.pem:ro + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://localhost:3002/collaboration/ready/v1').then(r=>process.exit(r.ok?0:1),()=>process.exit(1))"] + interval: 15s + timeout: 5s + retries: 20 + start_period: 10s + depends_on: + postgresql: + condition: service_healthy + restart: true + yhub-valkey: + condition: service_healthy + + # The conversion service (markdown, html, pdf, docx). + y-provider: + image: ${IMAGE_REGISTRY:-ghcr.io/typecellos/docs}/y-provider:${IMAGE_TAG} + user: "1000" + restart: unless-stopped + env_file: + - env.d/common + - env.d/yprovider + + # Serves the editor and routes everything else, on the same origin. + frontend: + image: ${IMAGE_REGISTRY:-ghcr.io/typecellos/docs}/frontend:${IMAGE_TAG} + user: "101" + restart: unless-stopped + entrypoint: + - /docker-entrypoint.sh + command: ["nginx", "-g", "daemon off;"] + env_file: + - env.d/common + volumes: + - ./default.conf.template:/etc/nginx/templates/docs.conf.template:ro + depends_on: + backend: + condition: service_healthy + yhub: + condition: service_healthy + +volumes: + caddy-data: + caddy-config: + postgresql: + yhub-valkey: + minio: + keycloak: diff --git a/deploy/dev-instance/env.d/backend b/deploy/dev-instance/env.d/backend new file mode 100644 index 0000000000..a4d3562137 --- /dev/null +++ b/deploy/dev-instance/env.d/backend @@ -0,0 +1,49 @@ +# Django +DJANGO_ALLOWED_HOSTS=${DOCS_HOST} +DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY} +DJANGO_SETTINGS_MODULE=impress.settings +DJANGO_CONFIGURATION=Production +DJANGO_CSRF_TRUSTED_ORIGINS=https://${DOCS_HOST} + +LOGGING_LEVEL_HANDLERS_CONSOLE=INFO +LOGGING_LEVEL_LOGGERS_ROOT=INFO +LOGGING_LEVEL_LOGGERS_APP=INFO + +PYTHONPATH=/app + +# Signs the tokens the backend presents to yhub and to y-provider +JWT_PRIVATE_KEY_FILE=/keys/private.pem + +# Collaboration server +YHUB_API_BASE_URL=http://yhub:3002 +COLLABORATION_API_URL=http://yhub:3002/collaboration +COLLABORATION_WS_URL=wss://${DOCS_HOST}/collaboration/ws/v1/docs + +# No mail server on the dev instance +DJANGO_EMAIL_BRAND_NAME="Docs dev" +DJANGO_EMAIL_URL_APP=https://${DOCS_HOST} + +# Media +AWS_S3_ENDPOINT_URL=https://${S3_HOST} +AWS_S3_ACCESS_KEY_ID=${S3_ACCESS_KEY} +AWS_S3_SECRET_ACCESS_KEY=${S3_SECRET_KEY} +AWS_STORAGE_BUCKET_NAME=${BUCKET_NAME} +MEDIA_BASE_URL=https://${DOCS_HOST} + +# OIDC (Keycloak on the same server, realm keycloak/realm.json) +OIDC_OP_JWKS_ENDPOINT=https://${KEYCLOAK_HOST}/realms/docs/protocol/openid-connect/certs +OIDC_OP_AUTHORIZATION_ENDPOINT=https://${KEYCLOAK_HOST}/realms/docs/protocol/openid-connect/auth +OIDC_OP_TOKEN_ENDPOINT=https://${KEYCLOAK_HOST}/realms/docs/protocol/openid-connect/token +OIDC_OP_USER_ENDPOINT=https://${KEYCLOAK_HOST}/realms/docs/protocol/openid-connect/userinfo +OIDC_OP_LOGOUT_ENDPOINT=https://${KEYCLOAK_HOST}/realms/docs/protocol/openid-connect/logout +OIDC_RP_CLIENT_ID=docs +OIDC_RP_CLIENT_SECRET=${OIDC_CLIENT_SECRET} +OIDC_RP_SIGN_ALGO=RS256 +OIDC_RP_SCOPES="openid email profile" +OIDC_USERINFO_FULLNAME_FIELDS=given_name,family_name +OIDC_USERINFO_SHORTNAME_FIELD=given_name + +LOGIN_REDIRECT_URL=https://${DOCS_HOST} +LOGIN_REDIRECT_URL_FAILURE=https://${DOCS_HOST} +LOGOUT_REDIRECT_URL=https://${DOCS_HOST} +OIDC_REDIRECT_ALLOWED_HOSTS=["https://${DOCS_HOST}"] diff --git a/deploy/dev-instance/env.d/common b/deploy/dev-instance/env.d/common new file mode 100644 index 0000000000..ed616f4c39 --- /dev/null +++ b/deploy/dev-instance/env.d/common @@ -0,0 +1,10 @@ +# Values come from the .env file on the server (scripts/init-env.sh). +DOCS_HOST=${DOCS_HOST} +KEYCLOAK_HOST=${KEYCLOAK_HOST} +S3_HOST=${S3_HOST} +BACKEND_HOST=backend +FRONTEND_HOST=frontend +YPROVIDER_HOST=y-provider +YHUB_HOST=yhub +BUCKET_NAME=${BUCKET_NAME} +REALM_NAME=docs diff --git a/deploy/dev-instance/env.d/postgresql b/deploy/dev-instance/env.d/postgresql new file mode 100644 index 0000000000..04db025bee --- /dev/null +++ b/deploy/dev-instance/env.d/postgresql @@ -0,0 +1,9 @@ +DB_HOST=postgresql +DB_NAME=docs +DB_USER=docs +DB_PASSWORD=${DB_PASSWORD} +DB_PORT=5432 + +POSTGRES_DB=docs +POSTGRES_USER=docs +POSTGRES_PASSWORD=${DB_PASSWORD} diff --git a/deploy/dev-instance/env.d/yhub b/deploy/dev-instance/env.d/yhub new file mode 100644 index 0000000000..138aa29d5c --- /dev/null +++ b/deploy/dev-instance/env.d/yhub @@ -0,0 +1,8 @@ +PORT=3002 +REDIS=redis://yhub-valkey:6379 +REDIS_PREFIX=yhub +POSTGRES=postgres://docs:${DB_PASSWORD}@postgresql:5432/yhub +COLLABORATION_BACKEND_BASE_URL=https://${DOCS_HOST} +COLLABORATION_SERVER_ORIGIN=https://${DOCS_HOST} +Y_PROVIDER_API_KEY=${Y_PROVIDER_API_KEY} +YHUB_JWT_PRIVATE_KEY_FILE=/keys/yhub-private.pem diff --git a/deploy/dev-instance/env.d/yprovider b/deploy/dev-instance/env.d/yprovider new file mode 100644 index 0000000000..70644cf900 --- /dev/null +++ b/deploy/dev-instance/env.d/yprovider @@ -0,0 +1,5 @@ +Y_PROVIDER_API_BASE_URL=http://y-provider:4444/api/ +Y_PROVIDER_API_KEY=${Y_PROVIDER_API_KEY} +COLLABORATION_SERVER_ORIGIN=https://${DOCS_HOST} +COLLABORATION_BACKEND_BASE_URL=https://${DOCS_HOST} +COLLABORATION_LOGGING=true diff --git a/deploy/dev-instance/keycloak/realm.json b/deploy/dev-instance/keycloak/realm.json new file mode 100644 index 0000000000..52d76ad454 --- /dev/null +++ b/deploy/dev-instance/keycloak/realm.json @@ -0,0 +1,108 @@ +{ + "realm": "docs", + "enabled": true, + "sslRequired": "external", + "registrationAllowed": false, + "loginWithEmailAllowed": true, + "resetPasswordAllowed": false, + "bruteForceProtected": true, + "clients": [ + { + "clientId": "docs", + "name": "Docs", + "enabled": true, + "protocol": "openid-connect", + "publicClient": false, + "clientAuthenticatorType": "client-secret", + "secret": "${OIDC_CLIENT_SECRET}", + "standardFlowEnabled": true, + "directAccessGrantsEnabled": false, + "redirectUris": [ + "https://${DOCS_HOST}/*" + ], + "webOrigins": [ + "https://${DOCS_HOST}" + ], + "attributes": { + "post.logout.redirect.uris": "https://${DOCS_HOST}/*" + } + } + ], + "users": [ + { + "username": "admin", + "email": "admin@example.com", + "emailVerified": true, + "firstName": "Admin", + "lastName": "Dev", + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "${SEED_PASSWORD_ADMIN}", + "temporary": false + } + ] + }, + { + "username": "alice", + "email": "alice@example.com", + "emailVerified": true, + "firstName": "Alice", + "lastName": "Martin", + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "alice", + "temporary": false + } + ] + }, + { + "username": "bob", + "email": "bob@example.com", + "emailVerified": true, + "firstName": "Bob", + "lastName": "Bernard", + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "bob", + "temporary": false + } + ] + }, + { + "username": "carol", + "email": "carol@example.com", + "emailVerified": true, + "firstName": "Carol", + "lastName": "Dubois", + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "carol", + "temporary": false + } + ] + }, + { + "username": "dave", + "email": "dave@example.com", + "emailVerified": true, + "firstName": "Dave", + "lastName": "Moreau", + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "dave", + "temporary": false + } + ] + } + ] +} diff --git a/deploy/dev-instance/scripts/deploy.sh b/deploy/dev-instance/scripts/deploy.sh new file mode 100755 index 0000000000..2c403e2c83 --- /dev/null +++ b/deploy/dev-instance/scripts/deploy.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Deploys the images of one commit: ./scripts/deploy.sh +# Every step is idempotent, so a failed deploy can be run again. +set -euo pipefail +cd "$(dirname "$0")/.." + +tag="${1:?usage: deploy.sh }" +./scripts/init-env.sh +sed -i "s/^IMAGE_TAG=.*/IMAGE_TAG=${tag}/" .env + +docker compose pull --quiet +docker compose up -d --wait postgresql redis yhub-valkey minio +docker compose run --rm createbuckets +# Schemas: the collaboration server's, then the backend's +docker compose run --rm --no-deps yhub yarn init-db +docker compose run --rm --no-deps backend python manage.py migrate --noinput +docker compose up -d --remove-orphans --wait --wait-timeout 600 + +set -a; . ./.env; set +a +docker compose exec -T backend python manage.py createsuperuser \ + --email admin@example.com --password "${SEED_PASSWORD_ADMIN}" + +# Demo users: the password is the user name. The realm import sets it on the +# first start only, so set it again for a Keycloak volume that already exists. +for attempt in $(seq 1 30); do + if docker compose exec -T -e KC_ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD}" keycloak sh -c ' + /opt/keycloak/bin/kcadm.sh config credentials --server http://localhost:8080 \ + --realm master --user admin --password "$KC_ADMIN_PASSWORD" >/dev/null && + for user in alice bob carol dave; do + /opt/keycloak/bin/kcadm.sh set-password -r docs --username "$user" --new-password "$user" + done'; then + break + fi + [ "${attempt}" = 30 ] && { echo "Keycloak is not ready" >&2; exit 1; } + sleep 4 +done + +./scripts/seed.sh + +docker image prune -f >/dev/null +docker compose ps --format 'table {{.Service}}\t{{.Image}}\t{{.Status}}' diff --git a/deploy/dev-instance/scripts/init-env.sh b/deploy/dev-instance/scripts/init-env.sh new file mode 100755 index 0000000000..8d3ea406ad --- /dev/null +++ b/deploy/dev-instance/scripts/init-env.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Creates the server configuration on the first deploy: the .env file with the +# host names and random secrets, and the two RS256 keys. It never changes +# existing values, so it is safe to run on every deploy. +# +# The host names derive from DEV_DOMAIN (default: .sslip.io). To use +# a real domain later, edit DOCS_HOST, KEYCLOAK_HOST and S3_HOST in .env and +# update the OIDC client of the "docs" realm in Keycloak. +set -euo pipefail +cd "$(dirname "$0")/.." + +random() { openssl rand -base64 48 | tr -dc 'A-Za-z0-9' | head -c "${1:-40}"; } + +if [ ! -f .env ]; then + domain="${DEV_DOMAIN:-$(curl -fsS4 https://ifconfig.me | tr . -).sslip.io}" + umask 077 + cat > .env </dev/null + echo "Created keys/${key}.pem" + fi +done +# Read by the containers that run as uid 1000 (the deploy user) +chmod 600 keys/*.pem diff --git a/deploy/dev-instance/scripts/seed.sh b/deploy/dev-instance/scripts/seed.sh new file mode 100755 index 0000000000..2524a33bd7 --- /dev/null +++ b/deploy/dev-instance/scripts/seed.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Puts the sample document on the instance: "07/10 meeting agenda (sample)", +# owned by alice, with 15 named versions by alice, bob, carol and dave. +# deploy.sh runs it on every deploy. It does nothing when the document is +# already there, and it takes about a minute at most: the history is built +# offline with its past timestamps, then stored in yhub. +# +# ./scripts/seed.sh [path to versions JSON] (default: seed-data/meetingVersions.json) +set -euo pipefail +cd "$(dirname "$0")/.." + +data="${1:-seed-data/meetingVersions.json}" +if [ ! -f "${data}" ]; then + echo "seed: no ${data} on the server, so no sample document" >&2 + exit 0 +fi +data="$(realpath "${data}")" +out="$(pwd)/seed-data/out" +rm -rf "${out}" && mkdir -p "${out}" && chmod 777 "${out}" + +host="$(grep '^DOCS_HOST=' .env | cut -d= -f2)" + +# 1. Sign in as the demo users, create the document, build its history +docker run --rm \ + -v "$(pwd)/seed:/seed:ro" \ + -v "${data}:/data/versions.json:ro" \ + -v "${out}:/out" \ + -v docs-seed-npm:/root/.npm \ + -e DOCS_URL="https://${host}" \ + -e SEED_USERS="alice:alice,bob:bob,carol:carol,dave:dave" \ + -e SEED_DATA=/data/versions.json \ + -e OUT=/out/history.bin \ + node:24-alpine sh -c ' + mkdir -p /work && cp /seed/package.json /seed/package-lock.json /seed/*.mjs /work/ && + cd /work && npm ci --prefer-offline --no-audit --no-fund --loglevel=error && node seed.mjs' + +if [ ! -f "${out}/history.bin" ]; then + exit 0 # already there +fi +docid="$(sed -E 's/.*"docid":"([^"]+)".*/\1/' "${out}/history.bin.json")" +from_ms="$(sed -E 's/.*"accessesFrom":([0-9]+).*/\1/' "${out}/history.bin.json")" + +# 2. Users only see the history after the date of their access: date the +# accesses before the recorded meeting. +docker compose exec -T backend python manage.py shell -c " +import datetime +from core.models import DocumentAccess +DocumentAccess.objects.filter(document_id='${docid}').update( + created_at=datetime.datetime.fromtimestamp(${from_ms} / 1000, datetime.timezone.utc)) +" + +# 3. Store the history in yhub +docker compose run --rm --no-deps \ + -v "$(pwd)/seed:/seed:ro" -v "${out}:/out:ro" \ + yhub node /seed/import.mjs /out/history.bin + +echo "seed: sample document at $(sed -E 's/.*"url":"([^"]+)".*/\1/' "${out}/history.bin.json")" diff --git a/deploy/dev-instance/seed/import.mjs b/deploy/dev-instance/seed/import.mjs new file mode 100644 index 0000000000..1ab3f528ef --- /dev/null +++ b/deploy/dev-instance/seed/import.mjs @@ -0,0 +1,41 @@ +// Stores the history built by seed.mjs in yhub: step 3 of scripts/seed.sh. +// +// It runs in the yhub image, with the @y/hub of the deployed server, and writes +// the document the way Docs' fullMigrate does: one persisted row at clock 0, +// and the named versions. It refuses a document that yhub already holds. +// +// Usage: node import.mjs +import fs from 'node:fs'; + +import { decodeAny, encodeAny } from '/app/node_modules/lib0/src/buffer.js'; +import { createPersistence } from '/app/node_modules/@y/hub/src/persistence.js'; + +const history = decodeAny(new Uint8Array(fs.readFileSync(process.argv[2]))); +const docRef = { org: history.org, docid: history.docid, branch: 'main' }; +const persistence = await createPersistence(process.env.POSTGRES, []); + +const current = await persistence.retrieveDoc(docRef, {}); +if (current.lastClock !== '0') { + console.log(`import: yhub already holds ${docRef.docid}, nothing to do`); +} else { + await persistence.store(docRef, { + lastClock: '0', + gcDoc: history.gcDoc, + nongcDoc: history.nongcDoc, + contentmap: history.contentmap, + contentids: history.contentids, + }); + for (const version of history.versions) { + await persistence.storeVersion(docRef, { + t: version.t, + name: version.name, + // what the version API stores for a version with no custom data + custom: encodeAny(null), + published: false, + at: version.t, + by: version.by, + }); + } + console.log(`import: stored ${history.versions.length} versions of ${docRef.docid}`); +} +process.exit(0); diff --git a/deploy/dev-instance/seed/package-lock.json b/deploy/dev-instance/seed/package-lock.json new file mode 100644 index 0000000000..79c3efb721 --- /dev/null +++ b/deploy/dev-instance/seed/package-lock.json @@ -0,0 +1,569 @@ +{ + "name": "docs-dev-instance-seed", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "docs-dev-instance-seed", + "dependencies": { + "@blocknote/core": "https://pkg.pr.new/TypeCellOS/BlockNote/@blocknote/core@bb192fe8fc", + "@y/prosemirror": "2.0.0-14", + "@y/protocols": "1.0.6-rc.1", + "@y/y": "14.0.0-rc.28", + "lib0": "1.0.0-rc.36" + } + }, + "node_modules/@blocknote/core": { + "version": "0.55.0", + "resolved": "https://pkg.pr.new/TypeCellOS/BlockNote/@blocknote/core@bb192fe8fc", + "integrity": "sha512-uxuA/QEckkOmI+Hye1kIIyyd9YIqcSm+mj95uccd7WeV6zAKyYkRcW69v/TObyfZ4c+k+sGcKBKzN7teiEhXkw==", + "license": "MPL-2.0", + "dependencies": { + "@emoji-mart/data": "^1.2.1", + "@handlewithcare/prosemirror-inputrules": "^0.1.4", + "@shikijs/types": "^4.4.3", + "@tiptap/core": "^3.31.3", + "@tiptap/extension-bold": "^3.31.3", + "@tiptap/extension-code": "^3.31.3", + "@tiptap/extension-italic": "^3.31.3", + "@tiptap/extension-strike": "^3.31.3", + "@tiptap/extension-text": "^3.31.3", + "@tiptap/extension-underline": "^3.31.3", + "@tiptap/extensions": "^3.31.3", + "@tiptap/pm": "^3.31.3", + "emoji-mart": "^5.6.0", + "fast-deep-equal": "^3.1.3", + "lib0": "1.0.0-rc.36", + "prosemirror-highlight": "^0.15.3", + "prosemirror-model": "^1.25.11", + "prosemirror-state": "^1.4.4", + "prosemirror-tables": "^1.8.5", + "prosemirror-transform": "^1.12.1", + "prosemirror-view": "^1.42.5" + }, + "peerDependencies": { + "@y/prosemirror": "^2.0.0-14", + "@y/protocols": "^1.0.6-rc.1", + "@y/y": "^14.0.0-rc.26", + "y-prosemirror": "^1.3.7", + "y-protocols": "^1.0.6", + "yjs": "^13.6.33" + }, + "peerDependenciesMeta": { + "@y/prosemirror": { + "optional": true + }, + "@y/protocols": { + "optional": true + }, + "@y/y": { + "optional": true + }, + "y-prosemirror": { + "optional": true + }, + "y-protocols": { + "optional": true + }, + "yjs": { + "optional": true + } + } + }, + "node_modules/@emoji-mart/data": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emoji-mart/data/-/data-1.2.1.tgz", + "integrity": "sha512-no2pQMWiBy6gpBEiqGeU77/bFejDqUTRY7KX+0+iur13op3bqUsXdnwoZs6Xb1zbv0gAj5VvS1PWoUUckSr5Dw==", + "license": "MIT" + }, + "node_modules/@handlewithcare/prosemirror-inputrules": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@handlewithcare/prosemirror-inputrules/-/prosemirror-inputrules-0.1.4.tgz", + "integrity": "sha512-GMqlBeG2MKM+tXEFd2N+wIv5z4VvJTg8JtfJUrdjvFq2W6v+AW8oTgiWyFw8L3iEQwvtQcVJxU873iB0LXUNNw==", + "license": "MIT", + "dependencies": { + "prosemirror-history": "^1.4.1", + "prosemirror-transform": "^1.0.0" + }, + "peerDependencies": { + "prosemirror-model": "^1.0.0", + "prosemirror-state": "^1.0.0", + "prosemirror-view": "^1.0.0" + } + }, + "node_modules/@shikijs/types": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-4.5.0.tgz", + "integrity": "sha512-drtPrtD8N0DENyD2cyt6Ajy+IWsiCLgMxODwLTpvH1UsN6a9jf90tNTPjqwHvA+0Lzu9e0txk8DjfqH1p9UJRw==", + "license": "MIT", + "dependencies": { + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/vscode-textmate": { + "version": "10.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/vscode-textmate/-/vscode-textmate-10.0.2.tgz", + "integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==", + "license": "MIT" + }, + "node_modules/@tiptap/core": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/core/-/core-3.31.4.tgz", + "integrity": "sha512-cIVUQYBycighzsk6GbmCef8ZiEDVxoT5EuAe4xg2q9dNkDqCXVqI4YGr9aavCQEiAS4uHiRwqvcsPqJHipFDfg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/pm": "3.31.4" + } + }, + "node_modules/@tiptap/extension-bold": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extension-bold/-/extension-bold-3.31.4.tgz", + "integrity": "sha512-QX1xoXvO0gz6uVsoLegumqQ4MJXL7lcUDQZ4J7la7LBqiIh4efrCmp2EXbhpr37M/YQL7w8q3g7H/VovSh7xIg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4" + } + }, + "node_modules/@tiptap/extension-code": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extension-code/-/extension-code-3.31.4.tgz", + "integrity": "sha512-pzi8GNDs5J1DEoO1HNYiJ5WZpgOZzlc/2VIa77mMpEuD4hZxuK0BQzlF/O6Ps/O5P1lGJT4xS0Yowpzb9mD9RA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4" + } + }, + "node_modules/@tiptap/extension-italic": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extension-italic/-/extension-italic-3.31.4.tgz", + "integrity": "sha512-pftTQBPYZocNDAj/u6OyDIp7YxmN0+N895cQPogJQ6qMHdLiRhzgawiEewJkniyN1w8k3Wx/Etp8pTKrxaOOBQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4" + } + }, + "node_modules/@tiptap/extension-strike": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extension-strike/-/extension-strike-3.31.4.tgz", + "integrity": "sha512-4MBr63TJIMqxCjE+gAPdlY1IdELgvbAP516K3Ntn/BkcpEbEoPMEYVz+XTxULyHQEuARDO6jsXPPxRXpA74jaw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4" + } + }, + "node_modules/@tiptap/extension-text": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extension-text/-/extension-text-3.31.4.tgz", + "integrity": "sha512-WL2ZRpFksgQcXJN05/UQH09YCH9LlhdLrSEcviASvEH/aAfsKiJlDrouN8PET+YAWK/F5xFc5/Q1z0xj+I23jw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4" + } + }, + "node_modules/@tiptap/extension-underline": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extension-underline/-/extension-underline-3.31.4.tgz", + "integrity": "sha512-0iFGPIqmqGkCrOFh1OPo7Dzv5s86dVN3bZaugsz0jK04otkb8bn04ZDKP2/6Wq+lcK0M9DfJ0QqfW5hC2nBZxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4" + } + }, + "node_modules/@tiptap/extensions": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extensions/-/extensions-3.31.4.tgz", + "integrity": "sha512-nIupIne/sHpV2Kw1M4MOmr4+G454KI8BXM5+LWKOU2EpzfALZWVSy7koF9lVRna4Zp5Gv8hfg5Z8kvmrYQvhkw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4", + "@tiptap/pm": "3.31.4" + } + }, + "node_modules/@tiptap/pm": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/pm/-/pm-3.31.4.tgz", + "integrity": "sha512-x7dwth79cQp4QGae66/RJqMDJQv0+n7b7dD7cyFQBWqzPFLJxN/4paHY4qbWedZjNLhRyIOAD9a5q4AYM1zmPA==", + "license": "MIT", + "dependencies": { + "prosemirror-changeset": "^2.4.1", + "prosemirror-commands": "^1.7.1", + "prosemirror-dropcursor": "^1.8.2", + "prosemirror-gapcursor": "^1.4.1", + "prosemirror-history": "^1.5.0", + "prosemirror-inputrules": "^1.5.1", + "prosemirror-keymap": "^1.2.3", + "prosemirror-model": "^1.25.11", + "prosemirror-schema-list": "^1.5.1", + "prosemirror-state": "^1.4.4", + "prosemirror-tables": "^1.8.5", + "prosemirror-transform": "^1.12.0", + "prosemirror-view": "^1.42.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + } + }, + "node_modules/@types/hast": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.5.tgz", + "integrity": "sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, + "node_modules/@y/prosemirror": { + "version": "2.0.0-14", + "resolved": "https://registry.npmjs.org/@y/prosemirror/-/prosemirror-2.0.0-14.tgz", + "integrity": "sha512-h0j2KWoVig9TxJ/1FjGMAPgLgugUffLCQL7bJvRzahfsNzGICiknGrQevvKpfTQkKcclKXsvmH+/lkFLMzhOhQ==", + "license": "MIT", + "dependencies": { + "@y/y": "^14.0.0-rc.26", + "lib0": "^1.0.0-rc.32" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "@y/protocols": "^1.0.6-rc.1", + "prosemirror-model": "^1.7.1", + "prosemirror-state": "^1.2.3", + "prosemirror-transform": "^1.8.0", + "prosemirror-view": "^1.9.10" + } + }, + "node_modules/@y/protocols": { + "version": "1.0.6-rc.1", + "resolved": "https://registry.npmjs.org/@y/protocols/-/protocols-1.0.6-rc.1.tgz", + "integrity": "sha512-e/qs7hXcLk/SeNitxMXv2ymozyWFTULwbJEi7cAf/K/iXw9nGwGXHrR5TNluQ/bMwOX1cwuUT0hjEojkfH0gsA==", + "license": "MIT", + "dependencies": { + "lib0": "^1.0.0-rc.1" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "@y/y": "*" + } + }, + "node_modules/@y/y": { + "version": "14.0.0-rc.28", + "resolved": "https://registry.npmjs.org/@y/y/-/y-14.0.0-rc.28.tgz", + "integrity": "sha512-Hzw2sSch5RpT4aKz0643LQnoN7qSm63VyWb5D3zzCDEgSe5G0CKu5W7zcAXKDlVwkXg5puKTNX5k/wdVg4LtMA==", + "license": "MIT", + "dependencies": { + "lib0": "^1.0.0-rc.35" + }, + "engines": { + "node": ">=22.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, + "node_modules/emoji-mart": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/emoji-mart/-/emoji-mart-5.6.0.tgz", + "integrity": "sha512-eJp3QRe79pjwa+duv+n7+5YsNhRcMl812EcFVwrnRvYKoNPoQb5qxU8DG6Bgwji0akHdp6D4Ln6tYLG58MFSow==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/lib0": { + "version": "1.0.0-rc.36", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-1.0.0-rc.36.tgz", + "integrity": "sha512-7t/I+PCQqakal22rhXq5giOUtDP/RfecQtdeKvzCtnAvPkxSLAE+/Zvzi/kJDopg+hBAYKo46qmxmlkhX0tYlQ==", + "license": "MIT", + "bin": { + "0ecdsa-generate-keypair": "src/bin/0ecdsa-generate-keypair.js", + "0gentesthtml": "src/bin/gentesthtml.js", + "0serve": "src/bin/0serve.js" + }, + "engines": { + "node": ">=22" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, + "node_modules/orderedmap": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/orderedmap/-/orderedmap-2.1.1.tgz", + "integrity": "sha512-TvAWxi0nDe1j/rtMcWcIj94+Ffe6n7zhow33h40SKxmsmozs6dz/e+EajymfoFcHd7sxNn8yHM8839uixMOV6g==", + "license": "MIT" + }, + "node_modules/prosemirror-changeset": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/prosemirror-changeset/-/prosemirror-changeset-2.4.4.tgz", + "integrity": "sha512-Gyuz9jYdrAeA9I7AholyOes9KUmv/lEwsC5O9s5xF5VB3EgTPsXMGUxz4+H1B+wKVPdZbvRcg1Cp3x8qX+8ofA==", + "license": "MIT", + "dependencies": { + "prosemirror-transform": "^1.0.0" + } + }, + "node_modules/prosemirror-commands": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/prosemirror-commands/-/prosemirror-commands-1.7.2.tgz", + "integrity": "sha512-q6Q6szxqdu9Xd6EcdKsqXghu5nQdZTpB4Q9yd04WRc7/jt763e/rT60Owh0L1GYY+T46o5rD+9lEN36dZS43tw==", + "license": "MIT", + "dependencies": { + "prosemirror-model": "^1.0.0", + "prosemirror-state": "^1.0.0", + "prosemirror-transform": "^1.10.2" + } + }, + "node_modules/prosemirror-dropcursor": { + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/prosemirror-dropcursor/-/prosemirror-dropcursor-1.8.4.tgz", + "integrity": "sha512-VmKxhcA6a+h5oPSYmfF1ZNeQqPbyYaIxqZsKqeYCyTlPtrjUgNaHL3P8vVaO19oqAV0zuaM/axkowZHrB1P3Tg==", + "license": "MIT", + "dependencies": { + "prosemirror-state": "^1.0.0", + "prosemirror-transform": "^1.1.0", + "prosemirror-view": "^1.1.0" + } + }, + "node_modules/prosemirror-gapcursor": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/prosemirror-gapcursor/-/prosemirror-gapcursor-1.4.1.tgz", + "integrity": "sha512-pMdYaEnjNMSwl11yjEGtgTmLkR08m/Vl+Jj443167p9eB3HVQKhYCc4gmHVDsLPODfZfjr/MmirsdyZziXbQKw==", + "license": "MIT", + "dependencies": { + "prosemirror-keymap": "^1.0.0", + "prosemirror-model": "^1.0.0", + "prosemirror-state": "^1.0.0", + "prosemirror-view": "^1.0.0" + } + }, + "node_modules/prosemirror-highlight": { + "version": "0.15.3", + "resolved": "https://registry.npmjs.org/prosemirror-highlight/-/prosemirror-highlight-0.15.3.tgz", + "integrity": "sha512-WVV2st0fX1w2TkAgmTmdbj77BlWYuLfW4BGXPo8JfIWsSly5xYcfID8QJQL+GT8kisMRuu4jgT6JAqj1DOwvvg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ocavue" + }, + "peerDependencies": { + "@lezer/common": "^1.0.0", + "@lezer/highlight": "^1.0.0", + "@shikijs/types": "^1.29.2 || ^2.0.0 || ^3.0.0 || ^4.0.0", + "@types/hast": "^3.0.0", + "highlight.js": "^11.9.0", + "lowlight": "^3.1.0", + "prosemirror-model": "^1.19.3", + "prosemirror-state": "^1.4.3", + "prosemirror-transform": "^1.8.0", + "prosemirror-view": "^1.32.4", + "refractor": "^5.0.0", + "sugar-high": "^0.6.1 || ^0.7.0 || ^0.8.0 || ^0.9.0 || ^1.0.0" + }, + "peerDependenciesMeta": { + "@lezer/common": { + "optional": true + }, + "@lezer/highlight": { + "optional": true + }, + "@shikijs/types": { + "optional": true + }, + "@types/hast": { + "optional": true + }, + "highlight.js": { + "optional": true + }, + "lowlight": { + "optional": true + }, + "prosemirror-model": { + "optional": true + }, + "prosemirror-state": { + "optional": true + }, + "prosemirror-transform": { + "optional": true + }, + "prosemirror-view": { + "optional": true + }, + "refractor": { + "optional": true + }, + "sugar-high": { + "optional": true + } + } + }, + "node_modules/prosemirror-history": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/prosemirror-history/-/prosemirror-history-1.5.1.tgz", + "integrity": "sha512-onlwnqKvgtFg6srZOmDOAhlAy1FIHQpOExORybX0S+DvJepE5tPALrB9iGet/2sqqqrelSnB10OoiOJqAvu2DA==", + "license": "MIT", + "dependencies": { + "prosemirror-state": "^1.2.2", + "prosemirror-transform": "^1.0.0", + "prosemirror-view": "^1.31.0", + "rope-sequence": "^1.3.0" + } + }, + "node_modules/prosemirror-inputrules": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/prosemirror-inputrules/-/prosemirror-inputrules-1.5.1.tgz", + "integrity": "sha512-7wj4uMjKaXWAQ1CDgxNzNtR9AlsuwzHfdFH1ygEHA2KHF2DOEaXl1CJfNPAKCg9qNEh4rum975QLaCiQPyY6Fw==", + "license": "MIT", + "dependencies": { + "prosemirror-state": "^1.0.0", + "prosemirror-transform": "^1.0.0" + } + }, + "node_modules/prosemirror-keymap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/prosemirror-keymap/-/prosemirror-keymap-1.2.3.tgz", + "integrity": "sha512-4HucRlpiLd1IPQQXNqeo81BGtkY8Ai5smHhKW9jjPKRc2wQIxksg7Hl1tTI2IfT2B/LgX6bfYvXxEpJl7aKYKw==", + "license": "MIT", + "dependencies": { + "prosemirror-state": "^1.0.0", + "w3c-keyname": "^2.2.0" + } + }, + "node_modules/prosemirror-model": { + "version": "1.25.12", + "resolved": "https://registry.npmjs.org/prosemirror-model/-/prosemirror-model-1.25.12.tgz", + "integrity": "sha512-Ue2gTmXMa7EhpLNhC7J+h4+ykD8ha12K6rrZFFKKJHBForfIStw5gJ6Zrf1mqqAa9NmpmEB4wp9bKmA4eUVjWg==", + "license": "MIT", + "dependencies": { + "orderedmap": "^2.0.0" + } + }, + "node_modules/prosemirror-schema-list": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/prosemirror-schema-list/-/prosemirror-schema-list-1.5.1.tgz", + "integrity": "sha512-927lFx/uwyQaGwJxLWCZRkjXG0p48KpMj6ueoYiu4JX05GGuGcgzAy62dfiV8eFZftgyBUvLx76RsMe20fJl+Q==", + "license": "MIT", + "dependencies": { + "prosemirror-model": "^1.0.0", + "prosemirror-state": "^1.0.0", + "prosemirror-transform": "^1.7.3" + } + }, + "node_modules/prosemirror-state": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/prosemirror-state/-/prosemirror-state-1.4.4.tgz", + "integrity": "sha512-6jiYHH2CIGbCfnxdHbXZ12gySFY/fz/ulZE333G6bPqIZ4F+TXo9ifiR86nAHpWnfoNjOb3o5ESi7J8Uz1jXHw==", + "license": "MIT", + "dependencies": { + "prosemirror-model": "^1.0.0", + "prosemirror-transform": "^1.0.0", + "prosemirror-view": "^1.27.0" + } + }, + "node_modules/prosemirror-tables": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/prosemirror-tables/-/prosemirror-tables-1.8.5.tgz", + "integrity": "sha512-V/0cDCsHKHe/tfWkeCmthNUcEp1IVO3p6vwN8XtwE9PZQLAZJigbw3QoraAdfJPir4NKJtNvOB8oYGKRl+t0Dw==", + "license": "MIT", + "dependencies": { + "prosemirror-keymap": "^1.2.3", + "prosemirror-model": "^1.25.4", + "prosemirror-state": "^1.4.4", + "prosemirror-transform": "^1.10.5", + "prosemirror-view": "^1.41.4" + } + }, + "node_modules/prosemirror-transform": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/prosemirror-transform/-/prosemirror-transform-1.12.2.tgz", + "integrity": "sha512-PE/aY0HEY4zczvmqrilgkUK/WautF0chvMqkmM/iN5/aPHvwrCWaQZjS6CIZp+K84YrvPFqrL+YArrfJX8xX7g==", + "license": "MIT", + "dependencies": { + "prosemirror-model": "^1.21.0" + } + }, + "node_modules/prosemirror-view": { + "version": "1.42.6", + "resolved": "https://registry.npmjs.org/prosemirror-view/-/prosemirror-view-1.42.6.tgz", + "integrity": "sha512-3g25f6hKV2bBxo3puH3EcYWmmVNSH0iWmNsMVT7sugDSpA025jgpOLarcDWr39SdzPjYMaiYtRdNtXialPSjEQ==", + "license": "MIT", + "dependencies": { + "prosemirror-model": "^1.25.12", + "prosemirror-state": "^1.0.0", + "prosemirror-transform": "^1.1.0" + } + }, + "node_modules/rope-sequence": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/rope-sequence/-/rope-sequence-1.3.4.tgz", + "integrity": "sha512-UT5EDe2cu2E/6O4igUr5PSFs23nvvukicWHx6GnOPlHAiiYbzNuCRQCuiUdHJQcqKalLKlrYJnjY0ySGsXNQXQ==", + "license": "MIT" + }, + "node_modules/w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", + "license": "MIT" + } + } +} diff --git a/deploy/dev-instance/seed/package.json b/deploy/dev-instance/seed/package.json new file mode 100644 index 0000000000..f51309a35e --- /dev/null +++ b/deploy/dev-instance/seed/package.json @@ -0,0 +1,13 @@ +{ + "name": "docs-dev-instance-seed", + "private": true, + "type": "module", + "description": "Builds the sample document of the dev instance (scripts/seed.sh).", + "dependencies": { + "@blocknote/core": "https://pkg.pr.new/TypeCellOS/BlockNote/@blocknote/core@bb192fe8fc", + "@y/prosemirror": "2.0.0-14", + "@y/protocols": "1.0.6-rc.1", + "@y/y": "14.0.0-rc.28", + "lib0": "1.0.0-rc.36" + } +} diff --git a/deploy/dev-instance/seed/replay.mjs b/deploy/dev-instance/seed/replay.mjs new file mode 100644 index 0000000000..2e972b3ee2 --- /dev/null +++ b/deploy/dev-instance/seed/replay.mjs @@ -0,0 +1,49 @@ +// Turns a recorded document history into Yjs updates for the Docs editor. +// +// The input is a list of versions. Each version is a list of steps, and each +// step is the full BlockNote document after a burst of editing. Each step is +// applied to a headless editor and written to Y the way the collaboration +// binding writes an edit (`docDiffToDelta`), so the history reflects the +// BlockNote build Docs runs. +import { BlockNoteEditor } from '@blocknote/core'; +import { docDiffToDelta } from '@blocknote/core/y'; +import { docToDelta } from '@y/prosemirror'; +import * as Y from '@y/y'; + +/** The shared type the Docs editor reads (`provider.doc.get('document-store')`). */ +export const FRAGMENT = 'document-store'; + +/** + * Prepares the replay. `next()` applies one step and returns the Yjs update + * (V1, the format yhub speaks) that the step adds to the document. + */ +export function createReplay() { + const editor = BlockNoteEditor.create(); + const ydoc = new Y.Doc({ gc: false }); + const yType = ydoc.get(FRAGMENT); + + function capture(write) { + const vector = Y.encodeStateVector(ydoc); + ydoc.transact(write); + return Y.encodeStateAsUpdate(ydoc, vector); + } + + return { + /** The update that writes the empty editor document. */ + base() { + return capture(() => { + yType.applyDelta(docToDelta(editor.prosemirrorState.doc)); + }); + }, + /** The update that turns the previous step into `blocks`. */ + next(blocks) { + const before = editor.prosemirrorState.doc; + editor.replaceBlocks(editor.document, blocks); + const delta = docDiffToDelta(before, editor.prosemirrorState.doc); + return capture(() => { + yType.applyDelta(delta); + }); + }, + ydoc, + }; +} diff --git a/deploy/dev-instance/seed/seed.mjs b/deploy/dev-instance/seed/seed.mjs new file mode 100644 index 0000000000..48bc0ace58 --- /dev/null +++ b/deploy/dev-instance/seed/seed.mjs @@ -0,0 +1,279 @@ +// Prepares the sample document of the dev instance: step 1 of scripts/seed.sh. +// +// It signs in as the demo users (which creates them in Docs), lets the first +// one create the document and give the others access, and builds the recorded +// history offline: every edit attributed to its author at a past timestamp, +// the way yhub records live edits (`insert`/`insertAt` in the contentmap). The +// result is written to OUT for import.mjs, which stores it in yhub. +// +// Nothing waits in real time: the timestamps are in the data, not the clock. +// +// Environment: +// DOCS_URL https:// +// SEED_USERS demo users, user:password pairs, comma separated. The first one +// owns the document; the authors of the versions take turns. +// SEED_DATA recorded versions (JSON: [{name, steps}]) +// SEED_DATE day of the recorded meeting, YYYY-MM-DD (default 2026-10-07) +// OUT where to write the history to import +import fs from 'node:fs'; + +import * as Y from '@y/y'; +import { decodeAny, encodeAny } from 'lib0/buffer'; + +import { FRAGMENT, createReplay } from './replay.mjs'; + +const ORG = 'docs'; +const DOCS_URL = required('DOCS_URL').replace(/\/$/, ''); +const TITLE = process.env.SEED_TITLE || '07/10 meeting agenda (sample)'; +const DATE = process.env.SEED_DATE || '2026-10-07'; + +function required(name) { + const value = process.env[name]; + if (!value) { + throw new Error(`${name} is not set`); + } + return value; +} + +function log(message) { + console.log(`seed: ${message}`); +} + +/** A cookie jar per host, enough for the sign-in redirects. */ +function createSession() { + const jars = new Map(); + + function jarFor(url) { + const { host } = new URL(url); + if (!jars.has(host)) { + jars.set(host, new Map()); + } + return jars.get(host); + } + + async function request(url, init = {}) { + const jar = jarFor(url); + const headers = new Headers(init.headers); + if (jar.size) { + headers.set( + 'cookie', + [...jar].map(([name, value]) => `${name}=${value}`).join('; '), + ); + } + const response = await fetch(url, { ...init, headers, redirect: 'manual' }); + for (const line of response.headers.getSetCookie()) { + const [pair] = line.split(';'); + const index = pair.indexOf('='); + const name = pair.slice(0, index).trim(); + const value = pair.slice(index + 1).trim(); + if (value === '' || /max-age=0/i.test(line)) { + jar.delete(name); + } else { + jar.set(name, value); + } + } + return response; + } + + async function follow(url, init = {}) { + let response = await request(url, init); + let current = url; + for ( + let i = 0; + i < 20 && [301, 302, 303, 307, 308].includes(response.status); + i++ + ) { + current = new URL(response.headers.get('location'), current).toString(); + await response.arrayBuffer(); + response = await request(current); + } + return { response, url: current }; + } + + return { + request, + follow, + cookie: (url, name) => jarFor(url).get(name), + }; +} + +/** Signs in through the OIDC flow of the backend and the Keycloak login form. */ +async function signIn(username, password) { + const session = createSession(); + const { response, url } = await session.follow( + `${DOCS_URL}/api/v1.0/authenticate/`, + ); + const html = await response.text(); + const action = html.match( + /]*id="kc-form-login"[^>]*action="([^"]+)"/, + )?.[1]; + if (!action) { + throw new Error(`No login form for ${username} at ${url} (${response.status})`); + } + const done = await session.follow(action.replaceAll('&', '&'), { + method: 'POST', + headers: { 'content-type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ username, password, credentialId: '' }), + }); + await done.response.arrayBuffer(); + + async function api(path, init = {}) { + const headers = new Headers(init.headers); + const csrf = session.cookie(DOCS_URL, 'csrftoken'); + if (csrf) { + headers.set('x-csrftoken', csrf); + } + headers.set('referer', `${DOCS_URL}/`); + return session.request(`${DOCS_URL}${path}`, { ...init, headers }); + } + + const me = await api('/api/v1.0/users/me/'); + if (!me.ok) { + throw new Error(`Sign-in failed for ${username}: ${me.status}`); + } + return { username, user: await me.json(), api }; +} + +async function json(response, what) { + if (!response.ok) { + throw new Error(`${what} failed: ${response.status} ${await response.text()}`); + } + return response.json(); +} + +async function hasContent(user, docId) { + const response = await user.api(`/collaboration/ydoc/v1/${ORG}/${docId}?gc=true`); + if (!response.ok) { + throw new Error(`Reading the document failed: ${response.status}`); + } + const { doc } = decodeAny(new Uint8Array(await response.arrayBuffer())); + const ydoc = new Y.Doc(); + Y.applyUpdate(ydoc, doc); + return ydoc.get(FRAGMENT).length > 0; +} + +async function ensureAccesses(owner, users, docId) { + const listed = await json( + await owner.api(`/api/v1.0/documents/${docId}/accesses/`), + 'Listing accesses', + ); + const existing = new Set(listed.map((access) => access.user?.id)); + for (const user of users) { + if (!existing.has(user.user.id)) { + await json( + await owner.api(`/api/v1.0/documents/${docId}/accesses/`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ user_id: user.user.id, role: 'editor' }), + }), + `Giving access to ${user.username}`, + ); + } + } +} + +/** "08:35:02–08:38:51 UTC" in a version name: when it was edited. */ +function timeRange(name) { + const [start, end] = name.match(/\d\d:\d\d:\d\d/g); + return [Date.parse(`${DATE}T${start}Z`), Date.parse(`${DATE}T${end}Z`)]; +} + +/** + * Replays the versions into one gc:false document and attributes what each + * edit added to its author at its time, like Docs' fullMigrate does. + */ +function buildHistory(versions, authors) { + const replay = createReplay(); + let seen = Y.createContentIds(); + const contentmaps = []; + + function attribute(by, at) { + const all = Y.createContentIdsFromDoc(replay.ydoc, true); + const fresh = Y.excludeContentIds(all, seen); + seen = all; + const attrs = (verb) => [ + Y.createContentAttribute(verb, by), + Y.createContentAttribute(`${verb}At`, at), + ]; + contentmaps.push( + Y.createContentMapFromContentIds(fresh, attrs('insert'), attrs('delete')), + ); + } + + const named = []; + versions.forEach((version, i) => { + const by = authors[i % authors.length]; + const [start, end] = timeRange(version.name); + if (i === 0) { + replay.base(); + attribute(by, start - 1000); + } + const count = version.steps.length; + let at = start; + version.steps.forEach((blocks, j) => { + replay.next(blocks); + at = count > 1 ? Math.round(start + ((end - start) * j) / (count - 1)) : start; + attribute(by, at); + }); + named.push({ t: at, name: version.name, by }); + }); + + const nongcDoc = Y.encodeStateAsUpdate(replay.ydoc); + const gc = new Y.Doc({ gc: true }); + Y.applyUpdate(gc, nongcDoc); + return { + nongcDoc, + gcDoc: Y.encodeStateAsUpdate(gc), + contentmap: Y.encodeContentMap(Y.mergeContentMaps(contentmaps)), + contentids: Y.encodeContentIds(seen), + versions: named, + first: timeRange(versions[0].name)[0], + }; +} + +const started = Date.now(); +const users = []; +for (const pair of required('SEED_USERS').split(',')) { + const index = pair.indexOf(':'); + users.push(await signIn(pair.slice(0, index), pair.slice(index + 1))); +} +const [owner, ...others] = users; + +const found = await json( + await owner.api( + `/api/v1.0/documents/?is_creator_me=true&title=${encodeURIComponent(TITLE)}`, + ), + 'Listing documents', +); +let doc = found.results.find((d) => d.title === TITLE); +if (doc && (await hasContent(owner, doc.id))) { + await ensureAccesses(owner, others, doc.id); + log(`already there: ${DOCS_URL}/docs/${doc.id}/`); + process.exit(0); +} +if (!doc) { + doc = await json( + await owner.api('/api/v1.0/documents/', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ title: TITLE }), + }), + 'Creating the document', + ); +} +await ensureAccesses(owner, others, doc.id); + +const versions = JSON.parse(fs.readFileSync(required('SEED_DATA'), 'utf8')); +const history = buildHistory( + versions, + users.map((u) => u.user.id), +); +fs.writeFileSync( + required('OUT'), + encodeAny({ org: ORG, docid: doc.id, ...history }), +); +fs.writeFileSync( + `${required('OUT')}.json`, + JSON.stringify({ docid: doc.id, url: `${DOCS_URL}/docs/${doc.id}/`, accessesFrom: history.first - 24 * 3600 * 1000 }), +); +log(`built ${versions.length} versions for ${DOCS_URL}/docs/${doc.id}/ in ${Date.now() - started} ms`); diff --git a/docker/files/production/etc/nginx/conf.d/default.conf.template b/docker/files/production/etc/nginx/conf.d/default.conf.template index 0b2fee1bd8..bf306f0be5 100644 --- a/docker/files/production/etc/nginx/conf.d/default.conf.template +++ b/docker/files/production/etc/nginx/conf.d/default.conf.template @@ -80,7 +80,7 @@ server { # Document routes, guarded by the same document authorization as the # websocket, and the public keys the collaboration server signs with. - location ~ ^/collaboration/(ydoc|rollback|prune|changeset|activity|jwks)/ { + location ~ ^/collaboration/(ydoc|rollback|prune|changeset|activity|version|jwks)/ { proxy_pass http://${YHUB_HOST}:3002; proxy_set_header X-Forwarded-Proto https; proxy_set_header Origin $http_origin;