MM-71467 Add Playwright E2E tests: contract-checked fake GitHub on every PR

.github/workflows/e2e.yml

@@ -0,0 +1,211 @@
+name: e2e
+on:
+  schedule:
+    - cron: "0 3 * * *"
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  playwright:
+    name: playwright (${{ matrix.name }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: min-server
+            image: mattermost/mattermost-enterprise-edition:release-12.0
+          - name: master
+            image: mattermostdevelopment/mattermost-enterprise-edition:master
+    env:
+      CI: true
+      MM_IMAGE: ${{ matrix.image }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+          cache-dependency-path: |
+            webapp/package-lock.json
+            e2e/integration/package-lock.json
+
+      - name: Build E2E plugin bundle
+        run: make e2e-dist
+        env:
+          # Builds only for the runner's OS/arch.
+          MM_SERVICESETTINGS_ENABLEDEVELOPER: true
+
+      - name: Install e2e dependencies
+        working-directory: e2e/integration
+        run: |
+          npm ci
+          npx playwright install --with-deps chromium
+
+      - name: Type-check e2e
+        working-directory: e2e/integration
+        run: npm run tsc
+
+      - name: Start Mattermost
+        working-directory: e2e/integration
+        run: npm run env:up
+
+      - name: Run Playwright
+        working-directory: e2e/integration
+        run: npm run test:ci
+
+      - name: Collect server logs
+        if: failure()
+        working-directory: e2e/integration
+        run: mkdir -p playwright-report && docker compose logs mattermost > playwright-report/mattermost.log 2>&1 || true
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report-${{ matrix.name }}
+          path: e2e/integration/playwright-report
+          retention-days: 14
+
+  # Non-blocking: a screenshot diff needs a person to judge whether the change is intended.
+  visual:
+    name: playwright visual
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    continue-on-error: true
+    env:
+      CI: true
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+          cache-dependency-path: |
+            webapp/package-lock.json
+            e2e/integration/package-lock.json
+
+      - name: Build E2E plugin bundle
+        run: make e2e-dist
+        env:
+          MM_SERVICESETTINGS_ENABLEDEVELOPER: true
+
+      - name: Install e2e dependencies
+        working-directory: e2e/integration
+        run: npm ci
+
+      - name: Compare screenshots
+        working-directory: e2e/integration
+        run: npm run test:visual
+
+      - uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: playwright-visual-diffs
+          path: |
+            e2e/integration/playwright-report
+            e2e/integration/test-results
+          retention-days: 14
+
+  contract-drift:
+    name: fake-github contract drift
+    if: github.event_name == 'schedule'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+          cache-dependency-path: e2e/integration/package-lock.json
+
+      - name: Install e2e dependencies
+        working-directory: e2e/integration
+        run: npm ci
+
+      - name: Compare committed specs with GitHub's published REST description and GraphQL schema
+        working-directory: e2e/integration
+        run: npm run contract:check
+
+  real-github:
+    name: real github.com journey
+    if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && vars.REAL_GITHUB_E2E_ENABLED == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    # The fixtures and hook on the test repository are shared between runs.
+    concurrency:
+      group: real-github-e2e
+      cancel-in-progress: false
+    env:
+      CI: true
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+          cache-dependency-path: |
+            webapp/package-lock.json
+            e2e/integration/package-lock.json
+
+      - name: Build plugin bundle
+        run: make dist
+        env:
+          MM_SERVICESETTINGS_ENABLEDEVELOPER: true
+
+      - name: Install e2e dependencies
+        working-directory: e2e/integration
+        run: |
+          npm ci
+          npx playwright install --with-deps chromium
+
+      - name: Start Mattermost
+        working-directory: e2e/integration
+        run: npm run env:up
+
+      # No report upload: it would carry real tokens and page content.
+      - name: Run journey
+        working-directory: e2e/integration
+        run: npm run test:real
+        env:
+          E2E_GITHUB_OAUTH_CLIENT_ID: ${{ secrets.E2E_GITHUB_OAUTH_CLIENT_ID }}
+          E2E_GITHUB_OAUTH_CLIENT_SECRET: ${{ secrets.E2E_GITHUB_OAUTH_CLIENT_SECRET }}
+          E2E_GITHUB_REPO: ${{ secrets.E2E_GITHUB_REPO }}
+          E2E_GITHUB_MACHINE_TOKEN: ${{ secrets.E2E_GITHUB_MACHINE_TOKEN }}
+          E2E_GITHUB_LOGIN_USERNAME: ${{ secrets.E2E_GITHUB_LOGIN_USERNAME }}
+          E2E_GITHUB_LOGIN_PASSWORD: ${{ secrets.E2E_GITHUB_LOGIN_PASSWORD }}
+          E2E_GITHUB_LOGIN_TOTP_SECRET: ${{ secrets.E2E_GITHUB_LOGIN_TOTP_SECRET }}

build/custom.mk

@@ -6,3 +6,30 @@ endif
 .PHONY: deploy-e2e
 deploy-e2e: dist
 	E2E_TESTING=true PLUGIN_E2E_MOCK_OAUTH_SERVER_URL=http://localhost:8080 ./build/bin/pluginctl deploy $(PLUGIN_ID) dist/$(BUNDLE_NAME)
+
+E2E_DIR = e2e/integration
+E2E_FAKE_GITHUB_URL ?= http://fake-github:8093
+
+.PHONY: e2e-dist
+e2e-dist: ## Builds dist/ with GitHub OAuth, REST and GraphQL pointed at the E2E fake. Never ship this bundle.
+	$(MAKE) dist PLUGIN_E2E_MOCK_OAUTH_SERVER_URL=$(E2E_FAKE_GITHUB_URL)
+
+$(E2E_DIR)/node_modules: $(E2E_DIR)/package-lock.json
+	cd $(E2E_DIR) && $(NPM) ci && npx playwright install chromium
+	touch $@
+
+.PHONY: e2e-env-up
+e2e-env-up: ## Starts the Mattermost stack used by the E2E suite
+	cd $(E2E_DIR) && docker compose up -d --wait
+
+.PHONY: e2e-env-down
+e2e-env-down: ## Stops the E2E Mattermost stack and drops its data
+	cd $(E2E_DIR) && docker compose --profile visual down -v
+
+.PHONY: e2e
+e2e: e2e-dist $(E2E_DIR)/node_modules ## Runs the fake-GitHub E2E suite against the stack from e2e-env-up
+	cd $(E2E_DIR) && npx playwright test
+
+.PHONY: e2e-real
+e2e-real: dist $(E2E_DIR)/node_modules ## Runs the real GitHub journey (needs e2e/integration/.env)
+	cd $(E2E_DIR) && npm run test:real

e2e/integration/.env.example

@@ -0,0 +1,22 @@
+# Copy to .env. docker compose and the tests both read it.
+# Values for the real lane live in 1Password (search "GitHub plugin E2E"); never paste them into chats, issues or PRs.
+
+# Stack (optional). Defaults avoid the ports other plugin E2E stacks use.
+# MM_PORT=8069
+# PW_BROWSER_PORT=3336
+# MM_IMAGE=mattermostdevelopment/mattermost-enterprise-edition:master
+
+# Real lane only (npm run test:real). The PR lane needs none of these.
+# A classic OAuth app whose callback URL is http://127.0.0.1:${MM_PORT}/plugins/github/oauth/complete
+E2E_GITHUB_OAUTH_CLIENT_ID=
+E2E_GITHUB_OAUTH_CLIENT_SECRET=
+# owner/name of a throwaway repository in a test organization. The machine account needs admin on it.
+E2E_GITHUB_REPO=
+# OAuth token (or classic personal access token) of the machine account.
+# Scopes: repo, notifications, read:org, admin:org_hook, admin:repo_hook
+E2E_GITHUB_MACHINE_TOKEN=
+
+# Optional: the real sign-in test skips without these. The account needs 2FA with an authenticator app.
+E2E_GITHUB_LOGIN_USERNAME=
+E2E_GITHUB_LOGIN_PASSWORD=
+E2E_GITHUB_LOGIN_TOTP_SECRET=

e2e/integration/.gitignore

@@ -0,0 +1,6 @@
+node_modules
+test-results
+playwright-report
+storage_state
+.env
+.auth

e2e/integration/README.md

@@ -0,0 +1,59 @@
+# GitHub plugin Playwright tests
+
+Two lanes. The fake lane runs on every PR, including forks, with no secrets. The real lane talks to github.com and runs nightly only when enabled.
+
+The older suite in `e2e/playwright` (and `playwright.yml`) is unchanged and still runs against its own mock.
+
+```sh
+make e2e-env-up    # Postgres + Mattermost on :8069
+make e2e           # builds the E2E bundle (make e2e-dist), installs the plugin, runs the suite
+make e2e-env-down
+
+make e2e-real      # normal bundle, against github.com. Needs e2e/integration/.env
+```
+
+`make e2e-dist` bakes `http://fake-github:8093` into `testOAuthServerURL`, which now moves the OAuth, REST, GraphQL and upload base URLs together, so the github.com code path is what gets tested. The plugin logs a warning on activation. Never ship that bundle.
+
+Overrides go in the shell or in `e2e/integration/.env` (copy `.env.example`; docker compose reads it too). Real values live in 1Password or GitHub Actions secrets.
+
+| Variable | Default | Purpose |
+|---|---|---|
+| `MM_IMAGE` | `mattermostdevelopment/mattermost-enterprise-edition:master` | Server image |
+| `MM_PORT` | `8069` | Host port; must match between `e2e-env-up` and `e2e` |
+| `PW_BROWSER_PORT` | `3336` | Host port of the dockerized visual browser |
+| `PW_HEADLESS` | `true` | `false` to watch the browser |
+
+## How GitHub is faked
+
+- `fake-github/server.ts` runs inside the Playwright process on port 8093. Mattermost reaches it as `fake-github` through `extra_hosts`; Chromium through `--host-resolver-rules`.
+- It is a classic OAuth app: tokens never expire and have no refresh token. Token errors come back as HTTP 200 with an `error` field, as form or JSON depending on `Accept`. Scopes from earlier grants are merged, and consent is skipped when an earlier grant already covers the request.
+- Permissions follow GitHub: private repos need `repo`; repo hooks need repo admin plus `repo` or a `*:repo_hook` scope, otherwise 404; org hooks need org admin plus `admin:org_hook`; notifications need `notifications` or `repo`; issue labels, assignees and milestone are silently dropped without push access; SAML-enforced orgs answer 403 with `X-GitHub-SSO`.
+- Creating an issue fires `issues.opened` to matching hooks and creating a hook sends `ping`, signed with `X-Hub-Signature` (HMAC-SHA1) and `X-Hub-Signature-256`.
+- Tests drive it through `support/fake_github.ts` (`reset`, `createRepo`, `createItem`, `webhook`, `fault`, `sso`, `revoke`, `requests`, ...).
+- Only `connect.spec.ts` drives the browser OAuth flow. Other specs connect with `seedUserInfo()` (`support/kv.ts`), which writes `<userID>_githubtoken` with the plugin's AES-CFB cipher. `installPlugin()` also seeds `mm34646_token_reset_done`, or the plugin resets every stored token a minute after it starts.
+- Don't use `pw.initSetup()`: it resets server config and disables the plugin.
+
+## Keeping the fake honest
+
+Every request and response is checked, and each test fails at teardown if anything broke the contract.
+
+- **REST:** validated against [github/rest-api-description](https://github.com/github/rest-api-description) (`api.github.com.json`), trimmed to the operations in `fake-github/contract/operations.json`. A route that isn't listed fails the run.
+- **GraphQL:** queries are validated and executed against GitHub's [public schema](https://docs.github.com/en/graphql/overview/public-schema), trimmed to the types the plugin uses. graphql-js rejects `... on Issue { state } ... on PullRequest { state }` as a field conflict while GitHub answers it, so that one message is ignored. The real lane checks GitHub still accepts it.
+- **Webhooks:** validated against the `x-webhooks` schemas in the same REST description. These replaced [octokit/webhooks](https://github.com/octokit/webhooks), which is deprecated and no longer updated.
+- Nightly, `npm run contract:check` fails if GitHub changed the committed specs. `npm run contract:update` refreshes them.
+
+## Real GitHub
+
+`npm run test:real` (or `make e2e-real`) needs the variables in `.env.example`. Global setup:
+
+- Creates long-lived fixtures once and reuses them: an `e2e-fixture` label, an open issue assigned to the machine account, and an open pull request from the `e2e-fixture` branch. It then polls search until both show up, because the plugin finds them through search and the index lags.
+- Points a repository hook at a public placeholder URL with a fresh secret. GitHub refuses hooks to private addresses, so tests fetch the recorded delivery from the hook deliveries API and post it to the plugin.
+- Seeds the machine account's token into KV, so most tests skip the browser login.
+
+The sign-in test logs in with password and TOTP, then revokes its grant so the next run starts clean. GitHub skips its emailed device check for accounts with 2FA.
+
+Writes go through `real-github/rate_limit.ts`, which stays under GitHub's content limits (80 a minute, 500 an hour) across runs on one machine. Retries are off so failures don't repeat writes. Traces, screenshots and video are off when `CI` is set. Don't put account, organization or repository names in this tree.
+
+## Screenshots
+
+`npm run test:visual` compares `@visual` shots in a dockerized browser against a pinned server image, in its own throwaway stack, and does not fail the PR job. Update baselines with `npm run test:visual-update-snapshots` and commit the PNGs.

e2e/integration/docker-compose.yml

@@ -0,0 +1,62 @@
+name: github-plugin-e2e
+
+services:
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: mmuser
+      POSTGRES_PASSWORD: mostest
+      POSTGRES_DB: mattermost_test
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U mmuser -d mattermost_test"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+    tmpfs:
+      - /var/lib/postgresql/data
+
+  mattermost:
+    image: ${MM_IMAGE:-mattermostdevelopment/mattermost-enterprise-edition:master}
+    # Server images are published for amd64 only; Apple Silicon runs them under emulation.
+    platform: linux/amd64
+    depends_on:
+      postgres:
+        condition: service_healthy
+    ports:
+      - "127.0.0.1:${MM_PORT:-8069}:${MM_PORT:-8069}"
+      - "127.0.0.1:${PW_BROWSER_PORT:-3336}:3333"
+    extra_hosts:
+      - "fake-github:host-gateway"
+    environment:
+      MM_SQLSETTINGS_DRIVERNAME: postgres
+      MM_SQLSETTINGS_DATASOURCE: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
+      MM_SERVICESETTINGS_SITEURL: http://127.0.0.1:${MM_PORT:-8069}
+      MM_SERVICESETTINGS_LISTENADDRESS: ":${MM_PORT:-8069}"
+      MM_SERVICESETTINGS_ENABLELOCALMODE: "true"
+      MM_PLUGINSETTINGS_ENABLE: "true"
+      MM_PLUGINSETTINGS_ENABLEUPLOADS: "true"
+      MM_PLUGINSETTINGS_AUTOMATICPREPACKAGEDPLUGINS: "false"
+      MM_TEAMSETTINGS_ENABLEOPENSERVER: "true"
+      MM_ANNOUNCEMENTSETTINGS_ADMINNOTICESENABLED: "false"
+      MM_ANNOUNCEMENTSETTINGS_USERNOTICESENABLED: "false"
+      MM_LOGSETTINGS_CONSOLELEVEL: INFO
+    healthcheck:
+      test: ["CMD", "/mattermost/bin/mmctl", "--local", "system", "status"]
+      interval: 5s
+      timeout: 5s
+      retries: 60
+
+  # Shares Mattermost's network so the page origin matches the URL the tests use from the host.
+  visual-browser:
+    profiles: [visual]
+    image: mcr.microsoft.com/playwright:v${PW_VERSION:-1.64.1}-noble
+    network_mode: service:mattermost
+    # Chromium rasterizes text differently per CPU architecture; CI runners are amd64.
+    platform: linux/amd64
+    init: true
+    command: npx -y playwright@${PW_VERSION:-1.64.1} run-server --port 3333 --host 0.0.0.0
+    healthcheck:
+      test: ["CMD", "node", "-e", "require('net').connect(3333, '127.0.0.1').on('connect', () => process.exit(0)).on('error', () => process.exit(1))"]
+      interval: 2s
+      timeout: 2s
+      retries: 60