Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
153 changes: 152 additions & 1 deletion .github/workflows/visual-tests-demos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -530,6 +530,27 @@ jobs:
CI_ENV: true # The `ignore` field in the visualtestrc.json should be disabled when running test locally
run: pnpm exec nx test-testcafe

- name: Set Playwright artifact name
if: ${{ !cancelled() && matrix.STRATEGY == 'screenshots' }}
run: echo "PLAYWRIGHT_JOB_NAME=$(echo "${{ matrix.CONSTEL }}-${{ matrix.THEME }}" | tr '/' '-')" >> $GITHUB_ENV

# The demo pages themselves are shot by Playwright; TestCafe above still runs the manual widget tests.
- name: Run Playwright tests (jQuery)
if: ${{ !cancelled() && matrix.STRATEGY == 'screenshots' }}
timeout-minutes: 10
working-directory: apps/demos
env:
NODE_OPTIONS: --max-old-space-size=8192
CHANGEDFILEINFOSPATH: changed-files.json
CONCURRENCY: 4
CONSTEL: ${{ matrix.CONSTEL }}
THEME: ${{ matrix.THEME }}
CI_ENV: true # The `ignore` field in the visualtestrc.json should be disabled when running test locally
# The default blob name is built from the test paths only, so jobs that differ by theme or
# shard alone would write the same zip and overwrite each other once merged.
PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ env.PLAYWRIGHT_JOB_NAME }}.zip
run: pnpm run test-playwright

- name: Show accessibility warnings
if: matrix.STRATEGY == 'accessibility'
working-directory: apps/demos
Expand All @@ -541,6 +562,48 @@ jobs:
echo "Accessibility report not found. Skipping warning generation."
fi

- name: Upload Playwright blob report
if: failure() && matrix.STRATEGY == 'screenshots'
uses: actions/upload-artifact@v7
with:
name: playwright-blob-${{ env.PLAYWRIGHT_JOB_NAME }}
path: ${{ github.workspace }}/apps/demos/testing/artifacts/playwright/blob-report
if-no-files-found: ignore

- name: Upload Playwright traces and screenshot diffs
if: always() && matrix.STRATEGY == 'screenshots'
uses: actions/upload-artifact@v7
with:
name: playwright-traces-${{ env.PLAYWRIGHT_JOB_NAME }}
path: ${{ github.workspace }}/apps/demos/testing/artifacts/playwright/test-results
if-no-files-found: ignore

- name: Collect new etalons
if: always() && matrix.STRATEGY == 'screenshots'
run: |
mkdir -p "$RUNNER_TEMP/new-etalons"
git ls-files --others --exclude-standard -z -- 'apps/demos/testing/etalons/*.png' \
| xargs -0 -r cp --parents -t "$RUNNER_TEMP/new-etalons"

- name: Upload new etalons
if: always() && matrix.STRATEGY == 'screenshots'
uses: actions/upload-artifact@v7
with:
name: playwright-etalons-${{ env.PLAYWRIGHT_JOB_NAME }}
path: ${{ runner.temp }}/new-etalons
if-no-files-found: ignore

# A retry passes against the etalon its first attempt wrote, so new etalons fail the job here.
- name: Fail on new etalons
if: always() && matrix.STRATEGY == 'screenshots'
run: |
new_etalons=$(git ls-files --others --exclude-standard -- 'apps/demos/testing/etalons/*.png')
if [ -n "$new_etalons" ]; then
echo "::error::The tests wrote etalons that are not in the repository. They are in the playwright-etalons artifact:"
echo "$new_etalons"
exit 1
fi

- name: Sanitize job name
if: ${{ failure() }}
run: echo "JOB_NAME=$(echo "${{ matrix.CONSTEL }}-${{ matrix.THEME }}" | tr '/' '-')" >> $GITHUB_ENV
Expand Down Expand Up @@ -728,9 +791,97 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 6
needs: [testcafe-jquery, testcafe-frameworks-all]
if: ${{ !cancelled() && (needs.testcafe-jquery.result == 'failure' || needs.testcafe-frameworks-all.result == 'failure') }}
if: ${{ !cancelled() }}

steps:
- name: Detect Playwright artifacts
id: playwright-artifacts
uses: actions/github-script@v9
with:
script: |
const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId,
per_page: 100,
});

const artifactGroups = {
blobs: /^playwright-blob-/,
traces: /^playwright-traces-/,
etalons: /^playwright-etalons-/,
};

for (const [outputName, pattern] of Object.entries(artifactGroups)) {
const names = artifacts
.map((artifact) => artifact.name)
.filter((name) => pattern.test(name));

core.setOutput(outputName, names.length > 0 ? 'true' : 'false');
core.info(`${outputName}: ${names.length ? names.join(', ') : '(none)'}`);
}

- name: Get sources
if: steps.playwright-artifacts.outputs.blobs == 'true'
uses: actions/checkout@v6

- name: Use Node.js
if: steps.playwright-artifacts.outputs.blobs == 'true'
uses: actions/setup-node@v7
with:
node-version-file: '.node-version'

- uses: pnpm/action-setup@v6
if: steps.playwright-artifacts.outputs.blobs == 'true'
with:
run_install: false

- name: Install dependencies
if: steps.playwright-artifacts.outputs.blobs == 'true'
run: pnpm install --frozen-lockfile

- name: Download Playwright blob reports
if: steps.playwright-artifacts.outputs.blobs == 'true'
uses: actions/download-artifact@v8
with:
pattern: playwright-blob-*
path: ${{ github.workspace }}/blob-reports
merge-multiple: true

- name: Merge Playwright blob reports into one report
if: steps.playwright-artifacts.outputs.blobs == 'true'
working-directory: apps/demos
env:
PLAYWRIGHT_HTML_OUTPUT_DIR: ${{ github.workspace }}/playwright-report
run: pnpm exec playwright merge-reports --reporter html ${{ github.workspace }}/blob-reports

- name: Upload merged Playwright report
if: steps.playwright-artifacts.outputs.blobs == 'true'
uses: actions/upload-artifact@v7
with:
name: playwright-report
path: ${{ github.workspace }}/playwright-report
if-no-files-found: error

- name: Merge Playwright traces
if: steps.playwright-artifacts.outputs.traces == 'true'
uses: actions/upload-artifact/merge@v7
continue-on-error: true
with:
name: playwright-traces
pattern: playwright-traces-*
separate-directories: true
delete-merged: true

- name: Merge new etalons
if: steps.playwright-artifacts.outputs.etalons == 'true'
uses: actions/upload-artifact/merge@v7
continue-on-error: true
with:
name: playwright-etalons
pattern: playwright-etalons-*
delete-merged: true

- name: Merge jQuery screenshot artifacts
if: needs.testcafe-jquery.result == 'failure'
uses: actions/upload-artifact/merge@v7
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
export default async (page) => {
await page.click('.dx-dropdowneditor-button');
await page.click('.dx-list-item:first-child');
};
53 changes: 53 additions & 0 deletions apps/demos/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,59 @@ pnpm run convert-to-js "JSDemos/Demos/Diagram/**/React"
```

The "Check generated JS demos" GitHub Action ensures that the React JavaScript and TypeScript sources remain in sync.
## Screenshot Tests

Every demo page is compared with an etalon in `testing/etalons`. jQuery demos run on [Playwright](https://playwright.dev/) (`testing/common.spec.ts`); React, Vue and Angular demos, the manual widget tests (`testing/widgets`) and the accessibility checks still run on TestCafe.

### Run locally

Build DevExtreme first (`pnpm run demos:prepare` from the monorepo root). The tests use the system Google Chrome and start their own web server.

```
cd apps/demos
pnpm run test-playwright --ignore-snapshots # every jQuery demo, fluent.blue.light
pnpm run test-playwright --ignore-snapshots --grep DataGrid- # demos of one component
THEME=material.blue.light pnpm run test-playwright --ignore-snapshots
CONSTEL="jquery(1/3)" CONCURRENCY=4 pnpm run test-playwright --ignore-snapshots # one CI shard
```

Everything after the script name goes to `playwright test`, so `--headed`, `--ui` and `--repeat-each` work too.

A test fails on a console error or warning outside `testing/known-warnings.js` and on an uncaught error in the page.

### Etalons

The etalons are Linux renders made by CI, where the comparison is strict: no pixel may differ. Text is drawn differently on macOS and Windows, so a run on your machine compares loosely and proves little: pass `--ignore-snapshots` and never commit an etalon written there.

To add or update an etalon, delete the file and push. The job fails on the missing etalon and uploads what it rendered:

```
gh run download <run-id> -n playwright-etalons -D ci-etalons
cp -R ci-etalons/. ../.. # from apps/demos: the paths start at the repository root
```

Look at the image before you commit it. One etalon serves all frameworks of a demo.

### Files next to a demo

| File | Purpose |
|---|---|
| `client-script.js` | runs in the page before the demo, for example to fix the current date with `MockDate` |
| `test-code.js` | runs in the page after it loads; a returned promise is awaited |
| `test-styles.css` | styles added for the test only |
| `playwright-test-code.js` | steps that need real input, as a function of the Playwright `page` |
| `visualtestrc.json` | `ignore` switches the demo off on CI, for all frameworks or for one |

### Read a failed CI run

```
gh run download <run-id> -n playwright-report -D ci-report
gh run download <run-id> -n playwright-traces -D ci-traces
pnpm exec playwright show-report ci-report
```

A failed comparison uploads its `-expected`, `-actual` and `-diff` images with the trace. On CI a failed test runs once more; a test that passes then is listed under "flaky" and does not fail the job.

## See Also

- [Technical demos online](https://js.devexpress.com/Demos/)
Expand Down
7 changes: 7 additions & 0 deletions apps/demos/eslint.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,13 @@ export default [
files: ['testing/**/*.{js,ts}', 'utils/visual-tests/**/*.*'],
})),

{
files: ['playwright.config.ts'],
rules: {
'import/no-extraneous-dependencies': ['error', { devDependencies: true }],
},
},

{
files: ['testing/**/*.{js,ts}'],
languageOptions: {
Expand Down
2 changes: 2 additions & 0 deletions apps/demos/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@
"@babel/preset-env": "7.29.7",
"@babel/preset-typescript": "7.29.7",
"@eslint/js": "catalog:",
"@playwright/test": "catalog:",
"@stylistic/eslint-plugin": "catalog:",
"@testcafe-community/axe": "3.5.0",
"@types/eslint": "8.56.12",
Expand Down Expand Up @@ -139,6 +140,7 @@
"lint-css": "stylelint **/*.{css,vue}",
"lint-non-demos": "pnpm exec nx run-many -t lint-js lint-css lint-html -p devextreme-demos",
"lint": "pnpm exec nx run-many -t lint-js lint-demos lint-css lint-html -p devextreme-demos",
"test-playwright": "playwright test",
"test-testcafe": "ts-node utils/visual-tests/testcafe-runner.ts",
"test-testcafe:csp": "cross-env CSP_REPORT=true ts-node utils/visual-tests/testcafe-runner.ts",
"test-testcafe:accessibility": "cross-env STRATEGY=accessibility CONSTEL=jquery node utils/visual-tests/testcafe-runner.ts",
Expand Down
76 changes: 76 additions & 0 deletions apps/demos/playwright.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import { defineConfig } from '@playwright/test';
import { isMaterial } from './utils/visual-tests/helpers/theme-utils';
import {
BASE_URL, SERVER_PORT, THEME, VIEWPORT,
} from './utils/visual-tests/playwright/const';

// The CI agent and the container draw the same pixels and are judged exactly. The same comparison
// decides when a page has stopped changing, so a tolerance there lets the last frames of an
// animation pass for the final picture. A run on the host is lenient: another OS rasterises text
// differently, so its comparison proves nothing anyway.
const rendersLikeCI = !!process.env.CI || !!process.env.DX_CONTAINER;

const screenshotBudget = rendersLikeCI
? { maxDiffPixels: 0, threshold: 0 }
: { maxDiffPixelRatio: 0.05, threshold: 0.2 };

// The flags pin the rendering pipeline so screenshots stay comparable between runs.
const chromeArgs = [
'--no-sandbox',
'--disable-dev-shm-usage',
'--disable-gpu',
'--disable-partial-raster',
'--disable-skia-runtime-opts',
'--run-all-compositor-stages-before-draw',
'--disable-new-content-rendering-timeout',
'--disable-threaded-animation',
'--disable-threaded-scrolling',
'--disable-checker-imaging',
'--disable-image-animation-resync',
'--use-gl=swiftshader',
'--disable-features=PaintHolding',
'--js-flags=--random-seed=2147483647',
];

// Material keeps font hinting and subpixel positioning: its text is unstable without them.
if (!isMaterial(THEME)) {
chromeArgs.push('--font-render-hinting=none', '--disable-font-subpixel-positioning');
}

export default defineConfig({
testDir: './testing',
testMatch: 'common.spec.ts',
snapshotPathTemplate: '{testDir}/etalons/{arg}{ext}',
outputDir: './testing/artifacts/playwright/test-results',
fullyParallel: true,
forbidOnly: !!process.env.CI,
retries: process.env.CI ? 1 : 0,
workers: process.env.CONCURRENCY ? Number(process.env.CONCURRENCY) : undefined,
reporter: process.env.CI
? [['list'], ['blob', { outputDir: './testing/artifacts/playwright/blob-report' }]]
: [['list']],
timeout: 60000,
expect: {
timeout: 5000,
toHaveScreenshot: screenshotBudget,
},
use: {
baseURL: BASE_URL,
channel: 'chrome',
headless: true,
viewport: VIEWPORT,
launchOptions: {
args: chromeArgs,
// Playwright hides scrollbars in headless by default, which changes the page layout.
ignoreDefaultArgs: ['--hide-scrollbars'],
},
trace: 'retain-on-failure',
screenshot: 'only-on-failure',
},
webServer: {
command: `node ./utils/visual-tests/playwright/serve.js ${SERVER_PORT}`,
url: `${BASE_URL}/apps/demos/index.html`,
reuseExistingServer: !process.env.CI,
timeout: 30000,
},
});
1 change: 1 addition & 0 deletions apps/demos/testing/apiMocks/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ export interface MockRequest {
url: string;
method: string;
body?: string | Buffer;
headers?: Record<string, string>;
}

export interface MockHandler {
Expand Down
Loading
Loading