Skip to content

Commit 2362bd5

Browse files
rubennortemeta-codesync[bot]
authored andcommitted
Preview failed Fantom screenshot snapshots in terminals (#59498)
Summary: Pull Request resolved: #59498 Make visual regressions easier to inspect with Expected | Received | Difference previews and file paths. A Jest reporter reuses the saved PNGs without changing snapshot comparison. Terminal formatting preserves the full results for logs and other reporters. Select the protocol from the detected terminal and confirm VS Code image support before sending graphics. Keep full text diagnostics whenever a required preview cannot be shown. Retain failing comparison images as GitHub job artifacts to diagnose rendering differences in CI. Keep captures from every failed retry, even if a later attempt passes. Changelog: [Internal] Show terminal previews and upload GitHub artifacts for failing Fantom screenshot snapshots. Differential Revision: D127129237
1 parent a5ca822 commit 2362bd5

20 files changed

Lines changed: 1805 additions & 10 deletions

‎.github/actions/run-fantom-tests/action.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,14 @@ runs:
3838
exit 1
3939
env:
4040
LD_LIBRARY_PATH: ${{ github.workspace }}/private/react-native-fantom/build/tester
41+
- name: Upload failed screenshot comparisons
42+
if: ${{ always() }}
43+
uses: ./.github/actions/upload-artifact
44+
with:
45+
name: fantom-screenshot-failures
46+
compression-level: 1
47+
if-no-files-found: ignore
48+
path: private/react-native-fantom/build/screenshot-failures
4149
- name: Upload test results
4250
if: ${{ always() }}
4351
uses: ./.github/actions/upload-artifact
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/**
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* This source code is licensed under the MIT license found in the
5+
* LICENSE file in the root directory of this source tree.
6+
*
7+
* @flow strict
8+
* @format
9+
*/
10+
11+
declare module '@jest/reporters' {
12+
declare export class DefaultReporter<
13+
TTestResult = unknown,
14+
TAggregatedResult = unknown,
15+
> {
16+
constructor(globalConfig: unknown): void;
17+
onRunStart(aggregatedResults: unknown, options: unknown): void;
18+
onTestResult(
19+
test: unknown,
20+
result: TTestResult,
21+
aggregatedResults: TAggregatedResult,
22+
): void | Promise<void>;
23+
onRunComplete(
24+
testContexts: unknown,
25+
aggregatedResults: TAggregatedResult,
26+
): void;
27+
}
28+
declare export class VerboseReporter<
29+
TTestResult = unknown,
30+
TAggregatedResult = unknown,
31+
> extends DefaultReporter<TTestResult, TAggregatedResult> {}
32+
declare export class SummaryReporter<TAggregatedResult = unknown> {
33+
constructor(globalConfig: unknown): void;
34+
onRunStart(aggregatedResults: unknown, options: unknown): void;
35+
onRunComplete(
36+
testContexts: unknown,
37+
aggregatedResults: TAggregatedResult,
38+
): void;
39+
}
40+
}

‎private/react-native-fantom/__docs__/rendering.md‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,23 @@ the baseline directory before cleaning up the temporary file. Unused images are
108108
removed only during a full successful snapshot update; empty, failed, skipped or
109109
filtered suites preserve their baselines.
110110

111+
Failing screenshot snapshots show expected, received and diff PNG previews in
112+
supported terminals instead of hash diffs, with file paths beneath the images.
113+
Wide terminals show Expected, Received and Difference side by side; narrow
114+
terminals stack them. Fantom selects the image protocol from the detected
115+
terminal. VS Code requires `terminal.integrated.enableImages` in its settings;
116+
Fantom checks the terminal's response before sending images. If that check
117+
cannot confirm support (including when another program is reading terminal
118+
input), it keeps text diagnostics and file paths without printing image headers.
119+
Unsupported terminals, terminal multiplexers, CI and redirected output retain
120+
text diagnostics and file paths. Full text diagnostics also remain when any
121+
required image preview is unavailable.
122+
123+
GitHub Fantom jobs upload failed comparisons as the `fantom-screenshot-failures`
124+
artifact, grouped by test configuration and snapshot. Each comparison includes
125+
the expected, received and difference PNGs when available. Failures from earlier
126+
retries are retained even if a later attempt passes.
127+
111128
### Painting
112129

113130
The painter traverses Fabric's mounted child order, including stacking changes.

‎private/react-native-fantom/config/jest.config.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ module.exports = {
5454
testPathIgnorePatterns: baseConfig.testPathIgnorePatterns,
5555
transformIgnorePatterns: ['.*'],
5656
testRunner: '<rootDir>/private/react-native-fantom/runner/index.js',
57+
reporters: ['<rootDir>/private/react-native-fantom/reporter/index.js'],
5758
watchPathIgnorePatterns: ['<rootDir>/private/react-native-fantom/build/'],
5859
globalSetup:
5960
'<rootDir>/private/react-native-fantom/runner/global-setup/setup.js',
Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
/**
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* This source code is licensed under the MIT license found in the
5+
* LICENSE file in the root directory of this source tree.
6+
*
7+
* @flow strict-local
8+
* @format
9+
*/
10+
11+
import type {ScreenshotResult} from '../runtime/ScreenshotContext';
12+
import type {TerminalEnvironment} from './terminalImages';
13+
14+
import createImageRow from './imageRow';
15+
import saveScreenshotArtifacts from './screenshotArtifacts';
16+
import queryImageSupport from './terminalCapabilities';
17+
import {encodeTerminalImage, getImageProtocol} from './terminalImages';
18+
import {
19+
DefaultReporter,
20+
SummaryReporter,
21+
VerboseReporter,
22+
} from '@jest/reporters';
23+
import fs from 'node:fs';
24+
import path from 'node:path';
25+
import tty from 'node:tty';
26+
27+
export type TestResult = {
28+
readonly failureMessage?: ?string,
29+
readonly screenshotFailureMessage?: ?string,
30+
readonly testResults: ReadonlyArray<
31+
Readonly<{
32+
fullName: string,
33+
status: string,
34+
screenshotResults?: ReadonlyArray<ScreenshotResult>,
35+
}>,
36+
>,
37+
};
38+
39+
type AggregatedResult = {
40+
readonly testResults: ReadonlyArray<TestResult>,
41+
...
42+
};
43+
44+
interface TerminalOutput {
45+
readonly isTTY?: boolean;
46+
readonly columns?: number;
47+
readonly rows?: number;
48+
write(string): unknown;
49+
}
50+
51+
export default function createScreenshotReporter(globalConfig: {
52+
readonly verbose?: boolean,
53+
...
54+
}): DefaultReporter<TestResult, AggregatedResult> {
55+
const Reporter: Class<DefaultReporter<TestResult, AggregatedResult>> =
56+
globalConfig.verbose === true ? VerboseReporter : DefaultReporter;
57+
const summaryReporter = new SummaryReporter<AggregatedResult>(globalConfig);
58+
let terminalResults = new WeakMap<TestResult, TestResult>();
59+
return new (class ScreenshotReporter extends Reporter {
60+
#vsCodeImages: ?Promise<boolean> = null;
61+
62+
onRunStart(aggregatedResults: unknown, options: unknown): void {
63+
this.#vsCodeImages = null;
64+
terminalResults = new WeakMap();
65+
super.onRunStart(aggregatedResults, options);
66+
summaryReporter.onRunStart(aggregatedResults, options);
67+
}
68+
69+
async onTestResult(
70+
test: unknown,
71+
result: TestResult,
72+
aggregatedResults: AggregatedResult,
73+
): Promise<void> {
74+
if (process.env.GITHUB_ACTIONS === 'true') {
75+
const errors = saveScreenshotArtifacts(
76+
result.testResults
77+
.filter(testResult => testResult.status === 'failed')
78+
.flatMap(testResult => testResult.screenshotResults ?? []),
79+
path.resolve(__dirname, '../build/screenshot-failures'),
80+
);
81+
for (const error of errors) {
82+
console.warn(error);
83+
}
84+
}
85+
const input = process.stdin;
86+
const output = process.stderr;
87+
let terminalResult = result;
88+
if (output instanceof tty.WriteStream) {
89+
let supportsVSCodeImages = false;
90+
if (
91+
process.env.TERM_PROGRAM === 'vscode' &&
92+
input instanceof tty.ReadStream &&
93+
getImageProtocol(output.isTTY, process.env, true) != null &&
94+
result.testResults.some(
95+
testResult =>
96+
testResult.status === 'failed' &&
97+
testResult.screenshotResults?.some(
98+
screenshot =>
99+
!screenshot.pass && screenshot.receivedPath != null,
100+
),
101+
)
102+
) {
103+
this.#vsCodeImages ??= queryImageSupport(input, output);
104+
supportsVSCodeImages = await this.#vsCodeImages;
105+
}
106+
terminalResult = reportScreenshotPreviews(
107+
result,
108+
output,
109+
process.env,
110+
supportsVSCodeImages,
111+
);
112+
}
113+
terminalResults.set(result, terminalResult);
114+
await super.onTestResult(test, terminalResult, aggregatedResults);
115+
}
116+
117+
onRunComplete(
118+
testContexts: unknown,
119+
aggregatedResults: AggregatedResult,
120+
): void {
121+
super.onRunComplete(testContexts, aggregatedResults);
122+
summaryReporter.onRunComplete(testContexts, {
123+
...aggregatedResults,
124+
testResults: aggregatedResults.testResults.map(
125+
result => terminalResults.get(result) ?? result,
126+
),
127+
});
128+
}
129+
})(globalConfig);
130+
}
131+
132+
export function reportScreenshotPreviews(
133+
result: TestResult,
134+
output: TerminalOutput,
135+
env: TerminalEnvironment,
136+
supportsVSCodeImages: boolean = false,
137+
): TestResult {
138+
const protocol = getImageProtocol(output.isTTY, env, supportsVSCodeImages);
139+
if (protocol == null) {
140+
return result;
141+
}
142+
const readErrors = new Map<string, string>();
143+
const readImage = (file: string): ?Buffer => {
144+
try {
145+
return fs.readFileSync(file);
146+
} catch (error: unknown) {
147+
const message = error instanceof Error ? error.message : String(error);
148+
readErrors.set(file, message.replace(/\p{Cc}/gu, ''));
149+
return null;
150+
}
151+
};
152+
const columns =
153+
output.columns != null && output.columns > 0 ? output.columns : 80;
154+
const rows = output.rows != null && output.rows > 0 ? output.rows : 30;
155+
const horizontal = columns >= 60;
156+
const preview = (png: Buffer) =>
157+
encodeTerminalImage(png, protocol, {
158+
columns: columns - 2,
159+
rows: rows - 9,
160+
tmux: env.TMUX != null,
161+
});
162+
let previews = '';
163+
let hasPreview = false;
164+
let allPreviewsRendered = true;
165+
for (const test of result.testResults) {
166+
if (test.status !== 'failed') {
167+
continue;
168+
}
169+
for (const screenshot of test.screenshotResults ?? []) {
170+
const receivedPath = screenshot.receivedPath;
171+
if (screenshot.pass || receivedPath == null) {
172+
continue;
173+
}
174+
const panels = [
175+
{
176+
label: 'Expected',
177+
path: screenshot.expectedPath,
178+
image: screenshot.isNew ? null : readImage(screenshot.expectedPath),
179+
unavailable: screenshot.isNew ? 'no baseline' : 'preview unavailable',
180+
required: !screenshot.isNew,
181+
},
182+
{
183+
label: 'Received',
184+
path: receivedPath,
185+
image: readImage(receivedPath),
186+
unavailable: 'preview unavailable',
187+
required: true,
188+
},
189+
{
190+
label: 'Difference',
191+
path: screenshot.differencePath,
192+
image:
193+
screenshot.differencePath == null
194+
? null
195+
: readImage(screenshot.differencePath),
196+
unavailable: screenshot.isNew ? 'no baseline' : 'preview unavailable',
197+
required: !screenshot.isNew,
198+
},
199+
];
200+
const title = test.fullName.replace(/\p{Cc}/gu, '');
201+
previews += `\nScreenshot snapshot: ${title}\n`;
202+
const images = panels
203+
.map(panel => panel.image)
204+
.filter(image => image != null);
205+
const row =
206+
horizontal && images.length === panels.length
207+
? createImageRow(images)
208+
: null;
209+
const rowPreview = row == null ? null : preview(row);
210+
if (rowPreview != null) {
211+
hasPreview = true;
212+
const labelWidth = Math.max(
213+
8,
214+
Math.floor((rowPreview.columns - 6) / 3),
215+
);
216+
previews +=
217+
panels.map(panel => panel.label.padEnd(labelWidth)).join(' | ') +
218+
'\n';
219+
previews += rowPreview.sequence + '\n';
220+
for (const panel of panels) {
221+
previews += `${panel.label}: ${panel.path ?? panel.unavailable}\n`;
222+
}
223+
} else {
224+
for (const panel of panels) {
225+
previews += `${panel.label}: ${panel.path ?? panel.unavailable}\n`;
226+
const image = panel.image == null ? null : preview(panel.image);
227+
hasPreview ||= image != null;
228+
allPreviewsRendered &&= image != null || !panel.required;
229+
const readError = readErrors.get(panel.path ?? '');
230+
previews +=
231+
(image?.sequence ??
232+
(readError == null
233+
? panel.unavailable
234+
: `preview unavailable: ${readError}`)) + '\n';
235+
}
236+
}
237+
}
238+
}
239+
if (previews === '') {
240+
return result;
241+
}
242+
try {
243+
output.write(previews);
244+
} catch {
245+
return result;
246+
}
247+
if (
248+
hasPreview &&
249+
allPreviewsRendered &&
250+
result.screenshotFailureMessage != null
251+
) {
252+
return {...result, failureMessage: result.screenshotFailureMessage};
253+
}
254+
return result;
255+
}

0 commit comments

Comments
 (0)