From be066aa9c9e37b13741b94f6125b0bef9e0098e4 Mon Sep 17 00:00:00 2001 From: Kamalpreet Kaur Date: Thu, 12 Nov 2026 14:56:37 +0530 Subject: [PATCH 1/5] fix(cli): send the per-test accessibility verdict on test events On the CLI flow the binary set product_map.accessibility from the platform check alone, so every test in a tag-scoped build was reported as an accessibility test. The accessibility module now sets test_accessibility (accessibility on, auto-scanning on, test in the include/exclude tag scope), matching the classic flow's setTestRunAccessibilityVar. Its TEST/PRE observer is prepended so the value is on the instance before TestRunStarted is sent. Co-Authored-By: Claude Opus 5.5 --- .../src/cli/eventDispatcher.ts | 9 ++- .../constants/testFrameworkConstants.ts | 1 + .../src/cli/frameworks/testFramework.ts | 5 +- .../src/cli/modules/accessibilityModule.ts | 25 +++++++++ .../cli/modules/accessibilityModule.test.ts | 56 +++++++++++++++++++ 5 files changed, 92 insertions(+), 4 deletions(-) diff --git a/packages/browserstack-service/src/cli/eventDispatcher.ts b/packages/browserstack-service/src/cli/eventDispatcher.ts index 75385d58..0b5672a3 100644 --- a/packages/browserstack-service/src/cli/eventDispatcher.ts +++ b/packages/browserstack-service/src/cli/eventDispatcher.ts @@ -29,13 +29,18 @@ class EventDispatcher { * Add event observer * @param {string} event - Event name * @param {Function} callback - Callback function + * @param {boolean} prepend - Run before observers already registered for this event */ - registerObserver(hookRegistryKey: string, callback: Function) { + registerObserver(hookRegistryKey: string, callback: Function, prepend = false) { if (!this.observers[hookRegistryKey]) { this.observers[hookRegistryKey] = [] } - this.observers[hookRegistryKey].push(callback) + if (prepend) { + this.observers[hookRegistryKey].unshift(callback) + } else { + this.observers[hookRegistryKey].push(callback) + } } /** diff --git a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts index 1dd45e24..9cc2657a 100644 --- a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts +++ b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts @@ -4,6 +4,7 @@ export const TestFrameworkConstants = { KEY_TEST_NAME : 'test_name', KEY_TEST_FILE_PATH : 'test_file_path', KEY_TEST_TAGS : 'test_tags', + KEY_TEST_ACCESSIBILITY : 'test_accessibility', KEY_TEST_RESULT : 'test_result', KEY_TEST_RESULT_AT : 'test_result_at', KEY_TEST_STARTED_AT : 'test_started_at', diff --git a/packages/browserstack-service/src/cli/frameworks/testFramework.ts b/packages/browserstack-service/src/cli/frameworks/testFramework.ts index f291505c..ce7d8c5f 100644 --- a/packages/browserstack-service/src/cli/frameworks/testFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/testFramework.ts @@ -103,10 +103,11 @@ export default class TestFramework { * @param {TestFrameworkState} testFrameworkState * @param {HookState} hookState * @param {*} callback + * @param {boolean} prepend - Run before observers already registered for this event * @returns {void} */ - static registerObserver(testFrameworkState: State, hookState: State, callback: Function) { - eventDispatcher.registerObserver(CLIUtils.getHookRegistryKey(testFrameworkState, hookState), callback) + static registerObserver(testFrameworkState: State, hookState: State, callback: Function, prepend = false) { + eventDispatcher.registerObserver(CLIUtils.getHookRegistryKey(testFrameworkState, hookState), callback, prepend) } /** diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index 245cf7cf..3e708de2 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -12,6 +12,7 @@ import { HookState } from '../states/hookState.js' import accessibilityScripts from '../../scripts/accessibility-scripts.js' import { _getParamsForAppAccessibility, formatString, getAppA11yResults, getAppA11yResultsSummary, shouldScanTestForAccessibility, validateCapsWithA11y, validateCapsWithAppA11y, validateCapsWithNonBstackA11y, setBrowserstackAnnotation } from '../../util.js' import { AutomationFrameworkConstants } from '../frameworks/constants/automationFrameworkConstants.js' +import { TestFrameworkConstants } from '../frameworks/constants/testFrameworkConstants.js' import util from 'node:util' import type { Accessibility } from '../../grpc/index.js' import type { Capabilities } from '@wdio/types' @@ -42,6 +43,8 @@ export default class AccessibilityModule extends BaseModule { this.name = 'AccessibilityModule' this.accessibilityConfig = accessibilityConfig AutomationFramework.registerObserver(AutomationFrameworkState.CREATE, HookState.POST, this.onBeforeExecute.bind(this)) + // Prepended so the verdict is on the instance before TestHubModule serializes TestRunStarted. + TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.setTestAccessibility.bind(this), true) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this)) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this)) this.accessibility = Boolean(accessibilityConfig) @@ -197,6 +200,28 @@ export default class AccessibilityModule extends BaseModule { } } + /** + * Per-test product_map.accessibility, as the classic handler's setTestRunAccessibilityVar: + * only tests in the include/exclude tag scope are accessibility tests. The binary falls back + * to its platform check when this is absent, which marks every test in the build. + */ + setTestAccessibility(args: Record) { + try { + const testInstance = (args.instance as TestFrameworkInstance) || TestFramework.getTrackedInstance() + if (!testInstance) { + return + } + const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' + const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} + const world = args.world as { [key: string]: unknown } | undefined + const inScope = Boolean(this.accessibility && this.autoScanning && + shouldScanTestForAccessibility(suiteTitle, test.title || '', this.config.accessibilityOptions as Record | undefined, world, Boolean(world))) + TestFramework.setState(testInstance, TestFrameworkConstants.KEY_TEST_ACCESSIBILITY, inScope) + } catch (error) { + this.logger.debug(`setTestAccessibility: ${error}`) + } + } + async onBeforeTest(args: any) { try { diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index 0cd78797..943bb137 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -304,6 +304,62 @@ describe('AccessibilityModule', () => { }) }) + describe('setTestAccessibility', () => { + const world = { pickle: { tags: [{ name: '@accessibility' }] } } + + beforeEach(() => { + accessibilityModule.accessibility = true + accessibilityModule.autoScanning = true + }) + + it('should be prepended to TEST/PRE so TestRunStarted carries the verdict', () => { + expect(TestFramework.registerObserver).toHaveBeenCalledWith( + TestFrameworkState.TEST, + HookState.PRE, + expect.any(Function), + true + ) + }) + + it('should mark an in-scope test as an accessibility test', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' }, suiteTitle: 'Feature', world }) + + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Feature', 'Homepage', {}, world, true) + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', true) + }) + + it('should mark an out-of-scope test as not an accessibility test', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(false) + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Basket' }, suiteTitle: 'Feature', world }) + + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) + }) + + it('should mark no test when auto-scanning is off', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.autoScanning = false + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } }) + + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) + }) + + it('should mark no test when the platform does not support accessibility', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.accessibility = false + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } }) + + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) + }) + + it('should use the 3-arg title match for mocha', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'logs in' }, suiteTitle: 'Login' }) + + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Login', 'logs in', {}, undefined, false) + }) + }) + describe('onBeforeTest', () => { beforeEach(() => { // Mock shouldScanTestForAccessibility to return true for proper test behavior From 20479af57a3e87247e1a2d59a4961b4282b14f51 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:30:38 +0000 Subject: [PATCH 2/5] chore(changeset): auto-generate from PR template (patch) --- .changeset/pr-361.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/pr-361.md diff --git a/.changeset/pr-361.md b/.changeset/pr-361.md new file mode 100644 index 00000000..666b016b --- /dev/null +++ b/.changeset/pr-361.md @@ -0,0 +1,5 @@ +--- +"@wdio/browserstack-service": patch +--- + +- Fixed tests outside `includeTagsInTestingScope` / `excludeTagsInTestingScope` being reported as accessibility tests when using the CLI. From 9c109bd458d29730ffac78d68be3bba8cd096cc4 Mon Sep 17 00:00:00 2001 From: Kamalpreet Kaur Date: Thu, 12 Nov 2026 16:21:40 +0530 Subject: [PATCH 3/5] fix(cli): mark cascade-skipped scenarios as not accessibility tests Scenarios abandoned by a failed BeforeAll are built straight into TEST/POST, so the TEST/PRE verdict never ran for them and the binary fell back to its platform check. They never ran or scanned, so they now carry test_accessibility: false. A failure to set the verdict is logged at error level. Co-Authored-By: Claude Opus 5.5 --- .../src/cli/frameworks/wdioCucumberTestFramework.ts | 2 ++ .../browserstack-service/src/cli/modules/accessibilityModule.ts | 2 +- .../tests/cli/frameworks/wdioCucumberTestFramework.test.ts | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index c5e1d507..45535427 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -518,6 +518,8 @@ export default class WdioCucumberTestFramework extends TestFramework { [TestFrameworkConstants.KEY_TEST_RESULT_AT]: now, ...this.featureFilePathEntries(), [KEY_TEST_SKIPPED_CASCADE]: true, + // Never ran, so never scanned: not an accessibility test whatever its tags. + [TestFrameworkConstants.KEY_TEST_ACCESSIBILITY]: false, [KEY_BDD_META_INFO]: { feature: { name: feature.name, path: this.featureUriForMeta(), description: feature.description }, scenario: { name: scenario.name }, diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index 3e708de2..ca0466aa 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -218,7 +218,7 @@ export default class AccessibilityModule extends BaseModule { shouldScanTestForAccessibility(suiteTitle, test.title || '', this.config.accessibilityOptions as Record | undefined, world, Boolean(world))) TestFramework.setState(testInstance, TestFrameworkConstants.KEY_TEST_ACCESSIBILITY, inScope) } catch (error) { - this.logger.debug(`setTestAccessibility: ${error}`) + this.logger.error(`setTestAccessibility: ${error}`) } } diff --git a/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts b/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts index 9e413902..75478f2e 100644 --- a/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts @@ -236,6 +236,7 @@ describe('WdioCucumberTestFramework', () => { expect(first[TestFrameworkConstants.KEY_TEST_SCOPE]).toBe('Plain one') expect(first[TestFrameworkConstants.KEY_TEST_RESULT]).toBe('skipped') expect(first.test_skipped_cascade).toBe(true) + expect(first[TestFrameworkConstants.KEY_TEST_ACCESSIBILITY]).toBe(false) expect(first[TestFrameworkConstants.KEY_TEST_TAGS]).toBeUndefined() expect(first.bdd_meta_info.steps).toEqual([{ id: 's1', text: 'I do a thing', keyword: 'Given ', result: 'skipped' }]) // the cascade builds its own meta blob, so it needs the same relative/absolute split From f77f118a988dcb03d7f5ad78f9bc0cd771d03fcc Mon Sep 17 00:00:00 2001 From: Kamalpreet Kaur Date: Thu, 12 Nov 2026 16:26:53 +0530 Subject: [PATCH 4/5] fix(cli): leave auto-scanning out of the per-test accessibility verdict With autoScanning off, tests scanned manually with performScan() still need their accessibility test row, so test_accessibility is now accessibility on and test in the include/exclude tag scope. Auto-scanning still gates the automatic scans. Co-Authored-By: Claude Opus 5.5 --- .../src/cli/modules/accessibilityModule.ts | 8 ++++---- .../tests/cli/modules/accessibilityModule.test.ts | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index ca0466aa..0d5437fa 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -201,9 +201,9 @@ export default class AccessibilityModule extends BaseModule { } /** - * Per-test product_map.accessibility, as the classic handler's setTestRunAccessibilityVar: - * only tests in the include/exclude tag scope are accessibility tests. The binary falls back - * to its platform check when this is absent, which marks every test in the build. + * Per-test product_map.accessibility: only tests in the include/exclude tag scope are + * accessibility tests. Auto-scanning is left out so tests scanned manually keep their row. + * The binary falls back to its platform check when this is absent, which marks every test. */ setTestAccessibility(args: Record) { try { @@ -214,7 +214,7 @@ export default class AccessibilityModule extends BaseModule { const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} const world = args.world as { [key: string]: unknown } | undefined - const inScope = Boolean(this.accessibility && this.autoScanning && + const inScope = Boolean(this.accessibility && shouldScanTestForAccessibility(suiteTitle, test.title || '', this.config.accessibilityOptions as Record | undefined, world, Boolean(world))) TestFramework.setState(testInstance, TestFrameworkConstants.KEY_TEST_ACCESSIBILITY, inScope) } catch (error) { diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index 943bb137..68e7452d 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -336,12 +336,12 @@ describe('AccessibilityModule', () => { expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) }) - it('should mark no test when auto-scanning is off', () => { + it('should keep an in-scope test as an accessibility test when auto-scanning is off', () => { vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) accessibilityModule.autoScanning = false accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } }) - expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', true) }) it('should mark no test when the platform does not support accessibility', () => { From a283be3a95a7a73690a724e4a85ce267fe198e4e Mon Sep 17 00:00:00 2001 From: Kamalpreet Kaur Date: Fri, 13 Nov 2026 22:03:27 +0530 Subject: [PATCH 5/5] test: expect prepend flag in TestFramework.registerObserver Co-Authored-By: Claude Opus 5.5 --- .../tests/cli/frameworks/testFramework.test.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/browserstack-service/tests/cli/frameworks/testFramework.test.ts b/packages/browserstack-service/tests/cli/frameworks/testFramework.test.ts index a14baff6..75bf0da6 100644 --- a/packages/browserstack-service/tests/cli/frameworks/testFramework.test.ts +++ b/packages/browserstack-service/tests/cli/frameworks/testFramework.test.ts @@ -140,7 +140,18 @@ describe('TestFramework', () => { TestFramework.registerObserver(mockState as any, mockHook as any, mockCallback) - expect(eventDispatcher.registerObserver).toHaveBeenCalledWith('TEST_STATE-PRE_HOOK', mockCallback) + expect(eventDispatcher.registerObserver).toHaveBeenCalledWith('TEST_STATE-PRE_HOOK', mockCallback, false) + }) + + it('should forward prepend to the event dispatcher', async () => { + const mockCallback = vi.fn() + const { eventDispatcher } = await import('../../../src/cli/eventDispatcher.js') + const { CLIUtils } = await import('../../../src/cli/cliUtils.js') + vi.mocked(CLIUtils.getHookRegistryKey).mockReturnValue('TEST_STATE-PRE_HOOK') + + TestFramework.registerObserver('TEST_STATE' as any, 'PRE_HOOK' as any, mockCallback, true) + + expect(eventDispatcher.registerObserver).toHaveBeenCalledWith('TEST_STATE-PRE_HOOK', mockCallback, true) }) }) })