From 454f2f3e5382767cbc1612253f02a98cb487ced6 Mon Sep 17 00:00:00 2001 From: Kamalpreet Kaur Date: Thu, 12 Nov 2026 14:56:37 +0530 Subject: [PATCH 1/4] fix(cli): send the per-test accessibility verdict on test events On the CLI flow the binary set product_map.accessibility from the platform check alone, so every test in a tag-scoped build was reported as an accessibility test. The accessibility module now sets test_accessibility (accessibility on, auto-scanning on, test in the include/exclude tag scope), matching the classic flow's setTestRunAccessibilityVar. Its TEST/PRE observer is prepended so the value is on the instance before TestRunStarted is sent. Co-Authored-By: Claude Opus 5.5 --- .../src/cli/eventDispatcher.ts | 9 ++- .../constants/testFrameworkConstants.ts | 1 + .../src/cli/frameworks/testFramework.ts | 5 +- .../src/cli/modules/accessibilityModule.ts | 24 ++++++++ .../tests/cli/eventDispatcher.test.ts | 11 ++++ .../cli/modules/accessibilityModule.test.ts | 56 +++++++++++++++++++ 6 files changed, 102 insertions(+), 4 deletions(-) diff --git a/packages/browserstack-service/src/cli/eventDispatcher.ts b/packages/browserstack-service/src/cli/eventDispatcher.ts index 6fb2def6..65fd3a4c 100644 --- a/packages/browserstack-service/src/cli/eventDispatcher.ts +++ b/packages/browserstack-service/src/cli/eventDispatcher.ts @@ -25,13 +25,18 @@ class EventDispatcher { * Add event observer * @param {string} event - Event name * @param {Function} callback - Callback function + * @param {boolean} prepend - Run before observers already registered for this event */ - registerObserver(hookRegistryKey: string, callback: Function) { + registerObserver(hookRegistryKey: string, callback: Function, prepend = false) { if (!this.observers[hookRegistryKey]) { this.observers[hookRegistryKey] = [] } - this.observers[hookRegistryKey].push(callback) + if (prepend) { + this.observers[hookRegistryKey].unshift(callback) + } else { + this.observers[hookRegistryKey].push(callback) + } } /** diff --git a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts index e78f6717..c688a503 100644 --- a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts +++ b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts @@ -4,6 +4,7 @@ export const TestFrameworkConstants = { KEY_TEST_NAME : 'test_name', KEY_TEST_FILE_PATH : 'test_file_path', KEY_TEST_TAGS : 'test_tags', + KEY_TEST_ACCESSIBILITY : 'test_accessibility', KEY_TEST_RESULT : 'test_result', KEY_TEST_RESULT_AT : 'test_result_at', KEY_TEST_STARTED_AT : 'test_started_at', diff --git a/packages/browserstack-service/src/cli/frameworks/testFramework.ts b/packages/browserstack-service/src/cli/frameworks/testFramework.ts index 3e9404b2..10f688a6 100644 --- a/packages/browserstack-service/src/cli/frameworks/testFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/testFramework.ts @@ -120,10 +120,11 @@ export default class TestFramework { * @param {TestFrameworkState} testFrameworkState * @param {HookState} hookState * @param {*} callback + * @param {boolean} prepend - Run before observers already registered for this event * @returns {void} */ - static registerObserver(testFrameworkState: State, hookState: State, callback: Function) { - eventDispatcher.registerObserver(CLIUtils.getHookRegistryKey(testFrameworkState, hookState), callback) + static registerObserver(testFrameworkState: State, hookState: State, callback: Function, prepend = false) { + eventDispatcher.registerObserver(CLIUtils.getHookRegistryKey(testFrameworkState, hookState), callback, prepend) } /** diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index 063b68b8..b728210f 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -47,6 +47,8 @@ export default class AccessibilityModule extends BaseModule { this.name = 'AccessibilityModule' this.accessibilityConfig = accessibilityConfig AutomationFramework.registerObserver(AutomationFrameworkState.CREATE, HookState.POST, this.onBeforeExecute.bind(this)) + // Prepended so the verdict is on the instance before TestHubModule serializes TestRunStarted. + TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.setTestAccessibility.bind(this), true) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this)) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this)) // Track the hook window for every hook state the framework supports. PRE captures the @@ -345,6 +347,28 @@ export default class AccessibilityModule extends BaseModule { } } + /** + * Per-test product_map.accessibility, as the classic handler's setTestRunAccessibilityVar: + * only tests in the include/exclude tag scope are accessibility tests. The binary falls back + * to its platform check when this is absent, which marks every test in the build. + */ + setTestAccessibility(args: Record) { + try { + const testInstance = (args.instance as TestFrameworkInstance) || TestFramework.getTrackedInstance() + if (!testInstance) { + return + } + const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' + const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} + const world = args.world as { [key: string]: unknown } | undefined + const inScope = Boolean(this.accessibility && this.autoScanning && + shouldScanTestForAccessibility(suiteTitle, test.title || '', this.config.accessibilityOptions as Record | undefined, world, Boolean(world))) + TestFramework.setState(testInstance, TestFrameworkConstants.KEY_TEST_ACCESSIBILITY, inScope) + } catch (error) { + this.logger.debug(`setTestAccessibility: ${error}`) + } + } + async onBeforeTest(args: Record) { try { this.logger.debug('Accessibility before test hook. Starting accessibility scan for this test case.') diff --git a/packages/browserstack-service/tests/cli/eventDispatcher.test.ts b/packages/browserstack-service/tests/cli/eventDispatcher.test.ts index c8ce4850..b43f129a 100644 --- a/packages/browserstack-service/tests/cli/eventDispatcher.test.ts +++ b/packages/browserstack-service/tests/cli/eventDispatcher.test.ts @@ -60,6 +60,17 @@ describe('EventDispatcher', () => { expect(eventDispatcher.observers[hookKey][1]).toBe(callback2) }) + it('should run a prepended observer before those already registered', () => { + const callback1 = vi.fn() + const callback2 = vi.fn() + const hookKey = 'testHook' + + eventDispatcher.registerObserver(hookKey, callback1) + eventDispatcher.registerObserver(hookKey, callback2, true) + + expect(eventDispatcher.observers[hookKey]).toEqual([callback2, callback1]) + }) + it('should register observers for different hooks', () => { const callback1 = vi.fn() const callback2 = vi.fn() diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index c9009604..7bf8ee43 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -501,6 +501,62 @@ describe('AccessibilityModule', () => { }) }) + describe('setTestAccessibility', () => { + const world = { pickle: { tags: [{ name: '@accessibility' }] } } + + beforeEach(() => { + accessibilityModule.accessibility = true + accessibilityModule.autoScanning = true + }) + + it('should be prepended to TEST/PRE so TestRunStarted carries the verdict', () => { + expect(TestFramework.registerObserver).toHaveBeenCalledWith( + TestFrameworkState.TEST, + HookState.PRE, + expect.any(Function), + true + ) + }) + + it('should mark an in-scope test as an accessibility test', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' }, suiteTitle: 'Feature', world }) + + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Feature', 'Homepage', {}, world, true) + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', true) + }) + + it('should mark an out-of-scope test as not an accessibility test', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(false) + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Basket' }, suiteTitle: 'Feature', world }) + + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) + }) + + it('should mark no test when auto-scanning is off', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.autoScanning = false + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } }) + + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) + }) + + it('should mark no test when the platform does not support accessibility', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.accessibility = false + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } }) + + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) + }) + + it('should use the 3-arg title match for mocha', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'logs in' }, suiteTitle: 'Login' }) + + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Login', 'logs in', {}, undefined, false) + }) + }) + describe('onBeforeTest', () => { it('should set up accessibility metadata for test', async () => { const mockArgs = { From 03d847a2115842aa04d3bf57ae414c681b487585 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:30:31 +0000 Subject: [PATCH 2/4] chore(changeset): auto-generate from PR template (patch) --- .changeset/pr-360.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/pr-360.md diff --git a/.changeset/pr-360.md b/.changeset/pr-360.md new file mode 100644 index 00000000..666b016b --- /dev/null +++ b/.changeset/pr-360.md @@ -0,0 +1,5 @@ +--- +"@wdio/browserstack-service": patch +--- + +- Fixed tests outside `includeTagsInTestingScope` / `excludeTagsInTestingScope` being reported as accessibility tests when using the CLI. From f01547bf705887a8652dee492a3516ff77d6b6cc Mon Sep 17 00:00:00 2001 From: Kamalpreet Kaur Date: Thu, 12 Nov 2026 16:20:33 +0530 Subject: [PATCH 3/4] fix(cli): mark cascade-skipped scenarios as not accessibility tests Scenarios abandoned by a failed BeforeAll are built straight into TEST/POST, so the TEST/PRE verdict never ran for them and the binary fell back to its platform check. They never ran or scanned, so they now carry test_accessibility: false. The in-scope rule is computed by one helper for both the reported verdict and the scan gate, and a failure to set the verdict is logged at error level. Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioCucumberTestFramework.ts | 2 ++ .../src/cli/modules/accessibilityModule.ts | 33 ++++++++++--------- .../tests/service.test.ts | 1 + 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index d225e6b2..8506384e 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -518,6 +518,8 @@ export default class WdioCucumberTestFramework extends TestFramework { [TestFrameworkConstants.KEY_TEST_RESULT_AT]: now, ...resolveFeatureFilePaths(featurePath), [KEY_TEST_SKIPPED_CASCADE]: true, + // Never ran, so never scanned: not an accessibility test whatever its tags. + [TestFrameworkConstants.KEY_TEST_ACCESSIBILITY]: false, [KEY_BDD_META_INFO]: { feature: { name: feature.name, path: this.featureUriForMeta(), description: feature.description }, scenario: { name: scenario.name }, diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index b728210f..55a216e5 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -358,21 +358,29 @@ export default class AccessibilityModule extends BaseModule { if (!testInstance) { return } - const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' - const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} - const world = args.world as { [key: string]: unknown } | undefined - const inScope = Boolean(this.accessibility && this.autoScanning && - shouldScanTestForAccessibility(suiteTitle, test.title || '', this.config.accessibilityOptions as Record | undefined, world, Boolean(world))) - TestFramework.setState(testInstance, TestFrameworkConstants.KEY_TEST_ACCESSIBILITY, inScope) + TestFramework.setState(testInstance, TestFrameworkConstants.KEY_TEST_ACCESSIBILITY, this.shouldScanTest(args)) } catch (error) { - this.logger.debug(`setTestAccessibility: ${error}`) + this.logger.error(`setTestAccessibility: ${error}`) } } + private shouldScanTest(args: Record): boolean { + const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' + const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} + // Cucumber filters scans by gherkin tag, which needs the world object and the 6-arg + // form of shouldScanTestForAccessibility; the 3-arg form matches include/exclude tags + // against the test title instead and so silently scans every scenario. `args.world` is + // only ever populated on the cucumber path, so mocha and jasmine keep the exact 3-arg + // behaviour — both extra args arrive undefined/false and the tag branch is not taken. + const world = args.world as { [key: string]: unknown } | undefined + return Boolean(this.autoScanning && + shouldScanTestForAccessibility(suiteTitle, test.title || '', this.config.accessibilityOptions as Record | undefined, world, Boolean(world)) && + this.accessibility) + } + async onBeforeTest(args: Record) { try { this.logger.debug('Accessibility before test hook. Starting accessibility scan for this test case.') - const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} this.currentTestName = test.title || null @@ -381,14 +389,7 @@ export default class AccessibilityModule extends BaseModule { const testInstance: TestFrameworkInstance = TestFramework.getTrackedInstance() const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) - const accessibilityOptions = this.config.accessibilityOptions - // Cucumber filters scans by gherkin tag, which needs the world object and the 6-arg - // form of shouldScanTestForAccessibility; the 3-arg form matches include/exclude tags - // against the test title instead and so silently scans every scenario. `args.world` is - // only ever populated on the cucumber path, so mocha and jasmine keep the exact 3-arg - // behaviour — both extra args arrive undefined/false and the tag branch is not taken. - const world = args.world as { [key: string]: unknown } | undefined - const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title || '', accessibilityOptions as Record | undefined, world, Boolean(world)) && this.accessibility + const shouldScanTest = this.shouldScanTest(args) this.accessibilityMap.set(sessionId, shouldScanTest) diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index 7ce9efd8..349f0e54 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -2876,6 +2876,7 @@ describe('BEFORE_ALL skip cascade + hook flag pass-through (SDK-7047)', () => { const data = args.instance.getAllData() expect(data.get('test_result')).toBe('skipped') expect(data.get('test_skipped_cascade')).toBe(true) + expect(data.get('test_accessibility')).toBe(false) // the cascade payload has no world, so no tags expect(data.get('test_tags')).toBeUndefined() expect(data.get('test_scopes')).toEqual(['Login']) From 703b4d3488079f82ae2fdef9fba708bce85554e4 Mon Sep 17 00:00:00 2001 From: Kamalpreet Kaur Date: Thu, 12 Nov 2026 16:26:07 +0530 Subject: [PATCH 4/4] fix(cli): leave auto-scanning out of the per-test accessibility verdict With autoScanning off, tests scanned manually with performScan() still need their accessibility test row, so test_accessibility is now accessibility on and test in the include/exclude tag scope. Auto-scanning still gates the automatic scans. Co-Authored-By: Claude Opus 5.5 --- .../src/cli/modules/accessibilityModule.ts | 17 ++++++++--------- .../cli/modules/accessibilityModule.test.ts | 4 ++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index 55a216e5..9255c2d8 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -348,9 +348,9 @@ export default class AccessibilityModule extends BaseModule { } /** - * Per-test product_map.accessibility, as the classic handler's setTestRunAccessibilityVar: - * only tests in the include/exclude tag scope are accessibility tests. The binary falls back - * to its platform check when this is absent, which marks every test in the build. + * Per-test product_map.accessibility: only tests in the include/exclude tag scope are + * accessibility tests. Auto-scanning is left out so tests scanned manually keep their row. + * The binary falls back to its platform check when this is absent, which marks every test. */ setTestAccessibility(args: Record) { try { @@ -358,13 +358,13 @@ export default class AccessibilityModule extends BaseModule { if (!testInstance) { return } - TestFramework.setState(testInstance, TestFrameworkConstants.KEY_TEST_ACCESSIBILITY, this.shouldScanTest(args)) + TestFramework.setState(testInstance, TestFrameworkConstants.KEY_TEST_ACCESSIBILITY, this.isTestInScope(args)) } catch (error) { this.logger.error(`setTestAccessibility: ${error}`) } } - private shouldScanTest(args: Record): boolean { + private isTestInScope(args: Record): boolean { const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} // Cucumber filters scans by gherkin tag, which needs the world object and the 6-arg @@ -373,9 +373,8 @@ export default class AccessibilityModule extends BaseModule { // only ever populated on the cucumber path, so mocha and jasmine keep the exact 3-arg // behaviour — both extra args arrive undefined/false and the tag branch is not taken. const world = args.world as { [key: string]: unknown } | undefined - return Boolean(this.autoScanning && - shouldScanTestForAccessibility(suiteTitle, test.title || '', this.config.accessibilityOptions as Record | undefined, world, Boolean(world)) && - this.accessibility) + return Boolean(this.accessibility && + shouldScanTestForAccessibility(suiteTitle, test.title || '', this.config.accessibilityOptions as Record | undefined, world, Boolean(world))) } async onBeforeTest(args: Record) { @@ -389,7 +388,7 @@ export default class AccessibilityModule extends BaseModule { const testInstance: TestFrameworkInstance = TestFramework.getTrackedInstance() const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) - const shouldScanTest = this.shouldScanTest(args) + const shouldScanTest = this.autoScanning && this.isTestInScope(args) this.accessibilityMap.set(sessionId, shouldScanTest) diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index 7bf8ee43..b9d0b138 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -533,12 +533,12 @@ describe('AccessibilityModule', () => { expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) }) - it('should mark no test when auto-scanning is off', () => { + it('should keep an in-scope test as an accessibility test when auto-scanning is off', () => { vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) accessibilityModule.autoScanning = false accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } }) - expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', true) }) it('should mark no test when the platform does not support accessibility', () => {