fix(cli): send the per-test accessibility verdict on test events

.changeset/pr-359.md

@@ -0,0 +1,5 @@
+---
+"@wdio/browserstack-service": patch
+---
+
+- Fixed tests outside `includeTagsInTestingScope` / `excludeTagsInTestingScope` being reported as accessibility tests when using the CLI.

packages/browserstack-service/src/cli/eventDispatcher.ts

@@ -25,13 +25,18 @@ class EventDispatcher {
    * Add event observer
    * @param {string} event - Event name
    * @param {Function} callback - Callback function
+   * @param {boolean} prepend - Run before observers already registered for this event
    */
-    registerObserver(hookRegistryKey: string, callback: Function) {
+    registerObserver(hookRegistryKey: string, callback: Function, prepend = false) {
         if (!this.observers[hookRegistryKey]) {
             this.observers[hookRegistryKey] = []
         }
 
-        this.observers[hookRegistryKey].push(callback)
+        if (prepend) {
+            this.observers[hookRegistryKey].unshift(callback)
+        } else {
+            this.observers[hookRegistryKey].push(callback)
+        }
     }
 
     /**

packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts

@@ -4,6 +4,7 @@ export const TestFrameworkConstants = {
     KEY_TEST_NAME : 'test_name',
     KEY_TEST_FILE_PATH : 'test_file_path',
     KEY_TEST_TAGS : 'test_tags',
+    KEY_TEST_ACCESSIBILITY : 'test_accessibility',
     KEY_TEST_RESULT : 'test_result',
     KEY_TEST_RESULT_AT : 'test_result_at',
     KEY_TEST_STARTED_AT : 'test_started_at',

packages/browserstack-service/src/cli/frameworks/testFramework.ts

@@ -120,10 +120,11 @@ export default class TestFramework {
    * @param {TestFrameworkState} testFrameworkState
    * @param {HookState} hookState
    * @param {*} callback
+   * @param {boolean} prepend - Run before observers already registered for this event
    * @returns {void}
    */
-    static registerObserver(testFrameworkState: State, hookState: State, callback: Function) {
-        eventDispatcher.registerObserver(CLIUtils.getHookRegistryKey(testFrameworkState, hookState), callback)
+    static registerObserver(testFrameworkState: State, hookState: State, callback: Function, prepend = false) {
+        eventDispatcher.registerObserver(CLIUtils.getHookRegistryKey(testFrameworkState, hookState), callback, prepend)
     }
 
     /**

packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts

@@ -518,6 +518,8 @@ export default class WdioCucumberTestFramework extends TestFramework {
             [TestFrameworkConstants.KEY_TEST_RESULT_AT]: now,
             ...resolveFeatureFilePaths(featurePath),
             [KEY_TEST_SKIPPED_CASCADE]: true,
+            // Never ran, so never scanned: not an accessibility test whatever its tags.
+            [TestFrameworkConstants.KEY_TEST_ACCESSIBILITY]: false,
             [KEY_BDD_META_INFO]: {
                 feature: { name: feature.name, path: this.featureUriForMeta(), description: feature.description },
                 scenario: { name: scenario.name },

packages/browserstack-service/src/cli/modules/accessibilityModule.ts

@@ -47,6 +47,8 @@ export default class AccessibilityModule extends BaseModule {
         this.name = 'AccessibilityModule'
         this.accessibilityConfig = accessibilityConfig
         AutomationFramework.registerObserver(AutomationFrameworkState.CREATE, HookState.POST, this.onBeforeExecute.bind(this))
+        // Prepended so the verdict is on the instance before TestHubModule serializes TestRunStarted.
+        TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.setTestAccessibility.bind(this), true)
         TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this))
         TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this))
         // Track the hook window for every hook state the framework supports. PRE captures the
@@ -345,10 +347,39 @@ export default class AccessibilityModule extends BaseModule {
         }
     }
 
+    /**
+     * Per-test product_map.accessibility: only tests in the include/exclude tag scope are
+     * accessibility tests. Auto-scanning is left out so tests scanned manually keep their row.
+     * The binary falls back to its platform check when this is absent, which marks every test.
+     */
+    setTestAccessibility(args: Record<string, unknown>) {
+        try {
+            const testInstance = (args.instance as TestFrameworkInstance) || TestFramework.getTrackedInstance()
+            if (!testInstance) {
+                return
+            }
+            TestFramework.setState(testInstance, TestFrameworkConstants.KEY_TEST_ACCESSIBILITY, this.isTestInScope(args))
+        } catch (error) {
+            this.logger.error(`setTestAccessibility: ${error}`)
+        }
+    }
+
+    private isTestInScope(args: Record<string, unknown>): boolean {
+        const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || ''
+        const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {}
+        // Cucumber filters scans by gherkin tag, which needs the world object and the 6-arg
+        // form of shouldScanTestForAccessibility; the 3-arg form matches include/exclude tags
+        // against the test title instead and so silently scans every scenario. `args.world` is
+        // only ever populated on the cucumber path, so mocha and jasmine keep the exact 3-arg
+        // behaviour — both extra args arrive undefined/false and the tag branch is not taken.
+        const world = args.world as { [key: string]: unknown } | undefined
+        return Boolean(this.accessibility &&
+            shouldScanTestForAccessibility(suiteTitle, test.title || '', this.config.accessibilityOptions as Record<string, string> | undefined, world, Boolean(world)))
+    }
+
     async onBeforeTest(args: Record<string, unknown>) {
         try {
             this.logger.debug('Accessibility before test hook. Starting accessibility scan for this test case.')
-            const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || ''
             const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {}
 
             this.currentTestName = test.title || null
@@ -357,14 +388,7 @@ export default class AccessibilityModule extends BaseModule {
             const testInstance: TestFrameworkInstance = TestFramework.getTrackedInstance()
 
             const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID)
-            const accessibilityOptions = this.config.accessibilityOptions
-            // Cucumber filters scans by gherkin tag, which needs the world object and the 6-arg
-            // form of shouldScanTestForAccessibility; the 3-arg form matches include/exclude tags
-            // against the test title instead and so silently scans every scenario. `args.world` is
-            // only ever populated on the cucumber path, so mocha and jasmine keep the exact 3-arg
-            // behaviour — both extra args arrive undefined/false and the tag branch is not taken.
-            const world = args.world as { [key: string]: unknown } | undefined
-            const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title || '', accessibilityOptions as Record<string, string> | undefined, world, Boolean(world)) && this.accessibility
+            const shouldScanTest = this.autoScanning && this.isTestInScope(args)
 
             this.accessibilityMap.set(sessionId, shouldScanTest)
 

packages/browserstack-service/tests/cli/eventDispatcher.test.ts

@@ -60,6 +60,17 @@ describe('EventDispatcher', () => {
             expect(eventDispatcher.observers[hookKey][1]).toBe(callback2)
         })
 
+        it('should run a prepended observer before those already registered', () => {
+            const callback1 = vi.fn()
+            const callback2 = vi.fn()
+            const hookKey = 'testHook'
+
+            eventDispatcher.registerObserver(hookKey, callback1)
+            eventDispatcher.registerObserver(hookKey, callback2, true)
+
+            expect(eventDispatcher.observers[hookKey]).toEqual([callback2, callback1])
+        })
+
         it('should register observers for different hooks', () => {
             const callback1 = vi.fn()
             const callback2 = vi.fn()

packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts

@@ -501,6 +501,62 @@ describe('AccessibilityModule', () => {
         })
     })
 
+    describe('setTestAccessibility', () => {
+        const world = { pickle: { tags: [{ name: '@accessibility' }] } }
+
+        beforeEach(() => {
+            accessibilityModule.accessibility = true
+            accessibilityModule.autoScanning = true
+        })
+
+        it('should be prepended to TEST/PRE so TestRunStarted carries the verdict', () => {
+            expect(TestFramework.registerObserver).toHaveBeenCalledWith(
+                TestFrameworkState.TEST,
+                HookState.PRE,
+                expect.any(Function),
+                true
+            )
+        })
+
+        it('should mark an in-scope test as an accessibility test', () => {
+            vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true)
+            accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' }, suiteTitle: 'Feature', world })
+
+            expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Feature', 'Homepage', {}, world, true)
+            expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', true)
+        })
+
+        it('should mark an out-of-scope test as not an accessibility test', () => {
+            vi.mocked(shouldScanTestForAccessibility).mockReturnValue(false)
+            accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Basket' }, suiteTitle: 'Feature', world })
+
+            expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false)
+        })
+
+        it('should keep an in-scope test as an accessibility test when auto-scanning is off', () => {
+            vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true)
+            accessibilityModule.autoScanning = false
+            accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } })
+
+            expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', true)
+        })
+
+        it('should mark no test when the platform does not support accessibility', () => {
+            vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true)
+            accessibilityModule.accessibility = false
+            accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } })
+
+            expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false)
+        })
+
+        it('should use the 3-arg title match for mocha', () => {
+            vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true)
+            accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'logs in' }, suiteTitle: 'Login' })
+
+            expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Login', 'logs in', {}, undefined, false)
+        })
+    })
+
     describe('onBeforeTest', () => {
         it('should set up accessibility metadata for test', async () => {
             const mockArgs = {

packages/browserstack-service/tests/service.test.ts

@@ -2876,6 +2876,7 @@ describe('BEFORE_ALL skip cascade + hook flag pass-through (SDK-7047)', () => {
             const data = args.instance.getAllData()
             expect(data.get('test_result')).toBe('skipped')
             expect(data.get('test_skipped_cascade')).toBe(true)
+            expect(data.get('test_accessibility')).toBe(false)
             // the cascade payload has no world, so no tags
             expect(data.get('test_tags')).toBeUndefined()
             expect(data.get('test_scopes')).toEqual(['Login'])