fix(cli): send the per-test accessibility verdict on test events (v8)

.changeset/pr-361.md

@@ -0,0 +1,5 @@
+---
+"@wdio/browserstack-service": patch
+---
+
+- Fixed tests outside `includeTagsInTestingScope` / `excludeTagsInTestingScope` being reported as accessibility tests when using the CLI.

packages/browserstack-service/src/cli/eventDispatcher.ts

@@ -29,13 +29,18 @@ class EventDispatcher {
    * Add event observer
    * @param {string} event - Event name
    * @param {Function} callback - Callback function
+   * @param {boolean} prepend - Run before observers already registered for this event
    */
-    registerObserver(hookRegistryKey: string, callback: Function) {
+    registerObserver(hookRegistryKey: string, callback: Function, prepend = false) {
         if (!this.observers[hookRegistryKey]) {
             this.observers[hookRegistryKey] = []
         }
 
-        this.observers[hookRegistryKey].push(callback)
+        if (prepend) {
+            this.observers[hookRegistryKey].unshift(callback)
+        } else {
+            this.observers[hookRegistryKey].push(callback)
+        }
     }
 
     /**

packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts

@@ -4,6 +4,7 @@ export const TestFrameworkConstants = {
     KEY_TEST_NAME : 'test_name',
     KEY_TEST_FILE_PATH : 'test_file_path',
     KEY_TEST_TAGS : 'test_tags',
+    KEY_TEST_ACCESSIBILITY : 'test_accessibility',
     KEY_TEST_RESULT : 'test_result',
     KEY_TEST_RESULT_AT : 'test_result_at',
     KEY_TEST_STARTED_AT : 'test_started_at',

packages/browserstack-service/src/cli/frameworks/testFramework.ts

@@ -103,10 +103,11 @@ export default class TestFramework {
    * @param {TestFrameworkState} testFrameworkState
    * @param {HookState} hookState
    * @param {*} callback
+   * @param {boolean} prepend - Run before observers already registered for this event
    * @returns {void}
    */
-    static registerObserver(testFrameworkState: State, hookState: State, callback: Function) {
-        eventDispatcher.registerObserver(CLIUtils.getHookRegistryKey(testFrameworkState, hookState), callback)
+    static registerObserver(testFrameworkState: State, hookState: State, callback: Function, prepend = false) {
+        eventDispatcher.registerObserver(CLIUtils.getHookRegistryKey(testFrameworkState, hookState), callback, prepend)
     }
 
     /**

packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts

@@ -518,6 +518,8 @@ export default class WdioCucumberTestFramework extends TestFramework {
             [TestFrameworkConstants.KEY_TEST_RESULT_AT]: now,
             ...this.featureFilePathEntries(),
             [KEY_TEST_SKIPPED_CASCADE]: true,
+            // Never ran, so never scanned: not an accessibility test whatever its tags.
+            [TestFrameworkConstants.KEY_TEST_ACCESSIBILITY]: false,
             [KEY_BDD_META_INFO]: {
                 feature: { name: feature.name, path: this.featureUriForMeta(), description: feature.description },
                 scenario: { name: scenario.name },

packages/browserstack-service/src/cli/modules/accessibilityModule.ts

@@ -12,6 +12,7 @@ import { HookState } from '../states/hookState.js'
 import accessibilityScripts from '../../scripts/accessibility-scripts.js'
 import { _getParamsForAppAccessibility, formatString, getAppA11yResults, getAppA11yResultsSummary, shouldScanTestForAccessibility, validateCapsWithA11y, validateCapsWithAppA11y, validateCapsWithNonBstackA11y, setBrowserstackAnnotation } from '../../util.js'
 import { AutomationFrameworkConstants } from '../frameworks/constants/automationFrameworkConstants.js'
+import { TestFrameworkConstants } from '../frameworks/constants/testFrameworkConstants.js'
 import util from 'node:util'
 import type { Accessibility } from '../../grpc/index.js'
 import type { Capabilities } from '@wdio/types'
@@ -42,6 +43,8 @@ export default class AccessibilityModule extends BaseModule {
         this.name = 'AccessibilityModule'
         this.accessibilityConfig = accessibilityConfig
         AutomationFramework.registerObserver(AutomationFrameworkState.CREATE, HookState.POST, this.onBeforeExecute.bind(this))
+        // Prepended so the verdict is on the instance before TestHubModule serializes TestRunStarted.
+        TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.setTestAccessibility.bind(this), true)
         TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this))
         TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this))
         this.accessibility = Boolean(accessibilityConfig)
@@ -197,6 +200,28 @@ export default class AccessibilityModule extends BaseModule {
         }
     }
 
+    /**
+     * Per-test product_map.accessibility: only tests in the include/exclude tag scope are
+     * accessibility tests. Auto-scanning is left out so tests scanned manually keep their row.
+     * The binary falls back to its platform check when this is absent, which marks every test.
+     */
+    setTestAccessibility(args: Record<string, unknown>) {
+        try {
+            const testInstance = (args.instance as TestFrameworkInstance) || TestFramework.getTrackedInstance()
+            if (!testInstance) {
+                return
+            }
+            const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || ''
+            const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {}
+            const world = args.world as { [key: string]: unknown } | undefined
+            const inScope = Boolean(this.accessibility &&
+                shouldScanTestForAccessibility(suiteTitle, test.title || '', this.config.accessibilityOptions as Record<string, string> | undefined, world, Boolean(world)))
+            TestFramework.setState(testInstance, TestFrameworkConstants.KEY_TEST_ACCESSIBILITY, inScope)
+        } catch (error) {
+            this.logger.error(`setTestAccessibility: ${error}`)
+        }
+    }
+
     async onBeforeTest(args: any) {
         try {
 

packages/browserstack-service/tests/cli/frameworks/testFramework.test.ts

@@ -140,7 +140,18 @@ describe('TestFramework', () => {
 
                 TestFramework.registerObserver(mockState as any, mockHook as any, mockCallback)
 
-                expect(eventDispatcher.registerObserver).toHaveBeenCalledWith('TEST_STATE-PRE_HOOK', mockCallback)
+                expect(eventDispatcher.registerObserver).toHaveBeenCalledWith('TEST_STATE-PRE_HOOK', mockCallback, false)
+            })
+
+            it('should forward prepend to the event dispatcher', async () => {
+                const mockCallback = vi.fn()
+                const { eventDispatcher } = await import('../../../src/cli/eventDispatcher.js')
+                const { CLIUtils } = await import('../../../src/cli/cliUtils.js')
+                vi.mocked(CLIUtils.getHookRegistryKey).mockReturnValue('TEST_STATE-PRE_HOOK')
+
+                TestFramework.registerObserver('TEST_STATE' as any, 'PRE_HOOK' as any, mockCallback, true)
+
+                expect(eventDispatcher.registerObserver).toHaveBeenCalledWith('TEST_STATE-PRE_HOOK', mockCallback, true)
             })
         })
     })

packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts

@@ -236,6 +236,7 @@ describe('WdioCucumberTestFramework', () => {
             expect(first[TestFrameworkConstants.KEY_TEST_SCOPE]).toBe('Plain one')
             expect(first[TestFrameworkConstants.KEY_TEST_RESULT]).toBe('skipped')
             expect(first.test_skipped_cascade).toBe(true)
+            expect(first[TestFrameworkConstants.KEY_TEST_ACCESSIBILITY]).toBe(false)
             expect(first[TestFrameworkConstants.KEY_TEST_TAGS]).toBeUndefined()
             expect(first.bdd_meta_info.steps).toEqual([{ id: 's1', text: 'I do a thing', keyword: 'Given ', result: 'skipped' }])
             // the cascade builds its own meta blob, so it needs the same relative/absolute split

packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts

@@ -304,6 +304,62 @@ describe('AccessibilityModule', () => {
         })
     })
 
+    describe('setTestAccessibility', () => {
+        const world = { pickle: { tags: [{ name: '@accessibility' }] } }
+
+        beforeEach(() => {
+            accessibilityModule.accessibility = true
+            accessibilityModule.autoScanning = true
+        })
+
+        it('should be prepended to TEST/PRE so TestRunStarted carries the verdict', () => {
+            expect(TestFramework.registerObserver).toHaveBeenCalledWith(
+                TestFrameworkState.TEST,
+                HookState.PRE,
+                expect.any(Function),
+                true
+            )
+        })
+
+        it('should mark an in-scope test as an accessibility test', () => {
+            vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true)
+            accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' }, suiteTitle: 'Feature', world })
+
+            expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Feature', 'Homepage', {}, world, true)
+            expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', true)
+        })
+
+        it('should mark an out-of-scope test as not an accessibility test', () => {
+            vi.mocked(shouldScanTestForAccessibility).mockReturnValue(false)
+            accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Basket' }, suiteTitle: 'Feature', world })
+
+            expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false)
+        })
+
+        it('should keep an in-scope test as an accessibility test when auto-scanning is off', () => {
+            vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true)
+            accessibilityModule.autoScanning = false
+            accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } })
+
+            expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', true)
+        })
+
+        it('should mark no test when the platform does not support accessibility', () => {
+            vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true)
+            accessibilityModule.accessibility = false
+            accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } })
+
+            expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false)
+        })
+
+        it('should use the 3-arg title match for mocha', () => {
+            vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true)
+            accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'logs in' }, suiteTitle: 'Login' })
+
+            expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Login', 'logs in', {}, undefined, false)
+        })
+    })
+
     describe('onBeforeTest', () => {
         beforeEach(() => {
             // Mock shouldScanTestForAccessibility to return true for proper test behavior