diff --git a/.changeset/pr-361.md b/.changeset/pr-361.md new file mode 100644 index 00000000..666b016b --- /dev/null +++ b/.changeset/pr-361.md @@ -0,0 +1,5 @@ +--- +"@wdio/browserstack-service": patch +--- + +- Fixed tests outside `includeTagsInTestingScope` / `excludeTagsInTestingScope` being reported as accessibility tests when using the CLI. diff --git a/packages/browserstack-service/src/cli/eventDispatcher.ts b/packages/browserstack-service/src/cli/eventDispatcher.ts index 75385d58..0b5672a3 100644 --- a/packages/browserstack-service/src/cli/eventDispatcher.ts +++ b/packages/browserstack-service/src/cli/eventDispatcher.ts @@ -29,13 +29,18 @@ class EventDispatcher { * Add event observer * @param {string} event - Event name * @param {Function} callback - Callback function + * @param {boolean} prepend - Run before observers already registered for this event */ - registerObserver(hookRegistryKey: string, callback: Function) { + registerObserver(hookRegistryKey: string, callback: Function, prepend = false) { if (!this.observers[hookRegistryKey]) { this.observers[hookRegistryKey] = [] } - this.observers[hookRegistryKey].push(callback) + if (prepend) { + this.observers[hookRegistryKey].unshift(callback) + } else { + this.observers[hookRegistryKey].push(callback) + } } /** diff --git a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts index 1dd45e24..9cc2657a 100644 --- a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts +++ b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts @@ -4,6 +4,7 @@ export const TestFrameworkConstants = { KEY_TEST_NAME : 'test_name', KEY_TEST_FILE_PATH : 'test_file_path', KEY_TEST_TAGS : 'test_tags', + KEY_TEST_ACCESSIBILITY : 'test_accessibility', KEY_TEST_RESULT : 'test_result', KEY_TEST_RESULT_AT : 'test_result_at', KEY_TEST_STARTED_AT : 'test_started_at', diff --git a/packages/browserstack-service/src/cli/frameworks/testFramework.ts b/packages/browserstack-service/src/cli/frameworks/testFramework.ts index f291505c..ce7d8c5f 100644 --- a/packages/browserstack-service/src/cli/frameworks/testFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/testFramework.ts @@ -103,10 +103,11 @@ export default class TestFramework { * @param {TestFrameworkState} testFrameworkState * @param {HookState} hookState * @param {*} callback + * @param {boolean} prepend - Run before observers already registered for this event * @returns {void} */ - static registerObserver(testFrameworkState: State, hookState: State, callback: Function) { - eventDispatcher.registerObserver(CLIUtils.getHookRegistryKey(testFrameworkState, hookState), callback) + static registerObserver(testFrameworkState: State, hookState: State, callback: Function, prepend = false) { + eventDispatcher.registerObserver(CLIUtils.getHookRegistryKey(testFrameworkState, hookState), callback, prepend) } /** diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index c5e1d507..45535427 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -518,6 +518,8 @@ export default class WdioCucumberTestFramework extends TestFramework { [TestFrameworkConstants.KEY_TEST_RESULT_AT]: now, ...this.featureFilePathEntries(), [KEY_TEST_SKIPPED_CASCADE]: true, + // Never ran, so never scanned: not an accessibility test whatever its tags. + [TestFrameworkConstants.KEY_TEST_ACCESSIBILITY]: false, [KEY_BDD_META_INFO]: { feature: { name: feature.name, path: this.featureUriForMeta(), description: feature.description }, scenario: { name: scenario.name }, diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index 245cf7cf..0d5437fa 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -12,6 +12,7 @@ import { HookState } from '../states/hookState.js' import accessibilityScripts from '../../scripts/accessibility-scripts.js' import { _getParamsForAppAccessibility, formatString, getAppA11yResults, getAppA11yResultsSummary, shouldScanTestForAccessibility, validateCapsWithA11y, validateCapsWithAppA11y, validateCapsWithNonBstackA11y, setBrowserstackAnnotation } from '../../util.js' import { AutomationFrameworkConstants } from '../frameworks/constants/automationFrameworkConstants.js' +import { TestFrameworkConstants } from '../frameworks/constants/testFrameworkConstants.js' import util from 'node:util' import type { Accessibility } from '../../grpc/index.js' import type { Capabilities } from '@wdio/types' @@ -42,6 +43,8 @@ export default class AccessibilityModule extends BaseModule { this.name = 'AccessibilityModule' this.accessibilityConfig = accessibilityConfig AutomationFramework.registerObserver(AutomationFrameworkState.CREATE, HookState.POST, this.onBeforeExecute.bind(this)) + // Prepended so the verdict is on the instance before TestHubModule serializes TestRunStarted. + TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.setTestAccessibility.bind(this), true) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this)) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this)) this.accessibility = Boolean(accessibilityConfig) @@ -197,6 +200,28 @@ export default class AccessibilityModule extends BaseModule { } } + /** + * Per-test product_map.accessibility: only tests in the include/exclude tag scope are + * accessibility tests. Auto-scanning is left out so tests scanned manually keep their row. + * The binary falls back to its platform check when this is absent, which marks every test. + */ + setTestAccessibility(args: Record) { + try { + const testInstance = (args.instance as TestFrameworkInstance) || TestFramework.getTrackedInstance() + if (!testInstance) { + return + } + const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' + const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} + const world = args.world as { [key: string]: unknown } | undefined + const inScope = Boolean(this.accessibility && + shouldScanTestForAccessibility(suiteTitle, test.title || '', this.config.accessibilityOptions as Record | undefined, world, Boolean(world))) + TestFramework.setState(testInstance, TestFrameworkConstants.KEY_TEST_ACCESSIBILITY, inScope) + } catch (error) { + this.logger.error(`setTestAccessibility: ${error}`) + } + } + async onBeforeTest(args: any) { try { diff --git a/packages/browserstack-service/tests/cli/frameworks/testFramework.test.ts b/packages/browserstack-service/tests/cli/frameworks/testFramework.test.ts index a14baff6..75bf0da6 100644 --- a/packages/browserstack-service/tests/cli/frameworks/testFramework.test.ts +++ b/packages/browserstack-service/tests/cli/frameworks/testFramework.test.ts @@ -140,7 +140,18 @@ describe('TestFramework', () => { TestFramework.registerObserver(mockState as any, mockHook as any, mockCallback) - expect(eventDispatcher.registerObserver).toHaveBeenCalledWith('TEST_STATE-PRE_HOOK', mockCallback) + expect(eventDispatcher.registerObserver).toHaveBeenCalledWith('TEST_STATE-PRE_HOOK', mockCallback, false) + }) + + it('should forward prepend to the event dispatcher', async () => { + const mockCallback = vi.fn() + const { eventDispatcher } = await import('../../../src/cli/eventDispatcher.js') + const { CLIUtils } = await import('../../../src/cli/cliUtils.js') + vi.mocked(CLIUtils.getHookRegistryKey).mockReturnValue('TEST_STATE-PRE_HOOK') + + TestFramework.registerObserver('TEST_STATE' as any, 'PRE_HOOK' as any, mockCallback, true) + + expect(eventDispatcher.registerObserver).toHaveBeenCalledWith('TEST_STATE-PRE_HOOK', mockCallback, true) }) }) }) diff --git a/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts b/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts index 9e413902..75478f2e 100644 --- a/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/frameworks/wdioCucumberTestFramework.test.ts @@ -236,6 +236,7 @@ describe('WdioCucumberTestFramework', () => { expect(first[TestFrameworkConstants.KEY_TEST_SCOPE]).toBe('Plain one') expect(first[TestFrameworkConstants.KEY_TEST_RESULT]).toBe('skipped') expect(first.test_skipped_cascade).toBe(true) + expect(first[TestFrameworkConstants.KEY_TEST_ACCESSIBILITY]).toBe(false) expect(first[TestFrameworkConstants.KEY_TEST_TAGS]).toBeUndefined() expect(first.bdd_meta_info.steps).toEqual([{ id: 's1', text: 'I do a thing', keyword: 'Given ', result: 'skipped' }]) // the cascade builds its own meta blob, so it needs the same relative/absolute split diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index 0cd78797..68e7452d 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -304,6 +304,62 @@ describe('AccessibilityModule', () => { }) }) + describe('setTestAccessibility', () => { + const world = { pickle: { tags: [{ name: '@accessibility' }] } } + + beforeEach(() => { + accessibilityModule.accessibility = true + accessibilityModule.autoScanning = true + }) + + it('should be prepended to TEST/PRE so TestRunStarted carries the verdict', () => { + expect(TestFramework.registerObserver).toHaveBeenCalledWith( + TestFrameworkState.TEST, + HookState.PRE, + expect.any(Function), + true + ) + }) + + it('should mark an in-scope test as an accessibility test', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' }, suiteTitle: 'Feature', world }) + + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Feature', 'Homepage', {}, world, true) + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', true) + }) + + it('should mark an out-of-scope test as not an accessibility test', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(false) + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Basket' }, suiteTitle: 'Feature', world }) + + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) + }) + + it('should keep an in-scope test as an accessibility test when auto-scanning is off', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.autoScanning = false + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } }) + + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', true) + }) + + it('should mark no test when the platform does not support accessibility', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.accessibility = false + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } }) + + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) + }) + + it('should use the 3-arg title match for mocha', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'logs in' }, suiteTitle: 'Login' }) + + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Login', 'logs in', {}, undefined, false) + }) + }) + describe('onBeforeTest', () => { beforeEach(() => { // Mock shouldScanTestForAccessibility to return true for proper test behavior