diff --git a/.changeset/pr-359.md b/.changeset/pr-359.md new file mode 100644 index 00000000..666b016b --- /dev/null +++ b/.changeset/pr-359.md @@ -0,0 +1,5 @@ +--- +"@wdio/browserstack-service": patch +--- + +- Fixed tests outside `includeTagsInTestingScope` / `excludeTagsInTestingScope` being reported as accessibility tests when using the CLI. diff --git a/packages/browserstack-service/src/cli/eventDispatcher.ts b/packages/browserstack-service/src/cli/eventDispatcher.ts index 6fb2def6..65fd3a4c 100644 --- a/packages/browserstack-service/src/cli/eventDispatcher.ts +++ b/packages/browserstack-service/src/cli/eventDispatcher.ts @@ -25,13 +25,18 @@ class EventDispatcher { * Add event observer * @param {string} event - Event name * @param {Function} callback - Callback function + * @param {boolean} prepend - Run before observers already registered for this event */ - registerObserver(hookRegistryKey: string, callback: Function) { + registerObserver(hookRegistryKey: string, callback: Function, prepend = false) { if (!this.observers[hookRegistryKey]) { this.observers[hookRegistryKey] = [] } - this.observers[hookRegistryKey].push(callback) + if (prepend) { + this.observers[hookRegistryKey].unshift(callback) + } else { + this.observers[hookRegistryKey].push(callback) + } } /** diff --git a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts index e78f6717..c688a503 100644 --- a/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts +++ b/packages/browserstack-service/src/cli/frameworks/constants/testFrameworkConstants.ts @@ -4,6 +4,7 @@ export const TestFrameworkConstants = { KEY_TEST_NAME : 'test_name', KEY_TEST_FILE_PATH : 'test_file_path', KEY_TEST_TAGS : 'test_tags', + KEY_TEST_ACCESSIBILITY : 'test_accessibility', KEY_TEST_RESULT : 'test_result', KEY_TEST_RESULT_AT : 'test_result_at', KEY_TEST_STARTED_AT : 'test_started_at', diff --git a/packages/browserstack-service/src/cli/frameworks/testFramework.ts b/packages/browserstack-service/src/cli/frameworks/testFramework.ts index 3e9404b2..10f688a6 100644 --- a/packages/browserstack-service/src/cli/frameworks/testFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/testFramework.ts @@ -120,10 +120,11 @@ export default class TestFramework { * @param {TestFrameworkState} testFrameworkState * @param {HookState} hookState * @param {*} callback + * @param {boolean} prepend - Run before observers already registered for this event * @returns {void} */ - static registerObserver(testFrameworkState: State, hookState: State, callback: Function) { - eventDispatcher.registerObserver(CLIUtils.getHookRegistryKey(testFrameworkState, hookState), callback) + static registerObserver(testFrameworkState: State, hookState: State, callback: Function, prepend = false) { + eventDispatcher.registerObserver(CLIUtils.getHookRegistryKey(testFrameworkState, hookState), callback, prepend) } /** diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index d225e6b2..8506384e 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -518,6 +518,8 @@ export default class WdioCucumberTestFramework extends TestFramework { [TestFrameworkConstants.KEY_TEST_RESULT_AT]: now, ...resolveFeatureFilePaths(featurePath), [KEY_TEST_SKIPPED_CASCADE]: true, + // Never ran, so never scanned: not an accessibility test whatever its tags. + [TestFrameworkConstants.KEY_TEST_ACCESSIBILITY]: false, [KEY_BDD_META_INFO]: { feature: { name: feature.name, path: this.featureUriForMeta(), description: feature.description }, scenario: { name: scenario.name }, diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index 063b68b8..9255c2d8 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -47,6 +47,8 @@ export default class AccessibilityModule extends BaseModule { this.name = 'AccessibilityModule' this.accessibilityConfig = accessibilityConfig AutomationFramework.registerObserver(AutomationFrameworkState.CREATE, HookState.POST, this.onBeforeExecute.bind(this)) + // Prepended so the verdict is on the instance before TestHubModule serializes TestRunStarted. + TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.setTestAccessibility.bind(this), true) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this)) TestFramework.registerObserver(TestFrameworkState.TEST, HookState.POST, this.onAfterTest.bind(this)) // Track the hook window for every hook state the framework supports. PRE captures the @@ -345,10 +347,39 @@ export default class AccessibilityModule extends BaseModule { } } + /** + * Per-test product_map.accessibility: only tests in the include/exclude tag scope are + * accessibility tests. Auto-scanning is left out so tests scanned manually keep their row. + * The binary falls back to its platform check when this is absent, which marks every test. + */ + setTestAccessibility(args: Record) { + try { + const testInstance = (args.instance as TestFrameworkInstance) || TestFramework.getTrackedInstance() + if (!testInstance) { + return + } + TestFramework.setState(testInstance, TestFrameworkConstants.KEY_TEST_ACCESSIBILITY, this.isTestInScope(args)) + } catch (error) { + this.logger.error(`setTestAccessibility: ${error}`) + } + } + + private isTestInScope(args: Record): boolean { + const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' + const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} + // Cucumber filters scans by gherkin tag, which needs the world object and the 6-arg + // form of shouldScanTestForAccessibility; the 3-arg form matches include/exclude tags + // against the test title instead and so silently scans every scenario. `args.world` is + // only ever populated on the cucumber path, so mocha and jasmine keep the exact 3-arg + // behaviour — both extra args arrive undefined/false and the tag branch is not taken. + const world = args.world as { [key: string]: unknown } | undefined + return Boolean(this.accessibility && + shouldScanTestForAccessibility(suiteTitle, test.title || '', this.config.accessibilityOptions as Record | undefined, world, Boolean(world))) + } + async onBeforeTest(args: Record) { try { this.logger.debug('Accessibility before test hook. Starting accessibility scan for this test case.') - const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} this.currentTestName = test.title || null @@ -357,14 +388,7 @@ export default class AccessibilityModule extends BaseModule { const testInstance: TestFrameworkInstance = TestFramework.getTrackedInstance() const sessionId = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) - const accessibilityOptions = this.config.accessibilityOptions - // Cucumber filters scans by gherkin tag, which needs the world object and the 6-arg - // form of shouldScanTestForAccessibility; the 3-arg form matches include/exclude tags - // against the test title instead and so silently scans every scenario. `args.world` is - // only ever populated on the cucumber path, so mocha and jasmine keep the exact 3-arg - // behaviour — both extra args arrive undefined/false and the tag branch is not taken. - const world = args.world as { [key: string]: unknown } | undefined - const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title || '', accessibilityOptions as Record | undefined, world, Boolean(world)) && this.accessibility + const shouldScanTest = this.autoScanning && this.isTestInScope(args) this.accessibilityMap.set(sessionId, shouldScanTest) diff --git a/packages/browserstack-service/tests/cli/eventDispatcher.test.ts b/packages/browserstack-service/tests/cli/eventDispatcher.test.ts index c8ce4850..b43f129a 100644 --- a/packages/browserstack-service/tests/cli/eventDispatcher.test.ts +++ b/packages/browserstack-service/tests/cli/eventDispatcher.test.ts @@ -60,6 +60,17 @@ describe('EventDispatcher', () => { expect(eventDispatcher.observers[hookKey][1]).toBe(callback2) }) + it('should run a prepended observer before those already registered', () => { + const callback1 = vi.fn() + const callback2 = vi.fn() + const hookKey = 'testHook' + + eventDispatcher.registerObserver(hookKey, callback1) + eventDispatcher.registerObserver(hookKey, callback2, true) + + expect(eventDispatcher.observers[hookKey]).toEqual([callback2, callback1]) + }) + it('should register observers for different hooks', () => { const callback1 = vi.fn() const callback2 = vi.fn() diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index c9009604..b9d0b138 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -501,6 +501,62 @@ describe('AccessibilityModule', () => { }) }) + describe('setTestAccessibility', () => { + const world = { pickle: { tags: [{ name: '@accessibility' }] } } + + beforeEach(() => { + accessibilityModule.accessibility = true + accessibilityModule.autoScanning = true + }) + + it('should be prepended to TEST/PRE so TestRunStarted carries the verdict', () => { + expect(TestFramework.registerObserver).toHaveBeenCalledWith( + TestFrameworkState.TEST, + HookState.PRE, + expect.any(Function), + true + ) + }) + + it('should mark an in-scope test as an accessibility test', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' }, suiteTitle: 'Feature', world }) + + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Feature', 'Homepage', {}, world, true) + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', true) + }) + + it('should mark an out-of-scope test as not an accessibility test', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(false) + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Basket' }, suiteTitle: 'Feature', world }) + + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) + }) + + it('should keep an in-scope test as an accessibility test when auto-scanning is off', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.autoScanning = false + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } }) + + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', true) + }) + + it('should mark no test when the platform does not support accessibility', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.accessibility = false + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'Homepage' } }) + + expect(TestFramework.setState).toHaveBeenCalledWith(mockTestInstance, 'test_accessibility', false) + }) + + it('should use the 3-arg title match for mocha', () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + accessibilityModule.setTestAccessibility({ instance: mockTestInstance, test: { title: 'logs in' }, suiteTitle: 'Login' }) + + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Login', 'logs in', {}, undefined, false) + }) + }) + describe('onBeforeTest', () => { it('should set up accessibility metadata for test', async () => { const mockArgs = { diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index 7ce9efd8..349f0e54 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -2876,6 +2876,7 @@ describe('BEFORE_ALL skip cascade + hook flag pass-through (SDK-7047)', () => { const data = args.instance.getAllData() expect(data.get('test_result')).toBe('skipped') expect(data.get('test_skipped_cascade')).toBe(true) + expect(data.get('test_accessibility')).toBe(false) // the cascade payload has no world, so no tags expect(data.get('test_tags')).toBeUndefined() expect(data.get('test_scopes')).toEqual(['Login'])