diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..798e449 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,37 @@ +# Integration test harnesses + +End-to-end harnesses that integrate the `a11y-scan` command plugin from this +repository into real consumer projects and run accessibility scans against +sample sources with intentional issues. Each harness uses a **path dependency** +on the repo root (`../..`), so it always exercises the local plugin sources. + +| Folder | Consumer type | How the plugin is integrated | +|---|---|---| +| [`spm/`](./spm) | SwiftPM package | Package dependency on `AccessibilityDevTools`; the command plugin is invoked with `swift package plugin … scan`. | +| [`xcode-app/`](./xcode-app) | Xcode iOS app (XcodeGen) | A pre-compile build phase runs the scan on every build — the official Xcode integration. | + +## Why two harnesses + +The plugin supports both project types the product targets — SwiftPM packages +and Xcode apps — and they integrate the **command** plugin differently: + +- **SwiftPM** consumers declare the package dependency and invoke the command + plugin directly (`swift package plugin … scan`). The plugin must **not** be + attached to a target's `plugins:` array — `a11y-scan` is a *command* plugin, + not a build-tool plugin, and attaching it breaks `swift build`. +- **Xcode** apps have no `Package.swift`, so the integration synthesizes a + minimal one to host the command plugin and runs it from a build phase. This + harness checks that minimal package in directly (`xcode-app/Package.swift`). + +## Authentication + +Both harnesses need BrowserStack credentials to actually run a scan (the plugin +downloads the CLI and makes authenticated calls): + +```bash +export BROWSERSTACK_USERNAME= +export BROWSERSTACK_ACCESS_KEY= +``` + +Without credentials, the SPM end-to-end test skips and the Xcode build phase +no-ops with a warning, so builds/tests stay green. diff --git a/tests/spm/.gitignore b/tests/spm/.gitignore new file mode 100644 index 0000000..0ca9c38 --- /dev/null +++ b/tests/spm/.gitignore @@ -0,0 +1,3 @@ +.build/ +.swiftpm/ +Package.resolved diff --git a/tests/spm/Package.swift b/tests/spm/Package.swift new file mode 100644 index 0000000..85a340f --- /dev/null +++ b/tests/spm/Package.swift @@ -0,0 +1,36 @@ +// swift-tools-version: 5.9 +import PackageDescription + +// Integration-test harness: a real SwiftPM package that consumes the +// `a11y-scan` command plugin from this repository. +// +// The dependency is a *path* dependency on the repo root (`../..`) so the +// harness always exercises the local plugin sources rather than a published +// tag. When this lands on `main`, `../..` resolves to the AccessibilityDevTools +// package at the repository root. +// +// NOTE: `a11y-scan` is a *command* plugin (manually invoked), not a build-tool +// plugin. It must therefore NOT be attached to a target's `plugins:` array — +// doing so makes SwiftPM treat it as a build tool and breaks `swift build`. +// Declaring the package dependency is enough to make the command plugin +// available; it is invoked explicitly via `scripts/run-a11y-scan.sh` +// (`swift package plugin ... scan`). +let package = Package( + name: "A11yScanSPMConsumer", + platforms: [ + .iOS(.v15), + .macOS(.v12), + ], + dependencies: [ + .package(name: "AccessibilityDevTools", path: "../.."), + ], + targets: [ + // Sample sources containing intentional accessibility issues for the + // scanner to flag. + .target(name: "A11yDemoLib"), + .testTarget( + name: "A11yDemoLibTests", + dependencies: ["A11yDemoLib"] + ), + ] +) diff --git a/tests/spm/README.md b/tests/spm/README.md new file mode 100644 index 0000000..3ee07fd --- /dev/null +++ b/tests/spm/README.md @@ -0,0 +1,46 @@ +# SwiftPM integration harness + +A SwiftPM package (`A11yScanSPMConsumer`) that consumes the `a11y-scan` command +plugin from this repository via a path dependency on the repo root. + +``` +spm/ +├── Package.swift # path dependency on ../.. (AccessibilityDevTools) +├── Sources/A11yDemoLib/ # sample SwiftUI views with intentional a11y issues +├── Tests/A11yDemoLibTests/ # unit test + gated end-to-end scan test +└── scripts/run-a11y-scan.sh # invokes the command plugin +``` + +## Build & test + +```bash +cd tests/spm +swift build # compiles the plugin + sample sources +swift test # unit test passes; the end-to-end scan test skips by default +``` + +## Run the accessibility scan + +```bash +export BROWSERSTACK_USERNAME= +export BROWSERSTACK_ACCESS_KEY= +./scripts/run-a11y-scan.sh # fails the run on issues +./scripts/run-a11y-scan.sh --non-strict # reports issues without failing +``` + +The script runs: + +```bash +swift package plugin \ + --allow-writing-to-directory ~/.cache \ + --allow-writing-to-package-directory \ + --allow-network-connections 'all(ports: [])' \ + scan --include "**/*.swift" --include "**/*.xib" --include "**/*.storyboard" +``` + +To run the scan as part of `swift test`, set `RUN_A11Y_SCAN=1` (with credentials); +otherwise `testA11yScanPluginRuns` is skipped. + +> `a11y-scan` is a **command** plugin, so it is invoked explicitly — it is not +> attached to a target's `plugins:` array (that would make `swift build` treat +> it as a build-tool plugin and fail). diff --git a/tests/spm/Sources/A11yDemoLib/SampleViews.swift b/tests/spm/Sources/A11yDemoLib/SampleViews.swift new file mode 100644 index 0000000..92b315e --- /dev/null +++ b/tests/spm/Sources/A11yDemoLib/SampleViews.swift @@ -0,0 +1,42 @@ +#if canImport(SwiftUI) +import SwiftUI + +/// Sample SwiftUI views that deliberately contain accessibility issues so the +/// `a11y-scan` plugin has something to report when run against this package. +/// +/// These are NOT examples of good practice — each view documents the WCAG-style +/// issue the BrowserStack rule engine is expected to flag. +@available(iOS 15, macOS 12, *) +struct SampleContentView: View { + @State private var isOn = false + + var body: some View { + VStack(spacing: 16) { + // Issue: image conveys meaning but has no accessibility label and is + // not marked decorative. + Image(systemName: "trash") + + // Issue: icon-only button with no accessible label — screen readers + // announce nothing actionable. + Button(action: deleteItem) { + Image(systemName: "plus.circle") + } + + // Issue: toggle with no label describing what it controls. + Toggle("", isOn: $isOn) + + // Issue: empty text element provides no information. + Text("") + } + .padding() + } + + private func deleteItem() {} +} +#endif + +/// Public marker so the test target has a concrete symbol to import and assert +/// against without depending on SwiftUI being available on the host. +public enum A11yDemoLib { + public static let name = "A11yScanSPMConsumer" +} diff --git a/tests/spm/Tests/A11yDemoLibTests/A11yDemoLibTests.swift b/tests/spm/Tests/A11yDemoLibTests/A11yDemoLibTests.swift new file mode 100644 index 0000000..ba847f4 --- /dev/null +++ b/tests/spm/Tests/A11yDemoLibTests/A11yDemoLibTests.swift @@ -0,0 +1,44 @@ +import XCTest + +@testable import A11yDemoLib + +final class A11yDemoLibTests: XCTestCase { + /// Sanity check that the sample target builds and links. + func testLibraryIdentity() { + XCTAssertEqual(A11yDemoLib.name, "A11yScanSPMConsumer") + } + + /// End-to-end check that the `a11y-scan` command plugin runs against this + /// package and reports the intentional issues in `SampleViews.swift`. + /// + /// Skipped by default: the plugin downloads the BrowserStack CLI and makes + /// authenticated network calls, so it only runs when `RUN_A11Y_SCAN=1` and + /// BrowserStack credentials are present in the environment. + func testA11yScanPluginRuns() throws { + let env = ProcessInfo.processInfo.environment + guard env["RUN_A11Y_SCAN"] == "1" else { + throw XCTSkip("Set RUN_A11Y_SCAN=1 (with BrowserStack creds) to run the plugin end-to-end.") + } + guard env["BROWSERSTACK_USERNAME"] != nil, env["BROWSERSTACK_ACCESS_KEY"] != nil else { + throw XCTSkip("BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY are required for the scan.") + } + + // tests/spm/Tests/A11yDemoLibTests/ -> tests/spm + let packageDir = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + let script = packageDir.appendingPathComponent("scripts/run-a11y-scan.sh") + + let process = Process() + process.executableURL = URL(fileURLWithPath: "/bin/bash") + process.arguments = [script.path, "--non-strict"] + process.currentDirectoryURL = packageDir + try process.run() + process.waitUntilExit() + + // --non-strict makes the scan exit 0 even when issues are found, so a + // clean exit means the plugin downloaded, authenticated, and ran. + XCTAssertEqual(process.terminationStatus, 0, "a11y-scan plugin failed to run") + } +} diff --git a/tests/spm/scripts/run-a11y-scan.sh b/tests/spm/scripts/run-a11y-scan.sh new file mode 100755 index 0000000..8bbb3f0 --- /dev/null +++ b/tests/spm/scripts/run-a11y-scan.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Runs the BrowserStack `a11y-scan` command plugin against this SwiftPM package. +# +# Requires BrowserStack credentials in the environment: +# export BROWSERSTACK_USERNAME= +# export BROWSERSTACK_ACCESS_KEY= +# +# Any extra arguments are forwarded to the scan (e.g. --non-strict). +set -euo pipefail + +cd "$(dirname "$0")/.." + +: "${BROWSERSTACK_USERNAME:?Set BROWSERSTACK_USERNAME before running the scan}" +: "${BROWSERSTACK_ACCESS_KEY:?Set BROWSERSTACK_ACCESS_KEY before running the scan}" + +swift package plugin \ + --allow-writing-to-directory "$HOME/.cache" \ + --allow-writing-to-package-directory \ + --allow-network-connections 'all(ports: [])' \ + scan \ + --include "**/*.swift" \ + --include "**/*.xib" \ + --include "**/*.storyboard" \ + "$@" diff --git a/tests/xcode-app/.gitignore b/tests/xcode-app/.gitignore new file mode 100644 index 0000000..275dfdd --- /dev/null +++ b/tests/xcode-app/.gitignore @@ -0,0 +1,6 @@ +# Generated by XcodeGen — regenerate with `xcodegen generate`. +*.xcodeproj/ +.build/ +.swiftpm/ +Package.resolved +DerivedData/ diff --git a/tests/xcode-app/Package.swift b/tests/xcode-app/Package.swift new file mode 100644 index 0000000..f550351 --- /dev/null +++ b/tests/xcode-app/Package.swift @@ -0,0 +1,20 @@ +// swift-tools-version: 5.9 +import PackageDescription + +// Scan driver for the Xcode-app harness. +// +// Pure Xcode app projects have no Package.swift, so the official BrowserStack +// integration synthesizes a minimal one to host the `a11y-scan` command plugin +// (see scripts/{bash,zsh,fish}/spm.sh in this repo). We check that minimal +// package in directly so the scan can run over the app's `Sources/` without any +// network self-update step. +// +// `targets: []` is intentional — the scanner selects files by `--include` +// globs, not by SwiftPM target membership, so no target wiring is required. +let package = Package( + name: "A11yScanDemoAppScan", + dependencies: [ + .package(name: "AccessibilityDevTools", path: "../.."), + ], + targets: [] +) diff --git a/tests/xcode-app/README.md b/tests/xcode-app/README.md new file mode 100644 index 0000000..037002c --- /dev/null +++ b/tests/xcode-app/README.md @@ -0,0 +1,56 @@ +# Xcode app integration harness + +An iOS app (`A11yScanDemoApp`) that integrates the `a11y-scan` command plugin +through a pre-compile **build phase** — the official integration path for Xcode +projects (which have no `Package.swift` of their own). The project is described +as an [XcodeGen](https://github.com/yonaskolb/XcodeGen) spec so the generated +`.xcodeproj` does not need to be checked in. + +``` +xcode-app/ +├── project.yml # XcodeGen spec (app + unit-test targets) +├── Package.swift # minimal scan driver hosting the command plugin +├── Sources/ # @main app + ContentView with intentional a11y issues +├── Tests/ # unit test target +└── scripts/run-a11y-scan.sh # build-phase scan runner +``` + +## Generate the project + +```bash +brew install xcodegen # if not already installed +cd tests/xcode-app +xcodegen generate # produces A11yScanDemoApp.xcodeproj +open A11yScanDemoApp.xcodeproj +``` + +## How the plugin is integrated + +The app target has a **pre-compile build phase**, "BrowserStack Accessibility +Linter", that runs `scripts/run-a11y-scan.sh` before sources compile. The script +invokes the command plugin (`swift package plugin … scan`) against the minimal +`Package.swift`, scanning `Sources/` by include globs. `ENABLE_USER_SCRIPT_SANDBOXING` +is disabled in the spec so the scan can write the CLI cache to `~/.cache`. + +## Build & test + +```bash +export BROWSERSTACK_USERNAME= +export BROWSERSTACK_ACCESS_KEY= + +xcodebuild \ + -project A11yScanDemoApp.xcodeproj \ + -scheme A11yScanDemoApp \ + -destination 'platform=iOS Simulator,name=iPhone 15' \ + test +``` + +The scan runs as part of the build (pre-compile phase). It is configured with +`--non-strict` so issues are reported without failing the build; remove that +flag in `project.yml` to make accessibility violations fail the build. Without +credentials the scan phase no-ops with a warning so the build still succeeds. + +> Requires `xcodegen` and Xcode; neither is exercised by `swift test`. This spec +> was authored to the documented integration but the generated project has not +> been built in this environment — generate and run it locally to validate on +> your toolchain. diff --git a/tests/xcode-app/Sources/A11yScanDemoApp.swift b/tests/xcode-app/Sources/A11yScanDemoApp.swift new file mode 100644 index 0000000..8f675e0 --- /dev/null +++ b/tests/xcode-app/Sources/A11yScanDemoApp.swift @@ -0,0 +1,10 @@ +import SwiftUI + +@main +struct A11yScanDemoApp: App { + var body: some Scene { + WindowGroup { + ContentView() + } + } +} diff --git a/tests/xcode-app/Sources/ContentView.swift b/tests/xcode-app/Sources/ContentView.swift new file mode 100644 index 0000000..e1f8bd8 --- /dev/null +++ b/tests/xcode-app/Sources/ContentView.swift @@ -0,0 +1,35 @@ +import SwiftUI + +/// Demo screen with intentional accessibility issues for the `a11y-scan` plugin +/// to report during the build's pre-compile scan phase. Each control documents +/// the issue the BrowserStack rule engine is expected to flag. +struct ContentView: View { + @State private var notificationsEnabled = false + + var body: some View { + VStack(spacing: 20) { + // Issue: meaningful image with no accessibility label. + Image(systemName: "bell.fill") + .font(.largeTitle) + + // Issue: icon-only button with no accessible label. + Button(action: refresh) { + Image(systemName: "arrow.clockwise") + } + + // Issue: toggle with no descriptive label. + Toggle("", isOn: $notificationsEnabled) + .labelsHidden() + + // Issue: empty text element conveys nothing to assistive tech. + Text("") + } + .padding() + } + + private func refresh() {} +} + +#Preview { + ContentView() +} diff --git a/tests/xcode-app/Tests/A11yScanDemoAppTests.swift b/tests/xcode-app/Tests/A11yScanDemoAppTests.swift new file mode 100644 index 0000000..5371f6f --- /dev/null +++ b/tests/xcode-app/Tests/A11yScanDemoAppTests.swift @@ -0,0 +1,13 @@ +import XCTest + +@testable import A11yScanDemoApp + +final class A11yScanDemoAppTests: XCTestCase { + /// Sanity check that the app target builds and the test bundle links against + /// it. The accessibility scan itself runs as the app target's pre-compile + /// build phase (see project.yml), so a successful `xcodebuild test` means the + /// scan ran during the build. + func testContentViewExists() { + _ = ContentView() + } +} diff --git a/tests/xcode-app/project.yml b/tests/xcode-app/project.yml new file mode 100644 index 0000000..eb8cd6a --- /dev/null +++ b/tests/xcode-app/project.yml @@ -0,0 +1,60 @@ +# XcodeGen spec for the Xcode-app integration harness. +# +# Generate the project with: +# brew install xcodegen # if not already installed +# xcodegen generate +# +# This produces A11yScanDemoApp.xcodeproj. The app target runs the BrowserStack +# `a11y-scan` command plugin via a pre-compile build phase (the official Xcode +# integration), so an accessibility scan runs on every build. +name: A11yScanDemoApp +options: + bundleIdPrefix: com.browserstack.a11yscan + createIntermediateGroups: true + deploymentTarget: + iOS: "15.0" + +settings: + base: + SWIFT_VERSION: "5.9" + # Required so the scan build phase can write the CLI cache to ~/.cache. + ENABLE_USER_SCRIPT_SANDBOXING: NO + +targets: + A11yScanDemoApp: + type: application + platform: iOS + sources: + - path: Sources + settings: + base: + PRODUCT_BUNDLE_IDENTIFIER: com.browserstack.a11yscan.demo + GENERATE_INFOPLIST_FILE: YES + INFOPLIST_KEY_UIApplicationSceneManifest_Generation: YES + INFOPLIST_KEY_UILaunchScreen_Generation: YES + TARGETED_DEVICE_FAMILY: "1,2" + preBuildScripts: + - name: BrowserStack Accessibility Linter + # Runs the a11y-scan command plugin over Sources/ before compilation. + # --non-strict keeps local/CI builds green even when issues are found; + # drop it to fail the build on accessibility violations. + script: | + "${SRCROOT}/scripts/run-a11y-scan.sh" --non-strict + basedOnDependencyAnalysis: false + + A11yScanDemoAppTests: + type: bundle.unit-test + platform: iOS + sources: + - path: Tests + dependencies: + - target: A11yScanDemoApp + +schemes: + A11yScanDemoApp: + build: + targets: + A11yScanDemoApp: all + test: + targets: + - A11yScanDemoAppTests diff --git a/tests/xcode-app/scripts/run-a11y-scan.sh b/tests/xcode-app/scripts/run-a11y-scan.sh new file mode 100755 index 0000000..0cfc296 --- /dev/null +++ b/tests/xcode-app/scripts/run-a11y-scan.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Runs the BrowserStack `a11y-scan` command plugin over this Xcode app's +# Sources/. Invoked as a pre-compile build phase by the generated project, and +# also runnable standalone. +# +# Requires BrowserStack credentials in the environment: +# export BROWSERSTACK_USERNAME= +# export BROWSERSTACK_ACCESS_KEY= +# +# Extra arguments are forwarded to the scan (e.g. --non-strict). +set -euo pipefail + +cd "$(dirname "$0")/.." + +if [[ -z "${BROWSERSTACK_USERNAME:-}" || -z "${BROWSERSTACK_ACCESS_KEY:-}" ]]; then + echo "warning: BROWSERSTACK_USERNAME / BROWSERSTACK_ACCESS_KEY not set; skipping accessibility scan." >&2 + exit 0 +fi + +swift package plugin \ + --allow-writing-to-directory "$HOME/.cache" \ + --allow-writing-to-package-directory \ + --allow-network-connections 'all(ports: [])' \ + scan \ + --include "**/*.swift" \ + --include "**/*.xib" \ + --include "**/*.storyboard" \ + "$@"