Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ @interface AgentDeviceRunnerViewController : UIViewController
@property(nonatomic, assign) NSUInteger textEntryBurstEdits;
@property(nonatomic, assign) NSTimeInterval textEntryBurstMinGap;
@property(nonatomic, assign) NSTimeInterval textEntryAcknowledgeWindowSeconds;
@property(nonatomic, strong) NSLayoutConstraint *textEntryFieldTop;
@property(nonatomic, assign) BOOL alertFixtureStarted;
@property(nonatomic, strong) NSTimer *alertActivationBusyBackstop;
@property(nonatomic, strong) NSTimer *alertBannerRepost;
Expand Down Expand Up @@ -279,6 +280,10 @@ static NSTimeInterval AgentDeviceTextEntryAcknowledgeWindow(void) {
static const NSTimeInterval AgentDeviceTextEntryBurstBreakSeconds = 1.0;
static const NSUInteger AgentDeviceTextEntryAutoSubmitLength = 6;

static const CGFloat AgentDeviceTextEntryFieldTopInset = 24;
static const CGFloat AgentDeviceTextEntryFieldHeight = 44;
static const CGFloat AgentDeviceTextEntryNeighbourGap = 16;

- (void)agentDeviceTextEntryDidChange:(UITextField *)textField {
// A field whose app owns its value, the way a controlled React Native `TextInput` does. A burst
// typed faster than the app renders loses the characters that arrived while a render was in
Expand Down Expand Up @@ -334,6 +339,15 @@ - (void)agentDeviceTextEntryDidChange:(UITextField *)textField {
[textField removeFromSuperview];
}
}

// Moves the field up by its own height plus the gap below it when it gains focus, the way keyboard
// avoidance or a bottom sheet extending above the keyboard does, so the neighbouring field slides
// into the point the focus tap hit.
- (void)agentDeviceTextEntryDidBeginEditing:(UITextField *)textField {
self.textEntryFieldTop.constant =
AgentDeviceTextEntryFieldTopInset - AgentDeviceTextEntryFieldHeight - AgentDeviceTextEntryNeighbourGap;
[self.view layoutIfNeeded];
}
#endif

- (void)viewDidLoad {
Expand Down Expand Up @@ -395,12 +409,33 @@ - (void)viewDidLoad {
forControlEvents:UIControlEventEditingChanged];
textField.translatesAutoresizingMaskIntoConstraints = NO;
[self.view addSubview:textField];
self.textEntryFieldTop = [textField.topAnchor constraintEqualToAnchor:label.bottomAnchor
constant:AgentDeviceTextEntryFieldTopInset];
[NSLayoutConstraint activateConstraints:@[
[textField.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
[textField.topAnchor constraintEqualToAnchor:label.bottomAnchor constant:24],
self.textEntryFieldTop,
[textField.widthAnchor constraintEqualToConstant:240],
[textField.heightAnchor constraintEqualToConstant:44],
[textField.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight],
]];
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-moves-on-focus"]) {
textField.text = @"stale";
[textField addTarget:self
action:@selector(agentDeviceTextEntryDidBeginEditing:)
forControlEvents:UIControlEventEditingDidBegin];
UITextField *neighbour = [[UITextField alloc] init];
neighbour.accessibilityIdentifier = @"agent-device-text-entry-neighbour";
neighbour.borderStyle = UITextBorderStyleRoundedRect;
neighbour.text = @"neighbour";
neighbour.translatesAutoresizingMaskIntoConstraints = NO;
[self.view addSubview:neighbour];
[NSLayoutConstraint activateConstraints:@[
[neighbour.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
[neighbour.topAnchor constraintEqualToAnchor:textField.bottomAnchor
constant:AgentDeviceTextEntryNeighbourGap],
[neighbour.widthAnchor constraintEqualToConstant:240],
[neighbour.heightAnchor constraintEqualToConstant:AgentDeviceTextEntryFieldHeight],
]];
}
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-app-owned-value"]) {
self.textEntryAcknowledgeWindowSeconds = AgentDeviceTextEntryAcknowledgeWindow();
// Reports how many edits this app rendered and how many writes it had to make because a
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,8 @@ extension RunnerTests {
/// already reported success. The expected value is the
/// final text itself, and a settled mismatch is always reported: see
/// `awaitSynthesizedReplacementCommitOutcome`'s doc comment. Text carrying a submit key is
/// skipped outright: the app may clear or rewrite the field on submit.
/// skipped outright: the app may clear or rewrite the field on submit. So is a secure field,
/// whose value is never readable.
func awaitSynthesizedReplacementCommit(
app: XCUIApplication,
target: TextEntryTarget,
Expand All @@ -82,6 +83,7 @@ extension RunnerTests {
let outcome = Self.awaitSynthesizedReplacementCommitOutcome(
expectedText: expectedText,
placeholder: ingredients.placeholder,
fieldIsSecure: ingredients.fieldIsSecure,
now: { Date() },
observe: ingredients.observe,
waitForNextObservation: ingredients.waitForNextObservation
Expand Down Expand Up @@ -132,13 +134,16 @@ extension RunnerTests {
expectedText: String
) -> (
placeholder: String?,
fieldIsSecure: Bool,
observe: () -> String?,
waitForNextObservation: () -> Void
) {
let placeholder = resolveTextEntryElement(app: app, target: target)?.placeholderValue
let field = resolveTextEntryElement(app: app, target: target)
let placeholder = field?.placeholderValue
let waitStartedAt = Date()
return (
placeholder: placeholder,
fieldIsSecure: field?.elementType == .secureTextField,
observe: {
let observedText = self.editableTextValue(
for: self.resolveTextEntryElement(app: app, target: target),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,8 @@ extension RunnerTests {
/// One post. `characterCount` characters of the text, taken in order, are what it types.
struct Step: Equatable {
let characterCount: Int
/// True when this post selects the field's existing value away first. That selection is its
/// own synthesize record, so the post costs one call more than typing characters.
/// True when this post clears the field's existing value before typing its text in a
/// record of its own; `maxClearPassCount` says how.
var replacesExistingText = false
/// Seconds charged for what follows this post: the `--delay-ms` gap before the next post, or
/// the warmup read-back after a peeled first character. The read-back costs one poll, because
Expand All @@ -100,9 +100,28 @@ extension RunnerTests {
/// Set on the post whose value the loop waits for before the rest is posted.
var warmsUpField = false

/// Most clear passes a replacing post runs, each a select-all record and a delete key record
/// followed by a read of the field. In a React Native app launched moments earlier (iOS 26.5
/// simulator) select-alls were dropped, so a pass deleted only the last character, and two
/// fixed passes still left "Clie" of "Client" in 3 of 5 runs. The post clears until the field
/// reads empty; typing over text still there would append to it.
static let maxClearPassCount = 4
/// Clear passes a field that never exposes its value gets before it is typed into unverified.
static let unreadableClearPassCount = 2

/// How many private synthesize records this post runs.
var synthesizeCallCount: Int {
replacesExistingText ? 2 : 1
replacesExistingText ? 2 * Self.maxClearPassCount + 1 : 1
}

/// Keystrokes this post types at the synthesized pace, the clears' delete keys included.
var keystrokeCount: Int {
characterCount + clearReadCount
}

/// Reads of the field this post makes between its clear passes.
var clearReadCount: Int {
replacesExistingText ? Self.maxClearPassCount : 0
}
}

Expand All @@ -129,8 +148,9 @@ extension RunnerTests {
var seconds: TimeInterval {
steps.reduce(0) { total, step in
total
+ Double(step.characterCount) * TextEntryTiming.synthesizedCharacterInterval
+ Double(step.keystrokeCount) * TextEntryTiming.synthesizedCharacterInterval
+ Double(step.synthesizeCallCount) * TextEntryTiming.synthesizeCallOverhead
+ Double(step.clearReadCount) * TextEntryTiming.synthesizedClearReadAllowance
Comment thread
pvedula7 marked this conversation as resolved.
+ step.pauseAfterSeconds
}
}
Expand Down Expand Up @@ -226,6 +246,48 @@ extension RunnerTests {
case stop
}

/// What the field reads between a replacement's clear passes.
enum ClearedFieldRead: Equatable {
case text(String)
/// The field never exposes its value: a secure field.
case unreadable
/// The read could not answer, or no input resolves.
case unavailable
}

/// How a replacing post's clear ended.
enum SynthesizedClearOutcome: Equatable {
case cleared
/// The field never exposes its value, so it is typed into unverified, as the commit wait leaves it.
case unverified
/// The field still held text after the last pass, or could not be read. Nothing is typed.
case notCleared
/// A clear post stopped the plan.
case stopped
}

/// Clears until the field reads empty, at most `maxClearPassCount` passes. A read that cannot
/// answer ends the clear, so the post never types over text it could not see removed.
static func clearForSynthesizedReplacement(
clearOnce: () -> SynthesizedStepDispatch,
read: () -> ClearedFieldRead
) -> SynthesizedClearOutcome {
for pass in 1...SynthesizedTextPlan.Step.maxClearPassCount {
guard clearOnce() == .posted else { return .stopped }
switch read() {
case .text(let value) where value.isEmpty:
return .cleared
case .text:
continue
case .unreadable:
if pass >= SynthesizedTextPlan.Step.unreadableClearPassCount { return .unverified }
case .unavailable:
return .notCleared
}
}
return .notCleared
}

struct SynthesizedPlanRun {
let postedCharacterCount: Int
/// True when a post stopped the plan before its last step.
Expand Down Expand Up @@ -298,32 +360,68 @@ extension RunnerTests {
)
)
}
var clearOutcome: SynthesizedClearOutcome?
// A private synthesis channel that is gone mid-plan leaves the command the same
// point-and-focus fallback it had before the plan existed.
let synthesisAvailable = runSynthesizedTextPlan(
plan,
text: request.text,
post: { slice, step in
switch request.synthesizer.enterText(
app: request.app,
text: slice,
replacingExistingText: step.replacesExistingText
) {
case .continueTyping:
return .posted
case .fallback:
return .stop
case .raise(let message):
NSException(
name: NSExceptionName.internalInconsistencyException,
reason: message ?? "private XCTest text synthesis failed"
).raise()
return .stop
func enter(_ text: String, replacingExistingText: Bool) -> SynthesizedStepDispatch {
switch request.synthesizer.enterText(
app: request.app,
text: text,
replacingExistingText: replacingExistingText
) {
case .continueTyping:
return .posted
case .fallback:
return .stop
case .raise(let message):
NSException(
name: NSExceptionName.internalInconsistencyException,
reason: message ?? "private XCTest text synthesis failed"
).raise()
return .stop
}
}
if step.replacesExistingText {
var passes = 0
let outcome = Self.clearForSynthesizedReplacement(
clearOnce: {
passes += 1
return enter(XCUIKeyboardKey.delete.rawValue, replacingExistingText: true)
},
read: { self.clearedFieldRead(app: request.app, target: request.target) }
)
clearOutcome = outcome
NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_CLEAR passes=%d outcome=%@", passes, String(describing: outcome))
guard outcome == .cleared || outcome == .unverified else { return .stop }
}
return enter(slice, replacingExistingText: false)
},
// A replacement never peels a warmup character, so no step asks for a read-back.
waitAfterWarmupCharacter: { _ in }
)
if clearOutcome == .notCleared {
logTextEntryPhase(
commandId: request.commandId,
phase: "total",
startedAt: request.startedAt,
chars: request.text.count,
mode: .replacement
)
return .completed(
TextEntryResult(
verified: nil,
repaired: false,
expectedText: request.text,
observedText: nil,
textEntryRoute: "synthesized-first-responder-replacement",
failure: .clearNotObserved
)
)
}
if synthesisAvailable.stoppedEarly {
NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=verified-fallback reason=synthesis-unavailable")
guard let point = request.target.refreshPoint else { return .notApplicable }
Expand Down Expand Up @@ -400,7 +498,7 @@ extension RunnerTests {
enum SynthesizedTextCommitOutcome: Equatable {
/// The field holds exactly the expected text.
case settled
/// There was nothing to wait for: the text carries a submit key.
/// There was nothing to wait for: the text carries a submit key, or the field is secure.
case unobservable
/// The deadline expired with the expected text still not observed.
case notObserved
Expand Down Expand Up @@ -432,12 +530,18 @@ extension RunnerTests {
static func awaitSynthesizedReplacementCommitOutcome(
expectedText: String,
placeholder: String?,
fieldIsSecure: Bool = false,
stallBudget: TimeInterval = TextEntryTiming.synthesizedCommitStallTimeout,
ceiling: TimeInterval = TextEntryTiming.synthesizedCommitCeiling,
now: () -> Date = { Date() },
observe: () -> String?,
waitForNextObservation: () -> Void
) -> SynthesizedTextCommitOutcome {
// A secure field never exposes its value, so every read is nil and the wait could only expire.
// The element route leaves such a field unverified rather than failed; so does this one.
if fieldIsSecure {
return .unobservable
}
// A placeholder-equal AX value cannot prove a commit: an input handler may clear the field
// after dispatch, making the empty field render the same value. Refuse before polling because
// no later read can distinguish those states.
Expand Down
Loading
Loading