feat(recording): cap the touch overlay frame rate at the caller's --fps

apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift

@@ -88,7 +88,10 @@ func run() throws {
   let renderSize = resolvedRenderSize(for: sourceVideoTrack)
   let videoComposition = AVMutableVideoComposition()
   videoComposition.renderSize = renderSize
-  videoComposition.frameDuration = resolvedFrameDuration(for: sourceVideoTrack)
+  videoComposition.frameDuration = resolvedFrameDuration(
+    for: sourceVideoTrack,
+    maximumFrameRate: parsedArgs.maximumFrameRate
+  )
 
   let instruction = AVMutableVideoCompositionInstruction()
   instruction.timeRange = fullRange
@@ -158,11 +161,15 @@ func run() throws {
 
 func parseArguments(
   _ arguments: [String]
-) throws -> (inputPath: String, outputPath: String, eventsPath: String, timeoutSeconds: Double) {
+) throws -> (
+  inputPath: String, outputPath: String, eventsPath: String, timeoutSeconds: Double,
+  maximumFrameRate: Int32
+) {
   var inputPath: String?
   var outputPath: String?
   var eventsPath: String?
   var timeoutSeconds = defaultRecordingExportTimeoutSeconds
+  var maximumFrameRate = maximumCompositedFrameRate
   var index = 0
 
   while index < arguments.count {
@@ -194,17 +201,24 @@ func parseArguments(
       }
       timeoutSeconds = milliseconds / 1000
       index += 2
+    case "--max-fps":
+      let rawValue = try recordingOptionValue(arguments, nextIndex, "--max-fps")
+      guard let framesPerSecond = Int32(rawValue), framesPerSecond > 0 else {
+        throw RecordingScriptError.invalidArgs("--max-fps must be a positive integer")
+      }
+      maximumFrameRate = min(framesPerSecond, maximumCompositedFrameRate)
+      index += 2
     default:
       throw RecordingScriptError.invalidArgs("Unknown argument: \(argument)")
     }
   }
 
   guard let inputPath, let outputPath, let eventsPath else {
     throw RecordingScriptError.invalidArgs(
-      "Usage: recording-overlay.swift --input <video> --output <video> --events <json> [--quality <medium|high>] [--timeout-ms <ms>]"
+      "Usage: recording-overlay.swift --input <video> --output <video> --events <json> [--quality <medium|high>] [--timeout-ms <ms>] [--max-fps <n>]"
     )
   }
-  return (inputPath, outputPath, eventsPath, timeoutSeconds)
+  return (inputPath, outputPath, eventsPath, timeoutSeconds, maximumFrameRate)
 }
 
 /// The composited overlay must keep the captured track's dimensions, so it can only use a preset
@@ -339,19 +353,19 @@ func frameMeanLuma(_ image: CGImage) -> Double? {
 }
 
 /// The composited export renders at most this many frames a second, which is smooth for a touch
-/// that stays visible for 0.45s.
+/// that stays visible for 0.45s. `--max-fps` lowers it, never raises it.
 let maximumCompositedFrameRate: Int32 = 30
 
 /// A variable-frame-rate capture (simctl recordVideo, adb screenrecord) reports one tick of its
 /// timescale as `minFrameDuration` (1/600s from simctl). Rendering at that asked the compositor for
 /// every frame the encoder could make, 75 a second from an iOS simulator and 60 from an Android
 /// emulator, about three times the frames the capture holds; a 5-minute recording's export then
 /// outlasted the `record stop` request. So the frame duration is the capture's, but never shorter
-/// than one frame at `maximumCompositedFrameRate`. A capture that reports none renders at that
+/// than one frame at `maximumFrameRate`. A capture that reports none renders at that
 /// rate: its average (`nominalFrameRate`) can be a couple of frames a second over a still screen,
 /// which would skip a touch.
-func resolvedFrameDuration(for track: AVAssetTrack) -> CMTime {
-  let shortest = CMTime(value: 1, timescale: maximumCompositedFrameRate)
+func resolvedFrameDuration(for track: AVAssetTrack, maximumFrameRate: Int32) -> CMTime {
+  let shortest = CMTime(value: 1, timescale: maximumFrameRate)
   let minFrameDuration = track.minFrameDuration
   if minFrameDuration.isValid && !minFrameDuration.isIndefinite && minFrameDuration.seconds > 0 {
     return CMTimeMaximum(minFrameDuration, shortest)

packages/capture-kit/src/recording/__tests__/overlay.test.ts

@@ -131,6 +131,21 @@ test('overlay forwards the requested high export preset', async () => {
   );
 });
 
+test('overlay caps the helper frame rate at the caller fps only when one was asked for', async () => {
+  const videoPath = path.join(tmpDir, 'recording.mp4');
+  const telemetryPath = path.join(tmpDir, 'recording.gesture-telemetry.json');
+  fs.writeFileSync(videoPath, 'original');
+  fs.writeFileSync(telemetryPath, '{"events":[]}');
+
+  await overlayRecordingTouches({ videoPath, telemetryPath });
+  expect(helperScriptArgs()).not.toContain('--max-fps');
+
+  mockRunCmd.mockClear();
+  await overlayRecordingTouches({ videoPath, telemetryPath, fps: 15 });
+  const args = helperScriptArgs();
+  expect(args[args.indexOf('--max-fps') + 1]).toBe('15');
+});
+
 test('overlay hands the helper what is left of its budget, inside the record request', async () => {
   const videoPath = path.join(tmpDir, 'recording.mp4');
   const telemetryPath = path.join(tmpDir, 'recording.gesture-telemetry.json');
@@ -173,3 +188,40 @@ test('overlay leaves the video as recorded when compiling the helper spent its b
   expect(mockRunCmd.mock.calls.filter(([cmd]) => cmd !== 'xcrun')).toHaveLength(0);
   expect(fs.readFileSync(videoPath, 'utf8')).toBe('original');
 });
+
+test('overlay names its budget when the export ran out of it', async () => {
+  vi.useFakeTimers({ toFake: ['Date'] });
+  const videoPath = path.join(tmpDir, 'recording.mp4');
+  const telemetryPath = path.join(tmpDir, 'recording.gesture-telemetry.json');
+  fs.writeFileSync(videoPath, 'original');
+  fs.writeFileSync(telemetryPath, '{"events":[]}');
+
+  await overlayRecordingTouches({ videoPath, telemetryPath });
+  fs.writeFileSync(videoPath, 'original');
+  mockRunCmd.mockImplementationOnce(async () => {
+    vi.setSystemTime(Date.now() + OVERLAY_BUDGET_MS);
+    throw new AppError('COMMAND_FAILED', 'recording-overlay exited with code 1');
+  });
+
+  await expect(overlayRecordingTouches({ videoPath, telemetryPath })).rejects.toMatchObject({
+    code: 'COMMAND_FAILED',
+    message: `Failed to add touch overlays to the recording: the export did not finish within its ${OVERLAY_BUDGET_MS}ms budget; a lower --fps renders faster`,
+  });
+  expect(fs.readFileSync(videoPath, 'utf8')).toBe('original');
+});
+
+test('overlay keeps the plain failure when the export failed with budget to spare', async () => {
+  const videoPath = path.join(tmpDir, 'recording.mp4');
+  const telemetryPath = path.join(tmpDir, 'recording.gesture-telemetry.json');
+  fs.writeFileSync(videoPath, 'original');
+  fs.writeFileSync(telemetryPath, '{"events":[]}');
+
+  await overlayRecordingTouches({ videoPath, telemetryPath });
+  mockRunCmd.mockImplementationOnce(async () => {
+    throw new AppError('COMMAND_FAILED', 'recording-overlay exited with code 1');
+  });
+
+  await expect(overlayRecordingTouches({ videoPath, telemetryPath })).rejects.toMatchObject({
+    message: 'Failed to add touch overlays to the recording',
+  });
+});

packages/capture-kit/src/recording/overlay.ts

@@ -67,6 +67,7 @@ async function exportProcessedVideo(params: {
   await waitForPlayableVideo(videoPath);
 
   const outputPath = temporarySiblingVideoPath(videoPath);
+  let exportOverran = false;
   try {
     const executablePath = await compileSwiftSourceFile({
       sourcePath: scriptPath,
@@ -82,22 +83,30 @@ async function exportProcessedVideo(params: {
         `No time was left for the export within the ${params.budgetMs}ms budget`,
       );
     }
-    await runCmd(
-      executablePath,
-      [
-        '--input',
-        videoPath,
-        '--output',
-        outputPath,
-        ...scriptArgs,
-        '--timeout-ms',
-        String(Math.round(exportMs)),
-      ],
-      { timeoutMs: helperMs, env: buildSwiftToolEnv() },
-    );
+    try {
+      await runCmd(
+        executablePath,
+        [
+          '--input',
+          videoPath,
+          '--output',
+          outputPath,
+          ...scriptArgs,
+          '--timeout-ms',
+          String(Math.round(exportMs)),
+        ],
+        { timeoutMs: helperMs, env: buildSwiftToolEnv() },
+      );
+    } catch (error) {
+      exportOverran = deadline.remainingMs() <= HELPER_EXIT_GRACE_MS;
+      throw error;
+    }
     await waitForPlayableVideo(outputPath);
     fs.renameSync(outputPath, videoPath);
   } catch (error) {
+    const description = exportOverran
+      ? `${commandDescription}: the export did not finish within its ${params.budgetMs}ms budget; a lower --fps renders faster`
+      : commandDescription;
     const cause =
       error instanceof AppError
         ? error
@@ -109,7 +118,7 @@ async function exportProcessedVideo(params: {
           );
     throw new AppError(
       'COMMAND_FAILED',
-      commandDescription,
+      description,
       {
         ...cause.details,
         videoPath,
@@ -132,6 +141,8 @@ export async function overlayRecordingTouches(params: {
   videoPath: string;
   telemetryPath: string;
   exportQuality?: RecordingExportQuality;
+  /** The caller's `--fps`; the helper renders at most this many frames a second, capped at 30. */
+  fps?: number;
   targetLabel?: string;
 }): Promise<void> {
   const {
@@ -143,7 +154,13 @@ export async function overlayRecordingTouches(params: {
   await exportProcessedVideo({
     videoPath,
     scriptPath: getOverlayScriptPath(),
-    scriptArgs: ['--events', telemetryPath, '--quality', exportQuality],
+    scriptArgs: [
+      '--events',
+      telemetryPath,
+      '--quality',
+      exportQuality,
+      ...(params.fps === undefined ? [] : ['--max-fps', String(params.fps)]),
+    ],
     commandDescription: `Failed to add touch overlays to the ${targetLabel}`,
     budgetMs: OVERLAY_BUDGET_MS,
   });

packages/capture-kit/src/recording/recording-facts.test.ts

@@ -21,6 +21,7 @@ test('accepts the facts a caller asked for, with or without the optional ones',
     true,
   );
   assert.equal(recordingFactsAreValid(facet({ exportQuality: 'high' })), true);
+  assert.equal(recordingFactsAreValid(facet({ fps: 15 })), true);
 });
 
 test('refuses a facet whose recording facts no start could have produced', () => {
@@ -32,6 +33,9 @@ test('refuses a facet whose recording facts no start could have produced', () =>
 
 test('refuses an optional fact that is present but unreadable rather than dropping it', () => {
   assert.equal(recordingFactsAreValid(facet({ exportQuality: 'ultra' })), false);
+  assert.equal(recordingFactsAreValid(facet({ fps: 0 })), false);
+  assert.equal(recordingFactsAreValid(facet({ fps: 12.5 })), false);
+  assert.equal(recordingFactsAreValid(facet({ fps: '15' })), false);
   assert.equal(recordingFactsAreValid(facet({ activeSessionApp: { bundleId: '' } })), false);
   assert.equal(recordingFactsAreValid(facet({ activeSessionApp: { name: 'Example' } })), false);
   assert.equal(recordingFactsAreValid(facet({ activeSessionApp: 'com.example.app' })), false);
@@ -42,6 +46,11 @@ test('refuses an optional fact that is present but unreadable rather than droppi
 });
 
 test('names every key of the facet it validates', () => {
-  const declared = Object.keys({ ...FACTS, activeSessionApp: undefined, exportQuality: undefined });
+  const declared = Object.keys({
+    ...FACTS,
+    activeSessionApp: undefined,
+    exportQuality: undefined,
+    fps: undefined,
+  });
   assert.deepEqual([...RECORDING_FACTS_KEYS].sort(), declared.sort());
 });

packages/capture-kit/src/recording/recording-facts.ts

@@ -19,6 +19,7 @@ export type RecordingFacts = Readonly<{
   recordOnlySession: boolean;
   activeSessionApp?: RecordingAppIdentity;
   exportQuality?: RecordingExportQuality;
+  fps?: number;
 }>;
 
 /** The keys of {@link RecordingFacts}, so a backend carries the facet without naming it twice. */
@@ -28,6 +29,7 @@ export const RECORDING_FACTS_KEYS = [
   'recordOnlySession',
   'activeSessionApp',
   'exportQuality',
+  'fps',
 ] as const satisfies readonly (keyof RecordingFacts)[];
 
 /**
@@ -41,10 +43,15 @@ export function recordingFactsAreValid(value: Record<string, unknown>): value is
     typeof value.showTouches === 'boolean' &&
     typeof value.recordOnlySession === 'boolean' &&
     (value.exportQuality === undefined || isRecordingExportQuality(value.exportQuality)) &&
+    isOptionalFps(value.fps) &&
     isOptionalAppIdentity(value.activeSessionApp)
   );
 }
 
+function isOptionalFps(value: unknown): value is number | undefined {
+  return value === undefined || (typeof value === 'number' && Number.isInteger(value) && value > 0);
+}
+
 function isOptionalAppIdentity(value: unknown): value is RecordingAppIdentity | undefined {
   if (value === undefined) return true;
   if (!isRecord(value) || !isNonemptyText(value.bundleId)) return false;

packages/command-registry/src/flag-definitions-action.ts

@@ -42,7 +42,8 @@ export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [
     min: 1,
     max: 120,
     usageLabel: '--fps <n>',
-    usageDescription: 'Record: target frames per second (iOS physical device runner)',
+    usageDescription:
+      'Record: target frames per second (iOS physical device runner); also lowers the touch-overlay rate, which never exceeds 30',
     projectConfig: true,
     recorded: true,
   },

packages/contracts/src/screen-recording-runtime-host.ts

@@ -253,6 +253,8 @@ export type ScreenRecordingFinalizer = Readonly<{
       showTouches: boolean;
       gestureEvents: readonly RecordingGestureEvent[];
       exportQuality?: RecordingExportQuality;
+      /** The touch overlay renders at no more frames a second than this. */
+      fps?: number;
       targetLabel: string;
     }>,
     signal?: AbortSignal,

packages/contracts/src/screen-recording-runtime.ts

@@ -72,6 +72,11 @@ export type ScreenRecordingLiveSnapshot = Readonly<{
   recordOnlySession: boolean;
   activeSessionApp?: RecordingAppIdentity;
   exportQuality?: RecordingExportQuality;
+  /**
+   * The caller's `--fps`: the capture rate where the recorder takes one. The touch overlay renders
+   * at most this many frames a second, and never more than 30.
+   */
+  fps?: number;
   gestureEvents: readonly RecordingGestureEvent[];
   touchReferenceFrame?: GestureReferenceFrame;
   gestureClockOriginAtMs?: number;

packages/platform-android/src/recording/finalize.ts

@@ -123,6 +123,7 @@ async function exportCollectedChunks(
       showTouches: chunked ? false : recording.showTouches,
       gestureEvents: recording.gestureEvents,
       exportQuality: recording.exportQuality ?? 'medium',
+      ...(recording.fps === undefined ? {} : { fps: recording.fps }),
       targetLabel: TARGET_LABEL,
     });
   } catch (error) {

packages/platform-android/src/recording/live-snapshot.test.ts

@@ -22,6 +22,7 @@ describe('snapshot', () => {
         ...recordingInput(),
         clientOutputPath: '/client/capture.mp4',
         exportQuality: 'high',
+        fps: 15,
         activeSessionApp: { bundleId: 'com.example.app' },
       },
       1,
@@ -30,6 +31,7 @@ describe('snapshot', () => {
     expect(started).toMatchObject({
       clientOutPath: '/client/capture.mp4',
       exportQuality: 'high',
+      fps: 15,
       activeSessionApp: { bundleId: 'com.example.app' },
     });
     expect(snapshot({ ...recordingInput(), showTouches: false }, 1).showTouches).toBe(false);

packages/platform-android/src/recording/live-snapshot.ts

@@ -18,6 +18,7 @@ export function snapshot(
     recordOnlySession: input.recordOnlySession,
     ...(input.activeSessionApp ? { activeSessionApp: input.activeSessionApp } : {}),
     ...(input.exportQuality ? { exportQuality: input.exportQuality } : {}),
+    ...(input.fps === undefined ? {} : { fps: input.fps }),
     gestureEvents: [],
   });
 }

packages/platform-android/src/recording/manifest.ts

@@ -21,6 +21,7 @@ export type AndroidRecordingDescriptor = Readonly<{
   recordOnlySession: boolean;
   activeSessionApp?: ScreenRecordingStartInput['activeSessionApp'];
   exportQuality?: ScreenRecordingStartInput['exportQuality'];
+  fps?: number;
   transportMode: 'local' | 'transport-composed';
 }>;
 
@@ -46,6 +47,7 @@ export type NativeManifest = Readonly<{
   recordOnlySession: boolean;
   activeSessionApp?: ScreenRecordingStartInput['activeSessionApp'];
   exportQuality?: ScreenRecordingStartInput['exportQuality'];
+  fps?: number;
   transportMode: 'local' | 'transport-composed';
   chunks: readonly NativeChunk[];
   pendingRemotePath?: string;
@@ -82,6 +84,7 @@ export const androidScreenRecordingDescriptorCodec = Object.freeze({
               : {
                   exportQuality: body.exportQuality as AndroidRecordingDescriptor['exportQuality'],
                 }),
+            ...(body.fps === undefined ? {} : { fps: body.fps }),
           }),
         } as const)
       : ({ status: 'invalid', message: 'Invalid Android screen-recording descriptor' } as const),
@@ -110,6 +113,7 @@ export function createNativeManifest(
     recordOnlySession: input.recordOnlySession,
     ...(input.activeSessionApp === undefined ? {} : { activeSessionApp: input.activeSessionApp }),
     ...(input.exportQuality === undefined ? {} : { exportQuality: input.exportQuality }),
+    ...(input.fps === undefined ? {} : { fps: input.fps }),
     chunks,
     ...(pendingRemotePath === undefined ? {} : { pendingRemotePath }),
     transportMode,
@@ -163,6 +167,7 @@ export function nativeManifestMatchesDescriptor(
     manifest.recordOnlySession === descriptor.recordOnlySession &&
     JSON.stringify(manifest.activeSessionApp) === JSON.stringify(descriptor.activeSessionApp) &&
     manifest.exportQuality === descriptor.exportQuality &&
+    manifest.fps === descriptor.fps &&
     manifest.transportMode === descriptor.transportMode &&
     (manifest.completion === undefined ||
       completionMatchesDescriptor(manifest.completion, descriptor))