Add support for connecting to the DevTools server from integration tests (#9190)

Author: kenzieschmoll

packages/devtools_app/integration_test/test/live_connection/devtools_extensions_test.dart

@@ -2,6 +2,10 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file or at https://developers.google.com/open-source/licenses/bsd.
 
+// Do not delete these arguments. They are parsed by test runner.
+// test-argument:startDevToolsServer=true
+// test-argument:appPath="../devtools_extensions/example/app_that_uses_foo"
+
 import 'package:devtools_app/devtools_app.dart';
 import 'package:devtools_app/src/extensions/embedded/view.dart';
 import 'package:devtools_app/src/extensions/extension_screen.dart';
@@ -40,32 +44,32 @@ void main() {
   testWidgets('end to end extensions flow', (tester) async {
     await pumpDevTools(tester);
 
-    logStatus(
-      'verify static extensions are available before connecting to an app',
-    );
-    expect(extensionService.availableExtensions.length, 3);
-    expect(extensionService.visibleExtensions.length, 3);
-    await _verifyExtensionsSettingsMenu(tester, [
-      ExtensionEnabledState.none, // bar
-      ExtensionEnabledState.none, // baz
-      ExtensionEnabledState.none, // foo
-    ]);
+    // TODO(https://github.com/flutter/devtools/issues/9196): re-enable this
+    // test verification once DTD can be started from the integration test
+    // harness.
+    // logStatus(
+    //   'verify static extensions are available before connecting to an app',
+    // );
+    // expect(extensionService.availableExtensions.length, 2);
+    // expect(extensionService.visibleExtensions.length, 2);
+    // await _verifyExtensionsSettingsMenu(tester, [
+    //   ExtensionEnabledState.none, // dart_foo
+    //   ExtensionEnabledState.none, // standalone_extension
+    // ]);
 
     await connectToTestApp(tester, testApp);
 
-    expect(extensionService.availableExtensions.length, 5);
-    expect(extensionService.visibleExtensions.length, 5);
+    expect(extensionService.availableExtensions.length, 3);
+    expect(extensionService.visibleExtensions.length, 3);
     await _verifyExtensionsSettingsMenu(tester, [
-      ExtensionEnabledState.none, // bar
-      ExtensionEnabledState.none, // baz
+      ExtensionEnabledState.none, // dart_foo
       ExtensionEnabledState.none, // foo
-      ExtensionEnabledState.none, // provider
-      ExtensionEnabledState.none, // some_tool
+      ExtensionEnabledState.none, // standalone_extension
     ], closeMenuWhenDone: false);
 
     await _verifyExtensionVisibilitySetting(tester);
 
-    // Bar extension.
+    // dart_foo extension.
     // Enable, test context menu actions, then disable from context menu.
     await _switchToExtensionScreen(
       tester,
@@ -74,72 +78,64 @@ void main() {
     );
     await _answerEnableExtensionPrompt(tester, enable: true);
     await _verifyExtensionsSettingsMenu(tester, [
-      ExtensionEnabledState.enabled, // bar
-      ExtensionEnabledState.none, // baz
+      ExtensionEnabledState.enabled, // dart_foo
       ExtensionEnabledState.none, // foo
-      ExtensionEnabledState.none, // provider
-      ExtensionEnabledState.none, // some_tool
+      ExtensionEnabledState.none, // standalone_extension
     ]);
 
     await _verifyContextMenuActionsAndDisable(tester);
 
-    expect(extensionService.availableExtensions.length, 5);
-    expect(extensionService.visibleExtensions.length, 4);
+    expect(extensionService.availableExtensions.length, 3);
+    expect(extensionService.visibleExtensions.length, 2);
     await _verifyExtensionTabVisibility(
       tester,
       extensionIndex: 0,
       visible: false,
     );
     await _verifyExtensionsSettingsMenu(tester, [
-      ExtensionEnabledState.disabled, // bar
-      ExtensionEnabledState.none, // baz
+      ExtensionEnabledState.disabled, // dart_foo
       ExtensionEnabledState.none, // foo
-      ExtensionEnabledState.none, // provider
-      ExtensionEnabledState.none, // some_tool
+      ExtensionEnabledState.none, // standalone_extension
     ]);
 
-    // Baz extension. Hide immediately.
+    // foo extension. Hide immediately.
     await _switchToExtensionScreen(
       tester,
       extensionIndex: 1,
       initialLoad: true,
     );
     await _answerEnableExtensionPrompt(tester, enable: false);
 
-    expect(extensionService.availableExtensions.length, 5);
-    expect(extensionService.visibleExtensions.length, 3);
+    expect(extensionService.availableExtensions.length, 3);
+    expect(extensionService.visibleExtensions.length, 1);
     await _verifyExtensionTabVisibility(
       tester,
       extensionIndex: 1,
       visible: false,
     );
     await _verifyExtensionsSettingsMenu(tester, [
-      ExtensionEnabledState.disabled, // bar
-      ExtensionEnabledState.disabled, // baz
-      ExtensionEnabledState.none, // foo
-      ExtensionEnabledState.none, // provider
-      ExtensionEnabledState.none, // some_tool
+      ExtensionEnabledState.disabled, // dart_foo
+      ExtensionEnabledState.disabled, // foo
+      ExtensionEnabledState.none, // standalone_extension
     ]);
 
-    // Re-enable Baz extension from the extensions settings menu.
+    // Re-enable foo extension from the extensions settings menu.
     logStatus('verify we can re-enable an extension from the settings menu');
     await _changeExtensionSetting(tester, extensionIndex: 1, enable: true);
 
-    expect(extensionService.availableExtensions.length, 5);
-    expect(extensionService.visibleExtensions.length, 4);
+    expect(extensionService.availableExtensions.length, 3);
+    expect(extensionService.visibleExtensions.length, 2);
     await _switchToExtensionScreen(tester, extensionIndex: 1);
     expect(find.byType(EnableExtensionPrompt), findsNothing);
     expect(find.byType(EmbeddedExtensionView), findsOneWidget);
     expect(find.byType(HtmlElementView), findsOneWidget);
     await _verifyExtensionsSettingsMenu(tester, [
-      ExtensionEnabledState.disabled, // bar
-      ExtensionEnabledState.enabled, // baz
-      ExtensionEnabledState.none, // foo
-      ExtensionEnabledState.none, // provider
-      ExtensionEnabledState.none, // some_tool
+      ExtensionEnabledState.disabled, // dart_foo
+      ExtensionEnabledState.enabled, // foo
+      ExtensionEnabledState.none, // standalone_extension
     ]);
 
-    // Foo extension. Disable directly from settings menu.
+    // standalone_extension. Disable directly from settings menu.
     logStatus(
       'verify we can disable an extension screen directly from the settings menu',
     );
@@ -151,19 +147,17 @@ void main() {
 
     logStatus('disable the extension from the settings menu');
     await _changeExtensionSetting(tester, extensionIndex: 2, enable: false);
-    expect(extensionService.availableExtensions.length, 5);
-    expect(extensionService.visibleExtensions.length, 3);
+    expect(extensionService.availableExtensions.length, 3);
+    expect(extensionService.visibleExtensions.length, 1);
     await _verifyExtensionTabVisibility(
       tester,
       extensionIndex: 2,
       visible: false,
     );
     await _verifyExtensionsSettingsMenu(tester, [
-      ExtensionEnabledState.disabled, // bar
-      ExtensionEnabledState.enabled, // baz
-      ExtensionEnabledState.disabled, // foo
-      ExtensionEnabledState.none, // provider
-      ExtensionEnabledState.none, // some_tool
+      ExtensionEnabledState.disabled, // dart_foo
+      ExtensionEnabledState.enabled, // foo
+      ExtensionEnabledState.disabled, // standalone_extension
     ]);
   });
 }
@@ -332,7 +326,7 @@ Future<void> _verifyExtensionVisibilitySetting(WidgetTester tester) async {
     preferences.devToolsExtensions.showOnlyEnabledExtensions.value,
     isFalse,
   );
-  expect(extensionService.visibleExtensions.length, 5);
+  expect(extensionService.visibleExtensions.length, 3);
   // No need to open the settings menu as it should already be open.
   await _toggleShowOnlyEnabledExtensions(tester);
   expect(
@@ -347,7 +341,7 @@ Future<void> _verifyExtensionVisibilitySetting(WidgetTester tester) async {
     preferences.devToolsExtensions.showOnlyEnabledExtensions.value,
     isFalse,
   );
-  expect(extensionService.visibleExtensions.length, 5);
+  expect(extensionService.visibleExtensions.length, 3);
 
   await _closeExtensionSettingsMenu(tester);
 }

packages/devtools_app/integration_test/test_infra/run/_in_file_args.dart

@@ -39,13 +39,18 @@ class TestFileArgs {
   TestFileArgs._parse(
     Map<_TestFileArgItems, dynamic> args, {
     required this.appPath,
-  }) : experimentsOn = args[_TestFileArgItems.experimentsOn] ?? false;
+  }) : experimentsOn = args[_TestFileArgItems.experimentsOn] ?? false,
+       startDevToolsServer =
+           args[_TestFileArgItems.startDevToolsServer] ?? false;
+
+  /// The path to the application to connect to.
+  final String appPath;
 
   /// Whether experiments are enabled in the test.
   final bool experimentsOn;
 
-  /// The path to the application to connect to.
-  final String appPath;
+  /// Whether the DevTools server should be started and connected to DevTools.
+  final bool startDevToolsServer;
 
   /// Parses 'test-argument' comments in [fileContent].
   static Map<_TestFileArgItems, dynamic> _parseFileContent(String fileContent) {
@@ -62,4 +67,15 @@ class TestFileArgs {
 }
 
 /// The different arguments accepted as "file args."
-enum _TestFileArgItems { experimentsOn, appPath }
+enum _TestFileArgItems {
+  /// The path to the Dart or Flutter application to run and connect to DevTools
+  /// for this test.
+  appPath,
+
+  /// Whether to enable DevTools experimental features for this test.
+  experimentsOn,
+
+  /// Whether to start the DevTools server and connect it to DevTools for this
+  /// test.
+  startDevToolsServer,
+}

packages/devtools_app/integration_test/test_infra/run/run_test.dart

@@ -4,6 +4,7 @@
 
 import 'dart:async';
 import 'dart:convert';
+import 'dart:io';
 
 import 'package:args/args.dart';
 import 'package:devtools_shared/devtools_test_utils.dart';
@@ -12,6 +13,10 @@ import '_in_file_args.dart';
 import '_test_app_driver.dart';
 import '_utils.dart';
 
+/// The identifier for the stdout line that contains the DevTools server
+/// address when starting from the `dart devtools` command.
+const _devToolsServerAddressLine = 'Serving DevTools at ';
+
 /// Runs one test.
 ///
 /// Do not use this method directly, but instead use the run_tests.dart
@@ -24,6 +29,58 @@ Future<void> runFlutterIntegrationTest(
   IntegrationTestApp? testApp;
   late String testAppUri;
 
+  String? devToolsServerAddress;
+  Process? devToolsServerProcess;
+  if (testFileArgs.startDevToolsServer) {
+    // TODO(https://github.com/flutter/devtools/issues/9196): support starting
+    // DTD and passing the URI to DevTools server. Workspace roots should be set
+    // on the DTD instance based on the connected test app.
+
+    // Start the DevTools server. This will use the DevTools server that is
+    // shipped with the Dart SDK.
+    // TODO(https://github.com/flutter/devtools/issues/9197): launch the
+    // DevTools server from source so that end to end changes (server + app) can
+    // be tested.
+    devToolsServerProcess = await Process.start('dart', [
+      'devtools',
+      // Do not launch DevTools app in the browser. This DevTools server
+      // instance will be used to connect to the DevTools app that is run from
+      // Flutter driver from the integration test runner.
+      '--no-launch-browser',
+      // Disable CORS restrictions so that we can connect to the server from
+      // DevTools app that is served on a different origin.
+      '--disable-cors',
+    ]);
+
+    final addressCompleter = Completer<void>();
+    final sub = devToolsServerProcess.stdout.transform(utf8.decoder).listen((
+      line,
+    ) {
+      if (line.startsWith(_devToolsServerAddressLine)) {
+        // This will pull the server address from a String like:
+        // "Serving DevTools at http://127.0.0.1:9104.".
+        final regexp = RegExp(
+          r'http:\/\/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+',
+        );
+        final match = regexp.firstMatch(line);
+        if (match != null) {
+          devToolsServerAddress = match.group(0);
+          addressCompleter.complete();
+        }
+      }
+    });
+
+    await addressCompleter.future.timeout(
+      const Duration(seconds: 10),
+      onTimeout: () async {
+        await sub.cancel();
+        devToolsServerProcess?.kill();
+        throw Exception('Timed out waiting for DevTools server to start.');
+      },
+    );
+    await sub.cancel();
+  }
+
   if (!offline) {
     if (testRunnerArgs.testAppUri == null) {
       // Create the test app and start it.
@@ -72,6 +129,9 @@ Future<void> runFlutterIntegrationTest(
         'test_args=${jsonEncode(testArgs)}',
         if (testFileArgs.experimentsOn) 'enable_experiments=true',
         if (testRunnerArgs.updateGoldens) 'update_goldens=true',
+        if (devToolsServerAddress != null)
+          // Add the trailing slash because this is what DevTools app expects.
+          'debug_devtools_server=$devToolsServerAddress/',
       ],
       debugLogging: debugTestScript,
     );
@@ -81,6 +141,11 @@ Future<void> runFlutterIntegrationTest(
       await testApp.stop();
     }
 
+    if (devToolsServerProcess != null) {
+      debugLog('killing the DevTools server');
+      devToolsServerProcess.kill();
+    }
+
     debugLog('cancelling stream subscriptions');
     await testRunner.cancelAllStreamSubscriptions();
   }

packages/devtools_app/lib/src/extensions/embedded/_view_web.dart

@@ -156,12 +156,6 @@ class _ExtensionIFrameController extends DisposableController
   }
 
   void _postMessage(DevToolsExtensionEvent event) async {
-    // In [integrationTestMode] we are loading a placeholder url
-    // (https://flutter.dev/) in the extension iFrame, so trying to post a
-    // message causes a cross-origin security error. Return early when
-    // [integrationTestMode] is true so that [_postMessage] calls are a no-op.
-    if (integrationTestMode) return;
-
     await _iFrameReady.future;
     final message = event.toJson();
     assert(

packages/devtools_app/lib/src/shared/development_helpers.dart

@@ -54,7 +54,7 @@ bool debugShowAnalyticsConsentMessage = false;
 /// This flag should never be checked in with a value of true - this is covered
 /// by a test.
 final debugDevToolsExtensions =
-    _debugDevToolsExtensions || integrationTestMode || testMode || stagerMode;
+    _debugDevToolsExtensions || testMode || stagerMode;
 const _debugDevToolsExtensions = false;
 
 List<DevToolsExtensionConfig> debugHandleRefreshAvailableExtensions({

packages/devtools_shared/lib/src/test/io_utils.dart

@@ -82,9 +82,15 @@ mixin IOMixin {
   }) async {
     final processId = process.pid;
     if (debugLogging) {
-      print('Sending SIGTERM to $processId.');
+      print(
+        'Cancelling all stream subscriptions for process $processId before '
+        'killing.',
+      );
     }
     await cancelAllStreamSubscriptions();
+    if (debugLogging) {
+      print('Sending SIGTERM to $processId.');
+    }
     Process.killPid(processId);
     return process.exitCode.timeout(
       killTimeout,