Feature: Record and Replay Backend Traffic (#49)

Author: aniket-k-mukherjee

package-lock.json

@@ -1,12 +1,13 @@
 {
   "name": "appium-interceptor",
-  "version": "1.0.0-beta.10",
+  "version": "1.0.0-beta.11",
   "description": "Appium 2.0 plugin to mock api calls for android apps",
   "main": "./lib/index.js",
   "types": "./lib/types/index.d.ts",
   "scripts": {
     "build": "npx tsc",
     "test": "mocha --require ts-node/register -p test/plugin.spec.js --exit --timeout 260000",
+    "record-replay-test": "mocha --require ts-node/register -p test/record.replay.js --exit",
     "prepublish": "npx tsc",
     "lint": "eslint '**/*.js' --fix",
     "prettier": "prettier '**/*.js' --write --single-quote",
@@ -104,6 +105,7 @@
   ],
   "dependencies": {
     "@appium/support": "^4.1.11",
+    "queue-typescript": "^1.0.1",
     "ajv": "^6.12.6",
     "appium-adb": "^11.0.9",
     "axios": "^0.27.0",

package.json

@@ -1,5 +1,6 @@
 import { RequestInfo, SniffConfig } from './types';
 import { doesUrlMatch } from './utils/proxy';
+import log from './logger';
 
 export class ApiSniffer {
   private readonly requests: RequestInfo[] = [];
@@ -23,7 +24,10 @@ export class ApiSniffer {
   private doesRequestMatchesConfig(request: RequestInfo) {
     const doesIncludeRuleMatches = !this.config.include
       ? true
-      : this.config.include.some((config) => doesUrlMatch(config.url, request.url));
+      : this.config.include.some((config) => {
+        log.info(`Matching include url ${request.url} with request ${config.url}`);
+        doesUrlMatch(config.url, request.url)
+      });
     const doesExcludeRuleMatches = !this.config.exclude
       ? true
       : !this.config.exclude.some((config) => doesUrlMatch(config.url, request.url));

src/api-sniffer.ts

@@ -3,6 +3,7 @@ import stream from 'stream';
 import { constructURLFromHttpRequest } from './utils/proxy';
 import responseDecoder from './response-decoder';
 import parseHeader from 'parse-headers';
+import log from './logger';
 
 function readBodyFromStream(writable: stream.Writable | undefined, callback: (value: any) => void) {
   if (!writable) {

src/interceptor.ts

@@ -1,7 +1,7 @@
 import { BasePlugin } from 'appium/plugin';
 import http from 'http';
 import { Application } from 'express';
-import { CliArg, ISessionCapability, MockConfig, RequestInfo, SniffConfig } from './types';
+import { CliArg, ISessionCapability, MockConfig, RecordConfig, RequestInfo, ReplayConfig, SniffConfig } from './types';
 import _ from 'lodash';
 import { configureWifiProxy, isRealDevice } from './utils/adb';
 import { cleanUpProxyServer, sanitizeMockConfig, setupProxyServer } from './utils/proxy';
@@ -40,6 +40,26 @@ export class AppiumInterceptorPlugin extends BasePlugin {
       command: 'stopListening',
       params: { optional: ['id'] },
     },
+
+    'interceptor: startRecording': {
+      command: 'startRecording',
+      params: { optional: ['config'] },
+    },
+
+    'interceptor: stopRecording': {
+      command: 'stopRecording',
+      params: { optional: ['id'] },
+    },
+
+    'interceptor: startReplaying': {
+      command: 'startReplaying',
+      params: { required: ['replayConfig'] },
+    },
+
+    'interceptor: stopReplaying': {
+      command: 'stopReplaying',
+      params: { optional: ['id'] },
+    },
   };
 
   constructor(name: string, cliArgs: CliArg) {
@@ -77,6 +97,7 @@ export class AppiumInterceptorPlugin extends BasePlugin {
       await configureWifiProxy(adb, deviceUDID, realDevice, proxy);
       proxyCache.add(sessionId, proxy);
     }
+    log.info("Creating session for appium interceptor");
     return response;
   }
 
@@ -162,10 +183,53 @@ export class AppiumInterceptorPlugin extends BasePlugin {
     }
 
     log.info(`Stopping listener with id: ${id}`);
-    return proxy.removeSniffer(id);
+    return proxy.removeSniffer(false, id);
+  }
+
+  async startRecording(next: any, driver: any, config: SniffConfig): Promise<string> {
+    const proxy = proxyCache.get(driver.sessionId);
+    if (!proxy) {
+      logger.error('Proxy is not running');
+      throw new Error('Proxy is not active for current session');
+    }
+
+    log.info(`Adding listener with config ${config}`);
+    return proxy?.addSniffer(config);
+  }
+
+  async stopRecording(next: any, driver: any, id: any): Promise<RecordConfig[]> {
+    const proxy = proxyCache.get(driver.sessionId);
+    if (!proxy) {
+      logger.error('Proxy is not running');
+      throw new Error('Proxy is not active for current session');
+    }
+
+    log.info(`Stopping recording with id: ${id}`);
+    return proxy.removeSniffer(true, id);
+  }
+
+  async startReplaying(next:any, driver:any, replayConfig: ReplayConfig) {
+    const proxy = proxyCache.get(driver.sessionId);
+    if (!proxy) {
+      logger.error('Proxy is not running');
+      throw new Error('Proxy is not active for current session');
+    }
+    log.info('Starting replay traffic');
+    proxy.startReplaying();
+    return proxy.getRecordingManager().replayTraffic(replayConfig);
+  }
+
+  async stopReplaying(next: any, driver:any, id:any) {
+    const proxy = proxyCache.get(driver.sessionId);
+    if (!proxy) {
+      logger.error('Proxy is not running');
+      throw new Error('Proxy is not active for current session');
+    }
+    log.info("Initiating stop replaying traffic");
+    proxy.getRecordingManager().stopReplay(id);
   }
 
   async execute(next: any, driver: any, script: any, args: any) {
     return await this.executeMethod(next, driver, script, args);
   }
-}
+}

src/plugin.ts

@@ -1,4 +1,4 @@
-import { MockConfig, RequestInfo, SniffConfig } from './types';
+import { MockConfig, RecordConfig, RequestInfo, SniffConfig } from './types';
 import { Proxy as HttpProxy, IContext, IProxyOptions } from 'http-mitm-proxy';
 import { v4 as uuid } from 'uuid';
 import {
@@ -19,7 +19,8 @@ import { Mock } from './mock';
 import { RequestInterceptor } from './interceptor';
 import { ApiSniffer } from './api-sniffer';
 import _ from 'lodash';
-import logger from './logger';
+import log from './logger';
+import { RecordingManager } from './recording-manager';
 
 export interface ProxyOptions {
   deviceUDID: string;
@@ -31,20 +32,35 @@ export interface ProxyOptions {
 
 export class Proxy {
   private _started = false;
+  private _replayStarted = false;
   private readonly mocks = new Map<string, Mock>();
   private readonly sniffers = new Map<string, ApiSniffer>();
 
   private readonly httpProxy: HttpProxy;
+  private readonly recordingManager: RecordingManager;
 
   public isStarted(): boolean {
     return this._started;
   }
 
+  public isReplayStarted(): boolean {
+    return this._replayStarted;
+  }
+
+  public startReplaying(): void {
+    this._replayStarted = true;
+  }
+
   constructor(private readonly options: ProxyOptions) {
     this.httpProxy = new HttpProxy();
+    this.recordingManager = new RecordingManager(options);
     addDefaultMocks(this);
   }
 
+  public getRecordingManager(): RecordingManager {
+    return this.recordingManager;
+  }
+
   public get port(): number {
     return this.options.port;
   }
@@ -81,7 +97,7 @@ export class Proxy {
     this.httpProxy.onRequest(this.handleMockApiRequest.bind(this));
 
     this.httpProxy.onError((context, error, errorType) => {
-      logger.error(`${errorType}: ${error}`);
+      log.error(`${errorType}: ${error}`);
     });
 
     await new Promise((resolve) => {
@@ -123,26 +139,37 @@ export class Proxy {
     return id;
   }
 
-  public removeSniffer(id?: string): RequestInfo[] {
-    const _sniffers = [...this.sniffers.values()];
-    if (id && !_.isNil(this.sniffers.get(id))) {
-      _sniffers.push(this.sniffers.get(id)!);
+    public removeSniffer(record: boolean, id?: string): RequestInfo[] {
+      const _sniffers = [...this.sniffers.values()];
+      if (id && !_.isNil(this.sniffers.get(id))) {
+        _sniffers.push(this.sniffers.get(id)!);
+      }
+      let apiRequests;
+      if (record) {
+        apiRequests = this.recordingManager.getCapturedTraffic(_sniffers);
+      }
+      else {
+        apiRequests = _sniffers.reduce((acc, sniffer) => {
+          acc.push(...sniffer.getRequests());
+          return acc;
+        }, [] as RequestInfo[]);
+      }
+      _sniffers.forEach((sniffer) => this.sniffers.delete(sniffer.getId()));
+      return apiRequests;
     }
-    const apiRequests = _sniffers.reduce((acc, sniffer) => {
-      acc.push(...sniffer.getRequests());
-      return acc;
-    }, [] as RequestInfo[]);
-    _sniffers.forEach((sniffer) => this.sniffers.delete(sniffer.getId()));
-    return apiRequests;
-  }
 
   private async handleMockApiRequest(ctx: IContext, next: () => void): Promise<void> {
-    const matchedMocks = await this.findMatchingMocks(ctx);
-    if (matchedMocks.length) {
-      const compiledMock = compileMockConfig(matchedMocks);
-      this.applyMockToRequest(ctx, compiledMock, next);
-    } else {
-      next();
+    if (this.isReplayStarted()) {
+      this.recordingManager.handleRecordingApiRequest(ctx, next);
+    } else if (!this.isReplayStarted()) {
+      const matchedMocks = await this.findMatchingMocks(ctx);
+      if (matchedMocks.length) {
+        const compiledMock = compileMockConfig(matchedMocks);
+        this.applyMockToRequest(ctx, compiledMock, next);
+      }
+      else {
+        next();
+      }
     }
   }
 
@@ -197,4 +224,4 @@ export class Proxy {
       next();
     }
   }
-}
+}

src/proxy.ts

@@ -0,0 +1,27 @@
+import { RecordConfig } from './types';
+
+export class Record {
+  private enabled = true;
+
+  constructor(private id: string, private config: RecordConfig) {}
+
+  getId() {
+    return this.id;
+  }
+
+  getConfig() {
+    return this.config;
+  }
+
+  isEnabled() {
+    return this.enabled;
+  }
+
+  setEnableStatus(enbaleStatus: boolean) {
+    this.enabled = enbaleStatus;
+  }
+
+  updateConfig(config: RecordConfig) {
+    this.config = config;
+  }
+}