Merge pull request #383 from ulixee/rtc

feat(plugins): mask public ip in webrtc

core/lib/CorePlugins.ts

@@ -10,14 +10,14 @@ import { IInteractionGroups, IInteractionStep } from '@secret-agent/interfaces/I
 import IInteractionsHelper from '@secret-agent/interfaces/IInteractionsHelper';
 import IPoint from '@secret-agent/interfaces/IPoint';
 import ICorePlugin, {
-  IHumanEmulator,
   IBrowserEmulator,
+  IBrowserEmulatorClass,
   IBrowserEmulatorConfig,
-  ISelectBrowserMeta,
   ICorePluginClass,
-  IOnClientCommandMeta,
-  IBrowserEmulatorClass,
+  IHumanEmulator,
   IHumanEmulatorClass,
+  IOnClientCommandMeta,
+  ISelectBrowserMeta,
 } from '@secret-agent/interfaces/ICorePlugin';
 import ICorePlugins from '@secret-agent/interfaces/ICorePlugins';
 import ICorePluginCreateOptions from '@secret-agent/interfaces/ICorePluginCreateOptions';
@@ -26,6 +26,7 @@ import { PluginTypes } from '@secret-agent/interfaces/IPluginTypes';
 import requirePlugins from '@secret-agent/plugin-utils/lib/utils/requirePlugins';
 import IHttp2ConnectSettings from '@secret-agent/interfaces/IHttp2ConnectSettings';
 import IDeviceProfile from '@secret-agent/interfaces/IDeviceProfile';
+import IHttpSocketAgent from '@secret-agent/interfaces/IHttpSocketAgent';
 import Core from '../index';
 
 const DefaultBrowserEmulatorId = 'default-browser-emulator';
@@ -136,6 +137,14 @@ export default class CorePlugins implements ICorePlugins {
     this.instances.filter(p => p.onTlsConfiguration).forEach(p => p.onTlsConfiguration(settings));
   }
 
+  public async onHttpAgentInitialized(agent: IHttpSocketAgent): Promise<void> {
+    await Promise.all(
+      this.instances
+        .filter(p => p.onHttpAgentInitialized)
+        .map(p => p.onHttpAgentInitialized(agent)),
+    );
+  }
+
   public async onNewPuppetPage(page: IPuppetPage): Promise<void> {
     await Promise.all(
       this.instances.filter(p => p.onNewPuppetPage).map(p => p.onNewPuppetPage(page)),

core/lib/Session.ts

@@ -267,6 +267,7 @@ export default class Session extends TypedEventEmitter<{
     requestSession.on('resource-state', this.onResourceStates.bind(this));
     requestSession.on('socket-close', this.onSocketClose.bind(this));
     requestSession.on('socket-connect', this.onSocketConnect.bind(this));
+    await this.plugins.onHttpAgentInitialized(requestSession.requestAgent);
   }
 
   public nextTabId(): number {

full-client/test/basic.test.ts

@@ -60,6 +60,10 @@ describe('basic Full Client tests', () => {
   it('should get unreachable proxy errors in the client', async () => {
     const agent = await handler.createAgent({
       upstreamProxyUrl: koaServer.baseUrl,
+      upstreamProxyIpMask: {
+        proxyIp: '127.0.0.1',
+        publicIp: '127.0.0.1',
+      },
     });
     Helpers.needsClosing.push(agent);
     await expect(agent.goto(`${koaServer.baseUrl}/`)).rejects.toThrow();

interfaces/ICorePlugin.ts

@@ -12,10 +12,10 @@ import ITlsSettings from './ITlsSettings';
 import IHttpResourceLoadDetails from './IHttpResourceLoadDetails';
 import { IPuppetPage } from './IPuppetPage';
 import { IPuppetWorker } from './IPuppetWorker';
-import IViewport from './IViewport';
-import IGeolocation from './IGeolocation';
 import IDeviceProfile from './IDeviceProfile';
 import IHttp2ConnectSettings from './IHttp2ConnectSettings';
+import IHttpSocketAgent from './IHttpSocketAgent';
+import ISessionCreateOptions from './ISessionCreateOptions';
 
 export default interface ICorePlugin
   extends ICorePluginMethods,
@@ -108,6 +108,7 @@ export interface IBrowserEmulatorMethods {
   onDnsConfiguration?(settings: IDnsSettings): Promise<any> | void;
   onTcpConfiguration?(settings: ITcpSettings): Promise<any> | void;
   onTlsConfiguration?(settings: ITlsSettings): Promise<any> | void;
+  onHttpAgentInitialized?(agent: IHttpSocketAgent): Promise<any> | void;
 
   onHttp2SessionConnect?(
     request: IHttpResourceLoadDetails,
@@ -122,12 +123,10 @@ export interface IBrowserEmulatorMethods {
   websiteHasFirstPartyInteraction?(url: URL): Promise<any> | void; // needed for implementing first-party cookies
 }
 
-export interface IBrowserEmulatorConfig {
-  viewport?: IViewport;
-  geolocation?: IGeolocation;
-  timezoneId?: string;
-  locale?: string;
-}
+export type IBrowserEmulatorConfig = Pick<
+  ISessionCreateOptions,
+  'viewport' | 'geolocation' | 'timezoneId' | 'locale' | 'upstreamProxyIpMask' | 'upstreamProxyUrl'
+>;
 
 // decorator for browser emulator classes. hacky way to check the class implements statics we need
 // eslint-disable-next-line @typescript-eslint/no-unused-vars

interfaces/IHttpSocketAgent.ts

@@ -0,0 +1,6 @@
+import IHttpSocketConnectOptions from './IHttpSocketConnectOptions';
+import IHttpSocketWrapper from './IHttpSocketWrapper';
+
+export default interface IHttpSocketAgent {
+  createSocketConnection(options: IHttpSocketConnectOptions): Promise<IHttpSocketWrapper>;
+}

interfaces/IHttpSocketConnectOptions.ts

@@ -0,0 +1,11 @@
+export default interface IHttpSocketConnectOptions {
+  host: string;
+  port: string;
+  isSsl: boolean;
+  keepAlive?: boolean;
+  debug?: boolean;
+  servername?: string;
+  isWebsocket?: boolean;
+  keylogPath?: string;
+  proxyUrl?: string;
+}

interfaces/IHttpSocketWrapper.ts

@@ -0,0 +1,23 @@
+import * as net from 'net';
+
+export default interface IHttpSocketWrapper {
+  id: number;
+  alpn: string;
+  socket: net.Socket;
+  dnsResolvedIp: string;
+  remoteAddress: string;
+  localAddress: string;
+  serverName: string;
+
+  createTime: Date;
+  dnsLookupTime: Date;
+  connectTime: Date;
+  errorTime: Date;
+  closeTime: Date;
+
+  isConnected: boolean;
+  isClosing: boolean;
+
+  isHttp2(): boolean;
+  close(): void;
+}

interfaces/ISessionCreateOptions.ts

@@ -14,6 +14,7 @@ export default interface ISessionCreateOptions extends ISessionOptions {
   timezoneId?: string;
   locale?: string;
   upstreamProxyUrl?: string;
+  upstreamProxyIpMask?: { publicIp?: string; proxyIp?: string; ipLookupService?: string };
   input?: { command?: string } & any;
   geolocation?: IGeolocation;
   dependencyMap?: { [clientPluginId: string]: string[] };

mitm-socket/index.ts

@@ -5,18 +5,22 @@ import Log from '@secret-agent/commons/Logger';
 import { TypedEventEmitter } from '@secret-agent/commons/eventUtils';
 import Resolvable from '@secret-agent/commons/Resolvable';
 import { createIpcSocketPath } from '@secret-agent/commons/IpcUtils';
+import IHttpSocketConnectOptions from '@secret-agent/interfaces/IHttpSocketConnectOptions';
+import IHttpSocketWrapper from '@secret-agent/interfaces/IHttpSocketWrapper';
 import MitmSocketSession from './lib/MitmSocketSession';
 
 const { log } = Log(module);
 
 let idCounter = 0;
 
-export default class MitmSocket extends TypedEventEmitter<{
-  connect: void;
-  dial: void;
-  eof: void;
-  close: void;
-}> {
+export default class MitmSocket
+  extends TypedEventEmitter<{
+    connect: void;
+    dial: void;
+    eof: void;
+    close: void;
+  }>
+  implements IHttpSocketWrapper {
   public get isWebsocket(): boolean {
     return this.connectOpts.isWebsocket === true;
   }
@@ -50,7 +54,7 @@ export default class MitmSocket extends TypedEventEmitter<{
   private socketReadyPromise = new Resolvable<void>();
   private readonly callStack: string;
 
-  constructor(readonly sessionId: string, readonly connectOpts: IGoTlsSocketConnectOpts) {
+  constructor(readonly sessionId: string, readonly connectOpts: IHttpSocketConnectOptions) {
     super();
     this.callStack = new Error().stack.replace('Error:', '').trim();
     this.serverName = connectOpts.servername;
@@ -226,18 +230,6 @@ export default class MitmSocket extends TypedEventEmitter<{
   }
 }
 
-export interface IGoTlsSocketConnectOpts {
-  host: string;
-  port: string;
-  isSsl: boolean;
-  keepAlive?: boolean;
-  debug?: boolean;
-  servername?: string;
-  isWebsocket?: boolean;
-  keylogPath?: string;
-  proxyUrl?: string;
-}
-
 class Socks5ProxyConnectError extends Error {}
 class HttpProxyConnectError extends Error {}
 class SocketConnectError extends Error {}

mitm/lib/MitmRequestAgent.ts

@@ -1,4 +1,4 @@
-import MitmSocket, { IGoTlsSocketConnectOpts } from '@secret-agent/mitm-socket';
+import MitmSocket from '@secret-agent/mitm-socket';
 import * as http2 from 'http2';
 import { ClientHttp2Session, Http2ServerRequest } from 'http2';
 import Log from '@secret-agent/commons/Logger';
@@ -10,6 +10,7 @@ import ITcpSettings from '@secret-agent/interfaces/ITcpSettings';
 import ITlsSettings from '@secret-agent/interfaces/ITlsSettings';
 import Resolvable from '@secret-agent/commons/Resolvable';
 import IHttp2ConnectSettings from '@secret-agent/interfaces/IHttp2ConnectSettings';
+import IHttpSocketConnectOptions from '@secret-agent/interfaces/IHttpSocketConnectOptions';
 import IMitmRequestContext from '../interfaces/IMitmRequestContext';
 import MitmRequestContext from './MitmRequestContext';
 import RequestSession from '../handlers/RequestSession';
@@ -135,27 +136,7 @@ export default class MitmRequestAgent {
     return await pool.isHttp2(false, () => this.createSocketConnection(options));
   }
 
-  private async assignSocket(
-    ctx: IMitmRequestContext,
-    options: IGoTlsSocketConnectOpts & { headers: IResourceHeaders },
-  ): Promise<MitmSocket> {
-    ctx.setState(ResourceState.GetSocket);
-    const pool = this.getSocketPoolByOrigin(ctx.url.origin);
-
-    options.isSsl = ctx.isSSL;
-    options.keepAlive = !(
-      (options.headers.connection ?? options.headers.Connection) as string
-    )?.match(/close/i);
-    options.isWebsocket = ctx.isUpgrade;
-
-    const mitmSocket = await pool.getSocket(options.isWebsocket, () =>
-      this.createSocketConnection(options),
-    );
-    MitmRequestContext.assignMitmSocket(ctx, mitmSocket);
-    return mitmSocket;
-  }
-
-  private async createSocketConnection(options: IGoTlsSocketConnectOpts): Promise<MitmSocket> {
+  public async createSocketConnection(options: IHttpSocketConnectOptions): Promise<MitmSocket> {
     const session = this.session;
 
     const dnsLookupTime = new Date();
@@ -187,6 +168,25 @@ export default class MitmRequestAgent {
     return mitmSocket;
   }
 
+  private async assignSocket(
+    ctx: IMitmRequestContext,
+    options: IHttpSocketConnectOptions & { headers: IResourceHeaders },
+  ): Promise<MitmSocket> {
+    ctx.setState(ResourceState.GetSocket);
+    const pool = this.getSocketPoolByOrigin(ctx.url.origin);
+
+    options.isSsl = ctx.isSSL;
+    options.keepAlive = !((options.headers.connection ??
+      options.headers.Connection) as string)?.match(/close/i);
+    options.isWebsocket = ctx.isUpgrade;
+
+    const mitmSocket = await pool.getSocket(options.isWebsocket, () =>
+      this.createSocketConnection(options),
+    );
+    MitmRequestContext.assignMitmSocket(ctx, mitmSocket);
+    return mitmSocket;
+  }
+
   private getSocketPoolByOrigin(origin: string): SocketPool {
     let lookup = origin.split('://').pop();
     if (!lookup.includes(':') && origin.includes('://')) {

plugins/default-browser-emulator/index.ts

@@ -15,6 +15,7 @@ import IUserAgentOption from '@secret-agent/interfaces/IUserAgentOption';
 import BrowserEngine from '@secret-agent/plugin-utils/lib/BrowserEngine';
 import IGeolocation from '@secret-agent/interfaces/IGeolocation';
 import IHttp2ConnectSettings from '@secret-agent/interfaces/IHttp2ConnectSettings';
+import IHttpSocketAgent from '@secret-agent/interfaces/IHttpSocketAgent';
 import Viewports from './lib/Viewports';
 import setWorkerDomOverrides from './lib/setWorkerDomOverrides';
 import setPageDomOverrides from './lib/setPageDomOverrides';
@@ -38,6 +39,7 @@ import loadDomOverrides from './lib/loadDomOverrides';
 import DomOverridesBuilder from './lib/DomOverridesBuilder';
 import configureDeviceProfile from './lib/helpers/configureDeviceProfile';
 import configureHttp2Session from './lib/helpers/configureHttp2Session';
+import lookupPublicIp, { IpLookupServices } from './lib/helpers/lookupPublicIp';
 
 const dataLoader = new DataLoader(__dirname);
 export const latestBrowserEngineId = 'chrome-88-0';
@@ -51,6 +53,8 @@ export default class DefaultBrowserEmulator extends BrowserEmulator {
   public locale: string;
   public viewport: IViewport;
   public geolocation: IGeolocation;
+  public upstreamProxyIpMask: IBrowserEmulatorConfig['upstreamProxyIpMask'];
+  public upstreamProxyUrl: string;
 
   protected readonly data: IBrowserData;
   private readonly domOverridesBuilder: DomOverridesBuilder;
@@ -80,10 +84,17 @@ export default class DefaultBrowserEmulator extends BrowserEmulator {
       config.timezoneId || this.timezoneId || Intl.DateTimeFormat().resolvedOptions().timeZone;
     config.geolocation = config.geolocation || this.geolocation;
 
+    if (config.upstreamProxyUrl) {
+      config.upstreamProxyIpMask ??= {};
+      config.upstreamProxyIpMask.ipLookupService ??= IpLookupServices.ipify;
+    }
+
     this.locale = config.locale;
     this.viewport = config.viewport;
     this.timezoneId = config.timezoneId;
     this.geolocation = config.geolocation;
+    this.upstreamProxyIpMask = config.upstreamProxyIpMask;
+    this.upstreamProxyUrl = config.upstreamProxyUrl;
   }
 
   public onDnsConfiguration(settings: IDnsSettings): void {
@@ -102,6 +113,26 @@ export default class DefaultBrowserEmulator extends BrowserEmulator {
     modifyHeaders(this, this.data, resource);
   }
 
+  public async onHttpAgentInitialized(agent: IHttpSocketAgent): Promise<void> {
+    if (this.upstreamProxyIpMask) {
+      this.upstreamProxyIpMask.publicIp ??= await lookupPublicIp(
+        this.upstreamProxyIpMask.ipLookupService,
+      );
+      this.upstreamProxyIpMask.proxyIp ??= await lookupPublicIp(
+        this.upstreamProxyIpMask.ipLookupService,
+        agent,
+        this.upstreamProxyUrl,
+      );
+      this.logger.info('PublicIp Lookup', {
+        ...this.upstreamProxyIpMask,
+      });
+      this.domOverridesBuilder.add('webrtc', {
+        localIp: this.upstreamProxyIpMask.publicIp,
+        proxyIp: this.upstreamProxyIpMask.proxyIp,
+      });
+    }
+  }
+
   public onHttp2SessionConnect(
     request: IHttpResourceLoadDetails,
     settings: IHttp2ConnectSettings,

plugins/default-browser-emulator/injected-scripts/webrtc.ts

@@ -0,0 +1,32 @@
+const maskLocalIp = args.localIp;
+const replacementIp = args.proxyIp;
+
+if ('RTCIceCandidate' in self && RTCIceCandidate.prototype) {
+  proxyGetter(RTCIceCandidate.prototype, 'candidate', function () {
+    const result = ReflectCached.apply(...arguments);
+    return result.replace(maskLocalIp, replacementIp);
+  });
+  if ('address' in RTCIceCandidate.prototype) {
+    // @ts-ignore
+    proxyGetter(RTCIceCandidate.prototype, 'address', function () {
+      const result: string = ReflectCached.apply(...arguments);
+      return result.replace(maskLocalIp, replacementIp);
+    });
+  }
+  proxyFunction(RTCIceCandidate.prototype, 'toJSON', function () {
+    const json = ReflectCached.apply(...arguments);
+    if ('address' in json) json.address = json.address.replace(maskLocalIp, replacementIp);
+    if ('candidate' in json) json.candidate = json.candidate.replace(maskLocalIp, replacementIp);
+    return json;
+  });
+}
+
+if ('RTCSessionDescription' in self && RTCSessionDescription.prototype) {
+  proxyGetter(RTCSessionDescription.prototype, 'sdp', function () {
+    let result = ReflectCached.apply(...arguments);
+    while (result.indexOf(maskLocalIp) !== -1) {
+      result = result.replace(maskLocalIp, replacementIp);
+    }
+    return result;
+  });
+}