feat(agent): add request/response body intercept

agent/examples/package.json

@@ -4,6 +4,7 @@
   "version": "2.0.0-alpha.23",
   "dependencies": {
     "@ulixee/chrome-113-0": "^5672.127.8",
+    "@ulixee/js-path": "2.0.0-alpha.23",
     "@ulixee/unblocked-agent": "2.0.0-alpha.23"
   }
 }

agent/main/lib/Plugins.ts

@@ -62,7 +62,9 @@ export default class Plugins implements IUnblockedPlugins {
     onHttp2SessionConnect: [],
     shouldBlockRequest: [],
     beforeHttpRequest: [],
+    beforeHttpRequestBody: [],
     beforeHttpResponse: [],
+    beforeHttpResponseBody: [],
     afterHttpResponse: [],
     websiteHasFirstPartyInteraction: [],
   };
@@ -228,10 +230,18 @@ export default class Plugins implements IUnblockedPlugins {
     await Promise.all(this.hooksByName.beforeHttpRequest.map(fn => fn(resource)));
   }
 
+  public async beforeHttpRequestBody(resource: IHttpResourceLoadDetails): Promise<void> {
+    await Promise.all(this.hooksByName.beforeHttpRequestBody.map(fn => fn(resource)));
+  }
+
   public async beforeHttpResponse(resource: IHttpResourceLoadDetails): Promise<any> {
     await Promise.all(this.hooksByName.beforeHttpResponse.map(fn => fn(resource)));
   }
 
+  public async beforeHttpResponseBody(resource: IHttpResourceLoadDetails): Promise<void> {
+    await Promise.all(this.hooksByName.beforeHttpResponseBody.map(fn => fn(resource)));
+  }
+
   public async afterHttpResponse(resource: IHttpResourceLoadDetails): Promise<any> {
     await Promise.all(this.hooksByName.afterHttpResponse.map(fn => fn(resource)));
   }

agent/main/package.json

@@ -10,7 +10,7 @@
     "@ulixee/js-path": "2.0.0-alpha.23",
     "@ulixee/unblocked-agent-mitm": "2.0.0-alpha.23",
     "@ulixee/unblocked-specification": "2.0.0-alpha.23",
-    "devtools-protocol": "^0.0.981744",
+    "devtools-protocol": "^0.0.1137505",
     "nanoid": "^3.3.6",
     "tough-cookie": "^4.0.0"
   },

agent/main/test/_pageTestUtils.ts

@@ -57,7 +57,7 @@ export async function navigateFrame(
   }
 }
 
-export async function waitForVisible(frame: Frame, selector: string): Promise<INodeVisibility> {
+export async function waitForVisible(frame: Frame, selector: string, timeoutMs = 10e3): Promise<INodeVisibility> {
   let visibility: INodeVisibility;
   await wait(
     async () => {
@@ -66,7 +66,7 @@ export async function waitForVisible(frame: Frame, selector: string): Promise<IN
         return true;
       }
     },
-    { loopDelayMs: 100, timeoutMs: 10e3 },
+    { loopDelayMs: 100, timeoutMs },
   );
   return visibility;
 }

agent/main/test/mitm.test.ts

@@ -4,7 +4,10 @@ import { createPromise } from '@ulixee/commons/lib/utils';
 import { LocationStatus } from '@ulixee/unblocked-specification/agent/browser/Location';
 import { ITestKoaServer } from '@ulixee/unblocked-agent-testing/helpers';
 import Resolvable from '@ulixee/commons/lib/Resolvable';
+import IHttpResourceLoadDetails from '@ulixee/unblocked-specification/agent/net/IHttpResourceLoadDetails';
+import { Readable } from 'stream';
 import { Pool } from '../index';
+import { waitForVisible } from './_pageTestUtils';
 
 const mocks = {
   MitmRequestContext: {
@@ -320,3 +323,70 @@ This is the main body
     'https://dataliberationfoundation.org/dlfSite.png',
   ]);
 });
+
+
+test('should be able to intercept http requests and responses', async () => {
+  const agent = pool.createAgent({ logger: TestLogger.forTest(module) });
+  agent.hook({
+    async beforeHttpRequest(request: IHttpResourceLoadDetails): Promise<any> {
+      if (request.url.pathname === '/intercept-post') {
+        // NOTE: need to delete the content length (or set to correct value)
+        delete request.requestHeaders['Content-Length'];
+      }
+    },
+    async beforeHttpRequestBody(request: IHttpResourceLoadDetails): Promise<any> {
+
+      if (request.url.pathname === '/intercept-post') {
+        // drain first
+        for await (const _ of request.requestPostDataStream) {}
+        // send body. NOTE: we had to change out the content length before the body step
+        request.requestPostDataStream = Readable.from(Buffer.from('Intercept request'));
+      }
+    },
+    async beforeHttpResponse(response: IHttpResourceLoadDetails): Promise<any> {
+      if (response.url.pathname === '/intercept-post') {
+        response.responseHeaders['Content-Length'] = 'Intercepted'.length.toString();
+      }
+    },
+    async beforeHttpResponseBody(response: IHttpResourceLoadDetails): Promise<any> {
+      if (response.url.pathname === '/intercept-post') {
+        for await (const _ of response.responseBodyStream) {
+        }
+        response.responseBodyStream = Readable.from(Buffer.from('Intercepted'));
+      }
+    }
+  });
+  const page = await agent.newPage();
+  const requestPost = new Resolvable<string>();
+  koa.post('/intercept-post', async ctx => {
+    let request = '';
+    for await (const chunk of ctx.req) {
+      request += chunk.toString();
+    }
+    requestPost.resolve(request);
+    ctx.body = 'Result';
+  });
+  koa.get('/intercept', async ctx => {
+    ctx.body = `<html>
+<body>
+<h1>Nothing</h1>
+</body>
+<script type='text/javascript'>
+ fetch('/intercept-post', {
+    method: 'POST',
+    body: 'Send',
+  })
+  .then(x => x.text())
+  .then(x => {
+    document.querySelector('h1').textContent = x;
+    document.body.classList.add('ready');
+  });
+</script>
+</html>`;
+  });
+  await page.goto(`${koa.baseUrl}/intercept`);
+  await page.waitForLoad(LocationStatus.AllContentLoaded);
+  await expect(requestPost).resolves.toBe('Intercept request');
+  await waitForVisible(page.mainFrame, 'body.ready', 5e3);
+  await expect(page.evaluate('document.querySelector("h1").textContent')).resolves.toBe('Intercepted');
+});

agent/mitm/handlers/HttpRequestHandler.ts

@@ -1,14 +1,14 @@
-import * as http from 'http';
+import { CanceledPromiseError } from '@ulixee/commons/interfaces/IPendingWaitEvent';
 import Log, { hasBeenLoggedSymbol } from '@ulixee/commons/lib/Logger';
+import * as http from 'http';
 import { ClientHttp2Stream, Http2ServerRequest, Http2ServerResponse } from 'http2';
-import { CanceledPromiseError } from '@ulixee/commons/interfaces/IPendingWaitEvent';
 import IMitmRequestContext from '../interfaces/IMitmRequestContext';
-import HeadersHandler from './HeadersHandler';
+import ResourceState from '../interfaces/ResourceState';
+import HttpResponseCache from '../lib/HttpResponseCache';
 import MitmRequestContext from '../lib/MitmRequestContext';
 import { parseRawHeaders } from '../lib/Utils';
 import BaseHttpHandler from './BaseHttpHandler';
-import HttpResponseCache from '../lib/HttpResponseCache';
-import ResourceState from '../interfaces/ResourceState';
+import HeadersHandler from './HeadersHandler';
 
 const { log } = Log(module);
 
@@ -117,7 +117,7 @@ export default class HttpRequestHandler extends BaseHttpHandler {
       return;
     }
 
-    await context.requestSession.willSendResponse(context);
+    await context.requestSession.willSendHttpResponse(context);
 
     try {
       this.writeResponseHead();
@@ -131,7 +131,7 @@ export default class HttpRequestHandler extends BaseHttpHandler {
       return this.onError('ServerToProxyToClient.ReadWriteResponseError', err);
     }
 
-    await context.requestSession.haveSentResponse(context);
+    await context.requestSession.didSendHttpResponse(context);
 
     context.setState(ResourceState.End);
     this.cleanup();
@@ -210,11 +210,15 @@ export default class HttpRequestHandler extends BaseHttpHandler {
       }
     };
 
+    this.context.requestPostDataStream = clientToProxyRequest;
+    await this.context.requestSession.willSendHttpRequestBody(this.context);
+
     const data: Buffer[] = [];
-    for await (const chunk of clientToProxyRequest) {
+    for await (const chunk of this.context.requestPostDataStream) {
       data.push(chunk);
       proxyToServerRequest.write(chunk, onWriteError);
     }
+    delete this.context.requestPostDataStream;
 
     HeadersHandler.sendRequestTrailers(this.context);
     await new Promise(resolve => proxyToServerRequest.end(resolve));
@@ -267,13 +271,15 @@ export default class HttpRequestHandler extends BaseHttpHandler {
     context.setState(ResourceState.WriteProxyToClientResponseBody);
 
     context.responseBodySize = 0;
-
-    for await (const chunk of serverToProxyResponse) {
+    context.responseBodyStream = serverToProxyResponse;
+    await this.context.requestSession.willSendHttpResponseBody(this.context);
+    for await (const chunk of context.responseBodyStream) {
       const buffer = chunk as Buffer;
       context.responseBodySize += buffer.length;
       const data = context.cacheHandler.onResponseData(buffer);
       this.safeWriteToClient(data);
     }
+    delete this.context.responseBodyStream;
 
     if (context.cacheHandler.shouldServeCachedData) {
       this.safeWriteToClient(context.cacheHandler.cacheData);

agent/mitm/handlers/HttpUpgradeHandler.ts

@@ -97,14 +97,14 @@ export default class HttpUpgradeHandler extends BaseHttpHandler {
       responseMessage += `${serverResponse.rawHeaders[i]}: ${serverResponse.rawHeaders[i + 1]}\r\n`;
     }
     this.context.responseBodySize = 0;
-    await requestSession.willSendResponse(this.context);
+    await requestSession.willSendHttpResponse(this.context);
 
     this.context.setState(ResourceState.WriteProxyToClientResponseBody);
     clientSocket.write(`${responseMessage}\r\n`, error => {
       if (error) this.onError('ProxyToClient.UpgradeWriteError', error);
     });
 
-    await requestSession.haveSentResponse(this.context);
+    await requestSession.didSendHttpResponse(this.context);
 
     if (!serverSocket.readable || !serverSocket.writable) {
       this.context.setState(ResourceState.PrematurelyClosed);

agent/mitm/handlers/RequestSession.ts

@@ -72,7 +72,7 @@ export default class RequestSession
     hooks: INetworkHooks,
     logger: IBoundLog,
     public upstreamProxyUrl?: string,
-    public upstreamProxyUseSystemDns?: boolean
+    public upstreamProxyUseSystemDns?: boolean,
   ) {
     super();
     this.logger = logger.createChild(module);
@@ -118,7 +118,14 @@ export default class RequestSession
     }
   }
 
-  public async willSendResponse(context: IMitmRequestContext): Promise<void> {
+  // NOTE: must change names from plugin callbacks or it will loop back here
+  public async willSendHttpRequestBody(context: IMitmRequestContext): Promise<void> {
+    for (const hook of this.hooks) {
+      await hook.beforeHttpRequestBody?.(context);
+    }
+  }
+
+  public async willSendHttpResponse(context: IMitmRequestContext): Promise<void> {
     context.setState(ResourceState.EmulationWillSendResponse);
 
     if (context.resourceType === 'Document' && context.status === 200) {
@@ -131,7 +138,13 @@ export default class RequestSession
     }
   }
 
-  public async haveSentResponse(context: IMitmRequestContext): Promise<void> {
+  public async willSendHttpResponseBody(context: IMitmRequestContext): Promise<void> {
+    for (const hook of this.hooks) {
+      await hook.beforeHttpResponseBody?.(context);
+    }
+  }
+
+  public async didSendHttpResponse(context: IMitmRequestContext): Promise<void> {
     for (const hook of this.hooks) {
       await hook.afterHttpResponse?.(context);
     }

agent/mitm/package.json

@@ -13,6 +13,7 @@
   "devDependencies": {
     "@types/node": "^16.18.31",
     "@types/ws": "^7.4.7",
+    "@ulixee/unblocked-agent": "2.0.0-alpha.23",
     "@ulixee/unblocked-agent-testing": "2.0.0-alpha.23",
     "http-proxy-agent": "^4.0.1",
     "proxy": "^1.0.1",

agent/testing/package.json

@@ -23,6 +23,7 @@
     "@types/koa": "^2.11.3",
     "@types/koa__multer": "^2.0.2",
     "@types/koa__router": "^8.0.2",
-    "@types/node": "^16.18.31"
+    "@types/node": "^16.18.31",
+    "@ulixee/unblocked-agent-mitm": "2.0.0-alpha.23"
   }
 }

browser-emulator-builder/package.json

@@ -12,7 +12,10 @@
     "@double-agent/collect": "2.0.0-alpha.23",
     "@double-agent/collect-browser-codecs": "2.0.0-alpha.23",
     "@double-agent/collect-browser-dom-environment": "2.0.0-alpha.23",
+    "@double-agent/collect-browser-fonts": "2.0.0-alpha.23",
+    "@double-agent/collect-browser-speech": "2.0.0-alpha.23",
     "@double-agent/collect-http-basic-headers": "2.0.0-alpha.23",
+    "@double-agent/collect-http-ua-hints": "2.0.0-alpha.23",
     "@double-agent/collect-http2-session": "2.0.0-alpha.23",
     "@double-agent/collect-tls-clienthello": "2.0.0-alpha.23",
     "@ulixee/commons": "2.0.0-alpha.23",

double-agent/collect/plugins/browser-dom-environment/package.json

@@ -4,7 +4,6 @@
   "main": "index.js",
   "private": true,
   "license": "MIT",
-  "scripts": {},
   "description": "Collects the browser's DOM environment such as object structure, class inheritance amd key order",
   "dependencies": {
     "@double-agent/collect": "2.0.0-alpha.23"

double-agent/collect/plugins/tls-clienthello/package.json

@@ -4,7 +4,6 @@
   "main": "index.js",
   "private": true,
   "license": "MIT",
-  "scripts": {},
   "description": "Collects the TLS clienthello handshake when initiating a secure connection",
   "dependencies": {
     "@double-agent/collect": "2.0.0-alpha.23",

package.json

@@ -35,9 +35,9 @@
   "devDependencies": {
     "@commitlint/cli": "^17.6.3",
     "@commitlint/config-conventional": "^17.6.3",
-    "@ulixee/repo-tools": "^1.0.26",
-    "@types/node": "^16.18.31",
     "@types/jest": "^29.5.1",
+    "@types/node": "^16.18.31",
+    "@ulixee/repo-tools": "^1.0.26",
     "concurrently": "^6.2.1",
     "cross-env": "^7.0.3",
     "husky": "^8.0.3",
@@ -71,6 +71,5 @@
     "*.json": [
       "prettier --write"
     ]
-  },
-  "dependencies": {}
+  }
 }

plugins/default-browser-emulator/lib/helpers/configureBrowserLaunchArgs.ts

@@ -69,7 +69,11 @@ export function configureBrowserLaunchArgs(
 
   if (options.showChrome) {
     if (options.showDevtools) engine.launchArguments.push('--auto-open-devtools-for-tabs');
-  } else {
+  } else { if (process.platform === 'darwin') {
+    if (process.arch === 'arm64') {
+      engine.launchArguments.push('--use-gl=any');
+    }
+  }
     engine.launchArguments.push(
       '--hide-scrollbars',
       '--mute-audio',