chore: fix double agent

.eslintignore

@@ -3,3 +3,6 @@ build
 *.config.js
 tsconfig*.json
 double-agent
+jest.*.js
+**/install.js
+**/updateSeccompProfile.js

.github/workflows/double-agent-probes.yml

@@ -39,6 +39,9 @@ jobs:
       - name: Yarn build
         run: yarn build:simple
 
+      - name: Import Profiles
+        run: yarn workspace @ulixee/double-agent-stacks sync-config
+
       - name: Import Profiles
         run: yarn workspace @ulixee/double-agent-stacks 0
 

.husky/commit-msg

@@ -1,4 +1 @@
-#!/bin/sh
-. "$(dirname "$0")/_/husky.sh"
-
-npx commitlint --edit $1
+yarn commitlint --edit $1

.husky/pre-commit

@@ -1,4 +1 @@
-#!/bin/sh
-. "$(dirname "$0")/_/husky.sh"
-
-npx lint-staged
+yarn lint-staged

.husky/pre-push

@@ -1,4 +1 @@
-#!/bin/sh
-. "$(dirname "$0")/_/husky.sh"
-
-npm test
+yarn test

.yarnrc

@@ -0,0 +1,5 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+yarn-path ".yarn/releases/yarn-1.22.22.cjs"

agent/main/index.mjs

@@ -1,6 +1,5 @@
 import cjsImport from './index.js';
 
-const { CoreProcess, BrowserEmulators, HumanEmulators } = cjsImport;
+const { Agent, Page, Pool, BrowserContext, Browser } = cjsImport;
 
-export { CoreProcess, BrowserEmulators, HumanEmulators };
-export default cjsImport.default;
+export { Agent, Page, Pool, BrowserContext, Browser };

agent/mitm-socket/test/proxy.test.ts

@@ -1,16 +1,16 @@
+import { createPromise } from '@ulixee/commons/lib/utils';
 import { Helpers, TestLogger } from '@ulixee/unblocked-agent-testing';
-import * as Proxy from 'proxy';
-import * as http from 'http';
 import {
   getTlsConnection,
   httpGetWithSocket,
   readableToBuffer,
 } from '@ulixee/unblocked-agent-testing/helpers';
-import WebSocket = require('ws');
-import * as socks5 from 'simple-socks';
-import { createPromise } from '@ulixee/commons/lib/utils';
+import * as http from 'http';
 import * as http2 from 'http2';
-import exp = require('node:constants');
+import { AddressInfo } from 'node:net';
+import { createProxy } from 'proxy';
+import * as socks5 from 'simple-socks';
+import WebSocket = require('ws');
 import MitmSocket from '../index';
 import MitmSocketSession from '../lib/MitmSocketSession';
 
@@ -35,7 +35,7 @@ beforeAll(() => {
 test('should be able to send a request through a proxy', async () => {
   const htmlString = 'Proxy proxy echo echo';
   const proxy = await startProxy();
-  const proxyPort = proxy.address().port;
+  const proxyPort = (proxy.address() as AddressInfo).port;
   const connect = jest.fn();
   proxy.once('connect', connect);
 
@@ -62,17 +62,10 @@ test('should be able to send a request through a secure proxy with auth', async
   const password = `u:password`;
   const pass64 = Buffer.from(password).toString('base64');
   const proxyServer = await Helpers.runHttpsServer((req, res) => res.end(htmlString));
-  const proxy = new Proxy(proxyServer.server);
-  proxy.authenticate = (
-    req: http.IncomingMessage,
-    cb: (req: http.IncomingMessage, success: boolean) => any,
-  ) => {
+  const proxy = createProxy(proxyServer.server);
+  proxy.authenticate = (req: http.IncomingMessage) => {
     const auth = req.headers['proxy-authorization'];
-    const isValid = auth === `Basic ${pass64}`;
-    if (!isValid) {
-      return cb(null, false);
-    }
-    cb(null, true);
+    return auth === `Basic ${pass64}`;
   };
   const connect = jest.fn();
   proxy.once('connect', connect);
@@ -92,17 +85,10 @@ test('should be able to send a request through a secure proxy with auth using sp
   const password = `u:abcDEF!123-_`;
   const pass64 = Buffer.from(password).toString('base64');
   const proxyServer = await Helpers.runHttpsServer((req, res) => res.end(htmlString));
-  const proxy = new Proxy(proxyServer.server);
-  proxy.authenticate = (
-    req: http.IncomingMessage,
-    cb: (req: http.IncomingMessage, success: boolean) => any,
-  ) => {
+  const proxy = createProxy(proxyServer.server);
+  proxy.authenticate = (req: http.IncomingMessage) => {
     const auth = req.headers['proxy-authorization'];
-    const isValid = auth === `Basic ${pass64}`;
-    if (!isValid) {
-      return cb(null, false);
-    }
-    cb(null, true);
+    return auth === `Basic ${pass64}`;
   };
   const connect = jest.fn();
   proxy.once('connect', connect);
@@ -122,7 +108,7 @@ test('should be able to use a socks5 proxy', async () => {
   await new Promise(resolve => proxy.listen(0, resolve));
   Helpers.needsClosing.push(proxy);
 
-  const proxyPort = proxy.address().port;
+  const proxyPort = (proxy.address() as AddressInfo).port;
   const htmlString = 'Proxy proxy echo echo';
   const connect = jest.fn();
   proxy.once('proxyConnect', connect);
@@ -190,7 +176,6 @@ test('should be able to use a socks5 proxy with auth', async () => {
   expect(auth).toHaveBeenCalledTimes(1);
 });
 
-
 test('should be able to use a socks5 proxy with auth, using special characters', async () => {
   const proxy = socks5.createServer({
     authenticate(username, password, socket, callback) {
@@ -240,7 +225,7 @@ test('should be able to use a socks5 proxy with auth, using special characters',
 
 test('should handle websockets over proxies', async () => {
   const proxy = await startProxy();
-  const proxyPort = proxy.address().port;
+  const proxyPort = (proxy.address() as AddressInfo).port;
   const connect = jest.fn();
   proxy.once('connect', connect);
 
@@ -274,7 +259,7 @@ test('should handle websockets over proxies', async () => {
 
 async function startProxy() {
   const proxyPromise = createPromise();
-  const proxy = new Proxy(http.createServer());
+  const proxy = createProxy(http.createServer());
   proxy.listen(0, () => {
     proxyPromise.resolve();
   });

agent/mitm/package.json

@@ -13,10 +13,9 @@
   "devDependencies": {
     "@types/node": "^18.19.10",
     "@types/ws": "^7.4.7",
-    "@ulixee/unblocked-agent": "2.0.0-alpha.29",
     "@ulixee/unblocked-agent-testing": "2.0.0-alpha.29",
-    "http-proxy-agent": "^4.0.1",
-    "proxy": "^1.0.1",
+    "http-proxy-agent": "^7.0.2",
+    "proxy": "^2.2.0",
     "ws": "^8.18.0"
   }
 }

agent/mitm/test/MitmRequestAgent.test.ts

@@ -6,7 +6,7 @@ import { Helpers, TestLogger } from '@ulixee/unblocked-agent-testing';
 import { getProxyAgent, runHttpsServer } from '@ulixee/unblocked-agent-testing/helpers';
 import CertificateGenerator from '@ulixee/unblocked-agent-mitm-socket/lib/CertificateGenerator';
 import WebSocket = require('ws');
-import HttpProxyAgent = require('http-proxy-agent');
+import { HttpProxyAgent } from 'http-proxy-agent';
 import MitmServer from '../lib/MitmProxy';
 import RequestSession from '../handlers/RequestSession';
 import HeadersHandler from '../handlers/HeadersHandler';
@@ -211,12 +211,9 @@ test('it should not put upgrade connections in a pool', async () => {
     });
   });
 
+  const proxyUrl = `http://${session.getProxyCredentials()}@localhost:${mitmServer.port}`;
   const wsClient = new WebSocket(`ws://localhost:${httpServer.port}`, {
-    agent: HttpProxyAgent({
-      host: 'localhost',
-      port: mitmServer.port,
-      auth: session.getProxyCredentials(),
-    }),
+    agent: new HttpProxyAgent(proxyUrl),
   });
   Helpers.onClose(async () => wsClient.close());
 

agent/mitm/test/basic.test.ts

@@ -1,22 +1,21 @@
+import { createPromise } from '@ulixee/commons/lib/utils';
+import CertificateGenerator from '@ulixee/unblocked-agent-mitm-socket/lib/CertificateGenerator';
+import { Helpers, TestLogger } from '@ulixee/unblocked-agent-testing';
+import IHttpResourceLoadDetails from '@ulixee/unblocked-specification/agent/net/IHttpResourceLoadDetails';
 import * as http from 'http';
 import { IncomingHttpHeaders } from 'http';
-import { Helpers, TestLogger } from '@ulixee/unblocked-agent-testing';
-import * as Url from 'url';
+import { HttpProxyAgent } from 'http-proxy-agent';
 import { AddressInfo, Socket } from 'net';
-import { createPromise } from '@ulixee/commons/lib/utils';
-import IHttpResourceLoadDetails from '@ulixee/unblocked-specification/agent/net/IHttpResourceLoadDetails';
-import CertificateGenerator from '@ulixee/unblocked-agent-mitm-socket/lib/CertificateGenerator';
-import HttpProxyAgent = require('http-proxy-agent');
 import WebSocket = require('ws');
+import env from '../env';
+import HeadersHandler from '../handlers/HeadersHandler';
 import HttpRequestHandler from '../handlers/HttpRequestHandler';
+import HttpUpgradeHandler from '../handlers/HttpUpgradeHandler';
 import RequestSession, { IRequestSessionRequestEvent } from '../handlers/RequestSession';
+import { MitmProxy } from '../index';
+import IBrowserRequestMatcher from '../interfaces/IBrowserRequestMatcher';
 import MitmServer from '../lib/MitmProxy';
-import HeadersHandler from '../handlers/HeadersHandler';
-import HttpUpgradeHandler from '../handlers/HttpUpgradeHandler';
 import { parseRawHeaders } from '../lib/Utils';
-import IBrowserRequestMatcher from '../interfaces/IBrowserRequestMatcher';
-import env from '../env';
-import { MitmProxy } from '../index';
 
 const mocks = {
   httpRequestHandler: {
@@ -312,11 +311,9 @@ describe('basic MitM tests', () => {
       });
     });
 
+    const proxyUrl = `http://${session.getProxyCredentials()}@localhost:${mitmServer.port}`;
     const wsClient = new WebSocket(`ws://localhost:${httpServer.port}`, {
-      agent: HttpProxyAgent({
-        ...Url.parse(`http://localhost:${mitmServer.port}`),
-        auth: session.getProxyCredentials(),
-      }),
+      agent: new HttpProxyAgent(proxyUrl),
     });
 
     Helpers.onClose(async () => wsClient.close());

agent/testing/helpers.ts

@@ -1,27 +1,28 @@
+import { CanceledPromiseError } from '@ulixee/commons/interfaces/IPendingWaitEvent';
+import { getSourcedir } from '@ulixee/commons/lib/dirUtils';
+import Logger from '@ulixee/commons/lib/Logger';
+import { createPromise } from '@ulixee/commons/lib/utils';
+import MitmSocket from '@ulixee/unblocked-agent-mitm-socket';
+import MitmSocketSession from '@ulixee/unblocked-agent-mitm-socket/lib/MitmSocketSession';
+import MitmEnv from '@ulixee/unblocked-agent-mitm/env'; // eslint-disable-line import/no-extraneous-dependencies
 import * as Fs from 'fs';
-import * as Path from 'path';
-import * as Url from 'url';
-import { URL } from 'url';
-import * as querystring from 'querystring';
 import * as http from 'http';
 import { IncomingMessage, RequestListener, Server } from 'http';
+import * as http2 from 'http2';
 import * as https from 'https';
 import { Agent } from 'https';
-import { createPromise } from '@ulixee/commons/lib/utils';
 import * as net from 'net';
-import * as tls from 'tls';
-import * as http2 from 'http2';
+import * as Path from 'path';
+import * as querystring from 'querystring';
 import * as stream from 'stream';
-import { CanceledPromiseError } from '@ulixee/commons/interfaces/IPendingWaitEvent';
-import MitmSocket from '@ulixee/unblocked-agent-mitm-socket';
-import MitmSocketSession from '@ulixee/unblocked-agent-mitm-socket/lib/MitmSocketSession';
-import MitmEnv from '@ulixee/unblocked-agent-mitm/env'; // eslint-disable-line import/no-extraneous-dependencies
-import Logger from '@ulixee/commons/lib/Logger';
-import HttpProxyAgent = require('http-proxy-agent');
-import HttpsProxyAgent = require('https-proxy-agent');
-import Koa = require('koa');
-import KoaRouter = require('@koa/router');
+import * as tls from 'tls';
+import * as Url from 'url';
+import { URL } from 'url';
 import KoaMulter = require('@koa/multer');
+import KoaRouter = require('@koa/router');
+import { HttpProxyAgent } from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import Koa = require('koa');
 import { Helpers, TestLogger } from './index';
 
 const { log } = Logger(module);
@@ -50,10 +51,12 @@ export interface ITestHttpServer<T> {
   server: T;
 }
 
+const sourcedir = getSourcedir(__dirname);
+
 export async function runKoaServer(onlyCloseOnFinal = true): Promise<ITestKoaServer> {
   const koa = new Koa();
   const router = new KoaRouter() as ITestKoaServer;
-  const exampleOrgPath = Path.join(__dirname, 'html', 'example.org.html');
+  const exampleOrgPath = Path.join(sourcedir, 'html', 'example.org.html');
   const exampleOrgHtml = Fs.readFileSync(exampleOrgPath, 'utf-8');
   const upload = KoaMulter(); // note you can pass `multer` options here
 
@@ -96,8 +99,8 @@ export async function runKoaServer(onlyCloseOnFinal = true): Promise<ITestKoaSer
 
 export function sslCerts() {
   return {
-    key: Fs.readFileSync(`${__dirname}/certs/key.pem`),
-    cert: Fs.readFileSync(`${__dirname}/certs/cert.pem`),
+    key: Fs.readFileSync(`${sourcedir}/certs/key.pem`),
+    cert: Fs.readFileSync(`${sourcedir}/certs/cert.pem`),
   };
 }
 
@@ -262,10 +265,12 @@ export function httpRequest(
 }
 
 export function getProxyAgent(url: URL, proxyHost: string, auth?: string): Agent {
-  const ProxyAgent = url.protocol === 'https:' ? HttpsProxyAgent : HttpProxyAgent;
-  const opts = Url.parse(proxyHost);
-  opts.auth = auth;
-  return ProxyAgent(opts);
+  const proxyUrl = new URL(proxyHost);
+  if (auth) {
+    proxyUrl.username = auth.split(':')[0];
+    proxyUrl.password = auth.split(':')[1];
+  }
+  return url.protocol === 'https:' ? new HttpsProxyAgent(proxyUrl) : new HttpProxyAgent(proxyUrl);
 }
 
 export function httpGet(
@@ -415,7 +420,7 @@ export function getTlsConnection(
 }
 
 export function getLogo(): Buffer {
-  return Fs.readFileSync(`${__dirname}/html/img.png`);
+  return Fs.readFileSync(`${sourcedir}/html/img.png`);
 }
 
 export async function readableToBuffer(res: stream.Readable): Promise<Buffer> {

agent/testing/package.json

@@ -8,22 +8,22 @@
     "node": ">=18"
   },
   "dependencies": {
-    "@koa/multer": "^3.0.0",
-    "@koa/router": "^8.0.8",
+    "@koa/multer": "^3.0.2",
+    "@koa/router": "^13.0.0",
     "@ulixee/commons": "2.0.0-alpha.29",
     "@ulixee/unblocked-agent": "2.0.0-alpha.29",
+    "@ulixee/unblocked-agent-mitm": "2.0.0-alpha.29",
     "@ulixee/unblocked-agent-mitm-socket": "2.0.0-alpha.29",
     "@ulixee/unblocked-specification": "2.0.0-alpha.29",
-    "http-proxy-agent": "^4.0.1",
-    "https-proxy-agent": "^5.0.0",
-    "koa": "^2.11.0",
+    "http-proxy-agent": "^7.0.2",
+    "https-proxy-agent": "^7.0.5",
+    "koa": "^2.15.3",
     "multer": "^1.4.5-lts.1"
   },
   "devDependencies": {
-    "@types/koa": "^2.11.3",
-    "@types/koa__multer": "^2.0.2",
-    "@types/koa__router": "^8.0.2",
-    "@types/node": "^18.19.10",
-    "@ulixee/unblocked-agent-mitm": "2.0.0-alpha.29"
+    "@types/koa": "^2.15.0",
+    "@types/koa__multer": "^2.0.7",
+    "@types/koa__router": "^12.0.4",
+    "@types/node": "^18.19.10"
   }
 }

double-agent-stacks/data/external/userAgentConfig.json

@@ -1,3 +1,3 @@
 {
-  "browserIds": ["chrome-105-0"]
+  "browserIds": ["chrome-124-0", "chrome-123-0"]
 }

double-agent-stacks/data/local/userAgentConfig.json

@@ -1,3 +1,3 @@
 {
-  "browserIds": ["chrome-105-0"]
+  "browserIds": []
 }

double-agent-stacks/scripts/4-analyzeAssignmentResults.ts

@@ -6,5 +6,6 @@ import { getExternalDataPath } from '../paths';
 const assignmentsDataDir = getExternalDataPath(`/3-assignments`);
 const resultsDir = getExternalDataPath(`/4-assignment-results`);
 
-Fs.rmSync(resultsDir, { recursive: true });
+if (Fs.existsSync(resultsDir)) Fs.rmSync(resultsDir, { recursive: true });
+
 analyzeAssignmentResults(assignmentsDataDir, resultsDir).catch(console.log);