refactor: splits provider proxy into services and routes to improve f…

…uture testing (#696)

apps/provider-proxy/jest.config.js

@@ -0,0 +1,23 @@
+const common = {
+  transform: {
+    "^.+\\.(t|j)s$": ["ts-jest", { tsconfig: "./tsconfig.json" }]
+  },
+  rootDir: "."
+};
+
+module.exports = {
+  collectCoverageFrom: ["./src/**/*.{js,ts}"],
+  projects: [
+    {
+      displayName: "unit",
+      ...common,
+      testMatch: ["<rootDir>/src/**/*.spec.ts"]
+    },
+    {
+      displayName: "functional",
+      ...common,
+      testMatch: ["<rootDir>/test/functional/**/*.spec.ts"],
+      setupFilesAfterEnv: ["./test/setup-functional-tests.ts"]
+    }
+  ]
+};

apps/provider-proxy/package.json

@@ -7,12 +7,15 @@
   "main": "main.js",
   "scripts": {
     "build": "npx tsc",
+    "dev": "npm run start",
+    "dev-nodc": "npm run start",
     "format": "prettier --write ./*.{ts,js,json} **/*.{ts,js,json}",
     "lint": "eslint .",
+    "prod": "node ./dist/server.js",
     "start": "npm run build && node ./dist/server.js",
-    "dev": "npm run start",
-    "dev-nodc": "npm run start",
-    "prod": "node ./dist/server.js"
+    "test:functional": "jest --selectProjects functional",
+    "test:functional:cov": "jest --selectProjects functional --coverage",
+    "test:functional:watch": "jest --selectProjects functional --watch"
   },
   "dependencies": {
     "axios": "^1.7.2",
@@ -33,6 +36,7 @@
     "eslint": "^8.57.0",
     "eslint-config-next": "^14.2.3",
     "eslint-plugin-simple-import-sort": "^12.1.0",
+    "jest": "^29.7.0",
     "prettier": "^3.3.0",
     "prettier-plugin-tailwindcss": "^0.6.1",
     "typescript": "5.1.3"

apps/provider-proxy/server.ts

@@ -1,128 +1,13 @@
-import cors from "cors";
-import express, { Express, Request, Response } from "express";
 import http from "http";
-import { Agent } from "https";
-import fetch, { Headers } from "node-fetch";
 import { v4 as uuidv4 } from "uuid";
 import WebSocket from "ws";
 
-import { ClientWebSocketStats, WebSocketUsage } from "./clientSocketStats";
-import packageJson from "./package.json";
-import { humanFileSize } from "./sizeUtils";
-
-const app: Express = express();
+import { app } from "./src/app";
+import { ClientWebSocketStats, WebSocketUsage } from "./src/ClientSocketStats";
+import { container } from "./src/container";
 
 const { PORT = 3040 } = process.env;
 
-const webSocketStats: ClientWebSocketStats[] = [];
-
-const whitelist = [
-  "http://localhost:3001",
-  "http://localhost:3000",
-  "https://cloudmos.grafana.net",
-  "https://console.akash.network",
-  "https://staging-console.akash.network",
-  "https://akashconsole.vercel.app",
-  "https://console-beta.akash.network"
-];
-
-app.use(
-  cors({
-    origin: function (origin, callback) {
-      if (!origin || whitelist.indexOf(origin) !== -1) {
-        callback(null, true);
-      } else {
-        console.log("Cors refused: " + origin);
-        callback(new Error("Not allowed by CORS"));
-      }
-    }
-  })
-);
-app.use(express.json());
-
-app.get("/status", async (req: Request, res: Response) => {
-  const openClientWebSocketCount = webSocketStats.filter(x => !x.isClosed()).length;
-  const totalRequestCount = webSocketStats.reduce((a, b) => a + b.getStats().totalStats.count, 0);
-  const totalTransferred = webSocketStats.reduce((a, b) => a + b.getStats().totalStats.data, 0);
-
-  const logStreaming = webSocketStats
-    .map(s => s.getStats().usageStats["StreamLogs"])
-    .reduce((a, b) => ({ count: a.count + b.count, data: a.data + b.data }), {
-      count: 0,
-      data: 0
-    });
-  const logDownload = webSocketStats
-    .map(s => s.getStats().usageStats["DownloadLogs"])
-    .reduce((a, b) => ({ count: a.count + b.count, data: a.data + b.data }), {
-      count: 0,
-      data: 0
-    });
-  const eventStreaming = webSocketStats
-    .map(s => s.getStats().usageStats["StreamEvents"])
-    .reduce((a, b) => ({ count: a.count + b.count, data: a.data + b.data }), {
-      count: 0,
-      data: 0
-    });
-  const shell = webSocketStats
-    .map(s => s.getStats().usageStats["Shell"])
-    .reduce((a, b) => ({ count: a.count + b.count, data: a.data + b.data }), {
-      count: 0,
-      data: 0
-    });
-
-  res.send({
-    openClientWebSocketCount,
-    totalRequestCount,
-    totalTransferred: humanFileSize(totalTransferred),
-    logStreaming: `${logStreaming.count} (${humanFileSize(logStreaming.data)})`,
-    logDownload: `${logDownload.count} (${humanFileSize(logDownload.data)})`,
-    eventStreaming: `${eventStreaming.count} (${humanFileSize(eventStreaming.data)})`,
-    shell: `${shell.count} (${humanFileSize(shell.data)})`,
-    version: packageJson.version
-  });
-});
-
-app.post("/", async (req: Request, res: Response, next) => {
-  const { certPem, keyPem, method, body, url } = req.body;
-
-  const myHeaders = new Headers();
-  myHeaders.append("Content-Type", "application/json");
-
-  try {
-    const httpsAgent = new Agent({
-      cert: certPem,
-      key: keyPem,
-      rejectUnauthorized: false
-    });
-
-    const response = await fetch(url, {
-      method: method,
-      body: body,
-      headers: myHeaders,
-      agent: httpsAgent
-    });
-
-    if (response.status === 200) {
-      const responseText = await response.text();
-      const contentType = response.headers.get("content-type");
-      if (contentType && contentType.indexOf("application/json") !== -1) {
-        res.contentType("application/json");
-      } else {
-        res.contentType("application/text");
-      }
-      res.send(responseText);
-    } else {
-      const _res = await response.text();
-      console.log("Status code was not success (" + response.status + ") : " + _res);
-
-      res.status(500);
-      res.send(_res);
-    }
-  } catch (error) {
-    next(error);
-  }
-});
-
 const httpServer = app.listen(PORT, () => {
   console.log(`Http server listening on port ${PORT}`);
 });
@@ -145,7 +30,7 @@ wss.on("connection", (ws: WebSocket, req: http.IncomingMessage) => {
   const id = uuidv4();
 
   const stats = new ClientWebSocketStats(id);
-  webSocketStats.push(stats);
+  container.wsStats.add(stats);
 
   console.log("Connection", req.url);
   ws.on("message", async (messageStr: string) => {

apps/provider-proxy/src/app.ts

@@ -0,0 +1,33 @@
+import cors from "cors";
+import express from "express";
+
+import { getAppStatus } from "./routes/getAppStatus";
+import { proxyProviderRequest } from "./routes/proxyProviderRequest";
+
+export const app = express();
+
+const whitelist = [
+  "http://localhost:3001",
+  "http://localhost:3000",
+  "https://cloudmos.grafana.net",
+  "https://console.akash.network",
+  "https://staging-console.akash.network",
+  "https://akashconsole.vercel.app",
+  "https://console-beta.akash.network"
+];
+
+app.use(
+  cors({
+    origin: function (origin, callback) {
+      if (!origin || whitelist.indexOf(origin) !== -1) {
+        callback(null, true);
+      } else {
+        console.log("Cors refused: " + origin);
+        callback(new Error("Not allowed by CORS"));
+      }
+    }
+  })
+);
+app.use(express.json());
+app.get("/status", getAppStatus);
+app.post("/", proxyProviderRequest);

apps/provider-proxy/src/container.ts

@@ -0,0 +1,7 @@
+import { ProviderProxy } from "./services/ProviderProxy";
+import { WebsocketStats } from "./services/WebsocketStats";
+
+export const container = {
+  wsStats: new WebsocketStats(),
+  providerProxy: new ProviderProxy()
+};

apps/provider-proxy/src/routes/getAppStatus.ts

@@ -0,0 +1,48 @@
+import { Request, Response } from "express";
+
+import packageJson from "../../package.json";
+import { container } from "../container";
+import { humanFileSize } from "../sizeUtils";
+
+export async function getAppStatus(_: Request, res: Response): Promise<void> {
+  const webSocketStats = container.wsStats.getItems();
+  const openClientWebSocketCount = webSocketStats.filter(x => !x.isClosed()).length;
+  const totalRequestCount = webSocketStats.reduce((a, b) => a + b.getStats().totalStats.count, 0);
+  const totalTransferred = webSocketStats.reduce((a, b) => a + b.getStats().totalStats.data, 0);
+
+  const logStreaming = webSocketStats
+    .map(s => s.getStats().usageStats["StreamLogs"])
+    .reduce((a, b) => ({ count: a.count + b.count, data: a.data + b.data }), {
+      count: 0,
+      data: 0
+    });
+  const logDownload = webSocketStats
+    .map(s => s.getStats().usageStats["DownloadLogs"])
+    .reduce((a, b) => ({ count: a.count + b.count, data: a.data + b.data }), {
+      count: 0,
+      data: 0
+    });
+  const eventStreaming = webSocketStats
+    .map(s => s.getStats().usageStats["StreamEvents"])
+    .reduce((a, b) => ({ count: a.count + b.count, data: a.data + b.data }), {
+      count: 0,
+      data: 0
+    });
+  const shell = webSocketStats
+    .map(s => s.getStats().usageStats["Shell"])
+    .reduce((a, b) => ({ count: a.count + b.count, data: a.data + b.data }), {
+      count: 0,
+      data: 0
+    });
+
+  res.send({
+    openClientWebSocketCount,
+    totalRequestCount,
+    totalTransferred: humanFileSize(totalTransferred),
+    logStreaming: `${logStreaming.count} (${humanFileSize(logStreaming.data)})`,
+    logDownload: `${logDownload.count} (${humanFileSize(logDownload.data)})`,
+    eventStreaming: `${eventStreaming.count} (${humanFileSize(eventStreaming.data)})`,
+    shell: `${shell.count} (${humanFileSize(shell.data)})`,
+    version: packageJson.version
+  });
+}

apps/provider-proxy/src/routes/proxyProviderRequest.ts

@@ -0,0 +1,38 @@
+import { NextFunction, Request as ExpressRequest, Response as ExpressResponse } from "express";
+
+import { container } from "../container";
+
+export async function proxyProviderRequest(req: ExpressRequest, res: ExpressResponse, next: NextFunction): Promise<void> {
+  const { certPem, keyPem, method, body, url } = req.body;
+
+  try {
+    const response = await container.providerProxy.fetch(url, {
+      headers: {
+        "Content-Type": "application/json"
+      },
+      method,
+      body,
+      cert: certPem,
+      key: keyPem
+    });
+
+    if (response.status >= 200 && response.status < 300) {
+      const responseText = await response.text();
+      const contentType = response.headers.get("content-type");
+      if (contentType && contentType.indexOf("application/json") !== -1) {
+        res.contentType("application/json");
+      } else {
+        res.contentType("application/text");
+      }
+      res.send(responseText);
+    } else {
+      const _res = await response.text();
+      console.log("Status code was not success (" + response.status + ") : " + _res);
+
+      res.status(500);
+      res.send(_res);
+    }
+  } catch (error) {
+    next(error);
+  }
+}

apps/provider-proxy/src/services/ProviderProxy.ts

@@ -0,0 +1,24 @@
+import https, { RequestOptions } from "https";
+import fetch, { BodyInit, Headers, Response } from "node-fetch";
+
+export class ProviderProxy {
+  fetch(url: string, options: FetchOptions): Promise<Response> {
+    const httpsAgent = new https.Agent({
+      cert: options.cert,
+      key: options.key,
+      rejectUnauthorized: false
+    });
+
+    return fetch(url, {
+      method: options.method,
+      body: options.body,
+      headers: new Headers(options.headers),
+      agent: httpsAgent
+    });
+  }
+}
+
+export interface FetchOptions extends Pick<RequestOptions, "cert" | "key" | "method"> {
+  body?: BodyInit;
+  headers?: Record<string, string>;
+}

apps/provider-proxy/src/services/WebsocketStats.ts

@@ -0,0 +1,13 @@
+import { ClientWebSocketStats } from "../ClientSocketStats";
+
+export class WebsocketStats {
+  private readonly items: ClientWebSocketStats[] = [];
+
+  add(item: ClientWebSocketStats): void {
+    this.items.push(item);
+  }
+
+  getItems(): ReadonlyArray<ClientWebSocketStats> {
+    return this.items;
+  }
+}

apps/provider-proxy/test/functional/provider-proxy.spec.ts

@@ -0,0 +1,29 @@
+import { request } from "../setup/apiClient";
+
+describe("Provider proxy", () => {
+  it("should proxy request if provider uses self-signed certificate", async () => {
+    const response = await request("/", {
+      method: "POST",
+      body: JSON.stringify({
+        method: "GET",
+        url: "https://provider.tevuicdasialmareny.es:8443/status"
+      })
+    });
+
+    const body = await response.json();
+    expect(body.cluster_public_hostname).toBe("provider.tevuicdasialmareny.es");
+  });
+
+  it("should proxy request if provider uses trusted CA issued certificate", async () => {
+    const response = await request("/", {
+      method: "POST",
+      body: JSON.stringify({
+        method: "GET",
+        url: "https://api.cloudmos.io/internal/gpu-prices"
+      })
+    });
+
+    const body = await response.json();
+    expect(body.availability).toBeTruthy();
+  });
+});