refactor: begin to use axios for cloud api requests

packages/network/lib/agent.ts

@@ -441,3 +441,7 @@ class HttpsAgent extends https.Agent {
 const agent = new CombinedAgent()
 
 export default agent
+
+export const httpsAgent = new HttpsAgent()
+
+export const httpAgent = new HttpAgent()

packages/server/lib/cloud/api/axios_middleware/logging.ts

@@ -0,0 +1,26 @@
+import type { InternalAxiosRequestConfig, AxiosResponse, AxiosError, AxiosInstance } from 'axios'
+import Debug from 'debug'
+
+const debug = Debug('cypress:server:cloud:api')
+
+const logRequest = (req: InternalAxiosRequestConfig): InternalAxiosRequestConfig => {
+  debug(`${req.method} ${req.url}`)
+
+  return req
+}
+
+const logResponse = (res: AxiosResponse): AxiosResponse => {
+  debug(`${res.config.method} ${res.config.url}: %d %s -> \n  %o`, res.status, res.statusText, res.data)
+
+  return res
+}
+
+const logResponseErr = (err: AxiosError): never => {
+  debug(`${err.config?.method} ${err.config?.url}: %d -> \n  %o`, err.response?.statusText, err.response?.statusText, err.response?.data)
+  throw err
+}
+
+export const installLogging = (axios: AxiosInstance) => {
+  axios.interceptors.request.use(logRequest)
+  axios.interceptors.response.use(logResponse, logResponseErr)
+}

packages/server/lib/cloud/api/axios_middleware/transform_error.ts

@@ -0,0 +1,28 @@
+import { isObject } from 'lodash'
+import axios, { AxiosError, AxiosInstance } from 'axios'
+
+declare module 'axios' {
+  export interface AxiosError {
+    isApiError?: boolean
+  }
+}
+
+export const transformError = (err: AxiosError | Error & { error?: any, statusCode: number, isApiError?: boolean }): never => {
+  const { data, status } = axios.isAxiosError(err) ?
+    { data: err.response?.data, status: err.status } :
+    { data: err.error, status: err.statusCode }
+
+  if (isObject(data)) {
+    const body = JSON.stringify(data, null, 2)
+
+    err.message = [status, body].join('\n\n')
+  }
+
+  err.isApiError = true
+
+  throw err
+}
+
+export const installErrorTransform = (axios: AxiosInstance) => {
+  axios.interceptors.response.use(undefined, transformError)
+}

packages/server/lib/cloud/api/cloud_request.ts

@@ -0,0 +1,43 @@
+import os from 'os'
+
+import axios, { AxiosInstance } from 'axios'
+
+import pkg from '@packages/root'
+import { httpAgent, httpsAgent } from '@packages/network/lib/agent'
+
+import app_config from '../../../config/app.json'
+import { installErrorTransform } from './axios_middleware/transform_error'
+import { installLogging } from './axios_middleware/logging'
+
+// initialized with an export for testing purposes
+export const _create = (): AxiosInstance => {
+  const cfgKey = process.env.CYPRESS_CONFIG_ENV || process.env.CYPRESS_INTERNAL_ENV || 'development'
+
+  const instance = axios.create({
+    baseURL: app_config[cfgKey].api_url,
+    httpAgent,
+    httpsAgent,
+    headers: {
+      'x-os-name': os.platform(),
+      'x-cypress-version': pkg.version,
+      'User-Agent': `cypress/${pkg.version}`,
+    },
+  })
+
+  installLogging(instance)
+  installErrorTransform(instance)
+
+  return instance
+}
+
+export const CloudRequest = _create()
+
+export const isRetryableCloudError = (error: unknown) => {
+  const axiosErr = axios.isAxiosError(error) ? error : undefined
+
+  if (axiosErr && axiosErr.status) {
+    return [408, 429, 502, 503, 504].includes(axiosErr.status)
+  }
+
+  return true
+}

packages/server/lib/cloud/api/create_instance.ts

@@ -0,0 +1,51 @@
+import { CloudRequest, isRetryableCloudError } from './cloud_request'
+import { asyncRetry, exponentialBackoff } from '../../util/async_retry'
+
+// TODO: generate these types like system-tests' cloudValidations
+type CreateInstanceResponse = {
+  instanceId: string
+  claimedInstances: number
+  estimatedWallClockDuration: number | null
+  spec: string | null
+  totalInstances: number
+}
+
+type CreateInstanceRequestData = {
+  groupId: string
+  machineId: string
+  platform: {
+    browserName: string
+    browserVersion: string
+    osCpus: any[]
+    osMemory: Record<string, any> | null
+    osName: string
+    osVersion: string
+  }
+}
+
+export const createInstance = async (runId: string, instanceData: CreateInstanceRequestData, timeout: number = 0): Promise<CreateInstanceResponse> => {
+  let attemptNumber = 0
+
+  return asyncRetry(async () => {
+    const { data } = await CloudRequest.post<CreateInstanceResponse>(
+      `/runs/${runId}/instances`,
+      instanceData,
+      {
+        headers: {
+          'x-route-version': '5',
+          'x-cypress-run-id': runId,
+          'x-cypress-request-attempt': `${attemptNumber}`,
+        },
+        timeout,
+      },
+    )
+
+    attemptNumber++
+
+    return data
+  }, {
+    maxAttempts: 3,
+    retryDelay: exponentialBackoff(),
+    shouldRetry: isRetryableCloudError,
+  })()
+}

packages/server/lib/cloud/api/index.ts

@@ -24,6 +24,9 @@ import ProtocolManager from '../protocol'
 import type { ProjectBase } from '../../project-base'
 import type { AfterSpecDurations } from '@packages/types'
 
+import { createInstance } from './create_instance'
+import { transformError } from './axios_middleware/transform_error'
+
 const THIRTY_SECONDS = humanInterval('30 seconds')
 const SIXTY_SECONDS = humanInterval('60 seconds')
 const TWO_MINUTES = humanInterval('2 minutes')
@@ -208,19 +211,6 @@ const retryWithBackoff = (fn) => {
   return attempt(0)
 }
 
-const formatResponseBody = function (err) {
-  // if the body is JSON object
-  if (_.isObject(err.error)) {
-    // transform the error message to include the
-    // stringified body (represented as the 'error' property)
-    const body = JSON.stringify(err.error, null, 2)
-
-    err.message = [err.statusCode, body].join('\n\n')
-  }
-
-  throw err
-}
-
 const tagError = function (err) {
   err.isApiError = true
   throw err
@@ -444,37 +434,11 @@ export default {
 
       return result
     })
-    .catch(RequestErrors.StatusCodeError, formatResponseBody)
+    .catch(RequestErrors.StatusCodeError, transformError)
     .catch(tagError)
   },
 
-  createInstance (options) {
-    const { runId, timeout } = options
-
-    const body = _.pick(options, [
-      'spec',
-      'groupId',
-      'machineId',
-      'platform',
-    ])
-
-    return retryWithBackoff((attemptIndex) => {
-      return rp.post({
-        body,
-        url: recordRoutes.instances(runId),
-        json: true,
-        encrypt: preflightResult.encrypt,
-        timeout: timeout ?? SIXTY_SECONDS,
-        headers: {
-          'x-route-version': '5',
-          'x-cypress-run-id': runId,
-          'x-cypress-request-attempt': attemptIndex,
-        },
-      })
-    })
-    .catch(RequestErrors.StatusCodeError, formatResponseBody)
-    .catch(tagError)
-  },
+  createInstance,
 
   postInstanceTests (options) {
     const { instanceId, runId, timeout, ...body } = options
@@ -492,7 +456,7 @@ export default {
         },
         body,
       })
-      .catch(RequestErrors.StatusCodeError, formatResponseBody)
+      .catch(RequestErrors.StatusCodeError, transformError)
       .catch(tagError)
     })
   },
@@ -512,7 +476,7 @@ export default {
 
         },
       })
-      .catch(RequestErrors.StatusCodeError, formatResponseBody)
+      .catch(RequestErrors.StatusCodeError, transformError)
       .catch(tagError)
     })
   },
@@ -532,7 +496,7 @@ export default {
           'x-cypress-request-attempt': attemptIndex,
         },
       })
-      .catch(RequestErrors.StatusCodeError, formatResponseBody)
+      .catch(RequestErrors.StatusCodeError, transformError)
       .catch(tagError)
     })
   },
@@ -559,7 +523,7 @@ export default {
           'metadata',
         ]),
       })
-      .catch(RequestErrors.StatusCodeError, formatResponseBody)
+      .catch(RequestErrors.StatusCodeError, transformError)
       .catch(tagError)
     })
   },

packages/server/lib/cloud/api/log_requests.ts

@@ -0,0 +1,21 @@
+import type { InternalAxiosRequestConfig, AxiosResponse, AxiosError } from 'axios'
+import Debug from 'debug'
+
+const debug = Debug('cypress:server:cloud:api')
+
+export const logRequest = (req: InternalAxiosRequestConfig): InternalAxiosRequestConfig => {
+  debug(`${req.method} ${req.url}`)
+
+  return req
+}
+
+export const logResponse = (res: AxiosResponse): AxiosResponse => {
+  debug(`${res.config.method} ${res.config.url}: %d %s -> \n  %o`, res.status, res.statusText, res.data)
+
+  return res
+}
+
+export const logResponseErr = (err: AxiosError): never => {
+  debug(`${err.config?.method} ${err.config?.url}: %d -> \n  %o`, err.response?.statusText, err.response?.statusText, err.response?.data)
+  throw err
+}

packages/server/lib/cloud/network/is_retryable_error.ts

@@ -4,14 +4,19 @@ import Debug from 'debug'
 
 const debug = Debug('cypress-verbose:server:is-retryable-error')
 
-export const isRetryableError = (error?: Error) => {
+export const isRetryableError = (error: unknown) => {
   debug('is retryable error? system error: %s, httperror: %s, status: %d',
-    error && SystemError.isSystemError(error),
-    error && HttpError.isHttpError(error),
+    error && SystemError.isSystemError(error as any),
+    error && HttpError.isHttpError(error as any),
     (error as HttpError)?.status)
 
-  return error ? (
-    SystemError.isSystemError(error) ||
-    HttpError.isHttpError(error) && [408, 429, 502, 503, 504].includes(error.status)
-  ) : false
+  if (SystemError.isSystemError(error as any)) {
+    return true
+  }
+
+  if (HttpError.isHttpError(error as any)) {
+    return [408, 429, 502, 503, 504].includes((error as HttpError).status)
+  }
+
+  return false
 }

packages/server/lib/modes/record.js

@@ -501,9 +501,8 @@ const createInstance = (options = {}) => {
 
   spec = getSpecRelativePath(spec)
 
-  return api.createInstance({
+  return api.createInstance(runId, {
     spec,
-    runId,
     groupId,
     platform,
     machineId,

packages/server/lib/util/async_retry.ts

@@ -1,7 +1,7 @@
 type RetryOptions = {
   maxAttempts: number
   retryDelay?: (attempt: number) => number
-  shouldRetry?: (err?: Error) => boolean
+  shouldRetry?: (err?: unknown) => boolean
 }
 
 export function asyncRetry <
@@ -52,3 +52,15 @@ export const linearDelay = (inc: number) => {
     return attempt * inc
   }
 }
+
+export const exponentialBackoff = ({ factor, fuzz } = {
+  factor: 100,
+  fuzz: 0.1,
+}) => {
+  return (attempt: number) => {
+    const exponentialComponent = 2 ** attempt * factor
+    const fuzzComponent = exponentialComponent * fuzz * Math.random()
+
+    return exponentialComponent + fuzzComponent
+  }
+}

packages/server/package.json

@@ -38,6 +38,7 @@
     "@types/mime": "^3.0.1",
     "ansi_up": "5.0.0",
     "ast-types": "0.13.3",
+    "axios": "^1.7.9",
     "base64url": "^3.0.1",
     "better-sqlite3": "11.5.0",
     "black-hole-stream": "0.0.1",
@@ -214,6 +215,7 @@
   "productName": "Cypress",
   "workspaces": {
     "nohoist": [
+      "axios",
       "devtools-protocol",
       "edgedriver",
       "geckodriver",

packages/server/patches/axios+1.7.9.patch

@@ -0,0 +1,18 @@
+diff --git a/node_modules/axios/lib/platform/common/utils.js b/node_modules/axios/lib/platform/common/utils.js
+index 52a3186..08b92cc 100644
+--- a/node_modules/axios/lib/platform/common/utils.js
++++ b/node_modules/axios/lib/platform/common/utils.js
+@@ -1,6 +1,11 @@
+-const hasBrowserEnv = typeof window !== 'undefined' && typeof document !== 'undefined';
++/**
++ * patched due to how v8 snapshots work - axios was incorrectly determining that
++ * it was running in a browser context, leading to runtime errors.
++ */
++
++const hasBrowserEnv = false;
+
+-const _navigator = typeof navigator === 'object' && navigator || undefined;
++const _navigator = undefined;
+
+ /**
+  * Determine if we're running in a standard browser environment