node and browser benchmark runner

README.md

@@ -6,17 +6,23 @@
 
 ## Install
 
-`npm i benchmark-runner`
+`npm i orbit-db-benchmark-runner`
 
-## Usage
+## CLI Usage
 
-TBD
+Check [cli.js](./src/cli.js) or use `npx benchmarker -h` for help
 
-## Testing
+```
+Options:
+  -V, --version             output the version number
+  -b, --benchmarks <path>   benchmark folder or file (default: "./benchmarks")
+  -o, --output <file path>  report output path (.html or .json)
+  -i, --baselines <path>    baselines to use for comparison (.json output)
+  --no-node                 skip nodejs benchmarks
+  --no-browser              skip browser benchmarks
+```
 
-Mocha is used as the testing framework, SinonJS for stubs and mocks and ChaiJS for assertions. To run tests:
-
-`npm run test`
+**benchmarks ran for comparison are best ran on their own machine or a machine with few other things happening in the background**
 
 ## Contributing
 

package.json

@@ -4,16 +4,39 @@
   "description": "OrbitDB Benchmark Runner",
   "main": "./src/index.js",
   "bin": {
-    "benchmark-runner": "./src/cli.js"
-  },
-  "scripts": {
-    "test": "nyc mocha"
+    "benchmarker": "./src/cli.js"
   },
+  "scripts": {},
   "author": "mistakia",
   "license": "MIT",
   "dependencies": {
-    "expose-gc": "^1.0.0",
-    "yargs": "^15.4.1"
+    "@babel/core": "^7.13.10",
+    "@babel/preset-env": "^7.13.12",
+    "@babel/preset-react": "^7.12.13",
+    "@nivo/core": "^0.67.0",
+    "@nivo/line": "^0.67.0",
+    "babel-loader": "^8.2.2",
+    "bootstrap": "^4.6.0",
+    "commander": "^7.1.0",
+    "css-loader": "^5.1.3",
+    "express": "^4.17.1",
+    "html-webpack-plugin": "^4.5.2",
+    "inline-assets-html-plugin": "^1.0.0",
+    "is-node": "^1.0.2",
+    "puppeteer": "^8.0.0",
+    "react": "^17.0.2",
+    "react-bootstrap": "^1.5.2",
+    "react-dom": "^17.0.2",
+    "style-loader": "^2.0.0",
+    "val-loader": "^2.1.2",
+    "webpack": "^4.46.0",
+    "webpack-dev-middleware": "^4.1.0",
+    "ws": "^7.4.4"
+  },
+  "devDependencies": {
+    "ipfs": "^0.54.4",
+    "orbit-db": "^0.26.1",
+    "standard": "^14.3.4"
   },
   "localMaintainers": [
     "hajamark <[email protected]>",
@@ -32,13 +55,6 @@
   "homepage": "https://github.com/orbitdb/benchmark-runner#readme",
   "bugs": "https://github.com/orbitdb/benchmark-runner/issues",
   "repository": "github:orbitdb/benchmark-runner",
-  "devDependencies": {
-    "chai": "^4.2.0",
-    "mocha": "^8.1.3",
-    "nyc": "^15.1.0",
-    "sinon": "^9.0.3",
-    "standard": "^14.3.4"
-  },
   "standard": {
     "env": "mocha"
   }

src/benchmarker/client.js

@@ -0,0 +1,116 @@
+'use strict'
+const isNode = require('is-node')
+const nodeDir = (dir) => require('path').join(dir, 'node')
+const getWebSocket = () => isNode
+  ? require('ws')
+  : window.WebSocket
+const { makeId, withInfo, creators } = require('./ws-action')
+const {
+  timeMetric,
+  cpuUsageMetric,
+  memoryUsedMetric,
+  memoryTotalMetric
+} = require('./metrics')
+
+class Benchmarker {
+  constructor (ws, dir) {
+    this._ws = ws
+    this.dir = isNode ? nodeDir(dir) : dir
+    this._timeout = null
+
+    this.isNode = isNode
+    this.id = makeId()
+    this.info = {
+      id: this.id,
+      name: `benchmark-${this.id}`,
+      env: isNode ? 'node' : 'browser',
+      metrics: []
+    }
+    this._interval = 1000 // record metrics every this many ms
+
+    this.metrics = []
+    this.addMetric(timeMetric)
+  }
+
+  static async create (host, dir) {
+    const ws = await new Promise(resolve => {
+      const ws = new (getWebSocket())(`ws://${host}`)
+      ws.onopen = () => resolve(ws)
+    })
+    return new Benchmarker(ws, dir)
+  }
+
+  async close () {
+    if (this._ws.readyState !== 3) {
+      await new Promise(resolve => {
+        this._ws.onclose = () => resolve()
+        this._ws.close()
+      })
+    }
+  }
+
+  trackMemory () {
+    this.addMetric(memoryUsedMetric)
+    this.addMetric(memoryTotalMetric)
+  }
+
+  trackCpu () {
+    if (isNode) this.addMetric(cpuUsageMetric)
+  }
+
+  addMetric ({ name, get }) {
+    if (this.info.metrics.includes(name)) {
+      throw new Error('a metric with that name already exists')
+    }
+    if (this._timeout) {
+      throw new Error('metrics have already started being recorded')
+    }
+    this.metrics.push({ name, get })
+    this.info.metrics = this.metrics.map(m => m.name)
+  }
+
+  setInterval (interval) {
+    if (typeof interval !== 'number') {
+      throw new Error('interval must be a number')
+    }
+    if (this._timeout) {
+      throw new Error('metrics have already started being recorded')
+    }
+    this._interval = interval
+  }
+
+  setBenchmarkName (name) {
+    this.info.name = name.toString()
+  }
+
+  log (msg) {
+    this._sendAction(creators.LOG(msg))
+  }
+
+  _sendAction (action) {
+    this._ws.send(JSON.stringify(withInfo(this.info)(action)))
+  }
+
+  _recordMetrics () {
+    this._sendAction(creators.SEGMENT(this.metrics.map(({ get }) => get())))
+  }
+
+  startRecording () {
+    if (!this._timeout) {
+      const interval = this._interval
+      const repeater = () => {
+        this._recordMetrics()
+        this._timeout = setTimeout(repeater.bind(this), interval)
+      }
+      repeater()
+    }
+  }
+
+  stopRecording () {
+    clearTimeout(this._timeout)
+    this._timeout = null
+    this._recordMetrics()
+  }
+}
+
+module.exports = Benchmarker

src/benchmarker/metrics/index.js

@@ -0,0 +1,71 @@
+'use strict'
+const isNode = require('is-node')
+const useMetricState = (state, get) => () => {
+  const { newState, next } = get(state)
+  state = newState
+  return next
+}
+
+const timeMetric = {
+  name: 'time',
+  get: useMetricState(0, (state) => {
+    const now = Date.now()
+    return {
+      newState: state || now,
+      next: now - (state || now) // on first metric sample: now - now, aka 0
+    }
+  })
+}
+
+const ns2ms = (ms) => ms / 1000
+const cpuUsageMetric = {
+  name: 'cpu usage',
+  get: useMetricState(undefined, (state) => {
+    const time = Date.now()
+    const { user, system } = process.cpuUsage()
+    const total = ns2ms(user) + ns2ms(system)
+    return {
+      newState: { total, time },
+      next: state
+        // cpu usage to percent
+        ? Math.round(100 * ((total - state.total) / (time - state.time)))
+        : 0
+    }
+  })
+}
+
+const memorySample = () => {
+  const sample = isNode
+    ? process.memoryUsage()
+    : window.performance.memory
+  const memory = {
+    total: null,
+    used: null
+  }
+  // denominated in bytes
+  if (isNode) {
+    memory.total = sample.heapTotal
+    memory.used = sample.heapUsed
+  } else {
+    memory.total = sample.totalJSHeapSize
+    memory.used = sample.usedJSHeapSize
+  }
+  return memory
+}
+const toMegabytes = (bytes) => bytes / 1000000
+const memoryUsedMetric = {
+  name: 'heap used',
+  get: () => toMegabytes(memorySample().used)
+}
+const memoryTotalMetric = {
+  name: 'heap total',
+  get: () => toMegabytes(memorySample().total)
+}
+
+module.exports = {
+  useMetricState,
+  timeMetric,
+  cpuUsageMetric,
+  memoryUsedMetric,
+  memoryTotalMetric
+}

src/benchmarker/server.js

@@ -0,0 +1,39 @@
+'use strict'
+const WebSocket = require('ws')
+const { parse, types } = require('./ws-action')
+const logMessage = (id, msg) =>
+`benchmark id:${id}
+${msg}
+`
+
+class BenchmarkerServer {
+  constructor ({ port } = {}) {
+    this._wss = new WebSocket.Server({ port: port || 0 })
+    this._wss.on('connection', this._handleWsConnection.bind(this))
+    this.address = this._wss.address.bind(this._wss)
+    this.results = {}
+  }
+
+  static create (opts) { return new BenchmarkerServer(opts) }
+
+  async _handleWsConnection (ws) {
+    ws.on('message', m => {
+      const { info, type, msg } = parse(m)
+      switch (type) {
+        case types.LOG:
+          console.log(logMessage(info.id, msg))
+          break
+        case types.SEGMENT: {
+          const { name, env } = info
+          if (!this.results[name]) this.results[name] = {}
+          if (!this.results[name][env]) this.results[name][env] = info
+          if (!this.results[name][env].recorded) this.results[name][env].recorded = []
+          this.results[name][env].recorded.push(msg)
+          break
+        }
+      }
+    })
+  }
+}
+
+module.exports = BenchmarkerServer

src/benchmarker/ws-action.js

@@ -0,0 +1,22 @@
+'use strict'
+
+const action = {}
+
+action.types = {
+  LOG: 'LOG',
+  SEGMENT: 'SEGMENT'
+}
+
+action.creators = {
+  [action.types.LOG]: (msg) =>
+    ({ type: action.types.LOG, msg }),
+  [action.types.SEGMENT]: (msg) =>
+    ({ type: action.types.SEGMENT, msg })
+}
+
+action.makeId = () => Date.now()
+action.withInfo = (info) => (action) => ({ info, ...action })
+
+action.parse = (action) => JSON.parse(action)
+
+module.exports = action