From 67d63495a036b6236384150f798d2784aa2a073b Mon Sep 17 00:00:00 2001 From: Stephane Hervochon Date: Fri, 8 Mar 2024 18:07:42 +0100 Subject: [PATCH] Add Redis Storage Add On --- docs/.vitepress/config.ts | 3 +- .../guide/api/modules/breaker/sliding-time.md | 2 +- .../customization/addons/redisStorage.md | 106 ++++++ package-lock.json | 129 ++++++- .../@mollitia/redis-storage/.eslintrc.cjs | 9 + packages/@mollitia/redis-storage/.gitignore | 3 + packages/@mollitia/redis-storage/README.md | 54 +++ packages/@mollitia/redis-storage/package.json | 55 +++ packages/@mollitia/redis-storage/src/index.ts | 113 ++++++ .../redis-storage/src/redis-storage.ts | 80 ++++ .../redis-storage/test/helper/redis-mock.ts | 94 +++++ .../module/breaker/sliding-breaker.spec.ts | 94 +++++ .../breaker/sliding-count-breaker.spec.ts | 351 ++++++++++++++++++ .../breaker/sliding-time-breaker.spec.ts | 262 +++++++++++++ .../test/unit/module/ratelimit.spec.ts | 179 +++++++++ .../redis-storage/tsconfig.eslint.json | 4 + .../@mollitia/redis-storage/tsconfig.json | 10 + .../@mollitia/redis-storage/vite.config.ts | 10 + packages/mollitia/src/helpers/serializable.ts | 6 + packages/mollitia/src/index.ts | 12 +- packages/mollitia/src/module/breaker/index.ts | 213 +++++++---- .../module/breaker/sliding-count-breaker.ts | 28 +- .../module/breaker/sliding-time-breaker.ts | 45 +-- packages/mollitia/src/module/ratelimit.ts | 43 ++- .../breaker/sliding-count-breaker.spec.ts | 22 +- .../breaker/sliding-time-breaker.spec.ts | 12 +- 26 files changed, 1813 insertions(+), 126 deletions(-) create mode 100644 docs/src/guide/customization/addons/redisStorage.md create mode 100644 packages/@mollitia/redis-storage/.eslintrc.cjs create mode 100644 packages/@mollitia/redis-storage/.gitignore create mode 100644 packages/@mollitia/redis-storage/README.md create mode 100644 packages/@mollitia/redis-storage/package.json create mode 100644 packages/@mollitia/redis-storage/src/index.ts create mode 100644 packages/@mollitia/redis-storage/src/redis-storage.ts create mode 100644 packages/@mollitia/redis-storage/test/helper/redis-mock.ts create mode 100644 packages/@mollitia/redis-storage/test/unit/module/breaker/sliding-breaker.spec.ts create mode 100644 packages/@mollitia/redis-storage/test/unit/module/breaker/sliding-count-breaker.spec.ts create mode 100644 packages/@mollitia/redis-storage/test/unit/module/breaker/sliding-time-breaker.spec.ts create mode 100644 packages/@mollitia/redis-storage/test/unit/module/ratelimit.spec.ts create mode 100644 packages/@mollitia/redis-storage/tsconfig.eslint.json create mode 100644 packages/@mollitia/redis-storage/tsconfig.json create mode 100644 packages/@mollitia/redis-storage/vite.config.ts create mode 100644 packages/mollitia/src/helpers/serializable.ts diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 9fffcf0..9df7698 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -84,7 +84,8 @@ export default withMermaid({ link: '/guide/customization/addons', collapsed: false, items: [ - { text: 'Prometheus', link: '/guide/customization/addons/prometheus' } + { text: 'Prometheus', link: '/guide/customization/addons/prometheus' }, + { text: 'Redis Storage', link: '/guide/customization/addons/redisStorage' } ] } ] diff --git a/docs/src/guide/api/modules/breaker/sliding-time.md b/docs/src/guide/api/modules/breaker/sliding-time.md index c5ca8b6..1171031 100644 --- a/docs/src/guide/api/modules/breaker/sliding-time.md +++ b/docs/src/guide/api/modules/breaker/sliding-time.md @@ -86,7 +86,7 @@ await circuit.fn(myFunction5).execute(); | `slowCallDurationThreshold` | Specifies the duration (in ms) threshold above which calls are considered as slow | `60000` | | `permittedNumberOfCallsInHalfOpenState` | Specifies the number of permitted calls when the circuit is half open | `2` | | `halfOpenStateMaxDelay` | Specifies the maximum wait (in ms) in Half Open State, before switching back to open. 0 deactivates this | `0` | -| `slidingWindowSize` | Specifies the sliding duration (in ms) used to calculate failure and slow call rate percentages | `10` | +| `slidingWindowSize` | Specifies the sliding duration (in ms) used to calculate failure and slow call rate percentages | `60` | | `minimumNumberOfCalls` | Specifies the minimum number of calls used to calculate failure and slow call rate percentages | `10` | | `openStateDelay` | Specifies the time (in ms) the circuit stay opened before switching to half-open | `60000` | | `onError` | Allows filtering of the error to report as a failure or not. | `None` | diff --git a/docs/src/guide/customization/addons/redisStorage.md b/docs/src/guide/customization/addons/redisStorage.md new file mode 100644 index 0000000..9123027 --- /dev/null +++ b/docs/src/guide/customization/addons/redisStorage.md @@ -0,0 +1,106 @@ +# Redis Storage + +The `Mollitia` [Redis Storage](https://redis.io/) addon adds redis storage for some modules of every circuit. The list of modules coming with redis support are RateLimit, SlidingCountBreaker and SlidingTimeBreaker. + +``` bash + +``` + +## Quick Start + +``` bash +# Install mollitia +npm install mollitia --save +# Install the Redis storage addon +npm install @mollitia/redis-storage --save +``` + +``` typescript +// Then add the addon +import * as Mollitia from 'mollitia'; +import { StorageAddon } from '@mollitia/redis-storage'; +// Adds the Redis Storage addon to Mollitia +Mollitia.use(new StorageAddOn({ host: , port: , password: })); +``` + +Then, add `storage` options when creating modules. Storage is only available for RateLimit, SlindingCountBreaker or SlidingTimeBreaker module. + +``` typescript +const rateLimitModule = new Mollitia.Ratelimit({ + name: 'myRateLimitModule', + limitForPeriod: 2, + limitPeriod: 20000, + storage: { + // Setting storage.use to true indicates Redis Storage should be used + use: true + } +}; +// Creates a circuit +const myCircuit = new Mollitia.Circuit({ + // Initializes a circuit with a handler + func: yourFunction, + options: { + modules: [ + rateLimit: rateLimitModule + ] + } +}); +// This will execute yourFunction('dummy') +await myCircuit.execute('dummy'); + +``` + +## API Reference + +### Options + +#### When Addon is created + +| Name | Description | Default | +|:-----------------|:----------------------------------------------------------------------------|:-----------| +| `getMaxDelay` | Specifies the maximum time, in milliseconds,to get data from Redis storage | `500` | +| `setMaxDelay` | Specifies the maximum time, in milliseconds,to set data to Redis storage | `500` | +| `ttl` | Specifies the maximum duration, in milliseconds, the data stays in Redis | `0` | + +#### At module level + +| Name | Description | Default | +|:-----------------|:----------------------------------------------------------------------------|:-----------| +| `use` | Specifies if the redis storage is used for the module | `false` | +| `getMaxDelay` | Specifies the maximum time, in milliseconds,to get data from Redis storage | `500` | +| `setMaxDelay` | Specifies the maximum time, in milliseconds,to set data to Redis storage | `500` | +| `ttl` | Specifies the maximum duration, in milliseconds, the data stays in Redis | `0` | + +#### Option priority + +When an option is defined both at AddOn level and at module level, the option value is taken from module + +Example: +``` typescript +Mollitia.use(new StorageAddOn({ host: , port: , password: , getMaxDelay: 1000, setMaxDelay: 1000 })); +const rateLimitModule = new Mollitia.Ratelimit({ + name: 'myRateLimitModule', + limitForPeriod: 2, + limitPeriod: 20000, + storage: { + use: true, + getMaxDelay: 500 + } +}; +```` +With such configuration, getMaxDelay is 500, setMaxDelay is 1000 and ttl is 0 (not set, so using default value) + + +#### Additional information related to the options + +* getMaxDelay and setMaxDelay + +These options are available to avoid blocking the operations for a long time when Redis is slow or unavailable. + +* ttl + +This option could be used to avoid keeping some keys in Redis storage for a long duration. Setting ttl to 0 deactivate the ttl. + +Please note that this option is only applicable when the Redis Storage is used with SlindingCountBreaker module, as SlidingTimeBreker module and RateLimit module come with existing ttl (slidingWindowSize for SlidingCoundBreaker, limitPeriod for RateLimit). + +This option is converted to a number of seconds, and rounded to the next integer. diff --git a/package-lock.json b/package-lock.json index 3c9246b..3c2ef69 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "mollitia", + "name": "mollitia_v2-new", "lockfileVersion": 3, "requires": true, "packages": { @@ -1696,6 +1696,10 @@ "resolved": "packages/@mollitia/prometheus", "link": true }, + "node_modules/@mollitia/redis-storage": { + "resolved": "packages/@mollitia/redis-storage", + "link": true + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -3101,6 +3105,59 @@ "node": ">=14" } }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.5.11", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.11.tgz", + "integrity": "sha512-cV7yHcOAtNQ5x/yQl7Yw1xf53kO0FNDTdDU6bFIMbW6ljB7U7ns0YRM+QIkpoqTAt6zK5k9Fq0QWlUbLcq9AvA==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/graph": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.0.tgz", + "integrity": "sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz", + "integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.5.tgz", + "integrity": "sha512-hPP8w7GfGsbtYEJdn4n7nXa6xt6hVZnnDktKW4ArMaFQ/m/aR7eFvsLQmG/mn1Upq99btPJk+F27IQ2dYpCoUg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz", + "integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@rollup/pluginutils": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.0.5.tgz", @@ -3501,6 +3558,15 @@ "integrity": "sha512-lqa4UEhhv/2sjjIQgjX8B+RBjj47eo0mzGasklVJ78UKGQY1r0VpB9XHDaZZO9qzEFDdy4MrXLuEaSmPrPSe/A==", "dev": true }, + "node_modules/@types/redis": { + "version": "4.0.11", + "resolved": "https://registry.npmjs.org/@types/redis/-/redis-4.0.11.tgz", + "integrity": "sha512-bI+gth8La8Wg/QCR1+V1fhrL9+LZUSWfcqpOj2Kc80ZQ4ffbdL173vQd5wovmoV9i071FU9oP2g6etLuEwb6Rg==", + "deprecated": "This is a stub types definition. redis provides its own type definitions, so you do not need this installed.", + "dependencies": { + "redis": "*" + } + }, "node_modules/@types/semver": { "version": "7.5.3", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.3.tgz", @@ -5111,6 +5177,14 @@ "node": ">=0.10.0" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/cmd-shim": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/cmd-shim/-/cmd-shim-6.0.1.tgz", @@ -7241,6 +7315,14 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "engines": { + "node": ">= 4" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -12739,6 +12821,19 @@ "node": ">=8" } }, + "node_modules/redis": { + "version": "4.6.10", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.10.tgz", + "integrity": "sha512-mmbyhuKgDiJ5TWUhiKhBssz+mjsuSI/lSZNPI9QvZOYzWvYGejtb+W3RlDDf8LD6Bdl5/mZeG8O1feUGhXTxEg==", + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.5.11", + "@redis/graph": "1.1.0", + "@redis/json": "1.0.6", + "@redis/search": "1.1.5", + "@redis/time-series": "1.0.5" + } + }, "node_modules/regexp.prototype.flags": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz", @@ -14950,8 +15045,7 @@ "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/yaml": { "version": "2.3.3", @@ -15039,6 +15133,19 @@ "commander": "^9.4.1" } }, + "packages/@mollitia/plop": { + "version": "0.0.1", + "extraneous": true, + "license": "MIT", + "devDependencies": { + "@shared/tsconfig": "*", + "@shared/vite": "*", + "eslint-config-mollitia": "*" + }, + "peerDependencies": { + "mollitia": "*" + } + }, "packages/@mollitia/prometheus": { "version": "0.0.8", "license": "MIT", @@ -15051,6 +15158,22 @@ "mollitia": "*" } }, + "packages/@mollitia/redis-storage": { + "version": "0.0.1", + "license": "MIT", + "dependencies": { + "@types/redis": "4.0.11", + "redis": "4.6.10" + }, + "devDependencies": { + "@shared/tsconfig": "*", + "@shared/vite": "*", + "eslint-config-mollitia": "*" + }, + "peerDependencies": { + "mollitia": "*" + } + }, "packages/mollitia": { "version": "0.1.0", "license": "MIT", diff --git a/packages/@mollitia/redis-storage/.eslintrc.cjs b/packages/@mollitia/redis-storage/.eslintrc.cjs new file mode 100644 index 0000000..9ccf267 --- /dev/null +++ b/packages/@mollitia/redis-storage/.eslintrc.cjs @@ -0,0 +1,9 @@ +/* eslint-env node */ +module.exports = { + root: true, + extends: ['mollitia/typescript'], + parserOptions: { + tsconfigRootDir: __dirname, + project: './tsconfig.eslint.json' + } +}; diff --git a/packages/@mollitia/redis-storage/.gitignore b/packages/@mollitia/redis-storage/.gitignore new file mode 100644 index 0000000..ae409af --- /dev/null +++ b/packages/@mollitia/redis-storage/.gitignore @@ -0,0 +1,3 @@ +/coverage +/dist +/node_modules diff --git a/packages/@mollitia/redis-storage/README.md b/packages/@mollitia/redis-storage/README.md new file mode 100644 index 0000000..6fc9dd7 --- /dev/null +++ b/packages/@mollitia/redis-storage/README.md @@ -0,0 +1,54 @@ +# Mollitia + +


Mollitia Icon

+ +> Mollitia - Redis Storage Addon + +The `Mollitia` [Redis Storage](https://redis.io/) addon adds redis storage for some modules of every circuit. The list of modules coming with redis support are RateLimit, SlidingCountBreaker and SlidingTimeBreaker. + +## 📄 Documentation + +Please check out the official documentation to get started using **Mollitia**, visit [genesys.github.io/mollitia](https://genesys.github.io/mollitia). + +## ⚙️ Installation + +``` bash +npm install --save @mollitia/redis-storage +``` + +## 🚀 Usage + +``` typescript +// Imports the library +import * as Mollitia from 'mollitia'; +import { StorageAddon } from '@mollitia/redis-storage'; +// Adds the Redis Storage addon to Mollitia +Mollitia.use(new StorageAddOn({ host: , port: , password: })); +// Creates the module that will be used in your circuit, using Redis Storage +// Redis Storage is only applicable for Modules: +// - RateLimit +// - SlidingCountBreaker +// - SlidingTimeBreaker +const rateLimit = new Mollitia.Ratelimit({ + name: 'myRateLimit', + limitForPeriod: 2, + limitPeriod: 20000, + storage: { + // Setting storage.use to true indicates Redis Storage should be used + use: true + } +}; +// Creates a circuit +const myCircuit = new Mollitia.Circuit({ + // Initializes a circuit with a handler + func: yourFunction, + options: { + modules: [ + rateLimit + ] + } +}); +// This will execute yourFunction('dummy') +await myCircuit.execute('dummy'); + +``` diff --git a/packages/@mollitia/redis-storage/package.json b/packages/@mollitia/redis-storage/package.json new file mode 100644 index 0000000..2c44e2a --- /dev/null +++ b/packages/@mollitia/redis-storage/package.json @@ -0,0 +1,55 @@ +{ + "name": "@mollitia/redis-storage", + "type": "module", + "version": "0.0.1", + "description": "Redis Storage Addon", + "author": "Stephane Hervochon ", + "license": "MIT", + "keywords": [ + "mollitia", + "mollitia-storage", + "resiliency", + "resilience", + "node", + "nodejs", + "javascript", + "typescript", + "storage" + ], + "homepage": "https://genesys.github.io/mollitia/", + "repository": { + "type": "git", + "url": "https://github.com/genesys/mollitia/blob/main/packages/@mollitia/redis-storage" + }, + "main": "./dist/index.cjs", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "unpkg": "./dist/index.umd.js", + "exports": { + ".": { + "import": "./dist/index.js", + "require": "./dist/index.cjs", + "types": "./dist/index.d.ts" + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "tsc && vite build", + "lint": "eslint .", + "test:unit": "vitest run" + }, + "devDependencies": { + "@shared/tsconfig": "*", + "@shared/vite": "*", + "eslint-config-mollitia": "*" + }, + "peerDependencies": { + "mollitia": "*" + }, + "dependencies": { + "@types/redis": "4.0.11", + "redis": "4.6.10" + } +} diff --git a/packages/@mollitia/redis-storage/src/index.ts b/packages/@mollitia/redis-storage/src/index.ts new file mode 100644 index 0000000..fcf33dc --- /dev/null +++ b/packages/@mollitia/redis-storage/src/index.ts @@ -0,0 +1,113 @@ +import * as Mollitia from 'mollitia'; +import { RedisStorage } from './redis-storage.js'; + +export const version = __VERSION__; + +// Declaration Overriding +declare module 'mollitia' { + interface ModuleOptions { + /** + * Storage Circuit helper. [Storage Addon] + */ + storage?: { + use: boolean; + getMaxDelay?: number; + setMaxDelay?: number; + ttl?: number; + } + } +} +/** + * Array containing every modules. + */ +export const modules: Mollitia.Module[] = []; + +/** + * The StorageAddon Class, that should be added to the core Mollitia module. [Storage Addon] + * @example + * Mollitia.use(new MollitiaStorage.StorageAddon()); + */ +// export class StorageAddon extends EventEmitter implements Mollitia.Addon { +export class StorageAddon implements Mollitia.Addon { + private storage; + private getMaxDelay: number; + private setMaxDelay: number; + private ttl: number; + constructor(configuration: { host: string, port: number, password: string, ttl?: number, getMaxDelay?: number, setMaxDelay?: number }) { + // super(); + this.storage = new RedisStorage(configuration.host, configuration.port, configuration.password); + this.getMaxDelay = configuration.getMaxDelay || 500; //0 for getMaxDelay is not a valid value + this.setMaxDelay = configuration.setMaxDelay ?? 500; + this.ttl = configuration.ttl || 0; + } + + async getStateWithStorage (moduleName: string, getMaxDelay: number): Promise { + return new Promise((resolve, reject) => { + const opTimeout = setTimeout(() => { + reject(); + }, getMaxDelay); + try { + this.storage.getState(moduleName).then((data) => { + clearTimeout(opTimeout); + resolve(data); + }); + } + catch (e) { + console.log('Error occurred while trying to get the state from Redis storage'); + clearTimeout(opTimeout); + reject(); + } + }); + } + + async setStateWithStorage(moduleName: string, state: Mollitia.SerializableRecord[], setMaxDelay: number, ttl: number): Promise { + return new Promise((resolve, reject) => { + const opTimeout = setTimeout(() => { + reject(); + }, setMaxDelay); + try { + this.storage.setState(moduleName, state, ttl).then(() => { + clearTimeout(opTimeout); + resolve(); + }); + } catch (e) { + console.log('Error occurred while trying to set the state in Redis storage'); + clearTimeout(opTimeout); + reject(); + } + }); + } + + moduleOverride(mod: Mollitia.Ratelimit | Mollitia.SlidingWindowBreaker, getMaxDelay: number, setMaxDelay: number, moduleTtl: number): void { + mod.getState = async (): Promise => { + return this.getStateWithStorage(mod.name, getMaxDelay); + }; + mod.setState = async (state: Mollitia.SerializableRecord[], ttl = 0): Promise => { + return this.setStateWithStorage(mod.name, state, setMaxDelay, ttl || moduleTtl); + } + mod.clearState = async (): Promise => { + return this.storage.clearState(mod.name); + } + } + onModuleCreate (module: Mollitia.Module, options: Mollitia.ModuleOptions): void { + if (options.storage?.use) { + const getMaxDelay = options.storage.getMaxDelay || this.getMaxDelay; + const setMaxDelay = options.storage.setMaxDelay ?? this.setMaxDelay; + const moduleTtl = options.storage.ttl || this.ttl; + switch (module.constructor.name) { + case Mollitia.Ratelimit.name: { + this.moduleOverride(module as Mollitia.Ratelimit, getMaxDelay, setMaxDelay, moduleTtl); + break; + } + case Mollitia.SlidingCountBreaker.name: + case Mollitia.SlidingTimeBreaker.name: { + this.moduleOverride(module as Mollitia.SlidingWindowBreaker, getMaxDelay, setMaxDelay, moduleTtl); + break; + } + default: + break; + } + } + } +} + diff --git a/packages/@mollitia/redis-storage/src/redis-storage.ts b/packages/@mollitia/redis-storage/src/redis-storage.ts new file mode 100644 index 0000000..531e167 --- /dev/null +++ b/packages/@mollitia/redis-storage/src/redis-storage.ts @@ -0,0 +1,80 @@ +import * as Mollitia from 'mollitia'; +import { RedisClientType, RedisModules, createClient } from 'redis'; + +export interface CircuitStorage { + getState(moduleName: string): Promise; + setState(moduleName: string, state: Mollitia.SerializableRecord[]): Promise; + clearState(moduleName: string): Promise; +} + +export class RedisStorage implements CircuitStorage { + private client: RedisClientType; + private prefix = 'mollitia'; + private initializePromise: Promise; + constructor(host: string, port: number, password = '') { + this.client = createClient({ + socket: { + host, + port + }, + disableOfflineQueue: true, //disableOfflineQueue should be set to true to avoid blocking requests when Redis is down + password + }); + this.client.on('error', () => { + //console.log(err); + }); + this.initializePromise = new Promise((resolve) => { + this.client.on('ready', () => { + resolve(); + }); + }); + this.client.connect(); + } + public async getState(moduleName: string): Promise { + const data: Mollitia.SerializableRecord = {}; + await this.initializePromise; + const keys = await this.client.keys(`${this.prefix}::module::${moduleName}::*`); + for await (const key of keys) { + const val = await this.client.get(key); + if (val) { + try { + data[key.substring(key.lastIndexOf('::') + 2)] = JSON.parse(val); + } catch { + // value is not a json + } + } + } + return data; + } + public async setState(moduleName: string, state: Mollitia.SerializableRecord[], ttl = 0): Promise { + await this.initializePromise; + for await (const stateElem of state) { + const keyName = this.getKeyName(moduleName, stateElem['key'] as string); + if (!stateElem['value']) { + await this.client.del(keyName); + } else { + await this.client.set(keyName, JSON.stringify(stateElem['value'])); + if (ttl) { + this.client.expire(keyName, Math.ceil(ttl / 1000)); + } + } + } + } + public async clearState(moduleName: string): Promise { + try { + const keys = await this.client.keys(`${this.prefix}::module::${moduleName}::*`); + for await (const key of keys) { + await this.client.del(key); + } + } catch { + // Redis is not available + } + } + private getKeyName(moduleName: string, key: string): string { + return key ? + `${this.prefix}::module::${moduleName}::${key}` : + `${this.prefix}::module::${moduleName}`; + } +} + + diff --git a/packages/@mollitia/redis-storage/test/helper/redis-mock.ts b/packages/@mollitia/redis-storage/test/helper/redis-mock.ts new file mode 100644 index 0000000..7ab353e --- /dev/null +++ b/packages/@mollitia/redis-storage/test/helper/redis-mock.ts @@ -0,0 +1,94 @@ +let mockGetDelay = 0; +let mockSetDelay = 0; +let redisCrash = false; + +export const setRedisOptions = (config: { getDelay?: number, setDelay?: number, crash?: boolean }) => { + mockGetDelay = config.getDelay || 0; + mockSetDelay = config.setDelay || 0; + redisCrash = config.crash || false; +} + +const delay = (delay = 1) => { + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, delay); + }); +}; + +export const redisMock = { + createClient: () => { + const keyValuesCollection: { key: string, value: any }[] = []; + return { + on: (): Promise | void => { + return; + }, + connect: () => { + return; + }, + keys: async (keyPrefix: string): Promise => { + return new Promise((resolve) => { + const keyPrefixAdjusted = keyPrefix.replaceAll('*', ''); + const matchingKeys = keyValuesCollection.reduce( + (acc: string[], current) => { + if (current.key && current.key.indexOf(keyPrefixAdjusted) === 0) { + acc.push(current.key); + } + return acc; + }, [] + ); + resolve(matchingKeys); + }); + }, + get: async (key: string): Promise => { + if (redisCrash) { + throw ('Redis Error'); + } + if (mockGetDelay) { + await(mockGetDelay); + } + return new Promise((resolve) => { + const elem = keyValuesCollection.find((elem) => elem.key === key); + resolve(elem?.value || null); + }); + }, + set: async (key: string, value: any): Promise => { + if (redisCrash) { + throw ('Redis Error'); + } + if (mockSetDelay) { + await delay(mockSetDelay); + } + return new Promise((resolve) => { + const elem = keyValuesCollection.find((elem) => elem.key === key); + if (!elem) { + keyValuesCollection.push({ key, value }); + } else { + elem.value = value; + } + resolve(); + }); + }, + del: async (key: string): Promise => { + return new Promise((resolve) => { + const elemIndex = keyValuesCollection.findIndex((elem) => elem.key === key); + if (elemIndex > -1) { + keyValuesCollection.splice(elemIndex, 1); + } + resolve(); + }); + }, + expire: async (key: string, delay: number): Promise => { + return new Promise((resolve) => { + setTimeout(() => { + const elemIndex = keyValuesCollection.findIndex((elem) => elem.key === key); + if (elemIndex > -1) { + keyValuesCollection.splice(elemIndex, 1); + } + resolve(); + }, delay * 1000) + }); + } + } + } +}; diff --git a/packages/@mollitia/redis-storage/test/unit/module/breaker/sliding-breaker.spec.ts b/packages/@mollitia/redis-storage/test/unit/module/breaker/sliding-breaker.spec.ts new file mode 100644 index 0000000..6171db2 --- /dev/null +++ b/packages/@mollitia/redis-storage/test/unit/module/breaker/sliding-breaker.spec.ts @@ -0,0 +1,94 @@ +import { redisMock } from '../../../helper/redis-mock.js'; +import { describe, afterEach, it, expect, vi } from 'vitest'; +import * as Mollitia from 'mollitia'; +import { successAsync, failureAsync } from '../../../../../../../shared/vite/utils/vitest.js'; +import { StorageAddon } from '../../../../src/index.js'; + + +vi.mock('redis', () => { + return redisMock +}); + +const storageAddOn = new StorageAddon({ host: 'localhost', port: 6379, password: '', ttl: 1000 }); +((storageAddOn as any).storage as any).initializePromise = new Promise((resolve) => resolve()); +Mollitia.use(storageAddOn); + +const delay = (delay = 1) => { + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, delay); + }); +}; + +describe('Sliding Count Breaker - Redis TTL - With Redis Storage TTL', () => { + afterEach(() => { + successAsync.mockClear(); + failureAsync.mockClear(); + }); + it('Should use module storage TTL if found - SB', async () => { + const moduleName = 'mySlidingCountBreaker9'; + const breakerData = { + slidingWindowSize: 3, + minimumNumberOfCalls: 2, + storage: { + use: true, + ttl: 2000 + }, + name: moduleName + }; + const slidingCountBreaker = new Mollitia.SlidingCountBreaker(breakerData); + const slidingCountBreaker2 = new Mollitia.SlidingCountBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { + modules: [slidingCountBreaker] + } + }); + const circuit2 = new Mollitia.Circuit({ + options: { + modules: [slidingCountBreaker2] + } + }); + await slidingCountBreaker.clearState(); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.CLOSED); + await delay(1200); + const res = await storageAddOn.getStateWithStorage(moduleName, 1000); + expect(JSON.stringify(res)).toContain('requests'); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.OPENED); + }); + it('Should use redis storage storage TTL if no ttl configured in module', async () => { + const moduleName = 'mySlidingCountBreaker10'; + const breakerData = { + slidingWindowSize: 3, + minimumNumberOfCalls: 2, + storage: { + use: true + }, + name: moduleName + }; + const slidingCountBreaker = new Mollitia.SlidingCountBreaker(breakerData); + const slidingCountBreaker2 = new Mollitia.SlidingCountBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { + modules: [slidingCountBreaker] + } + }); + const circuit2 = new Mollitia.Circuit({ + options: { + modules: [slidingCountBreaker2] + } + }); + await slidingCountBreaker.clearState(); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.CLOSED); + await delay(1200); + const res = await storageAddOn.getStateWithStorage(moduleName, 1000); + expect(JSON.stringify(res)).toEqual('{}'); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); + }); +}); diff --git a/packages/@mollitia/redis-storage/test/unit/module/breaker/sliding-count-breaker.spec.ts b/packages/@mollitia/redis-storage/test/unit/module/breaker/sliding-count-breaker.spec.ts new file mode 100644 index 0000000..145e0df --- /dev/null +++ b/packages/@mollitia/redis-storage/test/unit/module/breaker/sliding-count-breaker.spec.ts @@ -0,0 +1,351 @@ +import { redisMock } from '../../../helper/redis-mock.js'; +import { describe, afterEach, it, expect, vi } from 'vitest'; +import * as Mollitia from 'mollitia'; +import { successAsync, failureAsync } from '../../../../../../../shared/vite/utils/vitest.js'; +import * as RedisStorage from '../../../../src/index.js'; + +vi.mock('redis', () => { + return redisMock; +}); + +const storageAddOn = new RedisStorage.StorageAddon({ host: 'localhost', port: 6379, password: '', ttl: 1000}); +((storageAddOn as any).storage as any).initializePromise = new Promise((resolve) => resolve()); +Mollitia.use(storageAddOn); + +const delay = (delay = 1) => { + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, delay); + }); +}; + +describe('Sliding Count Breaker', () => { + afterEach(() => { + successAsync.mockClear(); + failureAsync.mockClear(); + }); + it('should go to half open state after delay - CB', async () => { + const slidingCountBreaker = new Mollitia.SlidingCountBreaker({ + state: Mollitia.BreakerState.OPENED, + openStateDelay: 20, + storage: { + use: true + } + }); + const circuit = new Mollitia.Circuit({ + options: { + modules: [ + slidingCountBreaker + ] + } + }); + await slidingCountBreaker.clearState(); + await expect(circuit.fn(successAsync).execute('dummy')).rejects.toThrowError('Circuit is opened'); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.OPENED); + await delay(300); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.HALF_OPENED); + }); + it('Switch to Open when failure rate exceeded - CB', async () => { + const breakerData = { + slidingWindowSize: 10, + minimumNumberOfCalls: 3, + failureRateThreshold: 60, + openStateDelay: 2000, + storage: { + use: true + }, + name: 'mySlidingCountBreaker1' + }; + const breakerData2 = { + slidingWindowSize: 10, + minimumNumberOfCalls: 3, + failureRateThreshold: 60, + openStateDelay: 2000, + storage: { + use: true + }, + name: 'mySlidingCountBreaker2' + }; + const slidingCountBreaker = new Mollitia.SlidingCountBreaker(breakerData); + const slidingCountBreaker2 = new Mollitia.SlidingCountBreaker(breakerData); + const slidingCountBreaker3 = new Mollitia.SlidingCountBreaker(breakerData2); + const circuit = new Mollitia.Circuit({ + options: { modules: [ slidingCountBreaker ] } + }); + const circuit2 = new Mollitia.Circuit({ + options: { modules: [slidingCountBreaker2] } + }); + const circuit3 = new Mollitia.Circuit({ + options: { modules: [slidingCountBreaker3] } + }); + await slidingCountBreaker.clearState(); + await slidingCountBreaker3.clearState(); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + await expect(circuit2.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + // Even if 66% of failed requests, circuit is kept closed as last request is success + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); + await expect(circuit2.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.OPENED); + await expect(circuit2.fn(successAsync).execute('dummy')).rejects.toThrowError('Circuit is opened'); + expect(slidingCountBreaker3.state).toEqual(Mollitia.BreakerState.CLOSED); + await expect(circuit3.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + await expect(circuit3.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + await expect(circuit3.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingCountBreaker3.state).toEqual(Mollitia.BreakerState.CLOSED); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.OPENED); + await expect(circuit3.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingCountBreaker3.state).toEqual(Mollitia.BreakerState.OPENED); + }); + it('Half Open State max duration - CB', async () => { + const breakerData = { + name: 'mySlidingCountBreaker3', + halfOpenStateMaxDelay: 200, + openStateDelay: 100, + failureRateThreshold: 40, + permittedNumberOfCallsInHalfOpenState: 1, + minimumNumberOfCalls: 2, + storage: { + use: true + } + }; + const slidingCountBreaker = new Mollitia.SlidingCountBreaker(breakerData); + const slidingCountBreaker2 = new Mollitia.SlidingCountBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { modules: [ slidingCountBreaker ] } + }); + const circuit2 = new Mollitia.Circuit({ + options: { modules: [ slidingCountBreaker2 ] } + }); + await slidingCountBreaker.clearState(); + await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + await expect(circuit2.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.OPENED); + await delay(100); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.HALF_OPENED); + await delay(100); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.HALF_OPENED); + await delay(100); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.OPENED); + await expect(circuit.fn(successAsync).execute('dummy')).rejects.toThrowError('Circuit is opened'); + await delay(150); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.HALF_OPENED); + await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); + }); + it('Half Open State switch to Closed/Opened - CB', async () => { + const breakerData = { + name: 'mySlidingCountBreaker4', + failureRateThreshold: 50, + openStateDelay: 10, + state: Mollitia.BreakerState.HALF_OPENED, + permittedNumberOfCallsInHalfOpenState: 2, + storage: { + use: true + } + }; + const slidingCountBreaker = new Mollitia.SlidingCountBreaker(breakerData); + const slidingCountBreaker2 = new Mollitia.SlidingCountBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { modules: [ slidingCountBreaker ] } + }); + const circuit2 = new Mollitia.Circuit({ + options: { modules: [ slidingCountBreaker2 ] } + }); + await slidingCountBreaker.clearState(); + await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.HALF_OPENED); + await expect(circuit2.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.OPENED); + await delay(10); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.HALF_OPENED); + await expect(circuit2.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); + }); + it('Slow Requests - CB', async () => { + const breakerData = { + failureRateThreshold: 50, + openStateDelay: 10, + slidingWindowSize: 10, + minimumNumberOfCalls: 2, + permittedNumberOfCallsInHalfOpenState: 1, + slowCallDurationThreshold: 100, + slowCallRateThreshold: 50, + name: 'mySlidingCountBreaker5', + storage: { + use: true + } + }; + const slidingCountBreaker = new Mollitia.SlidingCountBreaker(breakerData); + const slidingCountBreaker2 = new Mollitia.SlidingCountBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { modules: [ slidingCountBreaker ] } + }); + const circuit2 = new Mollitia.Circuit({ + options: { modules: [ slidingCountBreaker2 ] } + }); + await slidingCountBreaker.clearState(); + await expect(circuit.fn(successAsync).execute('dummy', 150)).resolves.toEqual('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + // Even if 50% of slow requests, circuit is kept closed as last request is success + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.CLOSED); + await expect(circuit.fn(successAsync).execute('dummy', 150)).resolves.toEqual('dummy'); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.OPENED); + await delay(10); + await expect(circuit2.fn(successAsync).execute('dummy', 150)).resolves.toEqual('dummy'); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.OPENED); + await delay(10); + await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); + }); + it('Nb Max Requests reached - CB', async () => { + const breakerData = { + failureRateThreshold: 50, + openStateDelay: 10, + slidingWindowSize: 5, + minimumNumberOfCalls: 3, + storage: { + use: true + }, + name: 'mySlidingCountBreaker6' + }; + const slidingCountBreaker = new Mollitia.SlidingCountBreaker(breakerData); + const slidingCountBreaker2 = new Mollitia.SlidingCountBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { modules: [ slidingCountBreaker ] } + }); + const circuit2 = new Mollitia.Circuit({ + options: { modules: [ slidingCountBreaker2 ] } + }); + await slidingCountBreaker.clearState(); + await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); + await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.CLOSED); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); + await expect(circuit2.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.OPENED); + }); + it('No switch to Open when failures but failure reported as success - CB', async () => { + const breakerData = { + slidingWindowSize: 2, + minimumNumberOfCalls: 2, + failureRateThreshold: 60, + openStateDelay: 20, + storage: { + use: true + }, + name: 'mySlidingCountBreaker7', + onError: (err: string) => { + if (err === 'credentials-issue') { + return false; + } else { + return true; + } + } + }; + const slidingCountBreaker = new Mollitia.SlidingCountBreaker(breakerData); + const slidingCountBreaker2 = new Mollitia.SlidingCountBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { modules: [ slidingCountBreaker ] } + }); + const circuit2 = new Mollitia.Circuit({ + options: { modules: [ slidingCountBreaker2 ] } + }); + await slidingCountBreaker.clearState(); + await circuit.fn(failureAsync).execute('credentials-issue').catch(()=>{ return; }); + await circuit2.fn(failureAsync).execute('credentials-issue').catch(()=>{ return; }); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.CLOSED); + await circuit.fn(failureAsync).execute('real-issue').catch(()=>{ return; }); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); + await circuit2.fn(failureAsync).execute('real-issue').catch(()=>{ return; }); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.OPENED); + }); + + describe('Redis TTL - Sliding Count', () => { + it('Should use module storage TTL if found - SCB', async () => { + const moduleName = 'mySlidingCountBreaker8'; + const breakerData = { + slidingWindowSize: 3, + minimumNumberOfCalls: 2, + openStateDelay: 2000, + failureRateThreshold: 60, + storage: { + use: true, + ttl: 2000 + }, + name: moduleName + }; + const slidingCountBreaker = new Mollitia.SlidingCountBreaker(breakerData); + const slidingCountBreaker2 = new Mollitia.SlidingCountBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { + modules: [slidingCountBreaker] + } + }); + const circuit2 = new Mollitia.Circuit({ + options: { + modules: [slidingCountBreaker2] + } + }); + await slidingCountBreaker.clearState(); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.CLOSED); + await delay(100); + let res = await storageAddOn.getStateWithStorage(moduleName, 1000); + expect(JSON.stringify(res)).toContain('requests'); + await delay(1400); + res = await storageAddOn.getStateWithStorage(moduleName, 1000); + expect(JSON.stringify(res)).toContain('requests'); + await delay(500); + res = await storageAddOn.getStateWithStorage(moduleName, 1000); + expect(JSON.stringify(res)).toEqual('{}'); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); + }); + it('Should use AddOn storage TTL if found and no module TTL- SCB', async () => { + const moduleName = 'mySlidingCountBreaker9'; + const breakerData = { + slidingWindowSize: 3, + minimumNumberOfCalls: 2, + openStateDelay: 2000, + failureRateThreshold: 60, + storage: { + use: true + }, + name: moduleName + }; + const slidingCountBreaker = new Mollitia.SlidingCountBreaker(breakerData); + const slidingCountBreaker2 = new Mollitia.SlidingCountBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { + modules: [slidingCountBreaker] + } + }); + const circuit2 = new Mollitia.Circuit({ + options: { + modules: [slidingCountBreaker2] + } + }); + await slidingCountBreaker.clearState(); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingCountBreaker2.state).toEqual(Mollitia.BreakerState.CLOSED); + await delay(100); + let res = await storageAddOn.getStateWithStorage(moduleName, 1000); + expect(JSON.stringify(res)).toContain('requests'); + await delay(1000); + res = await storageAddOn.getStateWithStorage(moduleName, 1000); + expect(JSON.stringify(res)).toEqual('{}'); + }); + }); +},{ + timeout: 300000, +}); diff --git a/packages/@mollitia/redis-storage/test/unit/module/breaker/sliding-time-breaker.spec.ts b/packages/@mollitia/redis-storage/test/unit/module/breaker/sliding-time-breaker.spec.ts new file mode 100644 index 0000000..4ebcadb --- /dev/null +++ b/packages/@mollitia/redis-storage/test/unit/module/breaker/sliding-time-breaker.spec.ts @@ -0,0 +1,262 @@ +import { redisMock } from '../../../helper/redis-mock.js'; +import { describe, afterEach, it, expect, vi } from 'vitest'; +import * as Mollitia from 'mollitia'; +import { successAsync, failureAsync } from '../../../../../../../shared/vite/utils/vitest.js'; +import * as RedisStorage from '../../../../src/index.js'; + +vi.mock('redis', () => { + return redisMock; +}); + +const storageAddOn = new RedisStorage.StorageAddon({ host: 'localhost', port: 6379, password: '' }); +((storageAddOn as any).storage as any).initializePromise = new Promise((resolve) => resolve()); +Mollitia.use(storageAddOn); + +const delay = (delay = 1) => { + return new Promise((resolve) => { + setTimeout(() => { + resolve(); + }, delay); + }); +}; + +describe('Sliding Time Breaker', () => { + afterEach(() => { + successAsync.mockClear(); + failureAsync.mockClear(); + }); + it('should go to half open state after delay - TB', async () => { + const slidingTimeBreaker = new Mollitia.SlidingTimeBreaker({ + state: Mollitia.BreakerState.OPENED, + openStateDelay: 20, + storage: { + use: true + } + }); + const circuit = new Mollitia.Circuit({ + options: { + modules: [ + slidingTimeBreaker + ] + } + }); + await slidingTimeBreaker.clearState(); + await expect(circuit.fn(successAsync).execute('dummy')).rejects.toThrowError('Circuit is opened'); + expect(slidingTimeBreaker.state).toEqual(Mollitia.BreakerState.OPENED); + await delay(30); + expect(slidingTimeBreaker.state).toEqual(Mollitia.BreakerState.HALF_OPENED); + }); + it('switch to Open when failure rate exceeded - TB', async () => { + const breakerData = { + slidingWindowSize: 100, + minimumNumberOfCalls: 3, + failureRateThreshold: 70, + openStateDelay: 2000, + storage: { + use: true + }, + name: 'mySlidingTimeBreaker1' + }; + const breakerData2 = { + slidingWindowSize: 100, + minimumNumberOfCalls: 3, + failureRateThreshold: 70, + openStateDelay: 2000, + storage: { + use: true + }, + name: 'mySlidingTimeBreaker2' + }; + const slidingTimeBreaker = new Mollitia.SlidingTimeBreaker(breakerData); + const slidingTimeBreaker2 = new Mollitia.SlidingTimeBreaker(breakerData); + const slidingTimeBreaker3 = new Mollitia.SlidingTimeBreaker(breakerData2); + const circuit = new Mollitia.Circuit({ + options: { + modules: [slidingTimeBreaker] + } + }); + const circuit2 = new Mollitia.Circuit({ + options: { + modules: [slidingTimeBreaker2] + } + }); + const circuit3 = new Mollitia.Circuit({ + options: { + modules: [slidingTimeBreaker3] + } + }); + await slidingTimeBreaker.clearState(); + await slidingTimeBreaker3.clearState(); + await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + await expect(circuit2.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingTimeBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); + await expect(circuit2.fn(failureAsync).execute('dummy', 150)).rejects.toEqual('dummy'); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingTimeBreaker2.state).toEqual(Mollitia.BreakerState.CLOSED); + await expect(circuit2.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingTimeBreaker2.state).toEqual(Mollitia.BreakerState.OPENED); + await expect(circuit2.fn(successAsync).execute('dummy')).rejects.toThrowError('Circuit is opened'); + await expect(circuit3.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + await expect(circuit3.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingTimeBreaker2.state).toEqual(Mollitia.BreakerState.OPENED); + expect(slidingTimeBreaker3.state).toEqual(Mollitia.BreakerState.CLOSED); + await expect(circuit3.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingTimeBreaker3.state).toEqual(Mollitia.BreakerState.OPENED); + }); + it('Half Open State switch to Closed/Opened - TB', async () => { + const breakerData = { + failureRateThreshold: 50, + openStateDelay: 10, + state: Mollitia.BreakerState.HALF_OPENED, + permittedNumberOfCallsInHalfOpenState: 2, + storage: { + use: true + }, + name: 'mySlidingTimeBreaker3' + }; + const slidingTimeBreaker = new Mollitia.SlidingTimeBreaker(breakerData); + const slidingTimeBreaker2 = new Mollitia.SlidingTimeBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { + modules: [slidingTimeBreaker] + } + }); + const circuit2 = new Mollitia.Circuit({ + options: { + modules: [slidingTimeBreaker2] + } + }); + await slidingTimeBreaker.clearState(); + await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + await expect(circuit2.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingTimeBreaker2.state).toEqual(Mollitia.BreakerState.OPENED); + await delay(20); + expect(slidingTimeBreaker2.state).toEqual(Mollitia.BreakerState.HALF_OPENED); + await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingTimeBreaker2.state).toEqual(Mollitia.BreakerState.CLOSED); + }); + it('Slow Requests - TB', async () => { + const breakerData = { + failureRateThreshold: 50, + openStateDelay: 10, + slidingWindowSize: 1000, + minimumNumberOfCalls: 2, + permittedNumberOfCallsInHalfOpenState: 1, + slowCallDurationThreshold: 100, + slowCallRateThreshold: 50, + storage: { + use: true + }, + name: 'mySlidingTimeBreaker4' + }; + const slidingTimeBreaker = new Mollitia.SlidingTimeBreaker(breakerData); + const slidingTimeBreaker2 = new Mollitia.SlidingTimeBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { + modules: [slidingTimeBreaker] + } + }); + const circuit2 = new Mollitia.Circuit({ + options: { + modules: [slidingTimeBreaker2] + } + }); + await slidingTimeBreaker.clearState(); + await expect(circuit.fn(successAsync).execute('dummy', 150)).resolves.toEqual('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + // Even if 50% of slow requests, circuit is kept closed as last request is success + expect(slidingTimeBreaker2.state).toEqual(Mollitia.BreakerState.CLOSED); + await expect(circuit.fn(successAsync).execute('dummy', 150)).resolves.toEqual('dummy'); + expect(slidingTimeBreaker.state).toEqual(Mollitia.BreakerState.OPENED); + await delay(150); + expect(slidingTimeBreaker.state).toEqual(Mollitia.BreakerState.HALF_OPENED); + await expect(circuit2.fn(successAsync).execute('dummy', 150)).resolves.toEqual('dummy'); + expect(slidingTimeBreaker2.state).toEqual(Mollitia.BreakerState.OPENED); + await delay(10); + await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingTimeBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); + }); + + describe('Redis TTL - Sliding Time', () => { + it('Should use slidingTimeBreaker TTL for SlidingTimeBreaker module', async () => { + const moduleName = 'mySlidingTimeBreaker5'; + const breakerData = { + openStateDelay: 1000, + slidingWindowSize: 1000, + minimumNumberOfCalls: 2, + failureRateThreshold: 60, + permittedNumberOfCallsInHalfOpenState: 1, + storage: { + use: true + }, + name: moduleName + }; + const slidingTimeBreaker = new Mollitia.SlidingTimeBreaker(breakerData); + const slidingTimeBreaker2 = new Mollitia.SlidingTimeBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { + modules: [slidingTimeBreaker] + } + }); + const circuit2 = new Mollitia.Circuit({ + options: { + modules: [slidingTimeBreaker2] + } + }); + await slidingTimeBreaker.clearState(); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingTimeBreaker2.state).toEqual(Mollitia.BreakerState.CLOSED); + await delay(500); + let res = await storageAddOn.getStateWithStorage(moduleName, 1000); + expect(JSON.stringify(res)).toContain('requests'); + await delay(500); + // Delay to wait TTL -> Redis storage is cleared + res = await storageAddOn.getStateWithStorage(moduleName, 1000); + expect(JSON.stringify(res)).toEqual('{}'); + // So even if a request fails, the circuit is closed because this is the only request stored + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingTimeBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); + }); + it('Should ignore module storage ttl for SlidingTimeBreaker module, and use TTL from SlidingTimeBreaker', async () => { + const moduleName = 'mySlidingTimeBreaker6'; + const breakerData = { + openStateDelay: 1000, + slidingWindowSize: 2000, + minimumNumberOfCalls: 2, + failureRateThreshold: 60, + permittedNumberOfCallsInHalfOpenState: 1, + storage: { + use: true, + ttl: 1000 + }, + name: moduleName + }; + const slidingTimeBreaker = new Mollitia.SlidingTimeBreaker(breakerData); + const slidingTimeBreaker2 = new Mollitia.SlidingTimeBreaker(breakerData); + const circuit = new Mollitia.Circuit({ + options: { + modules: [slidingTimeBreaker] + } + }); + const circuit2 = new Mollitia.Circuit({ + options: { + modules: [slidingTimeBreaker2] + } + }); + await slidingTimeBreaker.clearState(); + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); + expect(slidingTimeBreaker2.state).toEqual(Mollitia.BreakerState.CLOSED); + // Delay to wait is > module storage TTL but < slidingWindowSize, so Redis Storage is not cleared + await delay(1100); + const res = await storageAddOn.getStateWithStorage(moduleName, 1000); + expect(JSON.stringify(res)).toContain('requests'); + // Hence, another request failure leads to circuit being opened + await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); + expect(slidingTimeBreaker.state).toEqual(Mollitia.BreakerState.OPENED); + }); + }); +}); diff --git a/packages/@mollitia/redis-storage/test/unit/module/ratelimit.spec.ts b/packages/@mollitia/redis-storage/test/unit/module/ratelimit.spec.ts new file mode 100644 index 0000000..c21cbfa --- /dev/null +++ b/packages/@mollitia/redis-storage/test/unit/module/ratelimit.spec.ts @@ -0,0 +1,179 @@ +import { redisMock, setRedisOptions } from '../../helper/redis-mock.js'; +import { describe, it, vi, expect } from 'vitest'; +import * as Mollitia from 'mollitia'; +import * as RedisStorage from '../../../src/index.js'; + +vi.mock('redis', () => { + return redisMock; +}); + +const storageAddOn = new RedisStorage.StorageAddon({ host: 'localhost', port: 6379, password: '' }); +((storageAddOn as any).storage as any).initializePromise = new Promise((resolve) => resolve()); +Mollitia.use(storageAddOn); + +const successAsync = vi.fn().mockImplementation((res: unknown, delay = 1) => { + return new Promise((resolve) => { + setTimeout(() => { + resolve(res); + }, delay); + }); +}); + +describe('ratelimit with Redis storage', () => { + it('No latency on Redis - should check ratelimit module with storage', async () => { + setRedisOptions({ getDelay: 0, setDelay: 0}); + const rateLimitData = { + name: 'myRateLimit', + limitForPeriod: 2, + limitPeriod: 20000, + storage: { + use: true + } + }; + const rateLimitData2 = { + name: 'myRateLimit2', + limitForPeriod: 2, + limitPeriod: 20000, + storage: { + use: true + } + }; + const rateLimit = new Mollitia.Ratelimit(rateLimitData); + const rateLimitBis = new Mollitia.Ratelimit(rateLimitData); + const rateLimit2 = new Mollitia.Ratelimit(rateLimitData2); + const circuit1 = new Mollitia.Circuit({ + options: { + modules: [rateLimit] + } + }); + const circuit1Bis = new Mollitia.Circuit({ + options: { + modules: [rateLimitBis] + } + }); + const circuit2 = new Mollitia.Circuit({ + options: { + modules: [rateLimit2] + } + }); + await rateLimit.clearState(); + await rateLimit2.clearState(); + await circuit1.fn(successAsync).execute('dummy'); + await circuit1Bis.fn(successAsync).execute('dummy'); + await expect(circuit1.fn(successAsync).execute('dummy')).rejects.toThrowError('Ratelimited'); + await circuit2.fn(successAsync).execute('dummy'); + await circuit2.fn(successAsync).execute('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).rejects.toThrowError('Ratelimited'); + }); + it('Latency on Redis < max allowed latency - should check ratelimit module with storage', async () => { + setRedisOptions({ getDelay: 100, setDelay: 1, crash: false}); + const rateLimitData = { + name: 'myRateLimit', + limitForPeriod: 2, + limitPeriod: 20000, + storage: { + use: true, + getMaxDelay: 500, + setMaxDelay: 500 + } + }; + const rateLimitData2 = { + name: 'myRateLimit2', + limitForPeriod: 2, + limitPeriod: 20000, + storage: { + use: true, + getMaxDelay: 500, + setMaxDelay: 500 + } + }; + const rateLimit = new Mollitia.Ratelimit(rateLimitData); + const rateLimitBis = new Mollitia.Ratelimit(rateLimitData); + const rateLimit2 = new Mollitia.Ratelimit(rateLimitData2); + const circuit1 = new Mollitia.Circuit({ + options: { + modules: [rateLimit] + } + }); + const circuit1Bis = new Mollitia.Circuit({ + options: { + modules: [rateLimitBis] + } + }); + const circuit2 = new Mollitia.Circuit({ + options: { + modules: [rateLimit2] + } + }); + await rateLimit.clearState(); + await rateLimit2.clearState(); + await circuit1.fn(successAsync).execute('dummy'); + await circuit1Bis.fn(successAsync).execute('dummy'); + await expect(circuit1.fn(successAsync).execute('dummy')).rejects.toThrowError('Ratelimited'); + await circuit2.fn(successAsync).execute('dummy'); + await circuit2.fn(successAsync).execute('dummy'); + await expect(circuit2.fn(successAsync).execute('dummy')).rejects.toThrowError('Ratelimited'); + }); + it('Latency on Redis > max allowed latency - should behave as if no storage', async () => { + setRedisOptions({ getDelay: 500, setDelay: 500}); + const rateLimitData = { + name: 'myRateLimit', + limitForPeriod: 2, + limitPeriod: 20000, + storage: { + use: true, + getMaxDelay: 100, + setMaxDelay: 100 + } + }; + const rateLimit = new Mollitia.Ratelimit(rateLimitData); + const rateLimitBis = new Mollitia.Ratelimit(rateLimitData); + const circuit1 = new Mollitia.Circuit({ + options: { + modules: [rateLimit] + } + }); + const circuit1Bis = new Mollitia.Circuit({ + options: { + modules: [rateLimitBis] + } + }); + await rateLimit.clearState(); + await circuit1.fn(successAsync).execute('dummy'); + await circuit1Bis.fn(successAsync).execute('dummy'); + await circuit1.fn(successAsync).execute('dummy'); + await circuit1Bis.fn(successAsync).execute('dummy'); + await expect(circuit1.fn(successAsync).execute('dummy')).rejects.toThrowError('Ratelimited'); + await expect(circuit1.fn(successAsync).execute('dummy')).rejects.toThrowError('Ratelimited'); + }); + // it('Redis is no longer working - Should behave as if no storage', async () => { + // setRedisOptions({ getDelay: 0, setDelay: 0, crash: true}); + // const rateLimitData = { + // name: 'myRateLimit', + // limitForPeriod: 2, + // limitPeriod: 20000, + // storage: { + // use: true, + // } + // }; + // const rateLimit = new Mollitia.Ratelimit(rateLimitData); + // const rateLimitBis = new Mollitia.Ratelimit(rateLimitData); + // const circuit1 = new Mollitia.Circuit({ + // options: { + // modules: [rateLimit] + // } + // }); + // const circuit1Bis = new Mollitia.Circuit({ + // options: { + // modules: [rateLimitBis] + // } + // }); + // await rateLimit.clearState(); + // await circuit1.fn(successAsync).execute('dummy'); + // await circuit1Bis.fn(successAsync).execute('dummy'); + // await circuit1.fn(successAsync).execute('dummy'); + // await circuit1Bis.fn(successAsync).execute('dummy'); + // await expect(circuit1.fn(successAsync).execute('dummy')).rejects.toThrowError('Ratelimited'); + // await expect(circuit1.fn(successAsync).execute('dummy')).rejects.toThrowError('Ratelimited'); + // }); +}); diff --git a/packages/@mollitia/redis-storage/tsconfig.eslint.json b/packages/@mollitia/redis-storage/tsconfig.eslint.json new file mode 100644 index 0000000..30fc715 --- /dev/null +++ b/packages/@mollitia/redis-storage/tsconfig.eslint.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "include": [".", "./.eslintrc.cjs"] +} diff --git a/packages/@mollitia/redis-storage/tsconfig.json b/packages/@mollitia/redis-storage/tsconfig.json new file mode 100644 index 0000000..0943608 --- /dev/null +++ b/packages/@mollitia/redis-storage/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "@shared/tsconfig/tsconfig.lib.json", + "compilerOptions": { + "noEmit": true, + "paths": { + "mollitia": ["../../mollitia/src/index.ts"] + } + }, + "exclude": ["dist"] +} diff --git a/packages/@mollitia/redis-storage/vite.config.ts b/packages/@mollitia/redis-storage/vite.config.ts new file mode 100644 index 0000000..ada5f7b --- /dev/null +++ b/packages/@mollitia/redis-storage/vite.config.ts @@ -0,0 +1,10 @@ +// Helpers +import { defineLibConfig } from '../../../shared/vite/index.js'; +import { version } from './package.json'; + +export default defineLibConfig({ + name: 'Mollitia', + base: './src', + entry: ['./index.ts'], + version +}); diff --git a/packages/mollitia/src/helpers/serializable.ts b/packages/mollitia/src/helpers/serializable.ts new file mode 100644 index 0000000..b6d800a --- /dev/null +++ b/packages/mollitia/src/helpers/serializable.ts @@ -0,0 +1,6 @@ +export type SerializablePrimitive = string | number | boolean | null | undefined; +export type SerializableArray = Array; +export type SerializableRecord = { + [field: string]: SerializablePrimitive | SerializableArray | SerializableRecord; +}; +export type Serializable = SerializablePrimitive | SerializableRecord | SerializableArray; diff --git a/packages/mollitia/src/index.ts b/packages/mollitia/src/index.ts index 9a24bf8..2c08f7c 100644 --- a/packages/mollitia/src/index.ts +++ b/packages/mollitia/src/index.ts @@ -66,7 +66,10 @@ export { BreakerError, BreakerMaxAllowedRequestError, BreakerState, - SlidingWindowBreakerOptions + SlidingWindowBreaker, + SlidingWindowBreakerOptions, + SlidingWindowRequestResult, + type SlidingElem } from './module/breaker/index.js'; // Sliding Count Breaker @@ -78,3 +81,10 @@ export { export { SlidingTimeBreaker } from './module/breaker/sliding-time-breaker.js'; + +export { + type Serializable, + type SerializableRecord, + type SerializablePrimitive, + type SerializableArray +} from './helpers/serializable.js'; diff --git a/packages/mollitia/src/module/breaker/index.ts b/packages/mollitia/src/module/breaker/index.ts index 37eabc4..20e6659 100644 --- a/packages/mollitia/src/module/breaker/index.ts +++ b/packages/mollitia/src/module/breaker/index.ts @@ -1,6 +1,6 @@ import { Module, ModuleOptions } from '../index.js'; import { Circuit, CircuitFunction } from '../../circuit.js'; - +import { SerializableRecord } from '../../helpers/serializable.js'; type ErrorCallback = (err: any) => boolean; type BreakerResultResponse = { @@ -93,12 +93,21 @@ export enum SlidingWindowRequestResult { TIMEOUT = 2 } -export abstract class SlidingWindowBreaker extends Module { +export interface SlidingElem extends SerializableRecord{ + result: SlidingWindowRequestResult, + timestamp?: number +} + +export abstract class SlidingWindowBreaker extends Module { // Public Attributes /** * Specifies the circuit state */ public state: BreakerState; + /** + * Specifies when the circuit state was set + */ + public stateTimestamp: number; /** * Specifies the time (in ms) the circuit stay opened before switching to half-open */ @@ -137,22 +146,18 @@ export abstract class SlidingWindowBreaker extends Module { */ public onError: ErrorCallback; // Private Attributes - protected callsInClosedState: T[]; private halfOpenMaxDelayTimeout = 0; private openTimeout = 0; - private nbCallsInHalfOpenedState: number; - private callsInHalfOpenedState: SlidingWindowRequestResult[]; + public nbRequestsInHalfOpenedState: number; + public requests: SlidingElem[]; + private isInitialized = false; constructor (options?: SlidingWindowBreakerOptions) { super(options); this.state = (options?.state !== undefined) ? options.state : BreakerState.CLOSED; + this.stateTimestamp = Date.now(); this.openStateDelay = (options?.openStateDelay !== undefined) ? options.openStateDelay : 60 * 1000; this.halfOpenStateMaxDelay = (options?.halfOpenStateMaxDelay !== undefined) ? options.halfOpenStateMaxDelay : 0; - if (this.state === BreakerState.OPENED) { - this.setHalfDelay(); - } else if (this.state === BreakerState.HALF_OPENED) { - this.setOpenDelay(); - } this.slidingWindowSize = (options?.slidingWindowSize !== undefined) ? options.slidingWindowSize : 10; this.minimumNumberOfCalls = (options?.minimumNumberOfCalls !== undefined) ? options.minimumNumberOfCalls : 10; this.failureRateThreshold = (options?.failureRateThreshold !== undefined) ? options.failureRateThreshold : 50; @@ -160,16 +165,14 @@ export abstract class SlidingWindowBreaker extends Module { this.slowCallRateThreshold = (options?.slowCallRateThreshold !== undefined) ? options?.slowCallRateThreshold : 100; this.permittedNumberOfCallsInHalfOpenState = (options?.permittedNumberOfCallsInHalfOpenState !== undefined) ? options.permittedNumberOfCallsInHalfOpenState : 2; - this.nbCallsInHalfOpenedState = 0; - this.callsInHalfOpenedState = []; - this.callsInClosedState = []; + this.nbRequestsInHalfOpenedState = 0; + this.requests = []; this.onError = options?.onError || (() => true); } private reinitializeCounters (): void { - this.nbCallsInHalfOpenedState = 0; - this.callsInClosedState = []; - this.callsInHalfOpenedState = []; + this.nbRequestsInHalfOpenedState = 0; + this.requests = []; } public onOpened(): void { this.reinitializeCounters(); @@ -183,26 +186,54 @@ export abstract class SlidingWindowBreaker extends Module { this.reinitializeCounters(); } - public async execute (circuit: Circuit, promise: CircuitFunction, ...params: any[]): Promise { - const _exec = this._promiseBreaker(circuit, promise, ...params); + public async execute(circuit: Circuit, promise: CircuitFunction, ...params: any[]): Promise { + let data; + try { + data = await this.getState(); + } catch (e) { + console.warn('Cannot get state'); + } + if (data?.requests) { + this.requests = (data.requests as SerializableRecord[]).map((r) => { + if (r.timestamp) { + return { result: r.result as SlidingWindowRequestResult, timestamp: r.timestamp as number} + } + return { result: r.result as SlidingWindowRequestResult} + }); + } else { + this.requests = []; + } + if (data?.state) { + this.state = (data.state as SerializableRecord).state as BreakerState; + this.stateTimestamp = (data.state as SerializableRecord).timestamp as number; + } + if (!this.isInitialized) { + this.isInitialized = true; + if (this.state === BreakerState.OPENED) { + await this.setHalfDelay(); + } else if (this.state === BreakerState.HALF_OPENED) { + await this.setOpenDelay(); + } + } + const _exec = this._promiseBreaker(circuit, promise, ...params); const _params = this.getExecParams(circuit, params); this.emit('execute', circuit, _exec, _params); return _exec; } - private async _promiseBreaker (circuit: Circuit, promise: CircuitFunction, ...params: any[]): Promise { + private async _promiseBreaker (circuit: Circuit, promise: CircuitFunction, ...params: any[]): Promise { switch (this.state) { case BreakerState.OPENED: this.logger?.debug(`${circuit.name}/${this.name} - Circuit is opened`); return Promise.reject(new BreakerError()); case BreakerState.HALF_OPENED: - return this.executeInHalfOpened(promise, ...params); + return this.executeInHalfOpened(promise, ...params); case BreakerState.CLOSED: - default: - return this.executeInClosed(promise, ...params); + default: + return this.executeInClosed(promise, ...params); } } - abstract executeInClosed (promise: CircuitFunction, ...params: any[]): Promise; + abstract executeInClosed (promise: CircuitFunction, ...params: any[]): Promise; protected adjustRequestResult(requestResult: SlidingWindowRequestResult, shouldReportFailure: boolean): SlidingWindowRequestResult { if (!shouldReportFailure && requestResult === SlidingWindowRequestResult.FAILURE) { @@ -211,13 +242,21 @@ export abstract class SlidingWindowBreaker extends Module { return requestResult; } - protected async executeInHalfOpened (promise: CircuitFunction, ...params: any[]): Promise { - if (this.nbCallsInHalfOpenedState < this.permittedNumberOfCallsInHalfOpenState) { - this.nbCallsInHalfOpenedState++; - const {requestResult, response, shouldReportFailure } = await this.executePromise(promise, ...params); - this.callsInHalfOpenedState.push(this.adjustRequestResult(requestResult, shouldReportFailure)); + protected async setStateSecure(state: SerializableRecord[], ttl?: number): Promise { + try { + await this.setState(state, ttl); + } catch (e) { + console.warn('Cannot set the state'); + } + } - if (this.callsInHalfOpenedState.length == this.permittedNumberOfCallsInHalfOpenState) { + protected async executeInHalfOpened(promise: CircuitFunction, ...params: any[]): Promise { + if (this.nbRequestsInHalfOpenedState < this.permittedNumberOfCallsInHalfOpenState) { + this.nbRequestsInHalfOpenedState++; + const { requestResult, response, shouldReportFailure } = await this.executePromise(promise, ...params); + this.requests.push({ result: this.adjustRequestResult(requestResult, shouldReportFailure) }); + await this.setStateSecure([ { key: 'requests', value: this.requests } ]); + if (this.requests.length == this.permittedNumberOfCallsInHalfOpenState) { this.checkCallRatesHalfOpen(this.open.bind(this), this.close.bind(this)); } if (requestResult === SlidingWindowRequestResult.FAILURE) { @@ -248,25 +287,26 @@ export abstract class SlidingWindowBreaker extends Module { } protected checkCallRatesHalfOpen(callbackFailure: (() => void), callbackSuccess?: (() => void)): void { - const {nbSlow, nbFailure} = this.callsInHalfOpenedState.reduce(this.getNbSlowAndFailure, {nbSlow: 0, nbFailure: 0}); - this.checkResult(nbSlow, nbFailure, this.callsInHalfOpenedState.length, callbackFailure, callbackSuccess); - } - - protected checkResult(nbSlow: number, nbFailure: number, nbCalls: number, callbackFailure: (() => void), callbackSuccess?: (() => void)): void { - if ( - (this.slowCallRateThreshold < 100 && (((nbSlow / nbCalls) * 100) >= this.slowCallRateThreshold)) || - (this.failureRateThreshold < 100 && (((nbFailure / nbCalls) * 100) >= this.failureRateThreshold)) - ) { - callbackFailure(); - } else { + const { nbSlow, nbFailure } = this.requests.reduce(this.getNbSlowAndFailure, { nbSlow: 0, nbFailure: 0 }); + const result = this.checkResult(nbSlow, nbFailure, this.requests.length); + if (result) { if (callbackSuccess) { callbackSuccess(); } + } else { + callbackFailure(); } } - protected getNbSlowAndFailure(acc: {nbSlow: number, nbFailure: number}, current: SlidingWindowRequestResult): {nbSlow: number, nbFailure: number} { - switch(current) { + private checkResult(nbSlow: number, nbFailure: number, nbCalls: number): boolean { + return !( + (this.slowCallRateThreshold < 100 && (((nbSlow / nbCalls) * 100) >= this.slowCallRateThreshold)) || + (this.failureRateThreshold < 100 && (((nbFailure / nbCalls) * 100) >= this.failureRateThreshold)) + ); + } + + protected getNbSlowAndFailure(acc: {nbSlow: number, nbFailure: number}, current: SlidingElem): {nbSlow: number, nbFailure: number} { + switch(current.result) { case SlidingWindowRequestResult.FAILURE: acc.nbFailure++; break; @@ -276,57 +316,80 @@ export abstract class SlidingWindowBreaker extends Module { return acc; } - protected _open (circuit: Circuit): void { - if (this.state !== BreakerState.OPENED) { - this.logger?.debug(`${circuit.name}/${this.name} - Breaker: Open`); - this.open(); - } - } - protected _close (circuit: Circuit): void { - if (this.state !== BreakerState.CLOSED) { - this.logger?.debug(`${circuit.name}/${this.name} - Breaker: Close`); - this.close(); - } + protected checkCallRatesClosed(): boolean { + const {nbSlow, nbFailure} = this.requests.reduce(this.getNbSlowAndFailure, {nbSlow: 0, nbFailure: 0}); + return this.checkResult(nbSlow, nbFailure, this.requests.length); } - public open (): void { + public async open(): Promise { if (this.state !== BreakerState.OPENED) { this.clearHalfOpenTimeout(); this.state = BreakerState.OPENED; + this.stateTimestamp = Date.now(); this.setHalfDelay(); this.onOpened(); + await this.setStateSecure([ + { key: 'state', value: { state: BreakerState.OPENED, timestamp: Date.now() } }, + { key: 'requests', value: '' } + ]); this.emit('state-changed', this.state); } } - public halfOpen (): void { + public async halfOpen(): Promise { if (this.state !== BreakerState.HALF_OPENED) { this.clearHalfOpenTimeout(); this.state = BreakerState.HALF_OPENED; + this.stateTimestamp = Date.now(); this.setOpenDelay(); this.onHalfOpened(); + await this.setStateSecure([ + { key: 'state', value: { state: BreakerState.HALF_OPENED, timestamp: Date.now() } }, + { key: 'requests', value: '' } + ]); this.emit('state-changed', this.state); } } - public close (): void { + public async close(): Promise { if (this.state !== BreakerState.CLOSED) { this.clearHalfOpenTimeout(); this.state = BreakerState.CLOSED; + this.stateTimestamp = Date.now(); this.onClosed(); + try { + await this.setStateSecure([ + { key: 'state', value: { state: BreakerState.CLOSED, timestamp: Date.now() } }, + { key: 'requests', value: '' } + ]); + } catch (e) { + console.warn('Timeout while setting state'); + } this.emit('state-changed', this.state); } } - private setHalfDelay (): void { - this.openTimeout = setTimeout(() => { + private async setHalfDelay(): Promise { + const timeInCurrentState = Date.now() - this.stateTimestamp; + if (timeInCurrentState >= this.openStateDelay) { this.logger?.debug(`${this.name} - Breaker: Half Open`); - this.halfOpen(); - }, this.openStateDelay) as number; + await this.halfOpen(); + } else { + this.openTimeout = setTimeout(async () => { + this.logger?.debug(`${this.name} - Breaker: Half Open`); + await this.halfOpen(); + }, (this.openStateDelay - timeInCurrentState)) as number; + } } - private setOpenDelay (): void { + private async setOpenDelay(): Promise { if (this.halfOpenStateMaxDelay) { - this.halfOpenMaxDelayTimeout = setTimeout(() => { + const timeInCurrentState = Date.now() - this.stateTimestamp; + if (timeInCurrentState >= this.halfOpenStateMaxDelay) { this.halfOpenMaxDelayTimeout = 0; - this.open(); - }, this.halfOpenStateMaxDelay) as number; + await this.open(); + } else { + this.halfOpenMaxDelayTimeout = setTimeout(async () => { + this.halfOpenMaxDelayTimeout = 0; + await this.open(); + }, (this.halfOpenStateMaxDelay - timeInCurrentState)) as number; + } } } private clearHalfOpenTimeout (): void { @@ -344,4 +407,26 @@ export abstract class SlidingWindowBreaker extends Module { this.openTimeout = 0; } } + public async getState(): Promise { + return new Promise((resolve) => { + resolve({ + requests: this.requests, + state: { + state: this.state, + timestamp: this.stateTimestamp + } + }); + }); + } + public async setState(state: SerializableRecord[], ttl?: number): Promise { + return new Promise((resolve) => { + resolve(); + }); + } + public async clearState(): Promise { + return new Promise((resolve) => { + this.requests = []; + resolve(); + }); + } } diff --git a/packages/mollitia/src/module/breaker/sliding-count-breaker.ts b/packages/mollitia/src/module/breaker/sliding-count-breaker.ts index 400dfb6..74fe46d 100644 --- a/packages/mollitia/src/module/breaker/sliding-count-breaker.ts +++ b/packages/mollitia/src/module/breaker/sliding-count-breaker.ts @@ -4,7 +4,7 @@ import { SlidingWindowBreaker, SlidingWindowBreakerOptions, SlidingWindowRequest /** * The Sliding Count Breaker Module, that allows to break the circuit if it fails too often. */ -export class SlidingCountBreaker extends SlidingWindowBreaker { +export class SlidingCountBreaker extends SlidingWindowBreaker { constructor(options?: SlidingWindowBreakerOptions) { super(options); this.slidingWindowSize = (options?.slidingWindowSize !== undefined) ? options.slidingWindowSize : 10; @@ -12,29 +12,33 @@ export class SlidingCountBreaker extends SlidingWindowBreaker (promise: CircuitFunction, ...params: any[]): Promise { + + public async executeInClosed(promise: CircuitFunction, ...params: any[]): Promise { const {requestResult, response, shouldReportFailure } = await this.executePromise(promise, ...params); const adjustedRequestResult = this.adjustRequestResult(requestResult, shouldReportFailure); - this.callsInClosedState.push(adjustedRequestResult); - const nbCalls = this.callsInClosedState.length; + this.requests.push({ result: adjustedRequestResult }); + const nbCalls = this.requests.length; + let stateSet = false; if (nbCalls >= this.minimumNumberOfCalls) { if (nbCalls > this.slidingWindowSize) { - this.callsInClosedState.splice(0,(nbCalls - this.slidingWindowSize)); + this.requests.splice(0, nbCalls - this.slidingWindowSize); + stateSet = true; + await this.setStateSecure([ { key: 'requests', value: this.requests } ]); } if (adjustedRequestResult !== SlidingWindowRequestResult.SUCCESS) { - this.checkCallRatesClosed(this.open.bind(this)); + if (!this.checkCallRatesClosed()) { + await this.open(); + stateSet = true; + } } } + if (!stateSet) { + await this.setStateSecure([{ key: 'requests', value: this.requests } ]); + } if (requestResult === SlidingWindowRequestResult.FAILURE) { return Promise.reject(response); } else { return Promise.resolve(response); } } - - private checkCallRatesClosed(callbackFailure: (() => void)): void { - const {nbSlow, nbFailure} = this.callsInClosedState.reduce(this.getNbSlowAndFailure, {nbSlow: 0, nbFailure: 0}); - this.checkResult(nbSlow, nbFailure, this.callsInClosedState.length, callbackFailure); - } } diff --git a/packages/mollitia/src/module/breaker/sliding-time-breaker.ts b/packages/mollitia/src/module/breaker/sliding-time-breaker.ts index b5ba5fe..9ee84e7 100644 --- a/packages/mollitia/src/module/breaker/sliding-time-breaker.ts +++ b/packages/mollitia/src/module/breaker/sliding-time-breaker.ts @@ -1,15 +1,11 @@ import { CircuitFunction } from '../../circuit.js'; import { SlidingWindowBreaker, SlidingWindowBreakerOptions, SlidingWindowRequestResult } from './index.js'; - -interface SlidingTimeElem { - result: SlidingWindowRequestResult, - timestamp: number -} +import { SerializableRecord } from '../../helpers/serializable.js'; /** * The Sliding Time Breaker Module, that allows to break the circuit if it often fails on a time window. */ -export class SlidingTimeBreaker extends SlidingWindowBreaker { +export class SlidingTimeBreaker extends SlidingWindowBreaker { private maxSize: number; constructor(options?: SlidingWindowBreakerOptions) { @@ -19,16 +15,16 @@ export class SlidingTimeBreaker extends SlidingWindowBreaker { } private filterCalls(): void { - let nbCalls = this.callsInClosedState.length; + let nbCalls = this.requests.length; if (nbCalls >= this.maxSize) { - this.callsInClosedState.shift(); + this.requests.splice(0, 1); nbCalls--; } let stillOk = true; const now = (new Date()).getTime(); for (let i=0; i this.slidingWindowSize) { - this.callsInClosedState.shift(); + if ((now - this.requests[0].timestamp!) > this.slidingWindowSize) { + this.requests.splice(0, 1); } else { stillOk = false; } @@ -39,12 +35,19 @@ export class SlidingTimeBreaker extends SlidingWindowBreaker { const {requestResult, response, shouldReportFailure } = await this.executePromise(promise, ...params); this.filterCalls(); const adjustedRequestResult = this.adjustRequestResult(requestResult, shouldReportFailure); - this.callsInClosedState.push({ + this.requests.push({ result: adjustedRequestResult, timestamp: (new Date()).getTime() }); - if (this.callsInClosedState.length >= this.minimumNumberOfCalls && adjustedRequestResult !== SlidingWindowRequestResult.SUCCESS) { - this.checkCallRatesClosed(this.open.bind(this)); + let stateSet = false; + if (this.requests.length >= this.minimumNumberOfCalls && adjustedRequestResult !== SlidingWindowRequestResult.SUCCESS) { + if (!this.checkCallRatesClosed()) { + await this.open(); + stateSet = true; + } + } + if (!stateSet) { + await this.setStateSecure([ { key: 'requests', value: this.requests } ], this.slidingWindowSize); } if (requestResult === SlidingWindowRequestResult.FAILURE) { return Promise.reject(response); @@ -52,20 +55,4 @@ export class SlidingTimeBreaker extends SlidingWindowBreaker { return Promise.resolve(response); } } - - private checkCallRatesClosed(callbackFailure: (() => void)): void { - const {nbSlow, nbFailure} = this.callsInClosedState.reduce(this.getNbSlowAndFailureTimeElem, {nbSlow: 0, nbFailure: 0}); - this.checkResult(nbSlow, nbFailure, this.callsInClosedState.length, callbackFailure); - } - - public getNbSlowAndFailureTimeElem (acc: {nbSlow: number, nbFailure: number}, current: SlidingTimeElem): {nbSlow: number, nbFailure: number} { - switch(current.result) { - case SlidingWindowRequestResult.FAILURE: - acc.nbFailure++; - break; - case SlidingWindowRequestResult.TIMEOUT: - acc.nbSlow++; - } - return acc; - } } diff --git a/packages/mollitia/src/module/ratelimit.ts b/packages/mollitia/src/module/ratelimit.ts index efdf41e..0b86a77 100644 --- a/packages/mollitia/src/module/ratelimit.ts +++ b/packages/mollitia/src/module/ratelimit.ts @@ -1,5 +1,6 @@ import { Module, ModuleOptions } from './index.js'; import { Circuit, CircuitFunction } from '../circuit.js'; +import { SerializableRecord } from '../helpers/serializable.js'; /** * Properties that customizes the ratelimit behavior. @@ -51,13 +52,47 @@ export class Ratelimit extends Module { this.limitForPeriod = (options?.limitForPeriod !== undefined) ? options.limitForPeriod : Infinity; this.requestsTime = []; } + public async getState(): Promise { + return new Promise((resolve) => { + resolve({ requests: this.requestsTime }); + }); + } + public async setState(state: SerializableRecord[], ttl?: number): Promise { + return new Promise((resolve) => { + resolve(); + }); + } + public async clearState(): Promise { + return new Promise((resolve) => { + this.requestsTime = []; + resolve(); + }); + } // Public Methods - public async execute (circuit: Circuit, promise: CircuitFunction, ...params: any[]): Promise { + public async execute(circuit: Circuit, promise: CircuitFunction, ...params: any[]): Promise { + let currentState; + try { + currentState = await this.getState(); + } catch (e) { + console.warn('Cannot get state'); + } + if (currentState?.requests) { + this.requestsTime = currentState?.requests as number[]; + } const _exec = this._promiseRatelimit(circuit, promise, ...params); const _params = this.getExecParams(circuit, params); this.emit('execute', circuit, _exec, _params); return _exec; } + + private async addCurrentRequest(requestDate: number): Promise { + this.requestsTime.push(requestDate); + try { + await this.setState( [{ key: 'requests', value: this.requestsTime }], this.limitPeriod ); + } catch (e) { + console.warn('Cannot set the state'); + } + } // Private Methods private async _promiseRatelimit (circuit: Circuit, promise: CircuitFunction, ...params: any[]): Promise { if (!this.limitPeriod) { @@ -65,13 +100,13 @@ export class Ratelimit extends Module { } const now = (new Date()).getTime(); if (this.requestsTime.length < this.limitForPeriod) { - this.requestsTime.push(now); + await this.addCurrentRequest(now); return promise(...params); } else { const deltaSinceFirstRequest = now - this.requestsTime[0]; if (deltaSinceFirstRequest > this.limitPeriod) { - this.requestsTime.shift(); - this.requestsTime.push(now); + this.requestsTime.splice(0, 1); + await this.addCurrentRequest(now); return promise(...params); } else { this.logger?.debug(`${circuit.name}/${this.name} - Ratelimited`); diff --git a/packages/mollitia/test/unit/module/breaker/sliding-count-breaker.spec.ts b/packages/mollitia/test/unit/module/breaker/sliding-count-breaker.spec.ts index f8e5a34..fafafd8 100644 --- a/packages/mollitia/test/unit/module/breaker/sliding-count-breaker.spec.ts +++ b/packages/mollitia/test/unit/module/breaker/sliding-count-breaker.spec.ts @@ -13,7 +13,7 @@ describe('Sliding Count Breaker', () => { state: Mollitia.BreakerState.OPENED, openStateDelay: 20 }); - new Mollitia.Circuit({ + const circuit = new Mollitia.Circuit({ options: { modules: [ slidingCountBreaker @@ -21,7 +21,13 @@ describe('Sliding Count Breaker', () => { } }); expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.OPENED); - await delay(30); + try { + await circuit.fn(failureAsync).execute('dummy'); + } catch { + // Request is executed in opened state - Failing with exception, this is normal situation. + // The execute request is just there to start timeout to switch from inital state (Opened) to other state (Half Opened) after OpenStateDelay timeout + } + await delay(100); expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.HALF_OPENED); }); it('Switch to Open when failure rate exceeded', async () => { @@ -48,8 +54,8 @@ describe('Sliding Count Breaker', () => { }); it('Half Open State max duration', async () => { const slidingCountBreaker = new Mollitia.SlidingCountBreaker({ - halfOpenStateMaxDelay: 20, - openStateDelay: 10, + halfOpenStateMaxDelay: 200, + openStateDelay: 100, state: Mollitia.BreakerState.HALF_OPENED, permittedNumberOfCallsInHalfOpenState: 1, minimumNumberOfCalls: 1 @@ -63,13 +69,13 @@ describe('Sliding Count Breaker', () => { }); await expect(circuit.fn(failureAsync).execute('dummy')).rejects.toEqual('dummy'); expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.OPENED); - await delay(10); + await delay(100); expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.HALF_OPENED); - await delay(10); + await delay(100); expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.HALF_OPENED); - await delay(10); + await delay(100); expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.OPENED); - await delay(10); + await delay(100); await expect(circuit.fn(successAsync).execute('dummy')).resolves.toEqual('dummy'); expect(slidingCountBreaker.state).toEqual(Mollitia.BreakerState.CLOSED); await delay(100); diff --git a/packages/mollitia/test/unit/module/breaker/sliding-time-breaker.spec.ts b/packages/mollitia/test/unit/module/breaker/sliding-time-breaker.spec.ts index f81f680..afb135f 100644 --- a/packages/mollitia/test/unit/module/breaker/sliding-time-breaker.spec.ts +++ b/packages/mollitia/test/unit/module/breaker/sliding-time-breaker.spec.ts @@ -3,7 +3,7 @@ import * as Mollitia from '../../../../src/index.js'; import { delay } from '../../../../src/helpers/time.js'; import { successAsync, failureAsync } from '../../../../../../shared/vite/utils/vitest.js'; -describe('Sliding Count Breaker', () => { +describe('Sliding Time Breaker', () => { afterEach(() => { successAsync.mockClear(); failureAsync.mockClear(); @@ -13,7 +13,7 @@ describe('Sliding Count Breaker', () => { state: Mollitia.BreakerState.OPENED, openStateDelay: 20 }); - new Mollitia.Circuit({ + const circuit = new Mollitia.Circuit({ options: { modules: [ slidingTimeBreaker @@ -21,7 +21,13 @@ describe('Sliding Count Breaker', () => { } }); expect(slidingTimeBreaker.state).toEqual(Mollitia.BreakerState.OPENED); - await delay(30); + try { + await circuit.fn(failureAsync).execute('dummy'); + } catch { + // Request is executed in opened state - Failing with exception, this is normal situation. + // The execute request is just there to start timeout to switch from inital state (Opened) to other state (Half Opened) after OpenStateDelay timeout + } + await delay(100); expect(slidingTimeBreaker.state).toEqual(Mollitia.BreakerState.HALF_OPENED); }); it('switch to Open when failure rate exceeded', async () => {