feat: socket server with redis multinode cluster (#720)

* feat: clustering support for redis

* feat: add auth support

* feat: socket io clustering with adminui

* docs: new setup steps

packages/sdk-socket-server-next/.env.sample

@@ -2,8 +2,8 @@ SEGMENT_API_KEY_PRODUCTION=123456789
 SEGMENT_API_KEY_DEBUG=123456789
 # REDIS_URL=redis://redis:6379/0
 # Example REDIS_NODES format: "redis://host1:6379,redis://host2:6379"
-# REDIS_NODES=redis://localhost:6380,redis://localhost:6381,redis://localhost:6382
-REDIS_URL=redis://redis:6379/0
+REDIS_NODES=redis://localhost:6380,redis://localhost:6381,redis://localhost:6382
+REDIS_PASSWORD=redis_password
 RATE_LIMITER=false
 RATE_LIMITER_HTTP_WINDOW_MINUTE=1
 RATE_LIMITER_HTTP_LIMIT=100000

packages/sdk-socket-server-next/README.md

@@ -17,10 +17,17 @@ This guide provides instructions for setting up and debugging the SDK socket ser
    - Copy the sample environment file: `cp .env.sample .env`
    - Adjust the `.env` file with the correct settings as per your project requirements.
 
-2. **Start the SDK Server**:
+2. **Start the REDIS cluster**:
    - For standard development, use: `yarn start`
    - For debugging with more verbose output, use: `yarn debug`
 
+3. **Check cluster status**:
+   - Use the command: `yarn docker:redis:check`
+   - This command sets up a local redis cluster and connect to it to make sure everything is working.
+
+4. **Start the SDK Socket Server via docker**:
+    - Use the command: `yarn docker:debug`
+
 ### Using Ngrok for External Access
 
 To expose your local server to the internet, particularly for testing with mobile apps like MetaMask Mobile, use Ngrok.
@@ -37,22 +44,6 @@ To expose your local server to the internet, particularly for testing with mobil
 3. **Configure Your DApp**:
    - Set the `communicationServerUrl` in your DApp's SDK options to your local IP or `localhost` with port 4000. For example: `communicationServerUrl: "http://{yourLocalIP | localhost}:4000"`
 
-## Debugging with Docker Compose
-
-You can use Docker Compose to run the SDK socket server in either a development or production environment.
-
-### Running in Development Mode
-
-1. **Start in Development Mode**:
-   - Use the command: `yarn start:docker:dev`
-   - This command sets up the environment for development and starts the server along with any other necessary services, like Redis.
-
-### Running in Production Mode
-
-1. **Start in Production Mode**:
-   - Use the command: `yarn start:docker`
-   - This command sets up the environment for production. It's optimized for performance and stability.
-
 ### Ngrok Configuration
 
 Follow the same Ngrok setup as mentioned in the Local Setup section above to expose your Docker Compose-based server.

packages/sdk-socket-server-next/docker-compose.yml

@@ -2,14 +2,22 @@ version: '3.9'
 
 services:
 
-  app:
+  # This service is used to debug the redis cluster
+  check-redis:
     image: node:latest
     volumes:
       - ./:/usr/src/app
     working_dir: /usr/src/app
     command: yarn debug:redis
-    # ports:
-    #   - "4000:4000"
+
+  appdev:
+    image: node:latest
+    volumes:
+      - ./:/usr/src/app
+    working_dir: /usr/src/app
+    command: yarn debug
+    ports:
+      - '4000:4000'
 
   app1:
     build:
@@ -51,37 +59,28 @@ services:
       - cache
 
   redis-master1:
-    # build:
-    #   context: .
-    #   dockerfile: Dockerfile.redis
     image: redis:7.2-alpine
-    command: redis-server --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 6380
+    command: redis-server --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 6379
     environment:
       - REDIS_ROLE=master
     ports:
-      - "6380:6380"
+      - "6380:6379"
 
   redis-master2:
     image: redis:7.2-alpine
-    # build:
-    #   context: .
-    #   dockerfile: Dockerfile.redis
-    command: redis-server --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 6381
+    command: redis-server --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 6379
     environment:
       - REDIS_ROLE=master
     ports:
-      - "6381:6381"
+      - "6381:6379"
 
   redis-master3:
     image: redis:7.2-alpine
-    # build:
-    #   context: .
-    #   dockerfile: Dockerfile.redis
-    command: redis-server --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 6382
+    command: redis-server --appendonly yes --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --port 6379
     environment:
       - REDIS_ROLE=master
     ports:
-      - "6382:6382"
+      - "6382:6379"
 
   redis-cluster-init:
     image: redis:7.2-alpine
@@ -92,11 +91,6 @@ services:
       - redis-master2
       - redis-master3
     entrypoint: ["/bin/sh", "/usr/local/bin/init-redis-cluster.sh"]
-    # environment:
-    #   - "REDISCLI_AUTH=yourpassword" # Optional: Include if your Redis nodes are password-protected
-    # To connect and debug the cluster use
-    # docker run -it --network  sdk-socket-server-next_default  --rm redis redis-cli -c -p 6379 -h redis-master1
-    # set mykey "Hello, Redis Cluster!"
 
   # cache is used if want to simulate single node redis architecture
   cache:

packages/sdk-socket-server-next/init-redis-cluster.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -e
+
+# Function to check if a redis node is ready
+check_redis() {
+    echo "Checking if Redis node '$1' is ready..."
+    while true; do
+        # Attempt to get a PONG response from the Redis node
+        if redis-cli -h "$1" -p "$2" ping | grep -q "PONG"; then
+            echo "Redis node '$1:$2' is ready."
+            break
+        else
+            echo "Waiting for Redis node '$1:$2' to be ready..."
+            sleep 2
+        fi
+    done
+}
+
+
+# Wait for all Redis nodes to be ready
+echo "Checking readiness of Redis nodes..."
+
+# Since we can't use arrays in sh, we list the nodes directly
+check_redis "redis-master1" "6379"
+check_redis "redis-master2" "6379"
+check_redis "redis-master3" "6379"
+
+
+# Since we can't pass arrays to redis-cli, we build the cluster create command manually
+echo "All Redis nodes are ready. Creating Redis Cluster..."
+echo "yes" | redis-cli --cluster create \
+    redis-master1:6379 \
+    redis-master2:6379 \
+    redis-master3:6379 \
+    --cluster-replicas 0
+
+echo "Redis Cluster created successfully."

packages/sdk-socket-server-next/package.json

@@ -18,7 +18,11 @@
     "build:post-tsc": "echo 'N/A'",
     "build:pre-tsc": "echo 'N/A'",
     "clean": "rimraf dist",
+    "docker:redis": "docker compose up redis-cluster-init",
+    "docker:redis:check": "yarn docker:redis && docker compose up check-redis",
     "debug": "NODE_ENV=development ts-node --transpile-only src/index.ts",
+    "debug:redis": "cross-env NODE_ENV=development ts-node --transpile-only src/redis-check.ts",
+    "docker:debug": "yarn docker:redis && docker compose up appdev",
     "lint": "yarn lint:eslint && yarn lint:misc --check",
     "lint:changelog": "yarn auto-changelog validate",
     "lint:eslint": "eslint .",
@@ -71,6 +75,7 @@
     "@types/winston": "^2.4.4",
     "@typescript-eslint/eslint-plugin": "^4.20.0",
     "@typescript-eslint/parser": "^4.20.0",
+    "cross-env": "^7.0.3",
     "eslint": "^7.30.0",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-import": "^2.23.4",

packages/sdk-socket-server-next/src/api-config.ts

@@ -6,17 +6,90 @@ import cors from 'cors';
 import express from 'express';
 import { rateLimit } from 'express-rate-limit';
 import helmet from 'helmet';
-import { createClient } from 'redis';
+import { Cluster, ClusterOptions, Redis } from 'ioredis';
 import { logger } from './logger';
 import { isDevelopment, isDevelopmentServer } from '.';
 
-// Initialize Redis client
-// Provide a default URL if REDIS_URL is not set
-const redisUrl = process.env.REDIS_URL || 'redis://localhost:6379';
-export const redisClient = createClient({ url: redisUrl });
 const THIRTY_DAYS_IN_SECONDS = 30 * 24 * 60 * 60; // expiration time of entries in Redis
 const hasRateLimit = process.env.RATE_LIMITER === 'true';
 
+// Initialize Redis Cluster client
+let redisNodes: {
+  host: string;
+  port: number;
+}[] = [];
+
+if (process.env.REDIS_NODES) {
+  // format: REDIS_NODES=redis://rediscluster-redis-cluster-0.rediscluster-redis-cluster-headless.redis.svc.cluster.local:6379,redis://rediscluster-redis-cluster-1.rediscluster-redis-cluster-headless.redis.svc.cluster.local:6379,redis://rediscluster-redis-cluster-2.rediscluster-redis-cluster-headless.redis.svc.cluster.local:6379
+  redisNodes = process.env.REDIS_NODES.split(',').map((node) => {
+    const [host, port] = node.replace('redis://', '').split(':');
+    return {
+      host,
+      port: parseInt(port, 10),
+    };
+  });
+}
+logger.info('Redis nodes:', redisNodes);
+
+if (redisNodes.length === 0) {
+  logger.error('No Redis nodes found');
+  process.exit(1);
+}
+
+const redisCluster = process.env.REDIS_CLUSTER === 'true';
+let redisClient: Cluster | Redis | undefined;
+const redisClusterOptions: ClusterOptions = {
+  // slotsRefreshTimeout: 2000,
+  redisOptions: {
+    // tls: {}, // WARN: enabling tls would fail the client if not setup with correct params
+    password: process.env.REDIS_PASSWORD,
+  },
+};
+
+export const getRedisClient = () => {
+  if (!redisClient) {
+    if (redisCluster) {
+      logger.info('Connecting to Redis Cluster');
+      redisClient = new Cluster(redisNodes, redisClusterOptions);
+    } else {
+      logger.info('Connecting to single Redis node');
+      redisClient = new Redis(redisNodes[0]);
+    }
+  }
+
+  redisClient.on('error', (error) => {
+    logger.error('Redis error:', error);
+  });
+
+  redisClient.on('connect', () => {
+    logger.info('Connected to Redis Cluster successfully');
+  });
+
+  redisClient.on('close', () => {
+    logger.info('Disconnected from Redis Cluster');
+  });
+
+  redisClient.on('reconnecting', () => {
+    logger.info('Reconnecting to Redis Cluster');
+  });
+
+  redisClient.on('end', () => {
+    logger.info('Redis Cluster connection ended');
+  });
+
+  redisClient.on('wait', () => {
+    logger.info('Redis Cluster waiting for connection');
+  });
+
+  redisClient.on('select', (node) => {
+    logger.info('Redis Cluster selected node:', node);
+  });
+
+  return redisClient;
+};
+
+export const pubClient = getRedisClient();
+
 const app = express();
 
 app.use(bodyParser.urlencoded({ extended: true }));
@@ -62,7 +135,7 @@ if (hasRateLimit) {
 
 async function inspectRedis(key?: string) {
   if (key && typeof key === 'string') {
-    const value = await redisClient.get(key);
+    const value = await pubClient.get(key);
     logger.debug(`inspectRedis Key: ${key}, Value: ${value}`);
   }
 }
@@ -111,19 +184,24 @@ app.post('/evt', async (_req, res) => {
       return res.status(400).json({ status: 'error' });
     }
 
-    let userIdHash = await redisClient.get(id);
+    let userIdHash = await pubClient.get(id);
 
     if (!userIdHash) {
       userIdHash = crypto.createHash('sha1').update(id).digest('hex');
-      await redisClient.set(id, userIdHash, { EX: THIRTY_DAYS_IN_SECONDS });
+      await pubClient.set(
+        id,
+        userIdHash,
+        'EX',
+        THIRTY_DAYS_IN_SECONDS.toString(),
+      );
     }
 
     if (isDevelopment) {
       inspectRedis(id);
     }
 
     let userInfo;
-    const cachedUserInfo = await redisClient.get(userIdHash);
+    const cachedUserInfo = await pubClient.get(userIdHash);
 
     if (cachedUserInfo) {
       logger.debug(`Cached user info found for ${userIdHash}`, cachedUserInfo);
@@ -149,9 +227,12 @@ app.post('/evt', async (_req, res) => {
         sdkVersion: body.sdkVersion || '',
       };
 
-      await redisClient.set(userIdHash, JSON.stringify(userInfo), {
-        EX: THIRTY_DAYS_IN_SECONDS,
-      });
+      await pubClient.set(
+        userIdHash,
+        JSON.stringify(userInfo),
+        'EX',
+        THIRTY_DAYS_IN_SECONDS.toString(),
+      );
     }
 
     if (isDevelopment) {

packages/sdk-socket-server-next/src/index.ts

@@ -28,15 +28,21 @@ process.on('SIGTERM', async () => {
   await cleanupAndExit(server, analytics);
 });
 
+process.on('unhandledRejection', (reason, promise) => {
+  logger.error('Unhandled Rejection at:', promise, 'reason:', reason);
+});
+
 configureSocketServer(server)
   .then((ioServer) => {
     logger.info(
       `socker.io server started development=${isDevelopment} adminUI=${withAdminUI}`,
     );
 
-    if (isDevelopmentServer && withAdminUI) {
+    if (withAdminUI) {
+      logger.info(`Starting socket.io admin ui`);
       instrument(ioServer, {
         auth: false,
+        namespaceName: 'admin',
         mode: 'development',
       });
     }