From a596be6df74fffe1e8d8a02d3d50ed0fefc03cea Mon Sep 17 00:00:00 2001 From: anomit ghosh Date: Thu, 10 Oct 2024 19:45:17 +0530 Subject: [PATCH 1/6] feat: add larger address space with internal class A IP addresses for docker network --- build.sh | 43 ++---------------------- docker-compose.yaml | 3 +- snapshotter/tests/test_docker_subnets.py | 34 ------------------- 3 files changed, 5 insertions(+), 75 deletions(-) delete mode 100644 snapshotter/tests/test_docker_subnets.py diff --git a/build.sh b/build.sh index e252d09..091d3e1 100755 --- a/build.sh +++ b/build.sh @@ -45,48 +45,11 @@ fi source .env export DOCKER_NETWORK_NAME="snapshotter-lite-v2-${SLOT_ID}" -SUBNET_SECOND_OCTET=$((16 + (SLOT_ID / 256) % 240)) -SUBNET_THIRD_OCTET=$((SLOT_ID % 256)) -# Always use 0 for the fourth octet to ensure a valid subnet -export DOCKER_NETWORK_SUBNET="172.${SUBNET_SECOND_OCTET}.${SUBNET_THIRD_OCTET}.0/24" +# Clean up any existing network with this name +docker network rm ${DOCKER_NETWORK_NAME} 2>/dev/null echo "Selected DOCKER_NETWORK_NAME: ${DOCKER_NETWORK_NAME}" -echo "Selected DOCKER_NETWORK_SUBNET: ${DOCKER_NETWORK_SUBNET}" - -# Test function for subnet calculation -test_subnet_calculation() { - local test_slot_id=$1 - local expected_second_octet=$2 - local expected_third_octet=$3 - - SLOT_ID=$test_slot_id - SUBNET_SECOND_OCTET=$((16 + (SLOT_ID / 256) % 240)) - SUBNET_THIRD_OCTET=$((SLOT_ID % 256)) - SUBNET="172.${SUBNET_SECOND_OCTET}.${SUBNET_THIRD_OCTET}.0/24" - - if [ $SUBNET_SECOND_OCTET -eq $expected_second_octet ] && - [ $SUBNET_THIRD_OCTET -eq $expected_third_octet ]; then - echo "Test passed for SLOT_ID $test_slot_id: $SUBNET" - else - echo "Test failed for SLOT_ID $test_slot_id: Expected 172.$expected_second_octet.$expected_third_octet.0/24, got $SUBNET" - fi -} - -# Run tests -echo "Running subnet calculation tests..." -test_subnet_calculation 1 16 0 -test_subnet_calculation 255 16 0 -test_subnet_calculation 256 16 1 -test_subnet_calculation 1000 16 3 -test_subnet_calculation 10000 16 39 -test_subnet_calculation 65535 16 255 -test_subnet_calculation 65536 17 0 -test_subnet_calculation 100000 17 134 -test_subnet_calculation 1048575 31 255 -test_subnet_calculation 1048576 16 0 - -# Add this line to run tests before the main script logic -[ "$1" = "--test" ] && exit 0 + if [ -z "$OVERRIDE_DEFAULTS" ]; then echo "setting default values..."; diff --git a/docker-compose.yaml b/docker-compose.yaml index 61c3438..097eed8 100755 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -94,4 +94,5 @@ networks: driver: bridge ipam: config: - - subnet: ${DOCKER_NETWORK_SUBNET} \ No newline at end of file + - subnet: 10.0.0.0/8 + gateway: 10.0.0.1 \ No newline at end of file diff --git a/snapshotter/tests/test_docker_subnets.py b/snapshotter/tests/test_docker_subnets.py deleted file mode 100644 index f3ee11b..0000000 --- a/snapshotter/tests/test_docker_subnets.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 - -import time - -def calculate_subnet(slot_id): - second_octet = 16 + (slot_id // 256) % 240 - third_octet = slot_id % 256 - return f"172.{second_octet}.{third_octet}.0/24" - -def test_unique_subnets(): - print("Testing unique subnet assignments for slot IDs 1 to 10000...") - - used_subnets = set() - collisions = 0 - - for slot_id in range(1, 10001): - subnet = calculate_subnet(slot_id) - - if subnet in used_subnets: - print(f"Collision detected: Slot ID {slot_id} maps to existing subnet {subnet}") - collisions += 1 - else: - used_subnets.add(subnet) - - if collisions == 0: - print("Test passed: All 10000 slot IDs have unique subnet assignments.") - else: - print(f"Test failed: {collisions} collisions detected.") - -if __name__ == "__main__": - start_time = time.time() - test_unique_subnets() - end_time = time.time() - print(f"Execution time: {end_time - start_time:.4f} seconds") \ No newline at end of file From 1ebb8a6c138ab357affb0d9be685ffbc82c60c67 Mon Sep 17 00:00:00 2001 From: anomit ghosh Date: Thu, 10 Oct 2024 19:57:55 +0530 Subject: [PATCH 2/6] add ufw rule to allow connections to 50051 from docker network information --- build.sh | 49 +++++++++++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/build.sh b/build.sh index 091d3e1..082feca 100755 --- a/build.sh +++ b/build.sh @@ -77,12 +77,6 @@ if [ -z "$SIGNER_ACCOUNT_PRIVATE_KEY" ]; then exit 1; fi -if [ -z "$DOCKER_NETWORK_SUBNET" ]; then - echo "DOCKER_NETWORK_SUBNET not found, please set this in your .env!"; - exit 1; -fi - -echo "DOCKER NETWORK SUBNET: ${DOCKER_NETWORK_SUBNET}" echo "DOCKER NETWORK NAME: ${DOCKER_NETWORK_NAME}" echo "Found SOURCE RPC URL ${SOURCE_RPC_URL}" @@ -132,26 +126,6 @@ else echo "Found LOCAL_COLLECTOR_PORT ${LOCAL_COLLECTOR_PORT}"; fi -# check if ufw command exists -if [ -x "$(command -v ufw)" ]; then - # delete old blanket allow rule - ufw delete allow $LOCAL_COLLECTOR_PORT >> /dev/null - ufw allow from $DOCKER_NETWORK_SUBNET to any port $LOCAL_COLLECTOR_PORT - if [ $? -eq 0 ]; then - echo "ufw allow rule added for local collector port ${LOCAL_COLLECTOR_PORT} to allow connections from ${DOCKER_NETWORK_SUBNET}.\n" - else - echo "ufw firewall allow rule could not added for local collector port ${LOCAL_COLLECTOR_PORT} \ -Please attempt to add it manually with the following command with sudo privileges: \ -sudo ufw allow from $DOCKER_NETWORK_SUBNET to any port $LOCAL_COLLECTOR_PORT \ -Then run ./build.sh again." - # exit script if ufw rule not added - exit 1 - fi -else - echo "ufw command not found, skipping firewall rule addition for local collector port ${LOCAL_COLLECTOR_PORT}. \ -If you are on a Linux VPS, please ensure that the port is open for connections from ${DOCKER_NETWORK_SUBNET} manually to ${LOCAL_COLLECTOR_PORT}." -fi - #fetch current git branch name GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) @@ -200,3 +174,26 @@ else docker-compose -f docker-compose.yaml $COLLECTOR_PROFILE_STRING up -V --abort-on-container-exit fi fi + +NETWORK_INFO=$(docker network inspect ${DOCKER_NETWORK_NAME}) +DOCKER_NETWORK_SUBNET=$(echo $NETWORK_INFO | jq -r '.[0].IPAM.Config[0].Subnet') + +# check if ufw command exists +if [ -x "$(command -v ufw)" ]; then + # delete old blanket allow rule + ufw delete allow $LOCAL_COLLECTOR_PORT >> /dev/null + ufw allow from $DOCKER_NETWORK_SUBNET to any port $LOCAL_COLLECTOR_PORT + if [ $? -eq 0 ]; then + echo "ufw allow rule added for local collector port ${LOCAL_COLLECTOR_PORT} to allow connections from ${DOCKER_NETWORK_SUBNET}.\n" + else + echo "ufw firewall allow rule could not added for local collector port ${LOCAL_COLLECTOR_PORT} \ +Please attempt to add it manually with the following command with sudo privileges: \ +sudo ufw allow from $DOCKER_NETWORK_SUBNET to any port $LOCAL_COLLECTOR_PORT \ +Then run ./build.sh again." + # exit script if ufw rule not added + exit 1 + fi +else + echo "ufw command not found, skipping firewall rule addition for local collector port ${LOCAL_COLLECTOR_PORT}. \ +If you are on a Linux VPS, please ensure that the port is open for connections from ${DOCKER_NETWORK_SUBNET} manually to ${LOCAL_COLLECTOR_PORT}." +fi From 30d433a9921c173445a3945f0388d06a5249d03c Mon Sep 17 00:00:00 2001 From: anomit ghosh Date: Fri, 11 Oct 2024 00:20:21 +0530 Subject: [PATCH 3/6] feat: allow class A octet to be passed as env --- build.sh | 53 ++++++++++++++++++++++++--------------------- docker-compose.yaml | 3 +-- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/build.sh b/build.sh index 082feca..6ea4ae2 100755 --- a/build.sh +++ b/build.sh @@ -47,9 +47,36 @@ source .env export DOCKER_NETWORK_NAME="snapshotter-lite-v2-${SLOT_ID}" # Clean up any existing network with this name docker network rm ${DOCKER_NETWORK_NAME} 2>/dev/null - echo "Selected DOCKER_NETWORK_NAME: ${DOCKER_NETWORK_NAME}" +# check if DOCKER_SUBNET is set, if not set it to 10.0.0.0/16 +if [ -z "$NETWORK_OCTET" ]; then + export DOCKER_SUBNET="10.0.0.0/16" +else + export DOCKER_SUBNET="10.${NETWORK_OCTET}.0.0/16" +fi + + +# check if ufw command exists +if [ -x "$(command -v ufw)" ]; then + # delete old blanket allow rule + ufw delete allow $LOCAL_COLLECTOR_PORT >> /dev/null + ufw allow from $DOCKER_NETWORK_SUBNET to any port $LOCAL_COLLECTOR_PORT + if [ $? -eq 0 ]; then + echo "ufw allow rule added for local collector port ${LOCAL_COLLECTOR_PORT} to allow connections from ${DOCKER_NETWORK_SUBNET}.\n" + else + echo "ufw firewall allow rule could not added for local collector port ${LOCAL_COLLECTOR_PORT} \ +Please attempt to add it manually with the following command with sudo privileges: \ +sudo ufw allow from $DOCKER_NETWORK_SUBNET to any port $LOCAL_COLLECTOR_PORT \ +Then run ./build.sh again." + # exit script if ufw rule not added + exit 1 + fi +else + echo "ufw command not found, skipping firewall rule addition for local collector port ${LOCAL_COLLECTOR_PORT}. \ +If you are on a Linux VPS, please ensure that the port is open for connections from ${DOCKER_NETWORK_SUBNET} manually to ${LOCAL_COLLECTOR_PORT}." +fi + if [ -z "$OVERRIDE_DEFAULTS" ]; then echo "setting default values..."; @@ -77,8 +104,6 @@ if [ -z "$SIGNER_ACCOUNT_PRIVATE_KEY" ]; then exit 1; fi -echo "DOCKER NETWORK NAME: ${DOCKER_NETWORK_NAME}" - echo "Found SOURCE RPC URL ${SOURCE_RPC_URL}" echo "Found SIGNER ACCOUNT ADDRESS ${SIGNER_ACCOUNT_ADDRESS}"; @@ -175,25 +200,3 @@ else fi fi -NETWORK_INFO=$(docker network inspect ${DOCKER_NETWORK_NAME}) -DOCKER_NETWORK_SUBNET=$(echo $NETWORK_INFO | jq -r '.[0].IPAM.Config[0].Subnet') - -# check if ufw command exists -if [ -x "$(command -v ufw)" ]; then - # delete old blanket allow rule - ufw delete allow $LOCAL_COLLECTOR_PORT >> /dev/null - ufw allow from $DOCKER_NETWORK_SUBNET to any port $LOCAL_COLLECTOR_PORT - if [ $? -eq 0 ]; then - echo "ufw allow rule added for local collector port ${LOCAL_COLLECTOR_PORT} to allow connections from ${DOCKER_NETWORK_SUBNET}.\n" - else - echo "ufw firewall allow rule could not added for local collector port ${LOCAL_COLLECTOR_PORT} \ -Please attempt to add it manually with the following command with sudo privileges: \ -sudo ufw allow from $DOCKER_NETWORK_SUBNET to any port $LOCAL_COLLECTOR_PORT \ -Then run ./build.sh again." - # exit script if ufw rule not added - exit 1 - fi -else - echo "ufw command not found, skipping firewall rule addition for local collector port ${LOCAL_COLLECTOR_PORT}. \ -If you are on a Linux VPS, please ensure that the port is open for connections from ${DOCKER_NETWORK_SUBNET} manually to ${LOCAL_COLLECTOR_PORT}." -fi diff --git a/docker-compose.yaml b/docker-compose.yaml index 097eed8..8b7aea7 100755 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -94,5 +94,4 @@ networks: driver: bridge ipam: config: - - subnet: 10.0.0.0/8 - gateway: 10.0.0.1 \ No newline at end of file + - subnet: ${DOCKER_SUBNET} \ No newline at end of file From bd9e22fd5061e690649eeba840e8ab9b1f8cb8d4 Mon Sep 17 00:00:00 2001 From: anomit ghosh Date: Sun, 13 Oct 2024 11:02:23 +0530 Subject: [PATCH 4/6] feat: allocate docker subnet within the pool of default class B addresses used by Docker --- build.sh | 47 ++++++++++++++++++++++++++++++++++++--------- docker-compose.yaml | 2 +- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/build.sh b/build.sh index 6ea4ae2..ebbe16f 100755 --- a/build.sh +++ b/build.sh @@ -45,18 +45,47 @@ fi source .env export DOCKER_NETWORK_NAME="snapshotter-lite-v2-${SLOT_ID}" -# Clean up any existing network with this name -docker network rm ${DOCKER_NETWORK_NAME} 2>/dev/null -echo "Selected DOCKER_NETWORK_NAME: ${DOCKER_NETWORK_NAME}" +# remove any existing network with this name +sudo docker network rm ${DOCKER_NETWORK_NAME} 2>/dev/null +# Use 172.18.0.0/16 as the base, which is within Docker's default pool +SUBNET_THIRD_OCTET=$((SLOT_ID % 256)) +export DOCKER_NETWORK_SUBNET="172.18.${SUBNET_THIRD_OCTET}.0/24" -# check if DOCKER_SUBNET is set, if not set it to 10.0.0.0/16 -if [ -z "$NETWORK_OCTET" ]; then - export DOCKER_SUBNET="10.0.0.0/16" -else - export DOCKER_SUBNET="10.${NETWORK_OCTET}.0.0/16" +echo "Selected DOCKER_NETWORK_NAME: ${DOCKER_NETWORK_NAME}" +echo "Selected DOCKER_NETWORK_SUBNET: ${DOCKER_NETWORK_SUBNET}" + +# Check if the first argument is "test" +if [ "$1" = "test" ]; then + echo "Running subnet calculation tests..." + + # Test function for subnet calculation + test_subnet_calculation() { + local test_slot_id=$1 + local expected_third_octet=$2 + + SLOT_ID=$test_slot_id + SUBNET_THIRD_OCTET=$((SLOT_ID % 256)) + SUBNET="172.18.${SUBNET_THIRD_OCTET}.0/24" + + if [ $SUBNET_THIRD_OCTET -eq $expected_third_octet ]; then + echo "Test passed for SLOT_ID $test_slot_id: $SUBNET" + else + echo "Test failed for SLOT_ID $test_slot_id: Expected 172.18.$expected_third_octet.0/24, got $SUBNET" + fi + } + + # Run test cases + test_subnet_calculation 0 0 + test_subnet_calculation 1 1 + test_subnet_calculation 99 99 + test_subnet_calculation 100 100 + test_subnet_calculation 255 255 + test_subnet_calculation 256 0 + + echo "Subnet calculation tests completed." + exit 0 fi - # check if ufw command exists if [ -x "$(command -v ufw)" ]; then # delete old blanket allow rule diff --git a/docker-compose.yaml b/docker-compose.yaml index 8b7aea7..61c3438 100755 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -94,4 +94,4 @@ networks: driver: bridge ipam: config: - - subnet: ${DOCKER_SUBNET} \ No newline at end of file + - subnet: ${DOCKER_NETWORK_SUBNET} \ No newline at end of file From 824527311435129f71becb3a69fadacdcc7968b2 Mon Sep 17 00:00:00 2001 From: anomit ghosh Date: Sun, 13 Oct 2024 11:55:30 +0530 Subject: [PATCH 5/6] fix: dont attempt removal of docker network --- build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.sh b/build.sh index ebbe16f..fd7cab5 100755 --- a/build.sh +++ b/build.sh @@ -45,8 +45,6 @@ fi source .env export DOCKER_NETWORK_NAME="snapshotter-lite-v2-${SLOT_ID}" -# remove any existing network with this name -sudo docker network rm ${DOCKER_NETWORK_NAME} 2>/dev/null # Use 172.18.0.0/16 as the base, which is within Docker's default pool SUBNET_THIRD_OCTET=$((SLOT_ID % 256)) export DOCKER_NETWORK_SUBNET="172.18.${SUBNET_THIRD_OCTET}.0/24" From ac984d0795e5896b6333548df33152a2fb7f6635 Mon Sep 17 00:00:00 2001 From: anomit ghosh Date: Sun, 13 Oct 2024 15:51:04 +0530 Subject: [PATCH 6/6] feat: accept third octet as an environment --- build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index fd7cab5..dd2ba54 100755 --- a/build.sh +++ b/build.sh @@ -46,7 +46,10 @@ fi source .env export DOCKER_NETWORK_NAME="snapshotter-lite-v2-${SLOT_ID}" # Use 172.18.0.0/16 as the base, which is within Docker's default pool -SUBNET_THIRD_OCTET=$((SLOT_ID % 256)) +if [ -z "$SUBNET_THIRD_OCTET" ]; then + SUBNET_THIRD_OCTET=$((SLOT_ID % 256)) + echo "SUBNET_THIRD_OCTET not found in .env, setting to default value ${SUBNET_THIRD_OCTET}" +fi export DOCKER_NETWORK_SUBNET="172.18.${SUBNET_THIRD_OCTET}.0/24" echo "Selected DOCKER_NETWORK_NAME: ${DOCKER_NETWORK_NAME}"