From 31dde3aec7e42ce034dccda187e368c31f28839b Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Wed, 8 Nov 2023 14:27:29 -0800
Subject: [PATCH 01/31] Initial support for swarm commands in carma bash script

---
 engineering_tools/carma | 63 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/engineering_tools/carma b/engineering_tools/carma
index e3ef7f8a45..c27dd8fa6a 100755
--- a/engineering_tools/carma
+++ b/engineering_tools/carma
@@ -339,6 +339,69 @@ carma__stop() {
     fi
 }
 
+carma__swarm() {
+    if ! docker container inspect carma-config > /dev/null 2>&1; then
+        echo "No existing CARMA configuration found, nothing to report. Please set a config."
+        echo "Done."
+        exit 1
+    fi
+
+    if [ "$1" = "deploy" ]; then
+
+        echo "Checking Docker Swarm status..."
+        if [ "$(docker info | grep Swarm | sed 's/Swarm: //g')" == " inactive" ]; then
+            echo "Initalizing Docker Swarm..."
+            docker swarm init
+        else
+            echo "Docker Swarm active"
+        fi
+
+        # if carma-config-data volume doesn't exist exit or update
+
+        # for local testing a new image should be built based on development then the contents
+        # should be copied into carma-config-data
+
+        if [ "$2" = "single" ]; then 
+            echo "Deploying CARMA to single host Swarm stack..."
+
+            docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+            'cat /opt/carma/vehicle/config/docker-compose-background-pc1.yml' | \
+            docker stack deploy --compose-file - carma-background
+
+            docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+            'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \
+            docker stack deploy --compose-file - carma-pc1
+
+            docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+            'cat /opt/carma/vehicle/config/docker-compose-pc2.yml' | \
+            docker stack deploy --compose-file - carma-pc2
+
+
+        elif [ "$2" = "dual" ]; then
+            echo "Deploying CARMA to dual compute Swarm stack..."
+            # token exchange between pcs here
+        
+        elif [ ! -z "$2" ]; then
+            echo "Unrecognized argument \"swarm deploy $2\""
+
+        fi
+
+    elif [ "$1" = "update-config" ]; then
+        echo "Updating carma-config-data volume..."
+        docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d usdotfhwastoldev/carma-config:develop-development
+        # consider option for updating from local builds
+
+    elif [ "$1" = "down" ]; then
+        echo "Removing CARMA services from stack...shutting down swarm..."
+        # docker stack rm --all
+        docker swarm leave --force
+        # docker network prune
+
+    elif [ ! -z "$1" ]; then
+        echo "Unrecognized argument \"swarm $1\""
+    fi
+}
+
 carma__ps() {
     if ! docker container inspect carma-config > /dev/null 2>&1; then
         echo "No existing CARMA configuration found, nothing to report. Please set a config."

From 66a81001979e13c0473f65352b81c53a6ca5a53f Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Mon, 13 Nov 2023 13:43:21 -0800
Subject: [PATCH 02/31] Updated dual compute deploy commands and volume config
 based on testing

---
 engineering_tools/carma | 53 ++++++++++++++++++++++++++++-------------
 1 file changed, 36 insertions(+), 17 deletions(-)

diff --git a/engineering_tools/carma b/engineering_tools/carma
index c27dd8fa6a..455d0c63a7 100755
--- a/engineering_tools/carma
+++ b/engineering_tools/carma
@@ -349,24 +349,42 @@ carma__swarm() {
     if [ "$1" = "deploy" ]; then
 
         echo "Checking Docker Swarm status..."
-        if [ "$(docker info | grep Swarm | sed 's/Swarm: //g')" == " inactive" ]; then
+        if [ "$(docker info | grep Swarm | sed 's/Swarm:  //g')" == "inactive" ]; then
             echo "Initalizing Docker Swarm..."
-            docker swarm init
+            docker swarm init --advertise-addr 192.168.88.100
         else
             echo "Docker Swarm active"
         fi
 
-        # if carma-config-data volume doesn't exist exit or update
+        # if carma-config-data volume doesn't exist exit and request update
+        if [ "$(docker volume ls | grep carma-config-data | sed 's/local     //g')" ]; then
+            echo "Found carma-config-data volume"
+        else
+            echo "Missing carma-config-data volume, add with carma swarm update-config [image tag]"
+            exit 1
+
+        # carma-web-ui will always run as a background process regardless of deploy argument
+        docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+        'cat /opt/carma/vehicle/config/docker-compose-background-pc1.yml' | \
+        docker stack deploy --compose-file - carma-background
 
-        # for local testing a new image should be built based on development then the contents
-        # should be copied into carma-config-data
 
+        # Run part of the stack on a single host. Mostly for debugging purposes outside of the dual PC setup
         if [ "$2" = "single" ]; then 
             echo "Deploying CARMA to single host Swarm stack..."
 
             docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-            'cat /opt/carma/vehicle/config/docker-compose-background-pc1.yml' | \
-            docker stack deploy --compose-file - carma-background
+            'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \
+            docker stack deploy --compose-file - carma-pc1
+
+
+        # For use with two connected PCs with known IPs
+        # Saves the generated swarm join token as a local variable and ssh's into the worker to execute
+        elif [ "$2" = "dual" ]; then
+            echo "Deploying CARMA to dual compute Swarm stack..."
+
+            local TKN=$(docker swarm join-token -q worker)
+            ssh dev@192.168.88.101 'docker swarm join $TKN 192.168.88.100:2377'
 
             docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
             'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \
@@ -375,27 +393,28 @@ carma__swarm() {
             docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
             'cat /opt/carma/vehicle/config/docker-compose-pc2.yml' | \
             docker stack deploy --compose-file - carma-pc2
-
-
-        elif [ "$2" = "dual" ]; then
-            echo "Deploying CARMA to dual compute Swarm stack..."
-            # token exchange between pcs here
+            
         
         elif [ ! -z "$2" ]; then
             echo "Unrecognized argument \"swarm deploy $2\""
 
         fi
 
+    # consider updating this to match carma-config_set command arguments
+    # Updates the carma-config-data volume based on a input tag string given by the user
     elif [ "$1" = "update-config" ]; then
         echo "Updating carma-config-data volume..."
-        docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d usdotfhwastoldev/carma-config:develop-development
-        # consider option for updating from local builds
+
+        if [ "$(docker images | grep $2 )" ]; then
+            echo "Found image, applying to carma-config-volume"
+            docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d usdotfhwastoldev/carma-config:$2
+        else
+            echo "Could not find $2 in local images"
 
     elif [ "$1" = "down" ]; then
-        echo "Removing CARMA services from stack...shutting down swarm..."
-        # docker stack rm --all
+        echo "Shutting down swarm..."
+        ssh dev@192.168.88.101 "docker swarm leave"
         docker swarm leave --force
-        # docker network prune
 
     elif [ ! -z "$1" ]; then
         echo "Unrecognized argument \"swarm $1\""

From 2b9c1b6d49a8a51c9e1c7f6c504a2c15f4db7074 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Mon, 13 Nov 2023 14:08:01 -0800
Subject: [PATCH 03/31] Minor fixes to token command, added fi

---
 engineering_tools/carma | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/engineering_tools/carma b/engineering_tools/carma
index 455d0c63a7..868962ed3d 100755
--- a/engineering_tools/carma
+++ b/engineering_tools/carma
@@ -362,6 +362,7 @@ carma__swarm() {
         else
             echo "Missing carma-config-data volume, add with carma swarm update-config [image tag]"
             exit 1
+        fi
 
         # carma-web-ui will always run as a background process regardless of deploy argument
         docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
@@ -384,7 +385,7 @@ carma__swarm() {
             echo "Deploying CARMA to dual compute Swarm stack..."
 
             local TKN=$(docker swarm join-token -q worker)
-            ssh dev@192.168.88.101 'docker swarm join $TKN 192.168.88.100:2377'
+            ssh dev@192.168.88.101 'docker swarm join --token $TKN 192.168.88.100:2377'
 
             docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
             'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \

From a97e5ce53e21a47a7fdf8628a44e144fcf482ca5 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Mon, 13 Nov 2023 14:15:50 -0800
Subject: [PATCH 04/31] Double quotes and spaces, my enemy

---
 engineering_tools/carma | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/engineering_tools/carma b/engineering_tools/carma
index 868962ed3d..67d0d32076 100755
--- a/engineering_tools/carma
+++ b/engineering_tools/carma
@@ -349,7 +349,7 @@ carma__swarm() {
     if [ "$1" = "deploy" ]; then
 
         echo "Checking Docker Swarm status..."
-        if [ "$(docker info | grep Swarm | sed 's/Swarm:  //g')" == "inactive" ]; then
+        if [ "$(docker info | grep Swarm | sed 's/Swarm: //g')" == " inactive" ]; then
             echo "Initalizing Docker Swarm..."
             docker swarm init --advertise-addr 192.168.88.100
         else
@@ -385,7 +385,7 @@ carma__swarm() {
             echo "Deploying CARMA to dual compute Swarm stack..."
 
             local TKN=$(docker swarm join-token -q worker)
-            ssh dev@192.168.88.101 'docker swarm join --token $TKN 192.168.88.100:2377'
+            ssh dev@192.168.88.101 "docker swarm join --token $TKN 192.168.88.100:2377"
 
             docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
             'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \
@@ -411,6 +411,7 @@ carma__swarm() {
             docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d usdotfhwastoldev/carma-config:$2
         else
             echo "Could not find $2 in local images"
+        fi
 
     elif [ "$1" = "down" ]; then
         echo "Shutting down swarm..."

From 01a60e9772f9149a018acc696f51c9b7b3788713 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Thu, 16 Nov 2023 14:44:23 -0800
Subject: [PATCH 05/31] Added commands for log attachment, help, and initial
 registry setup

---
 engineering_tools/carma | 102 +++++++++++++++++++++++++++++++++++++---
 1 file changed, 95 insertions(+), 7 deletions(-)

diff --git a/engineering_tools/carma b/engineering_tools/carma
index 67d0d32076..a29b324743 100755
--- a/engineering_tools/carma
+++ b/engineering_tools/carma
@@ -143,7 +143,7 @@ carma__attach() {
     local CARMA_DOCKER_FILE="`docker run --rm --volumes-from carma-config:ro --entrypoint sh busybox:latest -c 'cat /opt/carma/vehicle/config/docker-compose.yml'`"
 
     echo "Attaching to CARMA container STDOUT..."
-    echo "$CARMA_DOCKER_FILE" | docker-compose -p carma -f - logs --follow --tail=10
+    echo "$CARMA_DOCKER_FILE" | docker-compose logs -f -t --tail=10
 }
 
 carma-config__edit() {
@@ -360,7 +360,7 @@ carma__swarm() {
         if [ "$(docker volume ls | grep carma-config-data | sed 's/local     //g')" ]; then
             echo "Found carma-config-data volume"
         else
-            echo "Missing carma-config-data volume, add with carma swarm update-config [image tag]"
+            echo "Missing carma-config-data volume, add with carma swarm update-config <image tag>"
             exit 1
         fi
 
@@ -374,13 +374,13 @@ carma__swarm() {
         if [ "$2" = "single" ]; then 
             echo "Deploying CARMA to single host Swarm stack..."
 
-            docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+            docker run --rm -a -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
             'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \
             docker stack deploy --compose-file - carma-pc1
 
 
         # For use with two connected PCs with known IPs
-        # Saves the generated swarm join token as a local variable and ssh's into the worker to execute
+        # Saves the generated swarm join token and ssh's into the worker to execute and deploy
         elif [ "$2" = "dual" ]; then
             echo "Deploying CARMA to dual compute Swarm stack..."
 
@@ -398,10 +398,9 @@ carma__swarm() {
         
         elif [ ! -z "$2" ]; then
             echo "Unrecognized argument \"swarm deploy $2\""
-
         fi
 
-    # consider updating this to match carma-config_set command arguments
+
     # Updates the carma-config-data volume based on a input tag string given by the user
     elif [ "$1" = "update-config" ]; then
         echo "Updating carma-config-data volume..."
@@ -413,13 +412,68 @@ carma__swarm() {
             echo "Could not find $2 in local images"
         fi
 
+
+    # Attaches the swarm's STDOUT logs to the current terminal
+    elif [ "$1" = "attach" ]; then
+        echo "Attaching STDOUT logs to terminal..."
+
+        SWARM_SERVICES=$(docker service ls --format "{{.Name}}")
+
+        # this seems to loop forever and cannot be escaped with ctrl+C
+        # performance will likely be poor due to combined log streams
+        for item in $SWARM_SERVICES; do
+            docker service logs --follow --details --tail=10 "$item" &
+        done
+
+        sleep infinity
+
+
+    # Takes n image tags that are saved locally and pushes them to the registry
+    elif [ "$1" = "set-image" ]; then
+        echo "Installing Docker images..."
+
+        if [ "$(docker volume ls | grep carma-registry | sed 's/local     //g')" ]; then
+            echo "Found carma-registry volume"
+        else
+            echo "Missing carma-registry volume, adding now..."
+            docker volume create carma-registry
+        fi
+
+        # this may need to deployed in the background compose file instead of here
+        docker service create --constraint 'node.role==manager' -d --volume="carma-registry:/var/lib/registry" -p 5000:5000 --name carma-registry-service registry:2
+
+        IMAGE_NAMES="$*"
+
+        for SOURCE_IMAGE_NAME in IMAGE_NAMES
+        do
+
+        echo "Pushing $SOURCE_IMAGE_NAME to localhost:5000"
+
+        # Should set pull or should it be an option where the normal behavior pushes a local image to the registry
+        # docker pull $SOURCE_IMAGE_NAME
+        docker tag "$SOURCE_IMAGE_NAME" "localhost:5000/$SOURCE_IMAGE_NAME"
+        docker push "localhost:5000/$SOURCE_IMAGE_NAME"
+        docker rmi "localhost:5000/$SOURCE_IMAGE_NAME"
+
+        done
+
+
     elif [ "$1" = "down" ]; then
         echo "Shutting down swarm..."
-        ssh dev@192.168.88.101 "docker swarm leave"
+
+        if [ "$(docker node ls | grep worker )" ]; then
+            ssh dev@192.168.88.101 "docker swarm leave"
+        fi
+
         docker swarm leave --force
 
+
     elif [ ! -z "$1" ]; then
         echo "Unrecognized argument \"swarm $1\""
+    
+    
+    elif [ $# -eq 0 ]; then
+        carma__swarm-help
     fi
 }
 
@@ -648,6 +702,40 @@ Please enter one of the following commands:
 HELP
 }
 
+carma__swarm-help() {
+    cat <<SHELP
+-------------------------------------------------------------------------------
+| USDOT FHWA STOL CARMA Swarm                                                 |
+-------------------------------------------------------------------------------
+
+Please enter one of the following commands:
+    deploy:
+        [empty]
+            - Will always deploy carma-web-ui as a background service on PC1 
+        single
+            - Deploys PC1 services as described in docker-compose-pc1.yml
+              to use for debugging on a single test system
+        dual
+            - Deploys PC1 services from docker-compose-pc1.yml and
+              PC2 services from docker-compose-pc2.yml to the PC1
+              manger node and PC2 worker node
+    update-config <tag/image>
+        - Uses the local carma-config image defined in
+          <tag/image> to update the Docker named volume
+          "carma-config-data" which can be found at
+          /var/lib/docker/volumes/carma-config-data/_data
+    set-image <tag/image>
+        - Pushes the image defined by <tag/image> to the
+          the registry service and associated volume
+    attach
+        - View STDOUT from all running CARMA processes
+    down
+        - Shuts down the worker node and manager node
+          and ends Swarm along with all services
+
+SHELP
+}
+
 carma__config() {
     local cmdname=$1; shift
     if type "carma-config__$cmdname" >/dev/null 2>&1; then

From 09f343b6ad62f0009697651d2514204739bbeb47 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Fri, 17 Nov 2023 08:04:43 -0800
Subject: [PATCH 06/31] Changed swarm update-config to support config set
 arguments, fixed arg passing

---
 engineering_tools/carma | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/engineering_tools/carma b/engineering_tools/carma
index a29b324743..95fdf86139 100755
--- a/engineering_tools/carma
+++ b/engineering_tools/carma
@@ -405,11 +405,30 @@ carma__swarm() {
     elif [ "$1" = "update-config" ]; then
         echo "Updating carma-config-data volume..."
 
-        if [ "$(docker images | grep $2 )" ]; then
+        if [ "$2" = "-d" ]; then
+            local USERNAME=usdotfhwastoldev
+            local TAG=$3
+        elif [ "$2" = "-c" ]; then
+            local USERNAME=usdotfhwastolcandidate
+            local TAG=$3
+        else
+            local USERNAME=usdotfhwastol
+            local TAG=$2
+        fi
+
+        if [[ -z $2 ]]; then
+            echo "Please specify a tag string to update carma-config-data volume."
+            echo "Done."
+            exit 1
+        fi
+
+        local IMAGE_NAME="$USERNAME/carma-config:$TAG"
+
+        if [ "$(docker images | grep $USERNAME/carma-config | grep $TAG )" ]; then
             echo "Found image, applying to carma-config-volume"
-            docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d usdotfhwastoldev/carma-config:$2
+            docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d $USERNAME/carma-config:$TAG
         else
-            echo "Could not find $2 in local images"
+            echo "Could not find $USERNAME/carma-config:$TAG in local images"
         fi
 
 

From 93e637fc5656e864d96702245166546244c3aec0 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Fri, 17 Nov 2023 11:43:16 -0800
Subject: [PATCH 07/31] Added edit arg and minor updates to config management

---
 engineering_tools/carma | 53 ++++++++++++++++++++++++++---------------
 1 file changed, 34 insertions(+), 19 deletions(-)

diff --git a/engineering_tools/carma b/engineering_tools/carma
index 95fdf86139..e20ecd5569 100755
--- a/engineering_tools/carma
+++ b/engineering_tools/carma
@@ -340,18 +340,13 @@ carma__stop() {
 }
 
 carma__swarm() {
-    if ! docker container inspect carma-config > /dev/null 2>&1; then
-        echo "No existing CARMA configuration found, nothing to report. Please set a config."
-        echo "Done."
-        exit 1
-    fi
 
     if [ "$1" = "deploy" ]; then
 
         echo "Checking Docker Swarm status..."
         if [ "$(docker info | grep Swarm | sed 's/Swarm: //g')" == " inactive" ]; then
             echo "Initalizing Docker Swarm..."
-            docker swarm init --advertise-addr 192.168.88.100
+            docker swarm init --advertise-addr 192.168.88.100 # consider making a variable
         else
             echo "Docker Swarm active"
         fi
@@ -360,7 +355,7 @@ carma__swarm() {
         if [ "$(docker volume ls | grep carma-config-data | sed 's/local     //g')" ]; then
             echo "Found carma-config-data volume"
         else
-            echo "Missing carma-config-data volume, add with carma swarm update-config <image tag>"
+            echo "Missing carma-config-data volume, add with \"carma swarm update-config <image tag>\""
             exit 1
         fi
 
@@ -370,7 +365,7 @@ carma__swarm() {
         docker stack deploy --compose-file - carma-background
 
 
-        # Run part of the stack on a single host. Mostly for debugging purposes outside of the dual PC setup
+        # Run part of the stack on a single host. Mostly for debugging purposes off of the dual PC setup
         if [ "$2" = "single" ]; then 
             echo "Deploying CARMA to single host Swarm stack..."
 
@@ -432,7 +427,26 @@ carma__swarm() {
         fi
 
 
-    # Attaches the swarm's STDOUT logs to the current terminal
+    elif [ "$1" = "edit-config" ]; then
+        echo "Opening carma-config-data volume shell with read/write privileges..."
+
+        if [ "$(docker volume ls | grep carma-config-data | sed 's/local     //g')" ]; then
+            echo "Found carma-config-data volume"
+        else
+            echo "Missing carma-config-data volume, add with \"carma swarm update-config <image tag>\""
+            exit 1
+        fi
+
+        local carma_base=$(__get_image_from_config carma-base:)
+        if [[ -z $carma_base ]]; then
+            __pull_newest_carma_base
+            carma_base=$(__get_most_recent_carma_base)
+        fi
+
+        docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base bash
+
+
+    # Attaches each carma service's STDOUT logs to the current terminal
     elif [ "$1" = "attach" ]; then
         echo "Attaching STDOUT logs to terminal..."
 
@@ -440,8 +454,8 @@ carma__swarm() {
 
         # this seems to loop forever and cannot be escaped with ctrl+C
         # performance will likely be poor due to combined log streams
-        for item in $SWARM_SERVICES; do
-            docker service logs --follow --details --tail=10 "$item" &
+        for TASK in $SWARM_SERVICES; do
+            docker service logs --follow --details --tail=10 "$TASK" &
         done
 
         sleep infinity
@@ -449,6 +463,7 @@ carma__swarm() {
 
     # Takes n image tags that are saved locally and pushes them to the registry
     elif [ "$1" = "set-image" ]; then
+        shift
         echo "Installing Docker images..."
 
         if [ "$(docker volume ls | grep carma-registry | sed 's/local     //g')" ]; then
@@ -458,18 +473,15 @@ carma__swarm() {
             docker volume create carma-registry
         fi
 
-        # this may need to deployed in the background compose file instead of here
         docker service create --constraint 'node.role==manager' -d --volume="carma-registry:/var/lib/registry" -p 5000:5000 --name carma-registry-service registry:2
-
+        
         IMAGE_NAMES="$*"
 
-        for SOURCE_IMAGE_NAME in IMAGE_NAMES
+        for SOURCE_IMAGE_NAME in $IMAGE_NAMES
         do
-
+        
         echo "Pushing $SOURCE_IMAGE_NAME to localhost:5000"
 
-        # Should set pull or should it be an option where the normal behavior pushes a local image to the registry
-        # docker pull $SOURCE_IMAGE_NAME
         docker tag "$SOURCE_IMAGE_NAME" "localhost:5000/$SOURCE_IMAGE_NAME"
         docker push "localhost:5000/$SOURCE_IMAGE_NAME"
         docker rmi "localhost:5000/$SOURCE_IMAGE_NAME"
@@ -483,7 +495,6 @@ carma__swarm() {
         if [ "$(docker node ls | grep worker )" ]; then
             ssh dev@192.168.88.101 "docker swarm leave"
         fi
-
         docker swarm leave --force
 
 
@@ -743,8 +754,12 @@ Please enter one of the following commands:
           <tag/image> to update the Docker named volume
           "carma-config-data" which can be found at
           /var/lib/docker/volumes/carma-config-data/_data
+    edit-config
+        - Requires sudo privilege. Opens a bash shell
+          in a carma-base image with carma-config-data
+          mounted so that files can be modified
     set-image <tag/image>
-        - Pushes the image defined by <tag/image> to the
+        - Pushes the images defined by <tag/image> to the
           the registry service and associated volume
     attach
         - View STDOUT from all running CARMA processes

From 4564bc6a88dfe8fee475d64f6ae84f0ce6ac0500 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Fri, 17 Nov 2023 13:35:30 -0800
Subject: [PATCH 08/31] Minor fixes following dual compute test

---
 engineering_tools/carma | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/engineering_tools/carma b/engineering_tools/carma
index e20ecd5569..d55b8cc57b 100755
--- a/engineering_tools/carma
+++ b/engineering_tools/carma
@@ -369,7 +369,7 @@ carma__swarm() {
         if [ "$2" = "single" ]; then 
             echo "Deploying CARMA to single host Swarm stack..."
 
-            docker run --rm -a -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+            docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
             'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \
             docker stack deploy --compose-file - carma-pc1
 
@@ -473,7 +473,7 @@ carma__swarm() {
             docker volume create carma-registry
         fi
 
-        docker service create --constraint 'node.role==manager' -d --volume="carma-registry:/var/lib/registry" -p 5000:5000 --name carma-registry-service registry:2
+        docker service create --constraint 'node.role==manager' -d --volume "carma-registry:/var/lib/registry" -p 5000:5000 --name carma-registry-service registry:2
         
         IMAGE_NAMES="$*"
 
@@ -739,7 +739,8 @@ carma__swarm-help() {
 -------------------------------------------------------------------------------
 
 Please enter one of the following commands:
-    deploy:
+  swarm:
+    deploy
         [empty]
             - Will always deploy carma-web-ui as a background service on PC1 
         single
@@ -750,6 +751,12 @@ Please enter one of the following commands:
               PC2 services from docker-compose-pc2.yml to the PC1
               manger node and PC2 worker node
     update-config <tag/image>
+        - [empty]
+            - Defaults to a usdotfhwastol image
+        - d 
+            - Defines a usdotfhwastoldev image
+        - c
+            - Defines a usdotfhwastolcandidate image
         - Uses the local carma-config image defined in
           <tag/image> to update the Docker named volume
           "carma-config-data" which can be found at

From 89ea7b42a17e9fd1adb80542201232f144b32790 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Tue, 21 Nov 2023 13:34:56 -0800
Subject: [PATCH 09/31] Migrated swarm code to extensions and changed from ifs
 to functions. Improved naming for commands

---
 engineering_tools/carma                       | 215 +------------
 .../carma_script_extensions/swarm             | 289 ++++++++++++++++++
 2 files changed, 290 insertions(+), 214 deletions(-)
 create mode 100755 engineering_tools/carma_script_extensions/swarm

diff --git a/engineering_tools/carma b/engineering_tools/carma
index d55b8cc57b..331863ffa7 100755
--- a/engineering_tools/carma
+++ b/engineering_tools/carma
@@ -143,7 +143,7 @@ carma__attach() {
     local CARMA_DOCKER_FILE="`docker run --rm --volumes-from carma-config:ro --entrypoint sh busybox:latest -c 'cat /opt/carma/vehicle/config/docker-compose.yml'`"
 
     echo "Attaching to CARMA container STDOUT..."
-    echo "$CARMA_DOCKER_FILE" | docker-compose logs -f -t --tail=10
+    echo "$CARMA_DOCKER_FILE" | docker-compose -p carma logs -f - logs --follow --tail=10
 }
 
 carma-config__edit() {
@@ -339,174 +339,6 @@ carma__stop() {
     fi
 }
 
-carma__swarm() {
-
-    if [ "$1" = "deploy" ]; then
-
-        echo "Checking Docker Swarm status..."
-        if [ "$(docker info | grep Swarm | sed 's/Swarm: //g')" == " inactive" ]; then
-            echo "Initalizing Docker Swarm..."
-            docker swarm init --advertise-addr 192.168.88.100 # consider making a variable
-        else
-            echo "Docker Swarm active"
-        fi
-
-        # if carma-config-data volume doesn't exist exit and request update
-        if [ "$(docker volume ls | grep carma-config-data | sed 's/local     //g')" ]; then
-            echo "Found carma-config-data volume"
-        else
-            echo "Missing carma-config-data volume, add with \"carma swarm update-config <image tag>\""
-            exit 1
-        fi
-
-        # carma-web-ui will always run as a background process regardless of deploy argument
-        docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-        'cat /opt/carma/vehicle/config/docker-compose-background-pc1.yml' | \
-        docker stack deploy --compose-file - carma-background
-
-
-        # Run part of the stack on a single host. Mostly for debugging purposes off of the dual PC setup
-        if [ "$2" = "single" ]; then 
-            echo "Deploying CARMA to single host Swarm stack..."
-
-            docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-            'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \
-            docker stack deploy --compose-file - carma-pc1
-
-
-        # For use with two connected PCs with known IPs
-        # Saves the generated swarm join token and ssh's into the worker to execute and deploy
-        elif [ "$2" = "dual" ]; then
-            echo "Deploying CARMA to dual compute Swarm stack..."
-
-            local TKN=$(docker swarm join-token -q worker)
-            ssh dev@192.168.88.101 "docker swarm join --token $TKN 192.168.88.100:2377"
-
-            docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-            'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \
-            docker stack deploy --compose-file - carma-pc1
-
-            docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-            'cat /opt/carma/vehicle/config/docker-compose-pc2.yml' | \
-            docker stack deploy --compose-file - carma-pc2
-            
-        
-        elif [ ! -z "$2" ]; then
-            echo "Unrecognized argument \"swarm deploy $2\""
-        fi
-
-
-    # Updates the carma-config-data volume based on a input tag string given by the user
-    elif [ "$1" = "update-config" ]; then
-        echo "Updating carma-config-data volume..."
-
-        if [ "$2" = "-d" ]; then
-            local USERNAME=usdotfhwastoldev
-            local TAG=$3
-        elif [ "$2" = "-c" ]; then
-            local USERNAME=usdotfhwastolcandidate
-            local TAG=$3
-        else
-            local USERNAME=usdotfhwastol
-            local TAG=$2
-        fi
-
-        if [[ -z $2 ]]; then
-            echo "Please specify a tag string to update carma-config-data volume."
-            echo "Done."
-            exit 1
-        fi
-
-        local IMAGE_NAME="$USERNAME/carma-config:$TAG"
-
-        if [ "$(docker images | grep $USERNAME/carma-config | grep $TAG )" ]; then
-            echo "Found image, applying to carma-config-volume"
-            docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d $USERNAME/carma-config:$TAG
-        else
-            echo "Could not find $USERNAME/carma-config:$TAG in local images"
-        fi
-
-
-    elif [ "$1" = "edit-config" ]; then
-        echo "Opening carma-config-data volume shell with read/write privileges..."
-
-        if [ "$(docker volume ls | grep carma-config-data | sed 's/local     //g')" ]; then
-            echo "Found carma-config-data volume"
-        else
-            echo "Missing carma-config-data volume, add with \"carma swarm update-config <image tag>\""
-            exit 1
-        fi
-
-        local carma_base=$(__get_image_from_config carma-base:)
-        if [[ -z $carma_base ]]; then
-            __pull_newest_carma_base
-            carma_base=$(__get_most_recent_carma_base)
-        fi
-
-        docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base bash
-
-
-    # Attaches each carma service's STDOUT logs to the current terminal
-    elif [ "$1" = "attach" ]; then
-        echo "Attaching STDOUT logs to terminal..."
-
-        SWARM_SERVICES=$(docker service ls --format "{{.Name}}")
-
-        # this seems to loop forever and cannot be escaped with ctrl+C
-        # performance will likely be poor due to combined log streams
-        for TASK in $SWARM_SERVICES; do
-            docker service logs --follow --details --tail=10 "$TASK" &
-        done
-
-        sleep infinity
-
-
-    # Takes n image tags that are saved locally and pushes them to the registry
-    elif [ "$1" = "set-image" ]; then
-        shift
-        echo "Installing Docker images..."
-
-        if [ "$(docker volume ls | grep carma-registry | sed 's/local     //g')" ]; then
-            echo "Found carma-registry volume"
-        else
-            echo "Missing carma-registry volume, adding now..."
-            docker volume create carma-registry
-        fi
-
-        docker service create --constraint 'node.role==manager' -d --volume "carma-registry:/var/lib/registry" -p 5000:5000 --name carma-registry-service registry:2
-        
-        IMAGE_NAMES="$*"
-
-        for SOURCE_IMAGE_NAME in $IMAGE_NAMES
-        do
-        
-        echo "Pushing $SOURCE_IMAGE_NAME to localhost:5000"
-
-        docker tag "$SOURCE_IMAGE_NAME" "localhost:5000/$SOURCE_IMAGE_NAME"
-        docker push "localhost:5000/$SOURCE_IMAGE_NAME"
-        docker rmi "localhost:5000/$SOURCE_IMAGE_NAME"
-
-        done
-
-
-    elif [ "$1" = "down" ]; then
-        echo "Shutting down swarm..."
-
-        if [ "$(docker node ls | grep worker )" ]; then
-            ssh dev@192.168.88.101 "docker swarm leave"
-        fi
-        docker swarm leave --force
-
-
-    elif [ ! -z "$1" ]; then
-        echo "Unrecognized argument \"swarm $1\""
-    
-    
-    elif [ $# -eq 0 ]; then
-        carma__swarm-help
-    fi
-}
-
 carma__ps() {
     if ! docker container inspect carma-config > /dev/null 2>&1; then
         echo "No existing CARMA configuration found, nothing to report. Please set a config."
@@ -732,51 +564,6 @@ Please enter one of the following commands:
 HELP
 }
 
-carma__swarm-help() {
-    cat <<SHELP
--------------------------------------------------------------------------------
-| USDOT FHWA STOL CARMA Swarm                                                 |
--------------------------------------------------------------------------------
-
-Please enter one of the following commands:
-  swarm:
-    deploy
-        [empty]
-            - Will always deploy carma-web-ui as a background service on PC1 
-        single
-            - Deploys PC1 services as described in docker-compose-pc1.yml
-              to use for debugging on a single test system
-        dual
-            - Deploys PC1 services from docker-compose-pc1.yml and
-              PC2 services from docker-compose-pc2.yml to the PC1
-              manger node and PC2 worker node
-    update-config <tag/image>
-        - [empty]
-            - Defaults to a usdotfhwastol image
-        - d 
-            - Defines a usdotfhwastoldev image
-        - c
-            - Defines a usdotfhwastolcandidate image
-        - Uses the local carma-config image defined in
-          <tag/image> to update the Docker named volume
-          "carma-config-data" which can be found at
-          /var/lib/docker/volumes/carma-config-data/_data
-    edit-config
-        - Requires sudo privilege. Opens a bash shell
-          in a carma-base image with carma-config-data
-          mounted so that files can be modified
-    set-image <tag/image>
-        - Pushes the images defined by <tag/image> to the
-          the registry service and associated volume
-    attach
-        - View STDOUT from all running CARMA processes
-    down
-        - Shuts down the worker node and manager node
-          and ends Swarm along with all services
-
-SHELP
-}
-
 carma__config() {
     local cmdname=$1; shift
     if type "carma-config__$cmdname" >/dev/null 2>&1; then
diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
new file mode 100755
index 0000000000..921c377013
--- /dev/null
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -0,0 +1,289 @@
+#!/bin/bash
+
+#  Copyright (C) 2023-2024 LEIDOS.
+# 
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may not
+#  use this file except in compliance with the License. You may obtain a copy of
+#  the License at
+# 
+#  http://www.apache.org/licenses/LICENSE-2.0
+# 
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations under
+#  the License.
+
+
+swarm__install() {
+    echo "Installing Swarm images..."
+}
+
+
+swarm__start() {
+    echo "Checking Docker Swarm status..."
+    if [ "$(docker info | grep Swarm | sed 's/Swarm: //g')" == " inactive" ]; then
+        echo "Initalizing Docker Swarm..."
+        docker swarm init --advertise-addr 192.168.88.100 # consider making a variable
+    else
+        echo "Docker Swarm active"
+    fi
+
+    # if carma-config-data volume doesn't exist exit and request update
+    if [ ! -z "$(docker volume ls | grep carma-config-data)" ]; then
+        echo "Found carma-config-data volume"
+    else
+        echo "Missing carma-config-data volume, add with \"carma swarm update-config <image tag>\""
+        exit 1
+    fi
+
+    # carma-web-ui will always run as a background process regardless of deploy argument
+    docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+    'cat /opt/carma/vehicle/config/docker-compose-background-pc1.yml' | \
+    docker stack deploy --compose-file - carma-background
+
+
+    # deploy services exclusively from docker-compose-pc1.yml on the manager node
+    # NOTE: This argument is used for debugging/developing Docker Swarm deployment on
+    #       a single host PC. To deploy all services to two connected PCs, use "dual"
+    if [ "$1" = "single" ]; then 
+        echo "Deploying CARMA to single host Swarm stack..."
+
+        docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+        'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \
+        docker stack deploy --compose-file - carma-pc1
+
+
+    # For use with two connected PCs with known IPs
+    # Saves the generated swarm join token and ssh's into the worker to execute and deploy
+    elif [ "$1" = "dual" ]; then
+        echo "Deploying CARMA to dual compute Swarm stack..."
+
+        local TKN=$(docker swarm join-token -q worker)
+        ssh dev@192.168.88.101 "docker swarm join --token $TKN 192.168.88.100:2377"
+
+        docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+        'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \
+        docker stack deploy --compose-file - carma-pc1
+
+        docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+        'cat /opt/carma/vehicle/config/docker-compose-pc2.yml' | \
+        docker stack deploy --compose-file - carma-pc2
+        
+    
+    elif [ ! -z "$1" ]; then
+        echo "Unrecognized argument \"swarm deploy $1\""
+    fi
+}
+
+
+swarm-config__set() {
+    echo "Updating carma-config-data volume..."
+
+    if [ "$1" = "-d" ]; then
+        local USERNAME=usdotfhwastoldev
+        local TAG=$2
+    elif [ "$1" = "-c" ]; then
+        local USERNAME=usdotfhwastolcandidate
+        local TAG=$2
+    else
+        local USERNAME=usdotfhwastol
+        local TAG=$1
+    fi
+
+    if [[ -z $2 ]]; then
+        echo "Please specify a tag string to update carma-config-data volume."
+        echo "Done."
+        exit 1
+    fi
+
+    local IMAGE_NAME="$USERNAME/carma-config:$TAG"
+
+    if [ "$(docker images | grep $USERNAME/carma-config | grep $TAG )" ]; then
+        echo "Found image, applying to carma-config-volume"
+        docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d $USERNAME/carma-config:$TAG
+    else
+        echo "Could not find $USERNAME/carma-config:$TAG in local images"
+    fi
+}
+
+
+swarm-config__edit() {
+    echo "Opening carma-config-data volume shell with read/write privileges..."
+
+    if [ "$(docker volume ls | grep carma-config-data | sed 's/local     //g')" ]; then
+        echo "Found carma-config-data volume"
+    else
+        echo "Missing carma-config-data volume, add with \"carma swarm update-config <image tag>\""
+        exit 1
+    fi
+
+    local carma_base=$(__get_image_from_config carma-base:)
+    if [[ -z $carma_base ]]; then
+        __pull_newest_carma_base
+        carma_base=$(__get_most_recent_carma_base)
+    fi
+
+    docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base bash
+}
+
+
+swarm-config__status() {
+    echo "Current configuration is loaded from image: <filler>"
+}
+
+
+swarm__attach() {
+    echo "Attaching STDOUT logs to terminal..."
+
+    SWARM_SERVICES=$(docker service ls --format "{{.Name}}")
+
+    # this seems to loop forever and cannot be escaped with ctrl+C
+    # performance will likely be poor due to combined log streams
+    for TASK in $SWARM_SERVICES; do
+        docker service logs --follow --details --tail=10 "$TASK" &
+    done
+
+    sleep infinity
+}
+
+
+swarm__register() {
+    echo "Installing Docker images..."
+
+    if [ "$(docker volume ls | grep carma-registry | sed 's/local     //g')" ]; then
+        echo "Found carma-registry volume"
+    else
+        echo "Missing carma-registry volume, adding now..."
+        docker volume create carma-registry
+    fi
+
+    # if [ "$2" = "-d" ]; then
+    #     local USERNAME=usdotfhwastoldev
+    #     local TAG=$3
+    # elif [ "$2" = "-c" ]; then
+    #     local USERNAME=usdotfhwastolcandidate
+    #     local TAG=$3
+    # else
+    #     local USERNAME=usdotfhwastol
+    #     local TAG=$2
+    # fi
+
+    if [[ -z $1 ]]; then
+        echo "Please specify a tag string to update carma-config-data volume."
+        echo "Done."
+        exit 1
+    fi
+
+    docker service create --constraint 'node.role==manager' -d --volume "carma-registry:/var/lib/registry" -p 5000:5000 --name carma-registry-service registry:2
+    
+    IMAGE_NAMES="$*"
+
+    for SOURCE_IMAGE_NAME in $IMAGE_NAMES
+    do
+    
+    echo "Pushing $SOURCE_IMAGE_NAME to localhost:5000"
+
+    docker tag "$SOURCE_IMAGE_NAME" "localhost:5000/$SOURCE_IMAGE_NAME"
+    docker push "localhost:5000/$SOURCE_IMAGE_NAME"
+    docker rmi "localhost:5000/$SOURCE_IMAGE_NAME"
+
+    done
+}
+
+
+swarm__stop() {
+    echo "Shutting down swarm..."
+
+    if [ "$(docker node ls | grep worker )" ]; then
+        ssh dev@192.168.88.101 "docker swarm leave"
+    fi
+    docker swarm leave --force
+
+    # if [ "$1" = "all" ]; then
+        # shuts down stacks on worker
+        # leaves worker
+
+        # shuts down stack on manager
+        # leaves manager
+
+    # elif [ ! -z "$1" ]; then
+    #     echo "Unrecognized argument \"swarm $1\""
+}
+
+
+swarm__help() {
+    cat <<HELP
+-------------------------------------------------------------------------------
+| USDOT FHWA STOL CARMA Swarm                                                 |
+-------------------------------------------------------------------------------
+
+Please enter one of the following commands:
+  swarm
+    start
+        [empty]
+            - Will always deploy carma-web-ui as a background service
+              on the manager from docker-compose-background-manager.yml
+        single
+            - Deploys manager services as described in 
+              docker-compose-manager.yml to use for debugging 
+              on a single host test system
+        dual
+            - Deploys manager services from docker-compose-manager.yml
+              and worker services from docker-compose-worker.yml to 
+              the manger node and worker node respectively
+    config
+        set <tag/image>
+            - Uses the local carma-config image defined in
+              <tag/image> to update the Docker named volume
+              "carma-config-data" which can be found at
+              /var/lib/docker/volumes/carma-config-data/_data
+            - [empty]
+                - Defaults to a usdotfhwastol image
+            - d 
+                - Defines a usdotfhwastoldev image
+            - c
+                - Defines a usdotfhwastolcandidate image
+        edit
+            - Requires sudo privilege. Opens a bash shell
+              in a carma-base image with carma-config-data
+              mounted so that files can be modified
+        status
+            - Reports the active Docker Compose files from
+              carma-config-data to the active terminal
+    register <images>
+        - Pushes the images defined by <images> to the
+          registry service. Images must have their full
+          identifier to be added.
+    attach
+        - View STDOUT from all running CARMA processes
+    stop
+        - Shuts down the worker node and manager node
+          and ends Swarm along with all services
+    help
+        - Display this information
+
+HELP
+}
+
+
+swarm__config() {
+    local cmdname=$1; shift
+    if type "swarm-config__$cmdname" >/dev/null 2>&1; then
+        "swarm-config__$cmdname" "$@"
+    else
+        swarm__help
+        exit 1
+    fi
+}
+
+
+carma__swarm() {
+    local cmdname=$1; shift
+    if type "swarm__$cmdname" >/dev/null 2>&1; then
+        "swarm__$cmdname" "$@"
+    else
+        swarm__help
+        exit 1
+    fi
+}
\ No newline at end of file

From 99d1ff4f8c50338c6047b61eef0e70f2cc1cb353 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Wed, 22 Nov 2023 09:22:14 -0800
Subject: [PATCH 10/31] Swarm install/config status added. Minor improvments to
 image-based argument handling

---
 .../carma_script_extensions/swarm             | 91 ++++++++++++++++---
 1 file changed, 77 insertions(+), 14 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index 921c377013..d21baf4084 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -16,7 +16,45 @@
 
 
 swarm__install() {
-    echo "Installing Swarm images..."
+    if [ "$1" = "-d" ]; then
+        local USERNAME=usdotfhwastoldev
+        local TAG=$2
+    elif [ "$1" = "-c" ]; then
+        local USERNAME=usdotfhwastolcandidate
+        local TAG=$2
+    else
+        local USERNAME=usdotfhwastol
+        local TAG=$1
+    fi
+
+    if [[ -z $TAG ]]; then
+        echo "Please specify a tag string to update carma-config-data volume."
+        exit 1
+    fi
+
+    if [ ! -z "$(echo $TAG | grep :)" ]; then
+        local IMAGE_NAME="$TAG"
+    else
+        local IMAGE_NAME="$USERNAME/carma-config:$TAG"
+    fi
+
+    echo "Downloading $IMAGE_NAME..."
+    docker pull $IMAGE_NAME
+
+    docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d usdotfhwastol/carma-config:carma-system-4.4.3-47-g40f710d-SNAPSHOT-development
+
+    local BACKGROUND_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+    'cat /opt/carma/vehicle/config/docker-compose-background-manager.yml'`"
+    local MANAGER_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+    'cat /opt/carma/vehicle/config/docker-compose-manager.yml'`"
+    local WORKER_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+    'cat /opt/carma/vehicle/config/docker-compose-worker.yml'`"
+
+    echo "Downloading $IMAGE_NAME dependencies..."
+    echo "$BACKGROUND_COMPOSE" | docker-compose -f - pull
+    echo "$MANAGER_COMPOSE" | docker-compose -f - pull
+    echo "$WORKER_COMPOSE" | docker-compose -f - pull
+
 }
 
 
@@ -39,19 +77,19 @@ swarm__start() {
 
     # carma-web-ui will always run as a background process regardless of deploy argument
     docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-    'cat /opt/carma/vehicle/config/docker-compose-background-pc1.yml' | \
+    'cat /opt/carma/vehicle/config/docker-compose-background-manager.yml' | \
     docker stack deploy --compose-file - carma-background
 
 
-    # deploy services exclusively from docker-compose-pc1.yml on the manager node
+    # deploy services exclusively from docker-compose-manager.yml on the manager node
     # NOTE: This argument is used for debugging/developing Docker Swarm deployment on
     #       a single host PC. To deploy all services to two connected PCs, use "dual"
     if [ "$1" = "single" ]; then 
         echo "Deploying CARMA to single host Swarm stack..."
 
         docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-        'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \
-        docker stack deploy --compose-file - carma-pc1
+        'cat /opt/carma/vehicle/config/docker-compose-manager.yml' | \
+        docker stack deploy --compose-file - carma-manager
 
 
     # For use with two connected PCs with known IPs
@@ -63,12 +101,12 @@ swarm__start() {
         ssh dev@192.168.88.101 "docker swarm join --token $TKN 192.168.88.100:2377"
 
         docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-        'cat /opt/carma/vehicle/config/docker-compose-pc1.yml' | \
-        docker stack deploy --compose-file - carma-pc1
+        'cat /opt/carma/vehicle/config/docker-compose-manager.yml' | \
+        docker stack deploy --compose-file - carma-manager
 
         docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-        'cat /opt/carma/vehicle/config/docker-compose-pc2.yml' | \
-        docker stack deploy --compose-file - carma-pc2
+        'cat /opt/carma/vehicle/config/docker-compose-worker.yml' | \
+        docker stack deploy --compose-file - carma-worker
         
     
     elif [ ! -z "$1" ]; then
@@ -91,13 +129,16 @@ swarm-config__set() {
         local TAG=$1
     fi
 
-    if [[ -z $2 ]]; then
+    if [[ -z $TAG ]]; then
         echo "Please specify a tag string to update carma-config-data volume."
-        echo "Done."
         exit 1
     fi
 
-    local IMAGE_NAME="$USERNAME/carma-config:$TAG"
+    if [ ! -z "$(echo $TAG | grep :)" ]; then
+        local IMAGE_NAME="$TAG"
+    else
+        local IMAGE_NAME="$USERNAME/carma-config:$TAG"
+    fi
 
     if [ "$(docker images | grep $USERNAME/carma-config | grep $TAG )" ]; then
         echo "Found image, applying to carma-config-volume"
@@ -111,7 +152,7 @@ swarm-config__set() {
 swarm-config__edit() {
     echo "Opening carma-config-data volume shell with read/write privileges..."
 
-    if [ "$(docker volume ls | grep carma-config-data | sed 's/local     //g')" ]; then
+    if [ ! -z "$(docker volume ls | grep carma-config-data)" ]; then
         echo "Found carma-config-data volume"
     else
         echo "Missing carma-config-data volume, add with \"carma swarm update-config <image tag>\""
@@ -129,7 +170,29 @@ swarm-config__edit() {
 
 
 swarm-config__status() {
-    echo "Current configuration is loaded from image: <filler>"
+    if [ ! -z "$(docker volume ls | grep carma-config-data)" ]; then
+        echo "Found carma-config-data volume"
+    else
+        echo "Missing carma-config-data volume, add with \"carma swarm update-config <image tag>\""
+        exit 1
+    fi
+
+    if [ -z "$1" ]; then
+        echo ""
+        docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+        'cat /opt/carma/vehicle/config/docker-compose-manager.yml <(echo)
+         cat /opt/carma/vehicle/config/docker-compose-worker.yml <(echo)
+         cat /opt/carma/vehicle/config/docker-compose-background-manager.yml <(echo)
+         cat /opt/carma/vehicle/config/carma.launch <(echo) 
+         cat /opt/carma/vehicle/config/carma.config.js <(echo)
+         cat /opt/carma/vehicle/config/carma.urdf <(echo)'
+        
+    else
+        echo " -- $1:"
+        docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+        "cat /opt/carma/vehicle/config/$1"
+    fi
+
 }
 
 

From 712d7afa702d30edb683354ee520d420d314d6d5 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Wed, 22 Nov 2023 13:43:07 -0800
Subject: [PATCH 11/31] Updated to support env variables

---
 .../carma_script_extensions/swarm             | 54 ++++++++++---------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index d21baf4084..e68cb0060f 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -41,7 +41,7 @@ swarm__install() {
     echo "Downloading $IMAGE_NAME..."
     docker pull $IMAGE_NAME
 
-    docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d usdotfhwastol/carma-config:carma-system-4.4.3-47-g40f710d-SNAPSHOT-development
+    docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d "$IMAGE_NAME"
 
     local BACKGROUND_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
     'cat /opt/carma/vehicle/config/docker-compose-background-manager.yml'`"
@@ -55,6 +55,22 @@ swarm__install() {
     echo "$MANAGER_COMPOSE" | docker-compose -f - pull
     echo "$WORKER_COMPOSE" | docker-compose -f - pull
 
+
+    local carma_base=$(__get_image_from_config carma-base:)
+    if [[ -z $carma_base ]]; then
+        __pull_newest_carma_base
+        carma_base=$(__get_most_recent_carma_base)
+    fi
+
+    MANAGER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$MANAGER_USER"; echo "@"; echo "$MANAGER_IP"; } | tr -d "\n"')
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+
+    export MANAGER_IP=${MANAGER_INFO#*@}
+    export MANAGER_USER=${MANAGER_INFO%%@*}
+
+    export WORKER_IP=${WORKER_INFO#*@}
+    export WORKER_USER=${WORKER_INFO%%@*}
+
 }
 
 
@@ -98,7 +114,7 @@ swarm__start() {
         echo "Deploying CARMA to dual compute Swarm stack..."
 
         local TKN=$(docker swarm join-token -q worker)
-        ssh dev@192.168.88.101 "docker swarm join --token $TKN 192.168.88.100:2377"
+        ssh $WORKER_INFO "docker swarm join --token $TKN $MANAGER_IP:2377"
 
         docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
         'cat /opt/carma/vehicle/config/docker-compose-manager.yml' | \
@@ -146,6 +162,7 @@ swarm-config__set() {
     else
         echo "Could not find $USERNAME/carma-config:$TAG in local images"
     fi
+
 }
 
 
@@ -166,6 +183,15 @@ swarm-config__edit() {
     fi
 
     docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base bash
+
+    MANAGER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$MANAGER_USER"; echo "@"; echo "$MANAGER_IP"; } | tr -d "\n"')
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+
+    export MANAGER_IP=${MANAGER_INFO#*@}
+    export MANAGER_USER=${MANAGER_INFO%%@*}
+
+    export WORKER_IP=${WORKER_INFO#*@}
+    export WORKER_USER=${WORKER_INFO%%@*}
 }
 
 
@@ -221,17 +247,6 @@ swarm__register() {
         docker volume create carma-registry
     fi
 
-    # if [ "$2" = "-d" ]; then
-    #     local USERNAME=usdotfhwastoldev
-    #     local TAG=$3
-    # elif [ "$2" = "-c" ]; then
-    #     local USERNAME=usdotfhwastolcandidate
-    #     local TAG=$3
-    # else
-    #     local USERNAME=usdotfhwastol
-    #     local TAG=$2
-    # fi
-
     if [[ -z $1 ]]; then
         echo "Please specify a tag string to update carma-config-data volume."
         echo "Done."
@@ -245,6 +260,7 @@ swarm__register() {
     for SOURCE_IMAGE_NAME in $IMAGE_NAMES
     do
     
+    # Consider default-value variables for host/port that can be modified
     echo "Pushing $SOURCE_IMAGE_NAME to localhost:5000"
 
     docker tag "$SOURCE_IMAGE_NAME" "localhost:5000/$SOURCE_IMAGE_NAME"
@@ -259,19 +275,9 @@ swarm__stop() {
     echo "Shutting down swarm..."
 
     if [ "$(docker node ls | grep worker )" ]; then
-        ssh dev@192.168.88.101 "docker swarm leave"
+        ssh $WORKER_INFO "docker swarm leave"
     fi
     docker swarm leave --force
-
-    # if [ "$1" = "all" ]; then
-        # shuts down stacks on worker
-        # leaves worker
-
-        # shuts down stack on manager
-        # leaves manager
-
-    # elif [ ! -z "$1" ]; then
-    #     echo "Unrecognized argument \"swarm $1\""
 }
 
 

From 08721742f37165c655effeeced431585369626e8 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Mon, 27 Nov 2023 07:10:26 -0800
Subject: [PATCH 12/31] Naming fixes and added clarity

---
 .../carma_script_extensions/swarm             | 34 +++++++++----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index e68cb0060f..4036720c39 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -28,7 +28,7 @@ swarm__install() {
     fi
 
     if [[ -z $TAG ]]; then
-        echo "Please specify a tag string to update carma-config-data volume."
+        echo "Please specify a tag string to update 'carma-config-data' volume."
         exit 1
     fi
 
@@ -78,16 +78,16 @@ swarm__start() {
     echo "Checking Docker Swarm status..."
     if [ "$(docker info | grep Swarm | sed 's/Swarm: //g')" == " inactive" ]; then
         echo "Initalizing Docker Swarm..."
-        docker swarm init --advertise-addr 192.168.88.100 # consider making a variable
+        docker swarm init --advertise-addr $MANAGER_IP
     else
         echo "Docker Swarm active"
     fi
 
     # if carma-config-data volume doesn't exist exit and request update
     if [ ! -z "$(docker volume ls | grep carma-config-data)" ]; then
-        echo "Found carma-config-data volume"
+        echo "Found 'carma-config-data' volume"
     else
-        echo "Missing carma-config-data volume, add with \"carma swarm update-config <image tag>\""
+        echo "Missing 'carma-config-data' volume, add with \"carma swarm update-config <image tag>\""
         exit 1
     fi
 
@@ -132,7 +132,7 @@ swarm__start() {
 
 
 swarm-config__set() {
-    echo "Updating carma-config-data volume..."
+    echo "Updating 'carma-config-data' volume..."
 
     if [ "$1" = "-d" ]; then
         local USERNAME=usdotfhwastoldev
@@ -146,7 +146,7 @@ swarm-config__set() {
     fi
 
     if [[ -z $TAG ]]; then
-        echo "Please specify a tag string to update carma-config-data volume."
+        echo "Please specify a tag string to update 'carma-config-data' volume."
         exit 1
     fi
 
@@ -167,12 +167,12 @@ swarm-config__set() {
 
 
 swarm-config__edit() {
-    echo "Opening carma-config-data volume shell with read/write privileges..."
+    echo "Opening 'carma-config-data' volume shell with read/write privileges..."
 
     if [ ! -z "$(docker volume ls | grep carma-config-data)" ]; then
-        echo "Found carma-config-data volume"
+        echo "Found 'carma-config-data' volume"
     else
-        echo "Missing carma-config-data volume, add with \"carma swarm update-config <image tag>\""
+        echo "Missing 'carma-config-data' volume, add with \"carma swarm update-config <image tag>\""
         exit 1
     fi
 
@@ -197,9 +197,9 @@ swarm-config__edit() {
 
 swarm-config__status() {
     if [ ! -z "$(docker volume ls | grep carma-config-data)" ]; then
-        echo "Found carma-config-data volume"
+        echo "Found 'carma-config-data' volume"
     else
-        echo "Missing carma-config-data volume, add with \"carma swarm update-config <image tag>\""
+        echo "Missing 'carma-config-data' volume, add with \"carma swarm update-config <image tag>\""
         exit 1
     fi
 
@@ -241,14 +241,14 @@ swarm__register() {
     echo "Installing Docker images..."
 
     if [ "$(docker volume ls | grep carma-registry | sed 's/local     //g')" ]; then
-        echo "Found carma-registry volume"
+        echo "Found 'carma-registry' volume"
     else
-        echo "Missing carma-registry volume, adding now..."
+        echo "Missing 'carma-registry' volume, adding now..."
         docker volume create carma-registry
     fi
 
     if [[ -z $1 ]]; then
-        echo "Please specify a tag string to update carma-config-data volume."
+        echo "Please specify a tag string to update 'carma-config-data' volume."
         echo "Done."
         exit 1
     fi
@@ -305,7 +305,7 @@ Please enter one of the following commands:
         set <tag/image>
             - Uses the local carma-config image defined in
               <tag/image> to update the Docker named volume
-              "carma-config-data" which can be found at
+              'carma-config-data' which can be found at
               /var/lib/docker/volumes/carma-config-data/_data
             - [empty]
                 - Defaults to a usdotfhwastol image
@@ -315,11 +315,11 @@ Please enter one of the following commands:
                 - Defines a usdotfhwastolcandidate image
         edit
             - Requires sudo privilege. Opens a bash shell
-              in a carma-base image with carma-config-data
+              in a carma-base image with 'carma-config-data'
               mounted so that files can be modified
         status
             - Reports the active Docker Compose files from
-              carma-config-data to the active terminal
+              'carma-config-data' to the active terminal
     register <images>
         - Pushes the images defined by <images> to the
           registry service. Images must have their full

From b71efdf682cb67681d5171e8149545ddc9b01864 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Mon, 27 Nov 2023 08:02:04 -0800
Subject: [PATCH 13/31] Registry updates for testing, minor fixes

---
 .../carma_script_extensions/swarm             | 32 +++++++++++++------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index 4036720c39..ef4c93ecd9 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -238,7 +238,7 @@ swarm__attach() {
 
 
 swarm__register() {
-    echo "Installing Docker images..."
+    echo "Adding Docker images to Manager's local registry..."
 
     if [ "$(docker volume ls | grep carma-registry | sed 's/local     //g')" ]; then
         echo "Found 'carma-registry' volume"
@@ -248,26 +248,36 @@ swarm__register() {
     fi
 
     if [[ -z $1 ]]; then
-        echo "Please specify a tag string to update 'carma-config-data' volume."
+        echo "Please specify an image string to add to the registry"
         echo "Done."
         exit 1
     fi
 
-    docker service create --constraint 'node.role==manager' -d --volume "carma-registry:/var/lib/registry" -p 5000:5000 --name carma-registry-service registry:2
+    local REGISTRY_PORT=5000
+    local REGISTRY_HOST="localhost"
+
+    docker service create --constraint 'node.role==manager' -d --volume "carma-registry:/var/lib/registry" -p "$REGISTRY_PORT:$REGISTRY_PORT" --name carma-registry-service registry:2
     
     IMAGE_NAMES="$*"
 
     for SOURCE_IMAGE_NAME in $IMAGE_NAMES
     do
-    
-    # Consider default-value variables for host/port that can be modified
-    echo "Pushing $SOURCE_IMAGE_NAME to localhost:5000"
-
-    docker tag "$SOURCE_IMAGE_NAME" "localhost:5000/$SOURCE_IMAGE_NAME"
-    docker push "localhost:5000/$SOURCE_IMAGE_NAME"
-    docker rmi "localhost:5000/$SOURCE_IMAGE_NAME"
+        echo "Pushing $SOURCE_IMAGE_NAME to $REGISTRY_HOST:$REGISTRY_PORT"
+        docker tag "$SOURCE_IMAGE_NAME" "$REGISTRY_HOST:$REGISTRY_PORT/$SOURCE_IMAGE_NAME"
+        docker push "$REGISTRY_HOST:$REGISTRY_PORT/$SOURCE_IMAGE_NAME"
+        docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/$SOURCE_IMAGE_NAME"
+    done
 
+    ssh -R "$REGISTRY_PORT:$REGISTRY_HOST:$REGISTRY_PORT" "$WORKER_INFO" sh
+    for TARGET_IMAGE_NAME in $IMAGE_NAMES
+    do
+        echo "Pulling \$TARGET_IMAGE_NAME onto $WORKER_INFO from $REGISTRY_HOST:$REGISTRY_PORT"
+        docker pull "$REGISTRY_HOST:$REGISTRY_PORT/\$TARGET_IMAGE_NAME" \
+          && docker tag "$REGISTRY_HOST:$REGISTRY_PORT/\$TARGET_IMAGE_NAME" "\$TARGET_IMAGE_NAME" \
+          && docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/\$TARGET_IMAGE_NAME"
     done
+
+    # May need to find the service and remove it after use since Swarm will likely keep it alive
 }
 
 
@@ -289,6 +299,8 @@ swarm__help() {
 
 Please enter one of the following commands:
   swarm
+    install
+        - Installs images locally based on the active Compose files
     start
         [empty]
             - Will always deploy carma-web-ui as a background service

From 42c3a4613d405ad76ff003b489f368f70bfdd236 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Mon, 27 Nov 2023 13:45:11 -0800
Subject: [PATCH 14/31] Test fix for env variable subshell usage

---
 .../carma_script_extensions/swarm             | 28 +++++++------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index ef4c93ecd9..6ccc3b1311 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -55,7 +55,11 @@ swarm__install() {
     echo "$MANAGER_COMPOSE" | docker-compose -f - pull
     echo "$WORKER_COMPOSE" | docker-compose -f - pull
 
+}
+
 
+swarm__start() {
+    echo "Checking Docker Swarm status..."
     local carma_base=$(__get_image_from_config carma-base:)
     if [[ -z $carma_base ]]; then
         __pull_newest_carma_base
@@ -65,17 +69,12 @@ swarm__install() {
     MANAGER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$MANAGER_USER"; echo "@"; echo "$MANAGER_IP"; } | tr -d "\n"')
     WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
 
-    export MANAGER_IP=${MANAGER_INFO#*@}
-    export MANAGER_USER=${MANAGER_INFO%%@*}
+    MANAGER_IP=${MANAGER_INFO#*@}
+    MANAGER_USER=${MANAGER_INFO%%@*}
 
-    export WORKER_IP=${WORKER_INFO#*@}
-    export WORKER_USER=${WORKER_INFO%%@*}
+    WORKER_IP=${WORKER_INFO#*@}
+    WORKER_USER=${WORKER_INFO%%@*}
 
-}
-
-
-swarm__start() {
-    echo "Checking Docker Swarm status..."
     if [ "$(docker info | grep Swarm | sed 's/Swarm: //g')" == " inactive" ]; then
         echo "Initalizing Docker Swarm..."
         docker swarm init --advertise-addr $MANAGER_IP
@@ -183,15 +182,6 @@ swarm-config__edit() {
     fi
 
     docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base bash
-
-    MANAGER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$MANAGER_USER"; echo "@"; echo "$MANAGER_IP"; } | tr -d "\n"')
-    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
-
-    export MANAGER_IP=${MANAGER_INFO#*@}
-    export MANAGER_USER=${MANAGER_INFO%%@*}
-
-    export WORKER_IP=${WORKER_INFO#*@}
-    export WORKER_USER=${WORKER_INFO%%@*}
 }
 
 
@@ -284,6 +274,8 @@ swarm__register() {
 swarm__stop() {
     echo "Shutting down swarm..."
 
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+
     if [ "$(docker node ls | grep worker )" ]; then
         ssh $WORKER_INFO "docker swarm leave"
     fi

From 6092f73f38ad5e425ca057fee359e4a5e6834620 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Tue, 28 Nov 2023 14:21:29 -0800
Subject: [PATCH 15/31] Added read only vars, PR changes, fixed stop and
 register

---
 .../carma_script_extensions/swarm             | 122 ++++++++++--------
 1 file changed, 69 insertions(+), 53 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index 6ccc3b1311..cf2cebf13b 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -14,17 +14,16 @@
 #  License for the specific language governing permissions and limitations under
 #  the License.
 
-
 swarm__install() {
     if [ "$1" = "-d" ]; then
-        local USERNAME=usdotfhwastoldev
-        local TAG=$2
+        local -r USERNAME=usdotfhwastoldev
+        local -r TAG=$2
     elif [ "$1" = "-c" ]; then
-        local USERNAME=usdotfhwastolcandidate
-        local TAG=$2
+        local -r USERNAME=usdotfhwastolcandidate
+        local -r TAG=$2
     else
-        local USERNAME=usdotfhwastol
-        local TAG=$1
+        local -r USERNAME=usdotfhwastol
+        local -r TAG=$1
     fi
 
     if [[ -z $TAG ]]; then
@@ -33,9 +32,9 @@ swarm__install() {
     fi
 
     if [ ! -z "$(echo $TAG | grep :)" ]; then
-        local IMAGE_NAME="$TAG"
+        local -r IMAGE_NAME="$TAG"
     else
-        local IMAGE_NAME="$USERNAME/carma-config:$TAG"
+        local -r IMAGE_NAME="$USERNAME/carma-config:$TAG"
     fi
 
     echo "Downloading $IMAGE_NAME..."
@@ -43,11 +42,11 @@ swarm__install() {
 
     docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d "$IMAGE_NAME"
 
-    local BACKGROUND_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+    local -r BACKGROUND_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
     'cat /opt/carma/vehicle/config/docker-compose-background-manager.yml'`"
-    local MANAGER_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+    local -r MANAGER_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
     'cat /opt/carma/vehicle/config/docker-compose-manager.yml'`"
-    local WORKER_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+    local -r WORKER_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
     'cat /opt/carma/vehicle/config/docker-compose-worker.yml'`"
 
     echo "Downloading $IMAGE_NAME dependencies..."
@@ -60,6 +59,14 @@ swarm__install() {
 
 swarm__start() {
     echo "Checking Docker Swarm status..."
+    # if carma-config-data volume doesn't exist exit and request update
+    if [ ! -z "$(docker volume ls | grep carma-config-data)" ]; then
+        echo "Found 'carma-config-data' volume"
+    else
+        echo "Missing 'carma-config-data' volume, add with \"carma swarm update-config <image tag>\""
+        exit 1
+    fi
+
     local carma_base=$(__get_image_from_config carma-base:)
     if [[ -z $carma_base ]]; then
         __pull_newest_carma_base
@@ -82,14 +89,6 @@ swarm__start() {
         echo "Docker Swarm active"
     fi
 
-    # if carma-config-data volume doesn't exist exit and request update
-    if [ ! -z "$(docker volume ls | grep carma-config-data)" ]; then
-        echo "Found 'carma-config-data' volume"
-    else
-        echo "Missing 'carma-config-data' volume, add with \"carma swarm update-config <image tag>\""
-        exit 1
-    fi
-
     # carma-web-ui will always run as a background process regardless of deploy argument
     docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
     'cat /opt/carma/vehicle/config/docker-compose-background-manager.yml' | \
@@ -112,7 +111,7 @@ swarm__start() {
     elif [ "$1" = "dual" ]; then
         echo "Deploying CARMA to dual compute Swarm stack..."
 
-        local TKN=$(docker swarm join-token -q worker)
+        local -r TKN=$(docker swarm join-token -q worker)
         ssh $WORKER_INFO "docker swarm join --token $TKN $MANAGER_IP:2377"
 
         docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
@@ -134,25 +133,25 @@ swarm-config__set() {
     echo "Updating 'carma-config-data' volume..."
 
     if [ "$1" = "-d" ]; then
-        local USERNAME=usdotfhwastoldev
-        local TAG=$2
+        local -r USERNAME=usdotfhwastoldev
+        local -r TAG=$2
     elif [ "$1" = "-c" ]; then
-        local USERNAME=usdotfhwastolcandidate
-        local TAG=$2
+        local -r USERNAME=usdotfhwastolcandidate
+        local -r TAG=$2
     else
-        local USERNAME=usdotfhwastol
-        local TAG=$1
+        local -r USERNAME=usdotfhwastol
+        local -r TAG=$1
     fi
 
     if [[ -z $TAG ]]; then
         echo "Please specify a tag string to update 'carma-config-data' volume."
-        exit 1
+        exit 1      
     fi
 
     if [ ! -z "$(echo $TAG | grep :)" ]; then
-        local IMAGE_NAME="$TAG"
+        local -r IMAGE_NAME="$TAG"
     else
-        local IMAGE_NAME="$USERNAME/carma-config:$TAG"
+        local -r IMAGE_NAME="$USERNAME/carma-config:$TAG"
     fi
 
     if [ "$(docker images | grep $USERNAME/carma-config | grep $TAG )" ]; then
@@ -186,6 +185,8 @@ swarm-config__edit() {
 
 
 swarm-config__status() {
+    # note the image being used
+    # remove old compose files not needed due to split can
     if [ ! -z "$(docker volume ls | grep carma-config-data)" ]; then
         echo "Found 'carma-config-data' volume"
     else
@@ -193,29 +194,30 @@ swarm-config__status() {
         exit 1
     fi
 
+    echo "Current carma-config image: "
+
     if [ -z "$1" ]; then
-        echo ""
+        echo -e " \n -- docker-compose-background-manager.yml\n"
         docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-        'cat /opt/carma/vehicle/config/docker-compose-manager.yml <(echo)
-         cat /opt/carma/vehicle/config/docker-compose-worker.yml <(echo)
-         cat /opt/carma/vehicle/config/docker-compose-background-manager.yml <(echo)
-         cat /opt/carma/vehicle/config/carma.launch <(echo) 
-         cat /opt/carma/vehicle/config/carma.config.js <(echo)
-         cat /opt/carma/vehicle/config/carma.urdf <(echo)'
+        'cat /opt/carma/vehicle/config/docker-compose-background-manager.yml    <(echo) <(echo) <(echo " -- docker-compose-manager.yml") <(echo)
+         cat /opt/carma/vehicle/config/docker-compose-manager.yml               <(echo) <(echo) <(echo " -- docker-compose-worker.yml") <(echo)
+         cat /opt/carma/vehicle/config/docker-compose-worker.yml                <(echo) <(echo) <(echo " -- carma.launch") <(echo)
+         cat /opt/carma/vehicle/config/carma.launch                             <(echo) <(echo) <(echo " -- carma.config.js") <(echo)
+         cat /opt/carma/vehicle/config/carma.config.js                          <(echo) <(echo) <(echo " -- carma.urdf") <(echo)
+         cat /opt/carma/vehicle/config/carma.urdf                               <(echo)'
         
     else
         echo " -- $1:"
         docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
         "cat /opt/carma/vehicle/config/$1"
     fi
-
 }
 
 
 swarm__attach() {
     echo "Attaching STDOUT logs to terminal..."
 
-    SWARM_SERVICES=$(docker service ls --format "{{.Name}}")
+    local SWARM_SERVICES=$(docker service ls --format "{{.Name}}")
 
     # this seems to loop forever and cannot be escaped with ctrl+C
     # performance will likely be poor due to combined log streams
@@ -243,11 +245,19 @@ swarm__register() {
         exit 1
     fi
 
-    local REGISTRY_PORT=5000
-    local REGISTRY_HOST="localhost"
+    local carma_base=$(__get_image_from_config carma-base:)
+    if [[ -z $carma_base ]]; then
+        __pull_newest_carma_base
+        carma_base=$(__get_most_recent_carma_base)
+    fi
+
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+
+    local -r REGISTRY_PORT=5000
+    local -r REGISTRY_HOST="localhost"
+
+    docker run -d --volume carma-registry:/var/lib/registry --publish "$REGISTRY_PORT:$REGISTRY_PORT" --name carma-registry-task registry:2.6
 
-    docker service create --constraint 'node.role==manager' -d --volume "carma-registry:/var/lib/registry" -p "$REGISTRY_PORT:$REGISTRY_PORT" --name carma-registry-service registry:2
-    
     IMAGE_NAMES="$*"
 
     for SOURCE_IMAGE_NAME in $IMAGE_NAMES
@@ -261,23 +271,22 @@ swarm__register() {
     ssh -R "$REGISTRY_PORT:$REGISTRY_HOST:$REGISTRY_PORT" "$WORKER_INFO" sh
     for TARGET_IMAGE_NAME in $IMAGE_NAMES
     do
-        echo "Pulling \$TARGET_IMAGE_NAME onto $WORKER_INFO from $REGISTRY_HOST:$REGISTRY_PORT"
-        docker pull "$REGISTRY_HOST:$REGISTRY_PORT/\$TARGET_IMAGE_NAME" \
-          && docker tag "$REGISTRY_HOST:$REGISTRY_PORT/\$TARGET_IMAGE_NAME" "\$TARGET_IMAGE_NAME" \
-          && docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/\$TARGET_IMAGE_NAME"
+        echo "Pulling $TARGET_IMAGE_NAME onto $WORKER_INFO from $REGISTRY_HOST:$REGISTRY_PORT"
+        docker pull "$REGISTRY_HOST:$REGISTRY_PORT/$TARGET_IMAGE_NAME" \
+          && docker tag "$REGISTRY_HOST:$REGISTRY_PORT/$TARGET_IMAGE_NAME" "$TARGET_IMAGE_NAME" \
+          && docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/$TARGET_IMAGE_NAME"
     done
-
-    # May need to find the service and remove it after use since Swarm will likely keep it alive
 }
 
 
 swarm__stop() {
     echo "Shutting down swarm..."
+    local carma_base=$(__get_image_from_config carma-base:)
 
     WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
 
-    if [ "$(docker node ls | grep worker )" ]; then
-        ssh $WORKER_INFO "docker swarm leave"
+    if [ "$(docker node ls -f "role=worker" | grep  "Ready" )" ]; then
+        ssh $WORKER_INFO "docker swarm leave" 
     fi
     docker swarm leave --force
 }
@@ -291,8 +300,15 @@ swarm__help() {
 
 Please enter one of the following commands:
   swarm
-    install
-        - Installs images locally based on the active Compose files
+    install <tag/image>
+        - Installs images to the local host based on Compose files
+          contained in the carma-config defined by <tag/image>
+        - [empty]
+            - Defaults to a usdotfhwastol image
+        - d 
+            - Defines a usdotfhwastoldev image
+        - c
+            - Defines a usdotfhwastolcandidate image
     start
         [empty]
             - Will always deploy carma-web-ui as a background service

From 35f7a2773e56ae8ab42bc0308b867bb430230063 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Wed, 29 Nov 2023 13:13:21 -0800
Subject: [PATCH 16/31] Added dual compute params and updated LaunchDescription

---
 carma/launch/carma_src.launch.py | 153 +++++++++++++++++++++++++------
 1 file changed, 124 insertions(+), 29 deletions(-)

diff --git a/carma/launch/carma_src.launch.py b/carma/launch/carma_src.launch.py
index 96f0f872bb..0784111c06 100644
--- a/carma/launch/carma_src.launch.py
+++ b/carma/launch/carma_src.launch.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from ament_index_python import get_package_share_directory
-from launch.actions import Shutdown
+from launch.actions import Shutdown, ExecuteProcess
 from launch import LaunchDescription
 from launch_ros.actions import Node
 from launch.actions import IncludeLaunchDescription
@@ -116,6 +116,22 @@ def generate_launch_description():
         default_value= 'False',
         description='Flag to enable opening http tunnesl to CARMA Cloud'
     )
+
+    # Declare system_architecture
+    system_architecture = LaunchConfiguration('system_architecture')
+    declare_system_architecture = DeclareLaunchArgument(
+        name = 'system_architecture',
+        default_value = 'single',
+        description = 'Flag to define whether a single compute system or a dual compute system is being used'
+    )
+
+    # Declare host_placement
+    host_placement = LaunchConfiguration('host_placement')
+    declare_host_placement = DeclareLaunchArgument(
+        name = 'host_placement',
+        default_value = 'manager',
+        description = 'Flag to define whether the current active host is a manager or worker for ROS node allocation'
+    )
     
     # Declare port
     port = LaunchConfiguration('port')
@@ -272,31 +288,110 @@ def generate_launch_description():
         ]
     )
 
-    return LaunchDescription([
-        declare_vehicle_calibration_dir_arg,
-        declare_vehicle_config_dir_arg,
-        declare_vehicle_characteristics_param_file_arg,
-        declare_vehicle_config_param_file_arg,
-        declare_route_file_folder,
-        declare_enable_guidance_plugin_validator,
-        declare_strategic_plugins_to_validate,
-        declare_tactical_plugins_to_validate,
-        declare_control_plugins_to_validate,
-        declare_enable_opening_tunnels,
-        declare_port,
-        declare_load_type,
-        declare_single_pcd_path,
-        declare_area,
-        declare_arealist_path,
-        declare_vector_map_file,
-        declare_simulation_mode,
-        drivers_group,
-        transform_group,
-        environment_group,
-        localization_group,
-        v2x_group,
-        guidance_group, 
-        ros2_rosbag_launch,
-        ui_group,
-        system_controller
-    ])
+
+    ExecuteProcess(
+        condition=IfCondition(
+            PythonExpression(
+                ["'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"]
+            )
+        ),
+        launch_description = LaunchDescription([
+            declare_vehicle_calibration_dir_arg,
+            declare_vehicle_config_dir_arg,
+            declare_vehicle_characteristics_param_file_arg,
+            declare_vehicle_config_param_file_arg,
+            declare_route_file_folder,
+            declare_enable_guidance_plugin_validator,
+            declare_strategic_plugins_to_validate,
+            declare_tactical_plugins_to_validate,
+            declare_control_plugins_to_validate,
+            declare_enable_opening_tunnels,
+            declare_port,
+            declare_load_type,
+            declare_single_pcd_path,
+            declare_area,
+            declare_arealist_path,
+            declare_vector_map_file,
+            declare_simulation_mode,
+            drivers_group,
+            transform_group,
+            environment_group,
+            localization_group,
+            v2x_group,
+            ros2_rosbag_launch,
+            ui_group,
+            system_controller
+        ])
+    )
+
+    ExecuteProcess(
+        condition=IfCondition(
+            PythonExpression(
+                ["'", system_architecture,"' == 'dual' and '", host_placement, "' == 'worker'"]
+            )
+        ),
+        launch_description = LaunchDescription([
+            declare_vehicle_calibration_dir_arg,
+            declare_vehicle_config_dir_arg,
+            declare_vehicle_characteristics_param_file_arg,
+            declare_vehicle_config_param_file_arg,
+            declare_route_file_folder,
+            declare_enable_guidance_plugin_validator,
+            declare_strategic_plugins_to_validate,
+            declare_tactical_plugins_to_validate,
+            declare_control_plugins_to_validate,
+            declare_enable_opening_tunnels,
+            declare_port,
+            declare_load_type,
+            declare_single_pcd_path,
+            declare_area,
+            declare_arealist_path,
+            declare_vector_map_file,
+            declare_simulation_mode,
+            drivers_group,
+            transform_group,
+            environment_group,
+            localization_group,
+            v2x_group,
+            guidance_group
+        ])
+    )
+
+    ExecuteProcess(
+        condition=IfCondition(
+            PythonExpression(
+                ["'", system_architecture,"' == 'single'"]
+            )
+        ),
+        launch_description = LaunchDescription([
+            declare_vehicle_calibration_dir_arg,
+            declare_vehicle_config_dir_arg,
+            declare_vehicle_characteristics_param_file_arg,
+            declare_vehicle_config_param_file_arg,
+            declare_route_file_folder,
+            declare_enable_guidance_plugin_validator,
+            declare_strategic_plugins_to_validate,
+            declare_tactical_plugins_to_validate,
+            declare_control_plugins_to_validate,
+            declare_enable_opening_tunnels,
+            declare_port,
+            declare_load_type,
+            declare_single_pcd_path,
+            declare_area,
+            declare_arealist_path,
+            declare_vector_map_file,
+            declare_simulation_mode,
+            drivers_group,
+            transform_group,
+            environment_group,
+            localization_group,
+            v2x_group,
+            guidance_group, 
+            ros2_rosbag_launch,
+            ui_group,
+            system_controller
+        ])
+    )
+
+
+    return launch_description
\ No newline at end of file

From b2c0b40feb832ad355a12328d9201fd21b4b9638 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Thu, 30 Nov 2023 10:05:06 -0800
Subject: [PATCH 17/31] Changed carma_src launch decription to better reflect
 conditionals

---
 carma/launch/carma_src.launch.py              | 154 ++++++------------
 .../carma_script_extensions/swarm             |   2 +-
 2 files changed, 48 insertions(+), 108 deletions(-)

diff --git a/carma/launch/carma_src.launch.py b/carma/launch/carma_src.launch.py
index 0784111c06..fd98e8b757 100644
--- a/carma/launch/carma_src.launch.py
+++ b/carma/launch/carma_src.launch.py
@@ -160,6 +160,29 @@ def generate_launch_description():
     simulation_mode = LaunchConfiguration('simulation_mode')
     declare_simulation_mode = DeclareLaunchArgument(name='simulation_mode', default_value = 'False', description = 'True if CARMA Platform is launched with CARLA Simulator')
 
+    # Declare initial configurations for the launch description
+    carma_launch_description = LaunchDescription([
+        declare_vehicle_calibration_dir_arg,
+        declare_vehicle_config_dir_arg,
+        declare_vehicle_characteristics_param_file_arg,
+        declare_vehicle_config_param_file_arg,
+        declare_route_file_folder,
+        declare_enable_guidance_plugin_validator,
+        declare_strategic_plugins_to_validate,
+        declare_tactical_plugins_to_validate,
+        declare_control_plugins_to_validate,
+        declare_enable_opening_tunnels,
+        declare_port,
+        declare_load_type,
+        declare_single_pcd_path,
+        declare_area,
+        declare_arealist_path,
+        declare_vector_map_file,
+        declare_simulation_mode  
+    ])
+
+    # condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"]))
+
     # Launch ROS2 rosbag logging
     ros2_rosbag_launch = GroupAction(
         actions=[
@@ -172,22 +195,26 @@ def generate_launch_description():
             )
         ]
     )
+    carma_launch_description.add_action(ros2_rosbag_launch)
 
     # Nodes
-
     transform_group = GroupAction(
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_TF_NS', default_value='/')),
             IncludeLaunchDescription(
+                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/transforms.launch.py'])
             ),
         ]
     )
+    carma_launch_description.add_action(transform_group)
+
 
     environment_group = GroupAction(
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_ENV_NS', default_value='environment')),
             IncludeLaunchDescription(
+                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/environment.launch.py']),
                 launch_arguments = { 
                     'subsystem_controller_param_file' : [vehicle_config_dir, '/SubsystemControllerParams.yaml'],
@@ -198,11 +225,14 @@ def generate_launch_description():
             ),
         ]
     )
+    carma_launch_description.add_action(environment_group)
+
 
     localization_group = GroupAction(
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_LOCZ_NS', default_value='localization')),
             IncludeLaunchDescription(
+                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/localization.launch.py']),
                 launch_arguments = { 
                     'subsystem_controller_param_file' : [vehicle_config_dir, '/SubsystemControllerParams.yaml'],
@@ -217,11 +247,14 @@ def generate_launch_description():
             )
         ]
     )
+    carma_launch_description.add_action(localization_group)
+
 
     v2x_group = GroupAction(
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_MSG_NS', default_value='message')),
             IncludeLaunchDescription(
+                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/message.launch.py']),
                 launch_arguments = { 
                     'vehicle_characteristics_param_file' : vehicle_characteristics_param_file,
@@ -232,12 +265,15 @@ def generate_launch_description():
             ),
         ]
     )
+    carma_launch_description.add_action(v2x_group)
+
 
     guidance_group = GroupAction(
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_GUIDE_NS', default_value='guidance')),
             
             IncludeLaunchDescription(
+                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/guidance.launch.py']),
                 launch_arguments={
                     'route_file_folder' : route_file_folder,
@@ -252,11 +288,14 @@ def generate_launch_description():
             ),
         ]
     )
+    carma_launch_description.add_action(guidance_group)
+
 
     drivers_group = GroupAction(
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_INTR_NS', default_value='hardware_interface')),
             IncludeLaunchDescription(
+                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/drivers.launch.py']),
                 launch_arguments = { 
                     'vehicle_config_param_file' : vehicle_config_param_file,
@@ -265,6 +304,8 @@ def generate_launch_description():
             ),
         ]
     )
+    carma_launch_description.add_action(drivers_group)
+
 
     system_controller = Node(
         package='system_controller',
@@ -274,12 +315,15 @@ def generate_launch_description():
         on_exit = Shutdown(), # Mark the subsystem controller as required for segfaults
         arguments=['--ros-args', '--log-level', GetLogLevel('system_controller', env_log_levels)]
     )
+    carma_launch_description.add_action(system_controller)
+
 
     ui_group = GroupAction(
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_UI_NS', default_value='ui')),
             
             IncludeLaunchDescription(
+                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/ui.launch.py']),
                 launch_arguments={
                 'port' : port
@@ -287,111 +331,7 @@ def generate_launch_description():
             ),
         ]
     )
+    carma_launch_description.add_action(ui_group)
 
 
-    ExecuteProcess(
-        condition=IfCondition(
-            PythonExpression(
-                ["'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"]
-            )
-        ),
-        launch_description = LaunchDescription([
-            declare_vehicle_calibration_dir_arg,
-            declare_vehicle_config_dir_arg,
-            declare_vehicle_characteristics_param_file_arg,
-            declare_vehicle_config_param_file_arg,
-            declare_route_file_folder,
-            declare_enable_guidance_plugin_validator,
-            declare_strategic_plugins_to_validate,
-            declare_tactical_plugins_to_validate,
-            declare_control_plugins_to_validate,
-            declare_enable_opening_tunnels,
-            declare_port,
-            declare_load_type,
-            declare_single_pcd_path,
-            declare_area,
-            declare_arealist_path,
-            declare_vector_map_file,
-            declare_simulation_mode,
-            drivers_group,
-            transform_group,
-            environment_group,
-            localization_group,
-            v2x_group,
-            ros2_rosbag_launch,
-            ui_group,
-            system_controller
-        ])
-    )
-
-    ExecuteProcess(
-        condition=IfCondition(
-            PythonExpression(
-                ["'", system_architecture,"' == 'dual' and '", host_placement, "' == 'worker'"]
-            )
-        ),
-        launch_description = LaunchDescription([
-            declare_vehicle_calibration_dir_arg,
-            declare_vehicle_config_dir_arg,
-            declare_vehicle_characteristics_param_file_arg,
-            declare_vehicle_config_param_file_arg,
-            declare_route_file_folder,
-            declare_enable_guidance_plugin_validator,
-            declare_strategic_plugins_to_validate,
-            declare_tactical_plugins_to_validate,
-            declare_control_plugins_to_validate,
-            declare_enable_opening_tunnels,
-            declare_port,
-            declare_load_type,
-            declare_single_pcd_path,
-            declare_area,
-            declare_arealist_path,
-            declare_vector_map_file,
-            declare_simulation_mode,
-            drivers_group,
-            transform_group,
-            environment_group,
-            localization_group,
-            v2x_group,
-            guidance_group
-        ])
-    )
-
-    ExecuteProcess(
-        condition=IfCondition(
-            PythonExpression(
-                ["'", system_architecture,"' == 'single'"]
-            )
-        ),
-        launch_description = LaunchDescription([
-            declare_vehicle_calibration_dir_arg,
-            declare_vehicle_config_dir_arg,
-            declare_vehicle_characteristics_param_file_arg,
-            declare_vehicle_config_param_file_arg,
-            declare_route_file_folder,
-            declare_enable_guidance_plugin_validator,
-            declare_strategic_plugins_to_validate,
-            declare_tactical_plugins_to_validate,
-            declare_control_plugins_to_validate,
-            declare_enable_opening_tunnels,
-            declare_port,
-            declare_load_type,
-            declare_single_pcd_path,
-            declare_area,
-            declare_arealist_path,
-            declare_vector_map_file,
-            declare_simulation_mode,
-            drivers_group,
-            transform_group,
-            environment_group,
-            localization_group,
-            v2x_group,
-            guidance_group, 
-            ros2_rosbag_launch,
-            ui_group,
-            system_controller
-        ])
-    )
-
-
-    return launch_description
\ No newline at end of file
+    return carma_launch_description
\ No newline at end of file
diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index cf2cebf13b..6953ab253a 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -124,7 +124,7 @@ swarm__start() {
         
     
     elif [ ! -z "$1" ]; then
-        echo "Unrecognized argument \"swarm deploy $1\""
+        echo "Unrecognized argument \"swarm start $1\""
     fi
 }
 

From 36283b8a8d0f0abbcb3611cd74ce4552420c569b Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Fri, 1 Dec 2023 07:49:29 -0800
Subject: [PATCH 18/31] Fixes to configuration params and string syntax
 following testing on dual compute

---
 carma/launch/carma_src.launch.py | 78 +++++++++++++++-----------------
 1 file changed, 37 insertions(+), 41 deletions(-)

diff --git a/carma/launch/carma_src.launch.py b/carma/launch/carma_src.launch.py
index fd98e8b757..0cb64855ed 100644
--- a/carma/launch/carma_src.launch.py
+++ b/carma/launch/carma_src.launch.py
@@ -160,28 +160,6 @@ def generate_launch_description():
     simulation_mode = LaunchConfiguration('simulation_mode')
     declare_simulation_mode = DeclareLaunchArgument(name='simulation_mode', default_value = 'False', description = 'True if CARMA Platform is launched with CARLA Simulator')
 
-    # Declare initial configurations for the launch description
-    carma_launch_description = LaunchDescription([
-        declare_vehicle_calibration_dir_arg,
-        declare_vehicle_config_dir_arg,
-        declare_vehicle_characteristics_param_file_arg,
-        declare_vehicle_config_param_file_arg,
-        declare_route_file_folder,
-        declare_enable_guidance_plugin_validator,
-        declare_strategic_plugins_to_validate,
-        declare_tactical_plugins_to_validate,
-        declare_control_plugins_to_validate,
-        declare_enable_opening_tunnels,
-        declare_port,
-        declare_load_type,
-        declare_single_pcd_path,
-        declare_area,
-        declare_arealist_path,
-        declare_vector_map_file,
-        declare_simulation_mode  
-    ])
-
-    # condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"]))
 
     # Launch ROS2 rosbag logging
     ros2_rosbag_launch = GroupAction(
@@ -195,26 +173,24 @@ def generate_launch_description():
             )
         ]
     )
-    carma_launch_description.add_action(ros2_rosbag_launch)
 
     # Nodes
     transform_group = GroupAction(
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_TF_NS', default_value='/')),
             IncludeLaunchDescription(
-                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/transforms.launch.py'])
             ),
         ]
     )
-    carma_launch_description.add_action(transform_group)
 
 
     environment_group = GroupAction(
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_ENV_NS', default_value='environment')),
             IncludeLaunchDescription(
-                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/environment.launch.py']),
                 launch_arguments = { 
                     'subsystem_controller_param_file' : [vehicle_config_dir, '/SubsystemControllerParams.yaml'],
@@ -225,14 +201,13 @@ def generate_launch_description():
             ),
         ]
     )
-    carma_launch_description.add_action(environment_group)
 
 
     localization_group = GroupAction(
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_LOCZ_NS', default_value='localization')),
             IncludeLaunchDescription(
-                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/localization.launch.py']),
                 launch_arguments = { 
                     'subsystem_controller_param_file' : [vehicle_config_dir, '/SubsystemControllerParams.yaml'],
@@ -247,14 +222,13 @@ def generate_launch_description():
             )
         ]
     )
-    carma_launch_description.add_action(localization_group)
 
 
     v2x_group = GroupAction(
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_MSG_NS', default_value='message')),
             IncludeLaunchDescription(
-                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/message.launch.py']),
                 launch_arguments = { 
                     'vehicle_characteristics_param_file' : vehicle_characteristics_param_file,
@@ -265,15 +239,13 @@ def generate_launch_description():
             ),
         ]
     )
-    carma_launch_description.add_action(v2x_group)
 
 
     guidance_group = GroupAction(
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_GUIDE_NS', default_value='guidance')),
-            
             IncludeLaunchDescription(
-                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/guidance.launch.py']),
                 launch_arguments={
                     'route_file_folder' : route_file_folder,
@@ -288,14 +260,13 @@ def generate_launch_description():
             ),
         ]
     )
-    carma_launch_description.add_action(guidance_group)
 
 
     drivers_group = GroupAction(
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_INTR_NS', default_value='hardware_interface')),
             IncludeLaunchDescription(
-                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/drivers.launch.py']),
                 launch_arguments = { 
                     'vehicle_config_param_file' : vehicle_config_param_file,
@@ -304,7 +275,6 @@ def generate_launch_description():
             ),
         ]
     )
-    carma_launch_description.add_action(drivers_group)
 
 
     system_controller = Node(
@@ -315,15 +285,13 @@ def generate_launch_description():
         on_exit = Shutdown(), # Mark the subsystem controller as required for segfaults
         arguments=['--ros-args', '--log-level', GetLogLevel('system_controller', env_log_levels)]
     )
-    carma_launch_description.add_action(system_controller)
 
 
     ui_group = GroupAction(
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_UI_NS', default_value='ui')),
-            
             IncludeLaunchDescription(
-                condition=IfCondition(PythonExpression(["'", system_architecture,"' == 'single' or'" "'", system_architecture,"' == 'dual' and '", host_placement, "' == 'manager'"])),
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/ui.launch.py']),
                 launch_arguments={
                 'port' : port
@@ -331,7 +299,35 @@ def generate_launch_description():
             ),
         ]
     )
-    carma_launch_description.add_action(ui_group)
 
 
-    return carma_launch_description
\ No newline at end of file
+    return LaunchDescription([
+        declare_vehicle_calibration_dir_arg,
+        declare_vehicle_config_dir_arg,
+        declare_vehicle_characteristics_param_file_arg,
+        declare_vehicle_config_param_file_arg,
+        declare_route_file_folder,
+        declare_enable_guidance_plugin_validator,
+        declare_strategic_plugins_to_validate,
+        declare_tactical_plugins_to_validate,
+        declare_control_plugins_to_validate,
+        declare_enable_opening_tunnels,
+        declare_port,
+        declare_load_type,
+        declare_single_pcd_path,
+        declare_area,
+        declare_arealist_path,
+        declare_vector_map_file,
+        declare_simulation_mode,
+        declare_system_architecture,
+        declare_host_placement,
+        drivers_group,
+        transform_group,
+        environment_group,
+        localization_group,
+        v2x_group,
+        guidance_group,
+        ros2_rosbag_launch,
+        ui_group,
+        system_controller
+    ])
\ No newline at end of file

From 0af165976f3fe07fa676f1cf7108f21597de5a05 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Fri, 1 Dec 2023 08:46:30 -0800
Subject: [PATCH 19/31] Fixed an issue that prevented setting a full image name
 in set config

---
 engineering_tools/carma_script_extensions/swarm | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index 6953ab253a..3a1ff3cf54 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -150,15 +150,22 @@ swarm-config__set() {
 
     if [ ! -z "$(echo $TAG | grep :)" ]; then
         local -r IMAGE_NAME="$TAG"
+        local -r ALT_USER=$(echo $TAG | sed 's/\/.*//')
+        local -r ALT_TAG=$(echo $TAG | sed 's/.*\://')
     else
         local -r IMAGE_NAME="$USERNAME/carma-config:$TAG"
     fi
 
+    echo "$ALT_USER /carma-config $ALT_TAG"
+
     if [ "$(docker images | grep $USERNAME/carma-config | grep $TAG )" ]; then
         echo "Found image, applying to carma-config-volume"
-        docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d $USERNAME/carma-config:$TAG
+        docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d $IMAGE_NAME
+    elif [ "$(docker images | grep $ALT_USER/carma-config | grep $ALT_TAG )" ]; then
+        echo "Found full image, applying to carma-config-volume"
+        docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d $IMAGE_NAME
     else
-        echo "Could not find $USERNAME/carma-config:$TAG in local images"
+        echo "Could not find $IMAGE_NAME in local images"
     fi
 
 }
@@ -328,7 +335,8 @@ Please enter one of the following commands:
               'carma-config-data' which can be found at
               /var/lib/docker/volumes/carma-config-data/_data
             - [empty]
-                - Defaults to a usdotfhwastol image
+                - Defaults to a usdotfhwastol image if only a tag,
+                  otherwise accepts full image names as an arg
             - d 
                 - Defines a usdotfhwastoldev image
             - c

From b001822c2366aeb645abfd25a84904b0d38b167b Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Fri, 1 Dec 2023 09:00:13 -0800
Subject: [PATCH 20/31] Removed unnecessary echo

---
 engineering_tools/carma_script_extensions/swarm | 2 --
 1 file changed, 2 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index 3a1ff3cf54..e468a3d9a3 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -156,8 +156,6 @@ swarm-config__set() {
         local -r IMAGE_NAME="$USERNAME/carma-config:$TAG"
     fi
 
-    echo "$ALT_USER /carma-config $ALT_TAG"
-
     if [ "$(docker images | grep $USERNAME/carma-config | grep $TAG )" ]; then
         echo "Found image, applying to carma-config-volume"
         docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d $IMAGE_NAME

From d15d4c8cb26a1eb9fb6923ac24605ba679b6a3a5 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Tue, 5 Dec 2023 07:03:04 -0800
Subject: [PATCH 21/31] Addressing PR comments related to variable naming and
 other issues in clarity

---
 carma/launch/carma_src.launch.py              | 18 ++--
 .../carma_script_extensions/swarm             | 95 ++++++++++---------
 2 files changed, 58 insertions(+), 55 deletions(-)

diff --git a/carma/launch/carma_src.launch.py b/carma/launch/carma_src.launch.py
index 0cb64855ed..c3f8d27f1d 100644
--- a/carma/launch/carma_src.launch.py
+++ b/carma/launch/carma_src.launch.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 from ament_index_python import get_package_share_directory
-from launch.actions import Shutdown, ExecuteProcess
+from launch.actions import Shutdown
 from launch import LaunchDescription
 from launch_ros.actions import Node
 from launch.actions import IncludeLaunchDescription
@@ -163,6 +163,7 @@ def generate_launch_description():
 
     # Launch ROS2 rosbag logging
     ros2_rosbag_launch = GroupAction(
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
         actions=[
             IncludeLaunchDescription(
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/ros2_rosbag.launch.py']),
@@ -176,7 +177,7 @@ def generate_launch_description():
 
     # Nodes
     transform_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_TF_NS', default_value='/')),
             IncludeLaunchDescription(
@@ -187,7 +188,7 @@ def generate_launch_description():
 
 
     environment_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_ENV_NS', default_value='environment')),
             IncludeLaunchDescription(
@@ -204,7 +205,7 @@ def generate_launch_description():
 
 
     localization_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_LOCZ_NS', default_value='localization')),
             IncludeLaunchDescription(
@@ -225,7 +226,7 @@ def generate_launch_description():
 
 
     v2x_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_MSG_NS', default_value='message')),
             IncludeLaunchDescription(
@@ -242,7 +243,7 @@ def generate_launch_description():
 
 
     guidance_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'worker')"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_GUIDE_NS', default_value='guidance')),
             IncludeLaunchDescription(
@@ -263,7 +264,7 @@ def generate_launch_description():
 
 
     drivers_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_INTR_NS', default_value='hardware_interface')),
             IncludeLaunchDescription(
@@ -278,6 +279,7 @@ def generate_launch_description():
 
 
     system_controller = Node(
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
         package='system_controller',
         name='system_controller',
         executable='system_controller',
@@ -288,7 +290,7 @@ def generate_launch_description():
 
 
     ui_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager''"])),
+        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_UI_NS', default_value='ui')),
             IncludeLaunchDescription(
diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index e468a3d9a3..9f9b0b7de8 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -31,23 +31,23 @@ swarm__install() {
         exit 1
     fi
 
-    if [ ! -z "$(echo $TAG | grep :)" ]; then
+    if [ -n "$(echo "$TAG" | grep :)" ]; then
         local -r IMAGE_NAME="$TAG"
     else
         local -r IMAGE_NAME="$USERNAME/carma-config:$TAG"
     fi
 
     echo "Downloading $IMAGE_NAME..."
-    docker pull $IMAGE_NAME
+    docker pull "$IMAGE_NAME"
 
     docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d "$IMAGE_NAME"
 
-    local -r BACKGROUND_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-    'cat /opt/carma/vehicle/config/docker-compose-background-manager.yml'`"
-    local -r MANAGER_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-    'cat /opt/carma/vehicle/config/docker-compose-manager.yml'`"
-    local -r WORKER_COMPOSE="`docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
-    'cat /opt/carma/vehicle/config/docker-compose-worker.yml'`"
+    local -r BACKGROUND_COMPOSE=$(docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+    'cat /opt/carma/vehicle/config/docker-compose-background-manager.yml')
+    local -r MANAGER_COMPOSE=$(docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+    'cat /opt/carma/vehicle/config/docker-compose-manager.yml')
+    local -r WORKER_COMPOSE=$(docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
+    'cat /opt/carma/vehicle/config/docker-compose-worker.yml')
 
     echo "Downloading $IMAGE_NAME dependencies..."
     echo "$BACKGROUND_COMPOSE" | docker-compose -f - pull
@@ -60,31 +60,31 @@ swarm__install() {
 swarm__start() {
     echo "Checking Docker Swarm status..."
     # if carma-config-data volume doesn't exist exit and request update
-    if [ ! -z "$(docker volume ls | grep carma-config-data)" ]; then
+    if [ -n "$(docker volume ls | grep carma-config-data)" ]; then
         echo "Found 'carma-config-data' volume"
     else
         echo "Missing 'carma-config-data' volume, add with \"carma swarm update-config <image tag>\""
         exit 1
     fi
 
-    local carma_base=$(__get_image_from_config carma-base:)
-    if [[ -z $carma_base ]]; then
+    local -r CARMA_BASE=$(__get_image_from_config carma-base:)
+    if [[ -z $CARMA_BASE ]]; then
         __pull_newest_carma_base
-        carma_base=$(__get_most_recent_carma_base)
+        CARMA_BASE=$(__get_most_recent_carma_base)
     fi
 
-    MANAGER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$MANAGER_USER"; echo "@"; echo "$MANAGER_IP"; } | tr -d "\n"')
-    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+    MANAGER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$MANAGER_USER"; echo "@"; echo "$MANAGER_IP"; } | tr -d "\n"')
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
 
     MANAGER_IP=${MANAGER_INFO#*@}
-    MANAGER_USER=${MANAGER_INFO%%@*}
+    # MANAGER_USER=${MANAGER_INFO%%@*}
 
-    WORKER_IP=${WORKER_INFO#*@}
-    WORKER_USER=${WORKER_INFO%%@*}
+    # WORKER_IP=${WORKER_INFO#*@}
+    # WORKER_USER=${WORKER_INFO%%@*}
 
     if [ "$(docker info | grep Swarm | sed 's/Swarm: //g')" == " inactive" ]; then
         echo "Initalizing Docker Swarm..."
-        docker swarm init --advertise-addr $MANAGER_IP
+        docker swarm init --advertise-addr "$MANAGER_IP"
     else
         echo "Docker Swarm active"
     fi
@@ -112,7 +112,7 @@ swarm__start() {
         echo "Deploying CARMA to dual compute Swarm stack..."
 
         local -r TKN=$(docker swarm join-token -q worker)
-        ssh $WORKER_INFO "docker swarm join --token $TKN $MANAGER_IP:2377"
+        ssh "$WORKER_INFO" "docker swarm join --token $TKN $MANAGER_IP:2377"
 
         docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
         'cat /opt/carma/vehicle/config/docker-compose-manager.yml' | \
@@ -123,7 +123,7 @@ swarm__start() {
         docker stack deploy --compose-file - carma-worker
         
     
-    elif [ ! -z "$1" ]; then
+    elif [ -n "$1" ]; then
         echo "Unrecognized argument \"swarm start $1\""
     fi
 }
@@ -148,20 +148,21 @@ swarm-config__set() {
         exit 1      
     fi
 
-    if [ ! -z "$(echo $TAG | grep :)" ]; then
+    if [ -n "$(echo "$TAG" | grep :)" ]; then
         local -r IMAGE_NAME="$TAG"
-        local -r ALT_USER=$(echo $TAG | sed 's/\/.*//')
-        local -r ALT_TAG=$(echo $TAG | sed 's/.*\://')
+        local -r ALT_USER=$(echo "$TAG" | sed 's/\/.*//')
+        local -r ALT_TAG=$(echo "$TAG" | sed 's/.*\://')
     else
         local -r IMAGE_NAME="$USERNAME/carma-config:$TAG"
     fi
 
-    if [ "$(docker images | grep $USERNAME/carma-config | grep $TAG )" ]; then
+
+    if [ "$(docker images | grep "$USERNAME"/carma-config | grep "$TAG" )" ]; then
         echo "Found image, applying to carma-config-volume"
-        docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d $IMAGE_NAME
-    elif [ "$(docker images | grep $ALT_USER/carma-config | grep $ALT_TAG )" ]; then
+        docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d "$IMAGE_NAME"
+    elif [ "$(docker images | grep "$ALT_USER"/carma-config | grep "$ALT_TAG" )" ]; then
         echo "Found full image, applying to carma-config-volume"
-        docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d $IMAGE_NAME
+        docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d "$IMAGE_NAME"
     else
         echo "Could not find $IMAGE_NAME in local images"
     fi
@@ -172,27 +173,27 @@ swarm-config__set() {
 swarm-config__edit() {
     echo "Opening 'carma-config-data' volume shell with read/write privileges..."
 
-    if [ ! -z "$(docker volume ls | grep carma-config-data)" ]; then
+    if [ -n "$(docker volume ls | grep carma-config-data)" ]; then
         echo "Found 'carma-config-data' volume"
     else
         echo "Missing 'carma-config-data' volume, add with \"carma swarm update-config <image tag>\""
         exit 1
     fi
 
-    local carma_base=$(__get_image_from_config carma-base:)
-    if [[ -z $carma_base ]]; then
+    local -r CARMA_BASE=$(__get_image_from_config carma-base:)
+    if [[ -z $CARMA_BASE ]]; then
         __pull_newest_carma_base
-        carma_base=$(__get_most_recent_carma_base)
+        CARMA_BASE=$(__get_most_recent_carma_base)
     fi
 
-    docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base bash
+    docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" bash
 }
 
 
 swarm-config__status() {
     # note the image being used
     # remove old compose files not needed due to split can
-    if [ ! -z "$(docker volume ls | grep carma-config-data)" ]; then
+    if [ -n "$(docker volume ls | grep carma-config-data)" ]; then
         echo "Found 'carma-config-data' volume"
     else
         echo "Missing 'carma-config-data' volume, add with \"carma swarm update-config <image tag>\""
@@ -222,7 +223,7 @@ swarm-config__status() {
 swarm__attach() {
     echo "Attaching STDOUT logs to terminal..."
 
-    local SWARM_SERVICES=$(docker service ls --format "{{.Name}}")
+    local -r SWARM_SERVICES=$(docker service ls --format "{{.Name}}")
 
     # this seems to loop forever and cannot be escaped with ctrl+C
     # performance will likely be poor due to combined log streams
@@ -233,7 +234,7 @@ swarm__attach() {
     sleep infinity
 }
 
-
+# Swarm has to be up to successfully register an image
 swarm__register() {
     echo "Adding Docker images to Manager's local registry..."
 
@@ -250,13 +251,13 @@ swarm__register() {
         exit 1
     fi
 
-    local carma_base=$(__get_image_from_config carma-base:)
-    if [[ -z $carma_base ]]; then
+    local -r CARMA_BASE=$(__get_image_from_config carma-base:)
+    if [[ -z $CARMA_BASE ]]; then
         __pull_newest_carma_base
-        carma_base=$(__get_most_recent_carma_base)
+        CARMA_BASE=$(__get_most_recent_carma_base)
     fi
 
-    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
 
     local -r REGISTRY_PORT=5000
     local -r REGISTRY_HOST="localhost"
@@ -286,12 +287,12 @@ swarm__register() {
 
 swarm__stop() {
     echo "Shutting down swarm..."
-    local carma_base=$(__get_image_from_config carma-base:)
+    local -r CARMA_BASE=$(__get_image_from_config carma-base:)
 
-    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config $carma_base /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
 
     if [ "$(docker node ls -f "role=worker" | grep  "Ready" )" ]; then
-        ssh $WORKER_INFO "docker swarm leave" 
+        ssh "$WORKER_INFO" "docker swarm leave" 
     fi
     docker swarm leave --force
 }
@@ -300,16 +301,16 @@ swarm__stop() {
 swarm__help() {
     cat <<HELP
 -------------------------------------------------------------------------------
-| USDOT FHWA STOL CARMA Swarm                                                 |
+| USDOT FHWA STOL CARMA Platform Swarm                                        |
 -------------------------------------------------------------------------------
 
 Please enter one of the following commands:
   swarm
     install <tag/image>
-        - Installs images to the local host based on Compose files
-          contained in the carma-config defined by <tag/image>
-        - [empty]
-            - Defaults to a usdotfhwastol image
+        - Install a configuration identified by <tag> and download 
+          dependencies. Initializes/updates carma-config-data volume
+          for Swarm use. If <tag> is bare (no :), it is assumed to 
+          be a usdotfhwastol/carma-config tag.
         - d 
             - Defines a usdotfhwastoldev image
         - c

From 0453dc87cc03c66308de0a83d8a5fa0c9cb5e4dc Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Tue, 5 Dec 2023 09:20:43 -0800
Subject: [PATCH 22/31] Removed commented-out items, minor registry fix

---
 .../carma_script_extensions/swarm             | 27 ++++++++++---------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index 9f9b0b7de8..3b24eea61b 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -77,10 +77,6 @@ swarm__start() {
     WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
 
     MANAGER_IP=${MANAGER_INFO#*@}
-    # MANAGER_USER=${MANAGER_INFO%%@*}
-
-    # WORKER_IP=${WORKER_INFO#*@}
-    # WORKER_USER=${WORKER_INFO%%@*}
 
     if [ "$(docker info | grep Swarm | sed 's/Swarm: //g')" == " inactive" ]; then
         echo "Initalizing Docker Swarm..."
@@ -274,19 +270,24 @@ swarm__register() {
         docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/$SOURCE_IMAGE_NAME"
     done
 
-    ssh -R "$REGISTRY_PORT:$REGISTRY_HOST:$REGISTRY_PORT" "$WORKER_INFO" sh
-    for TARGET_IMAGE_NAME in $IMAGE_NAMES
-    do
-        echo "Pulling $TARGET_IMAGE_NAME onto $WORKER_INFO from $REGISTRY_HOST:$REGISTRY_PORT"
-        docker pull "$REGISTRY_HOST:$REGISTRY_PORT/$TARGET_IMAGE_NAME" \
-          && docker tag "$REGISTRY_HOST:$REGISTRY_PORT/$TARGET_IMAGE_NAME" "$TARGET_IMAGE_NAME" \
-          && docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/$TARGET_IMAGE_NAME"
-    done
+    ssh -R "$REGISTRY_PORT:$REGISTRY_HOST:$REGISTRY_PORT" "$WORKER_INFO" sh <<EOF
+        for TARGET_IMAGE_NAME in $IMAGE_NAMES
+        do
+            echo "Pulling $TARGET_IMAGE_NAME onto $WORKER_INFO from $REGISTRY_HOST:$REGISTRY_PORT"
+            docker pull "$REGISTRY_HOST:$REGISTRY_PORT/$TARGET_IMAGE_NAME" \
+              && docker tag "$REGISTRY_HOST:$REGISTRY_PORT/$TARGET_IMAGE_NAME" "$TARGET_IMAGE_NAME" \
+              && docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/$TARGET_IMAGE_NAME"
+        done
+
+    EOF
+
+    docker stop carma-registry-task
 }
 
 
 swarm__stop() {
     echo "Shutting down swarm..."
+
     local -r CARMA_BASE=$(__get_image_from_config carma-base:)
 
     WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
@@ -350,7 +351,7 @@ Please enter one of the following commands:
     register <images>
         - Pushes the images defined by <images> to the
           registry service. Images must have their full
-          identifier to be added.
+          identifier to be added and Swarm must be active.
     attach
         - View STDOUT from all running CARMA processes
     stop

From fc90a3958812fb56e832a5a79aa8389ed00b0f7a Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Tue, 5 Dec 2023 12:08:34 -0800
Subject: [PATCH 23/31] Fixed EOF on registry

---
 engineering_tools/carma_script_extensions/swarm | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index 3b24eea61b..573689c135 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -270,17 +270,15 @@ swarm__register() {
         docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/$SOURCE_IMAGE_NAME"
     done
 
-    ssh -R "$REGISTRY_PORT:$REGISTRY_HOST:$REGISTRY_PORT" "$WORKER_INFO" sh <<EOF
+    ssh -R "$REGISTRY_PORT:$REGISTRY_HOST:$REGISTRY_PORT" "$WORKER_INFO" sh
         for TARGET_IMAGE_NAME in $IMAGE_NAMES
         do
             echo "Pulling $TARGET_IMAGE_NAME onto $WORKER_INFO from $REGISTRY_HOST:$REGISTRY_PORT"
-            docker pull "$REGISTRY_HOST:$REGISTRY_PORT/$TARGET_IMAGE_NAME" \
-              && docker tag "$REGISTRY_HOST:$REGISTRY_PORT/$TARGET_IMAGE_NAME" "$TARGET_IMAGE_NAME" \
-              && docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/$TARGET_IMAGE_NAME"
+            docker pull "$REGISTRY_HOST:$REGISTRY_PORT/\$TARGET_IMAGE_NAME" \
+              && docker tag "$REGISTRY_HOST:$REGISTRY_PORT/\$TARGET_IMAGE_NAME" "\$TARGET_IMAGE_NAME" \
+              && docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/\$TARGET_IMAGE_NAME"
         done
 
-    EOF
-
     docker stop carma-registry-task
 }
 

From 1bcfa093b9f7c74ddfc0b15eb4a959443063b030 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Tue, 5 Dec 2023 14:07:35 -0800
Subject: [PATCH 24/31] Corrected EOF issue on register from tab, added domain
 environment variable to main carma.env

---
 carma/launch/carma.env                          | 6 ++++++
 engineering_tools/carma_script_extensions/swarm | 4 +++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/carma/launch/carma.env b/carma/launch/carma.env
index d63ab04eb7..1cf57a0827 100644
--- a/carma/launch/carma.env
+++ b/carma/launch/carma.env
@@ -33,3 +33,9 @@ export CARMA_LOCZ_NS="/localization"
 
 # Namespace of nodes in the web ui stack
 export CARMA_UI_NS="/ui"
+
+###
+# Dual compute configuration
+###
+# Link ROS 2 instances across hosts
+export ROS_DOMAIN_ID=2
diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index 573689c135..4ce7256842 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -270,7 +270,7 @@ swarm__register() {
         docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/$SOURCE_IMAGE_NAME"
     done
 
-    ssh -R "$REGISTRY_PORT:$REGISTRY_HOST:$REGISTRY_PORT" "$WORKER_INFO" sh
+    ssh -R "$REGISTRY_PORT:$REGISTRY_HOST:$REGISTRY_PORT" "$WORKER_INFO" sh <<ENDSSH
         for TARGET_IMAGE_NAME in $IMAGE_NAMES
         do
             echo "Pulling $TARGET_IMAGE_NAME onto $WORKER_INFO from $REGISTRY_HOST:$REGISTRY_PORT"
@@ -278,8 +278,10 @@ swarm__register() {
               && docker tag "$REGISTRY_HOST:$REGISTRY_PORT/\$TARGET_IMAGE_NAME" "\$TARGET_IMAGE_NAME" \
               && docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/\$TARGET_IMAGE_NAME"
         done
+ENDSSH
 
     docker stop carma-registry-task
+    docker rm carma-registry-task
 }
 
 

From 70502e42b5e78fc9121c509ab51e152c19a5d752 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Mon, 18 Dec 2023 13:23:17 -0800
Subject: [PATCH 25/31] Added a check for carma-config-data volume existence on
 worker

---
 .../carma_script_extensions/swarm             | 44 ++++++++++++++++---
 1 file changed, 39 insertions(+), 5 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index 4ce7256842..ac9194b337 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -40,6 +40,8 @@ swarm__install() {
     echo "Downloading $IMAGE_NAME..."
     docker pull "$IMAGE_NAME"
 
+    # Set up the carma-config-data volume on the manager
+    docker volume create carma-config-data
     docker run --rm --name carma-config-data -v carma-config-data:/opt/carma/vehicle/config -d "$IMAGE_NAME"
 
     local -r BACKGROUND_COMPOSE=$(docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
@@ -54,6 +56,26 @@ swarm__install() {
     echo "$MANAGER_COMPOSE" | docker-compose -f - pull
     echo "$WORKER_COMPOSE" | docker-compose -f - pull
 
+
+    if [ -z $(docker volume ls | grep carma-config-data) ]; then
+        echo "Missing 'carma-config-data' volume, add with \"carma swarm update-config <image tag>\""
+        exit 1
+    fi
+
+    local -r CARMA_BASE=$(__get_image_from_config carma-base:)
+    if [[ -z $CARMA_BASE ]]; then
+        __pull_newest_carma_base
+        CARMA_BASE=$(__get_most_recent_carma_base)
+    fi
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+
+    if [ -z $(ssh "$WORKER_INFO" "docker volume ls | grep carma-config-data") ]; then
+        ssh "$WORKER_INFO" "docker volume create --driver local \
+            --opt type=nfs \
+            --opt o=addr="$WORKER_INFO",nfsvers=4 \
+            --opt device=:/export/carma_volume \
+            carma-config-data"
+    fi
 }
 
 
@@ -124,10 +146,26 @@ swarm__start() {
     fi
 }
 
-
+# add a check to make sure carma-config-data exists on the Worker PC
 swarm-config__set() {
     echo "Updating 'carma-config-data' volume..."
 
+    # Check to make sure the worker system has carma-config-data initalized
+    local -r CARMA_BASE=$(__get_image_from_config carma-base:)
+    if [[ -z $CARMA_BASE ]]; then
+        __pull_newest_carma_base
+        CARMA_BASE=$(__get_most_recent_carma_base)
+    fi
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+
+    if [ -z $(ssh "WORKER_INFO" "docker volume ls | grep carma-config-data") ]; then
+        ssh "$WORKER_INFO" "docker volume create --driver local \
+            --opt type=nfs \
+            --opt o=addr="$WORKER_INFO",nfsvers=4 \
+            --opt device=:/export/carma_volume \
+            carma-config-data"
+    fi
+
     if [ "$1" = "-d" ]; then
         local -r USERNAME=usdotfhwastoldev
         local -r TAG=$2
@@ -187,8 +225,6 @@ swarm-config__edit() {
 
 
 swarm-config__status() {
-    # note the image being used
-    # remove old compose files not needed due to split can
     if [ -n "$(docker volume ls | grep carma-config-data)" ]; then
         echo "Found 'carma-config-data' volume"
     else
@@ -196,8 +232,6 @@ swarm-config__status() {
         exit 1
     fi
 
-    echo "Current carma-config image: "
-
     if [ -z "$1" ]; then
         echo -e " \n -- docker-compose-background-manager.yml\n"
         docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \

From ec782639aa0b91aed102a6188458d0b40d68c2e2 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Tue, 19 Dec 2023 10:07:58 -0800
Subject: [PATCH 26/31] Improved IP/USER fetching by removing carma-base
 dependency

---
 .../carma_script_extensions/swarm             | 66 +++++++++----------
 1 file changed, 32 insertions(+), 34 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index ac9194b337..aeb323c11e 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -62,12 +62,9 @@ swarm__install() {
         exit 1
     fi
 
-    local -r CARMA_BASE=$(__get_image_from_config carma-base:)
-    if [[ -z $CARMA_BASE ]]; then
-        __pull_newest_carma_base
-        CARMA_BASE=$(__get_most_recent_carma_base)
-    fi
-    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config busybox:latest /bin/sh -c '
+        source ./opt/carma/vehicle/config/carma.env; eval $(cat /opt/carma/vehicle/config/carma.env);
+        ( echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; ) | tr -d "\n"')
 
     if [ -z $(ssh "$WORKER_INFO" "docker volume ls | grep carma-config-data") ]; then
         ssh "$WORKER_INFO" "docker volume create --driver local \
@@ -89,14 +86,15 @@ swarm__start() {
         exit 1
     fi
 
-    local -r CARMA_BASE=$(__get_image_from_config carma-base:)
-    if [[ -z $CARMA_BASE ]]; then
-        __pull_newest_carma_base
-        CARMA_BASE=$(__get_most_recent_carma_base)
-    fi
-
-    MANAGER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$MANAGER_USER"; echo "@"; echo "$MANAGER_IP"; } | tr -d "\n"')
-    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+    # This mounts carma-config-data to a busybox container then sources/exports the contents of 'carma.env'
+    # The exported env variables are then echo'd with new lines removed and loaded into the "INFO" var in the
+    # format: <user>@<ip address>
+    MANAGER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config busybox:latest /bin/sh -c '
+        source ./opt/carma/vehicle/config/carma.env; eval $(cat /opt/carma/vehicle/config/carma.env);
+        ( echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; ) | tr -d "\n"')
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config busybox:latest /bin/sh -c '
+        source ./opt/carma/vehicle/config/carma.env; eval $(cat /opt/carma/vehicle/config/carma.env);
+        ( echo "$MANAGER_USER"; echo "@"; echo "$MANAGER_IP"; ) | tr -d "\n"')
 
     MANAGER_IP=${MANAGER_INFO#*@}
 
@@ -150,15 +148,11 @@ swarm__start() {
 swarm-config__set() {
     echo "Updating 'carma-config-data' volume..."
 
-    # Check to make sure the worker system has carma-config-data initalized
-    local -r CARMA_BASE=$(__get_image_from_config carma-base:)
-    if [[ -z $CARMA_BASE ]]; then
-        __pull_newest_carma_base
-        CARMA_BASE=$(__get_most_recent_carma_base)
-    fi
-    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config busybox:latest /bin/sh -c '
+        source ./opt/carma/vehicle/config/carma.env; eval $(cat /opt/carma/vehicle/config/carma.env);
+        ( echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; ) | tr -d "\n"')
 
-    if [ -z $(ssh "WORKER_INFO" "docker volume ls | grep carma-config-data") ]; then
+    if [ -z $(ssh "$WORKER_INFO" "docker volume ls | grep carma-config-data") ]; then
         ssh "$WORKER_INFO" "docker volume create --driver local \
             --opt type=nfs \
             --opt o=addr="$WORKER_INFO",nfsvers=4 \
@@ -224,6 +218,8 @@ swarm-config__edit() {
 }
 
 
+# Currently does not track image version due to how volumes are used
+# Consider adding a way to track current image used/if modified
 swarm-config__status() {
     if [ -n "$(docker volume ls | grep carma-config-data)" ]; then
         echo "Found 'carma-config-data' volume"
@@ -232,6 +228,7 @@ swarm-config__status() {
         exit 1
     fi
 
+    # Reports the contents of several files from carma-config to the active terminal window
     if [ -z "$1" ]; then
         echo -e " \n -- docker-compose-background-manager.yml\n"
         docker run --rm -v carma-config-data:/opt/carma/vehicle/config --entrypoint sh busybox:latest -c \
@@ -255,8 +252,8 @@ swarm__attach() {
 
     local -r SWARM_SERVICES=$(docker service ls --format "{{.Name}}")
 
-    # this seems to loop forever and cannot be escaped with ctrl+C
-    # performance will likely be poor due to combined log streams
+    # This seems to loop forever and cannot be escaped with Ctrl+C
+    # Performance will likely be poor due to combined log streams
     for TASK in $SWARM_SERVICES; do
         docker service logs --follow --details --tail=10 "$TASK" &
     done
@@ -265,6 +262,7 @@ swarm__attach() {
 }
 
 # Swarm has to be up to successfully register an image
+# Most functionality based off of the previous work in https://github.com/kjrush/docker-pushmi-pullyu
 swarm__register() {
     echo "Adding Docker images to Manager's local registry..."
 
@@ -281,21 +279,20 @@ swarm__register() {
         exit 1
     fi
 
-    local -r CARMA_BASE=$(__get_image_from_config carma-base:)
-    if [[ -z $CARMA_BASE ]]; then
-        __pull_newest_carma_base
-        CARMA_BASE=$(__get_most_recent_carma_base)
-    fi
-
-    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config busybox:latest /bin/sh -c '
+        source ./opt/carma/vehicle/config/carma.env; eval $(cat /opt/carma/vehicle/config/carma.env);
+        ( echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; ) | tr -d "\n"')
 
     local -r REGISTRY_PORT=5000
     local -r REGISTRY_HOST="localhost"
 
+    # Create the registry task on the host and expose the ports
     docker run -d --volume carma-registry:/var/lib/registry --publish "$REGISTRY_PORT:$REGISTRY_PORT" --name carma-registry-task registry:2.6
 
     IMAGE_NAMES="$*"
 
+    # Each argument should contain a full image name. Goes through each and creates a new tagged version
+    # on the manager, adds it to the registry, and removes it
     for SOURCE_IMAGE_NAME in $IMAGE_NAMES
     do
         echo "Pushing $SOURCE_IMAGE_NAME to $REGISTRY_HOST:$REGISTRY_PORT"
@@ -304,6 +301,7 @@ swarm__register() {
         docker rmi "$REGISTRY_HOST:$REGISTRY_PORT/$SOURCE_IMAGE_NAME"
     done
 
+    # Registry images are then sent via an SSH call and the same process is followed as before but in reverse
     ssh -R "$REGISTRY_PORT:$REGISTRY_HOST:$REGISTRY_PORT" "$WORKER_INFO" sh <<ENDSSH
         for TARGET_IMAGE_NAME in $IMAGE_NAMES
         do
@@ -322,9 +320,9 @@ ENDSSH
 swarm__stop() {
     echo "Shutting down swarm..."
 
-    local -r CARMA_BASE=$(__get_image_from_config carma-base:)
-
-    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config "$CARMA_BASE" /bin/sh -c '{ echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; } | tr -d "\n"')
+    WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config busybox:latest /bin/sh -c '
+        source ./opt/carma/vehicle/config/carma.env; eval $(cat /opt/carma/vehicle/config/carma.env);
+        ( echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; ) | tr -d "\n"')
 
     if [ "$(docker node ls -f "role=worker" | grep  "Ready" )" ]; then
         ssh "$WORKER_INFO" "docker swarm leave" 

From d6c3db217b7251fe1310d9e28c312447c549e17a Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Tue, 19 Dec 2023 10:24:42 -0800
Subject: [PATCH 27/31] Updated settings to reflect working config on the dual
 compute test system

---
 carma/launch/carma_src.launch.py     | 18 +++++++++---------
 system_controller/config/config.yaml |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/carma/launch/carma_src.launch.py b/carma/launch/carma_src.launch.py
index c3f8d27f1d..cd73f1a902 100644
--- a/carma/launch/carma_src.launch.py
+++ b/carma/launch/carma_src.launch.py
@@ -163,7 +163,7 @@ def generate_launch_description():
 
     # Launch ROS2 rosbag logging
     ros2_rosbag_launch = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
+        condition=IfCondition(PythonExpression(["'", host_placement, "' == 'manager'"])),
         actions=[
             IncludeLaunchDescription(
                 PythonLaunchDescriptionSource([ThisLaunchFileDir(), '/ros2_rosbag.launch.py']),
@@ -177,7 +177,7 @@ def generate_launch_description():
 
     # Nodes
     transform_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
+        condition=IfCondition(PythonExpression(["'", host_placement, "' == 'manager'"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_TF_NS', default_value='/')),
             IncludeLaunchDescription(
@@ -188,7 +188,7 @@ def generate_launch_description():
 
 
     environment_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
+        condition=IfCondition(PythonExpression(["'", host_placement, "' == 'manager'"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_ENV_NS', default_value='environment')),
             IncludeLaunchDescription(
@@ -205,7 +205,7 @@ def generate_launch_description():
 
 
     localization_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
+        condition=IfCondition(PythonExpression(["'", host_placement, "' == 'manager'"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_LOCZ_NS', default_value='localization')),
             IncludeLaunchDescription(
@@ -226,7 +226,7 @@ def generate_launch_description():
 
 
     v2x_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
+        condition=IfCondition(PythonExpression(["'", host_placement, "' == 'manager'"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_MSG_NS', default_value='message')),
             IncludeLaunchDescription(
@@ -243,7 +243,7 @@ def generate_launch_description():
 
 
     guidance_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'worker')"])),
+        condition=IfCondition(PythonExpression(["'", host_placement, "' == 'worker'"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_GUIDE_NS', default_value='guidance')),
             IncludeLaunchDescription(
@@ -264,7 +264,7 @@ def generate_launch_description():
 
 
     drivers_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
+        condition=IfCondition(PythonExpression(["'", host_placement, "' == 'manager'"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_INTR_NS', default_value='hardware_interface')),
             IncludeLaunchDescription(
@@ -279,7 +279,7 @@ def generate_launch_description():
 
 
     system_controller = Node(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
+        condition=IfCondition(PythonExpression(["'", host_placement, "' == 'manager'"])),
         package='system_controller',
         name='system_controller',
         executable='system_controller',
@@ -290,7 +290,7 @@ def generate_launch_description():
 
 
     ui_group = GroupAction(
-        condition=IfCondition(PythonExpression(["'", system_architecture, "' == 'single' or '(", system_architecture, "' == 'dual' and '", host_placement, "' == 'manager')"])),
+        condition=IfCondition(PythonExpression(["'", host_placement, "' == 'manager'"])),
         actions=[
             PushRosNamespace(EnvironmentVariable('CARMA_UI_NS', default_value='ui')),
             IncludeLaunchDescription(
diff --git a/system_controller/config/config.yaml b/system_controller/config/config.yaml
index 4d5acbc91f..138513eabe 100644
--- a/system_controller/config/config.yaml
+++ b/system_controller/config/config.yaml
@@ -6,7 +6,7 @@
 
     # Long: Timeout for each service to be detected as available in milliseconds
     # Units: milliseconds
-    service_timeout_ms : 200
+    service_timeout_ms : 18000
 
     # Long: Timeout for each service call in milliseconds
     # Units: milliseconds

From 37e56aee03f21428e2fe3c41e556045b9540f85e Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Tue, 26 Dec 2023 08:32:25 -0800
Subject: [PATCH 28/31] Added rsync files to engineering_tools for tracking,
 may need to be moved

---
 engineering_tools/rsync-carma-config.service | 12 ++++++++++++
 engineering_tools/rsync_carma_config.sh      |  6 ++++++
 2 files changed, 18 insertions(+)
 create mode 100644 engineering_tools/rsync-carma-config.service
 create mode 100644 engineering_tools/rsync_carma_config.sh

diff --git a/engineering_tools/rsync-carma-config.service b/engineering_tools/rsync-carma-config.service
new file mode 100644
index 0000000000..6ad7fde284
--- /dev/null
+++ b/engineering_tools/rsync-carma-config.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Rsync service to sync the contents of the carma-config-data volume to NFS export/carma_volume
+
+[Service]
+ExecStart=/home/dev/carma_ws/src/carma-platform/engineering_tools/rsync_carma_config.sh
+Restart=always
+RestartSec=1
+StandardOutput=syslog
+StandardError=syslog
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/engineering_tools/rsync_carma_config.sh b/engineering_tools/rsync_carma_config.sh
new file mode 100644
index 0000000000..fe0412e307
--- /dev/null
+++ b/engineering_tools/rsync_carma_config.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+while true; do
+inotifywait -e modify,create,delete,move -r /var/lib/docker/volumes/carma-config-data && \
+rsync -av --delete /var/lib/docker/volumes/carma-config-data/_data /export/carma_volume
+
+done
\ No newline at end of file

From e801637a55508eea307efcf30c7968089221b052 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Tue, 2 Jan 2024 14:34:32 -0800
Subject: [PATCH 29/31] Cleared shell check warnings

---
 engineering_tools/carma_script_extensions/swarm | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index aeb323c11e..0862579ef0 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -57,7 +57,7 @@ swarm__install() {
     echo "$WORKER_COMPOSE" | docker-compose -f - pull
 
 
-    if [ -z $(docker volume ls | grep carma-config-data) ]; then
+    if [ -z "$(docker volume ls | grep carma-config-data)" ]; then
         echo "Missing 'carma-config-data' volume, add with \"carma swarm update-config <image tag>\""
         exit 1
     fi
@@ -66,10 +66,10 @@ swarm__install() {
         source ./opt/carma/vehicle/config/carma.env; eval $(cat /opt/carma/vehicle/config/carma.env);
         ( echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; ) | tr -d "\n"')
 
-    if [ -z $(ssh "$WORKER_INFO" "docker volume ls | grep carma-config-data") ]; then
+    if [ -z "$(ssh "$WORKER_INFO" "docker volume ls | grep carma-config-data")" ]; then
         ssh "$WORKER_INFO" "docker volume create --driver local \
             --opt type=nfs \
-            --opt o=addr="$WORKER_INFO",nfsvers=4 \
+            --opt o=addr="'$WORKER_INFO'",nfsvers=4 \
             --opt device=:/export/carma_volume \
             carma-config-data"
     fi
@@ -152,10 +152,10 @@ swarm-config__set() {
         source ./opt/carma/vehicle/config/carma.env; eval $(cat /opt/carma/vehicle/config/carma.env);
         ( echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; ) | tr -d "\n"')
 
-    if [ -z $(ssh "$WORKER_INFO" "docker volume ls | grep carma-config-data") ]; then
+    if [ -z "$(ssh "$WORKER_INFO" "docker volume ls | grep carma-config-data")" ]; then
         ssh "$WORKER_INFO" "docker volume create --driver local \
             --opt type=nfs \
-            --opt o=addr="$WORKER_INFO",nfsvers=4 \
+            --opt o=addr="'$WORKER_INFO'",nfsvers=4 \
             --opt device=:/export/carma_volume \
             carma-config-data"
     fi
@@ -302,7 +302,7 @@ swarm__register() {
     done
 
     # Registry images are then sent via an SSH call and the same process is followed as before but in reverse
-    ssh -R "$REGISTRY_PORT:$REGISTRY_HOST:$REGISTRY_PORT" "$WORKER_INFO" sh <<ENDSSH
+    ssh -R "$REGISTRY_PORT:$REGISTRY_HOST:$REGISTRY_PORT" "$WORKER_INFO" sh <<"ENDSSH"
         for TARGET_IMAGE_NAME in $IMAGE_NAMES
         do
             echo "Pulling $TARGET_IMAGE_NAME onto $WORKER_INFO from $REGISTRY_HOST:$REGISTRY_PORT"

From 6a29f4ce7c620e69b0e0e7f19656b81d13205b86 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Thu, 4 Jan 2024 14:10:24 -0800
Subject: [PATCH 30/31] Fixed swapped Manager/Worker variables

---
 engineering_tools/carma_script_extensions/swarm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/engineering_tools/carma_script_extensions/swarm b/engineering_tools/carma_script_extensions/swarm
index 0862579ef0..c7ebe9a48d 100755
--- a/engineering_tools/carma_script_extensions/swarm
+++ b/engineering_tools/carma_script_extensions/swarm
@@ -91,10 +91,10 @@ swarm__start() {
     # format: <user>@<ip address>
     MANAGER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config busybox:latest /bin/sh -c '
         source ./opt/carma/vehicle/config/carma.env; eval $(cat /opt/carma/vehicle/config/carma.env);
-        ( echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; ) | tr -d "\n"')
+        ( echo "$MANAGER_USER"; echo "@"; echo "$MANAGER_IP"; ) | tr -d "\n"')
     WORKER_INFO=$(docker run -it --rm -v carma-config-data:/opt/carma/vehicle/config busybox:latest /bin/sh -c '
         source ./opt/carma/vehicle/config/carma.env; eval $(cat /opt/carma/vehicle/config/carma.env);
-        ( echo "$MANAGER_USER"; echo "@"; echo "$MANAGER_IP"; ) | tr -d "\n"')
+        ( echo "$WORKER_USER"; echo "@"; echo "$WORKER_IP"; ) | tr -d "\n"')
 
     MANAGER_IP=${MANAGER_INFO#*@}
 

From 623f636dd6b2ccbacc6b9cd97546de9f1f92d7e9 Mon Sep 17 00:00:00 2001
From: CARMA Developer <willjohnsonk@gmail.com>
Date: Fri, 5 Jan 2024 09:52:18 -0800
Subject: [PATCH 31/31] Minor directory access change

---
 engineering_tools/rsync_carma_config.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/engineering_tools/rsync_carma_config.sh b/engineering_tools/rsync_carma_config.sh
index fe0412e307..1287443ac4 100644
--- a/engineering_tools/rsync_carma_config.sh
+++ b/engineering_tools/rsync_carma_config.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 while true; do
 inotifywait -e modify,create,delete,move -r /var/lib/docker/volumes/carma-config-data && \
-rsync -av --delete /var/lib/docker/volumes/carma-config-data/_data /export/carma_volume
+rsync -av --delete /var/lib/docker/volumes/carma-config-data/_data/ /export/carma_volume
 
 done
\ No newline at end of file