Added a bunch of extra documentation

.gitignore

@@ -37,3 +37,13 @@ ENV/
 env.bak/
 venv.bak/
 __pycache__/
+
+# Docker
+docker/runner/ssl-game-controller-config/20*
+docker/runner/ssl-game-controller-config/state-store.json.stream
+
+# RL
+roboteam_ai/src/RL/src/tmp/eval_monitor.monitor.csv
+roboteam_ai/src/RL/src/tmp/monitor.monitor.csv
+roboteam_ai/src/RL/src/ppo_tensorboard_main
+roboteam_ai/src/RL/src/best_model_PPO.zip

.gitmodules

@@ -11,3 +11,6 @@
 [submodule "roboteam_robothub/roboteam_embedded_messages"]
 	path = roboteam_robothub/roboteam_embedded_messages
 	url = https://github.com/RoboTeamTwente/roboteam_embedded_messages.git
+[submodule "tigers_sumatra"]
+	path = tigers_sumatra
+	url = [email protected]:RoboTeamTwente/tigers_sumatra.git

Dockerfile

@@ -0,0 +1,31 @@
+FROM roboteamtwente/roboteam:development
+
+# Create symbolic link from /home/roboteam to /home/roboteamtwente
+USER root
+RUN ln -s /home/roboteamtwente /home/roboteam
+
+# Install Java
+RUN apk add --no-cache openjdk11
+
+# Install Java 21
+RUN wget https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.1%2B12/OpenJDK21U-jdk_x64_alpine-linux_hotspot_21.0.1_12.tar.gz \
+    && tar -xzf OpenJDK21U-jdk_x64_alpine-linux_hotspot_21.0.1_12.tar.gz \
+    && mv jdk-21* /usr/lib/jvm/java-21-openjdk
+
+# Set up Java environment variables to point to Java 21
+ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk
+ENV PATH="${JAVA_HOME}/bin:${PATH}"
+
+WORKDIR /home/roboteam
+
+# Copy the entire current directory into the container
+COPY --chown=roboteamtwente:roboteamtwente . /home/roboteam/
+
+# Make sure build.sh is executable
+RUN chmod +x build.sh
+
+# Add the lib directory to LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/roboteam/build/release/lib
+
+# Switch back to the roboteamtwente user
+USER roboteamtwente

Dockerfile.ray

@@ -0,0 +1,16 @@
+# This dockerfile uses the ray project's official ray image as a base.
+# It also adds the torch and gymnasium libraries to the image.
+# It also adds the roboteam RL code to the image.
+
+FROM rayproject/ray:latest-py310
+
+# Install dependencies in a single layer to keep it cached
+RUN pip install torch==2.5.1 gymnasium numpy==1.24.3 ray[rllib] pyzmq
+
+# Copy the entire roboteam root folder (including the roboteam_ai and roboteam_networking folders)
+COPY roboteam_ai /roboteam/roboteam_ai
+COPY roboteam_networking /roboteam/roboteam_networking
+
+# Set working directory and Python path
+WORKDIR /roboteam
+ENV PYTHONPATH="/roboteam:${PYTHONPATH}"

README.md

@@ -36,3 +36,13 @@ To enable Tracy
 - Compile Tracy Server and run
   - Information is in the tracy [docs](https://github.com/wolfpld/tracy)
 - Run AI
+
+
+### Use of Ray
+The dockerimage Dockerfile.ray is used to build a docker image with the ray project's official ray image as a base. It also adds the necessary libraries to the image and the roboteam RL code to the image. Only build it if you want to deploy it to a cluster.
+
+Build the docker image using the following command from the root folder:
+- docker build -t roboteamtwente/ray:development -f Dockerfile.ray .
+
+Push it using the following command:
+- docker push roboteamtwente/ray:development

docker/README.md

@@ -100,6 +100,10 @@ docker run -itd --name rtt-release-env -h rtt-release-env roboteamtwente/robotea
 #### TODO
 https://lemariva.com/blog/2020/10/vscode-c-development-and-debugging-containers
 
+
+### Profile explanation
+If you want to run certain docker containers with a single command, you create a profile. This will run every docker container that has this command.
+
 ### Docker geeks
 We highly discourage executing command under this section unless you know what you are doing and for some reason you need to edit dockerfiles or composes structure.
 

docker/runner/README.md

@@ -0,0 +1,34 @@
+
+In a ray or distributed computing cluster, the terms "head node" and "worker nodes" refer to different roles that containers play in the cluster. The head node is the master node in a Ray cluster. You typically have one head node. Worker nodes are the containers that execute the jobs, in parallel. You can have as many worker nodes as you want.
+
+-----------------------------------------------------------
+## Installing Kuberay:
+curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+helm repo add kuberay https://ray-project.github.io/kuberay-helm/
+helm repo update
+helm install kuberay-operator kuberay/kuberay-operator --namespace ray-system --create-namespace
+
+https://docs.ray.io/en/latest/cluster/kubernetes/user-guides/config.html
+The above source was used for creating the ray-cluster.yaml
+
+Installing kubernetes and minikubernetes, you can follow this guide to check out how to install and run them: https://medium.com/@areesmoon/installing-minikube-on-ubuntu-20-04-lts-focal-fossa-b10fad9d0511
+
+Use 'pip install ray' and then 'pip show ray' to get your version of ray.
+
+----------------------------------------------------------------------------------
+After you have both kubernetes and ray, use the following command to create a cluster: kubectl apply -f ray-cluster.yaml
+This cluster launches a ray head node and one worker node. Launch the external simulator using kubectl apply -f simulator.yaml
+
+'kubectl get pods'-> this is will give you the cluster name
+
+Use to forward the needed port to the ray service: kubectl port-forward svc/<cluster name> 8265:8265
+This is the port that will be used inside ray_jobs.py, where we submit the jobs to ray.
+
+
+-----------------------------------------------------
+## Useful commands
+kubectl apply -f ray-cluster.yaml
+kubectl delete -f ray-cluster.yaml
+helm install kuberay-operator ray/kuberay-operator
+helm uninstall kuberay-operator
+kubectl port-forward svc/roboteam-ray-cluster-head-nodeport 8265:8265 6379:6379 10001:10001 8000:8000 &

docker/runner/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
 
   roboteam_primary_ai:
@@ -18,7 +16,7 @@ services:
       - LD_LIBRARY_PATH=/home/roboteamtwente/lib/
     volumes:
       - ../../build/release/:/home/roboteamtwente/
-    profiles: ["simulator","diff","game", "robocup"]
+    profiles: ["simulator","diff","game", "robocup", "RL"]
 
   roboteam_secondary_ai:
     image: roboteamtwente/roboteam:development
@@ -72,7 +70,7 @@ services:
       - LD_LIBRARY_PATH=/home/roboteamtwente/lib/
     volumes:
       - ../../build/release/:/home/roboteamtwente/
-    profiles: ["simulator","diff"]
+    profiles: ["simulator","diff", "RL"]
 
   roboteam_observer_game:
     image: roboteamtwente/roboteam:development
@@ -111,7 +109,7 @@ services:
       - LD_LIBRARY_PATH=/home/roboteamtwente/lib/
     volumes:
       - ../../build/release/:/home/roboteamtwente/
-    profiles: ["simulator","diff"]
+    profiles: ["simulator","diff", "RL"]
 
   roboteam_robothub_game:
     image: roboteamtwente/roboteam:development
@@ -147,7 +145,7 @@ services:
       - 8080:8080
     volumes:
       - ../../roboteam_interface/:/home/roboteamtwente/
-    profiles: ["simulator","diff","game", "robocup"]
+    profiles: ["simulator","diff","game", "robocup", "RL"]
 
   roboteam_autoref:
     image: gradle:8.4.0-jdk17
@@ -164,7 +162,7 @@ services:
       - GRADLE_USER_HOME=/home/roboteamtwente/.cache # Cache gradle dependencies
     volumes:
       - ../../roboteam_autoref/:/home/roboteamtwente/
-    profiles: ["simulator","diff","game","autoref"]
+    profiles: ["simulator","diff","game","autoref", "RL"]
 
   erforce_autoref_sim:
     image: roboteamtwente/roboteam:development
@@ -179,7 +177,7 @@ services:
       - 10010:10010 # Tracker port
     volumes:
       - ../../:/home/roboteamtwente/
-    profiles: ["simulator","diff"]
+    profiles: ["simulator","diff", "RL"]
 
   erforce_autoref_game:
     image: roboteamtwente/roboteam:development
@@ -212,13 +210,17 @@ services:
       - 30013:30013 #Simulation Feedback Port
     volumes:
       - ../../:/home/roboteamtwente/
-    profiles: ["simulator","diff"]
+    profiles: ["simulator","diff", "RL"]
 
   ssl-game-controller:
     image: robocupssl/ssl-game-controller:latest
     container_name: RTT_ssl-game-controller
     restart: always
-    network_mode: "host" # Workaround to connect from interface on host to AI websocket, please fix
+    network_mode: "host"
     ports:
-      - "8081:8081/tcp" # UI port
-    profiles: ["simulator","diff","game"]
+      - "8081:8081/tcp"
+    volumes:
+      - ./ssl-game-controller-config:/config
+      - ./ssl-game-controller-data:/data
+    command: -address :8081
+    profiles: ["simulator","diff","game", "RL"]

docker/runner/ray-cluster.yaml

@@ -0,0 +1,133 @@
+apiVersion: ray.io/v1alpha1
+kind: RayCluster
+metadata:
+  name: roboteam-ray-cluster
+spec:
+  rayVersion: "2.38.0"  
+  # Head node configuration
+  headGroupSpec:
+    rayStartParams:
+      dashboard-host: "0.0.0.0"
+    template:
+      metadata:
+        labels:
+          app: ray-head
+      spec:
+        hostNetwork: false
+        containers:
+        - name: ray-head
+          image: roboteamtwente/ray:development
+          imagePullPolicy: Always # Always pull the latest image
+          ports:
+          - containerPort: 8265  # dashboard port
+          - containerPort: 6379  # redis port 
+          - containerPort: 10001  # GCS server port
+          - containerPort: 8000   # Serve port
+          resources:
+            requests:
+              cpu: "500m"
+              memory: "1Gi"    # Increased from 256Mi
+            limits:
+              cpu: "1"         # Changed from 600 (which was too high)
+              memory: "2Gi"    # Increased from 512Mi
+
+          env:
+          - name: SIMULATION_HOST
+            value: "127.0.0.1"  # Using localhost since we're on host network
+          - name: VISION_PORT
+            value: "10020"      # Match your simulator's vision port
+          - name: REFEREE_PORT
+            value: "10003"      # Match your simulator's referee port
+
+          command: ["/bin/bash", "-c", "--"]
+          args: ["ray start --head --port=6379 --dashboard-host=0.0.0.0 --block"]
+          livenessProbe:
+            exec:
+              command:
+              - bash
+              - -c
+              - "wget -T 2 -q -O- http://localhost:52365/api/local_raylet_healthz | grep success"
+            initialDelaySeconds: 90
+            timeoutSeconds: 10
+            periodSeconds: 30
+            failureThreshold: 5
+          readinessProbe:
+            exec:
+              command:
+              - bash
+              - -c
+              - "wget -T 10 -q -O- http://localhost:8265/api/gcs_healthz | grep success"
+            initialDelaySeconds: 90
+            timeoutSeconds: 10
+            periodSeconds: 30
+            failureThreshold: 5
+
+
+  # Worker node configuration
+  workerGroupSpecs:
+    - groupName: worker-group
+      replicas: 1  # Number of worker nodes
+      rayStartParams:
+        num-cpus: "1" 
+      template:
+        metadata:
+          labels:
+            app: ray-worker
+        spec:
+          hostNetwork: true
+          containers:
+          - name: ray-worker
+            image: roboteamtwente/ray:development
+            imagePullPolicy: Always # Always pull the latest image
+            resources:
+              requests:
+                cpu: 500m
+                memory: 1Gi
+              limits:
+                cpu: 1000m
+                memory: 2Gi
+            env:
+            - name: RAY_HEAD_IP
+              value: "roboteam-ray-cluster-head-svc.default.svc.cluster.local"
+            - name: LD_LIBRARY_PATH
+              value: /home/roboteam/build/release/lib
+            - name: VISION_ADDRESS
+              value: "224.5.23.2"
+            - name: VISION_PORT
+              value: "10020"
+            - name: REFEREE_ADDRESS
+              value: "224.5.23.1"
+            - name: REFEREE_PORT
+              value: "10003"
+            - name: GC_PORT
+              value: "8081"
+            command: ["/bin/bash", "-c", "--"]
+            args: ["ray start --address='roboteam-ray-cluster-head-svc.default.svc.cluster.local:6379' --block"]
+
+#Add service for port forwarding
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: roboteam-ray-cluster-head-nodeport  # Changed name to avoid conflict
+spec:
+  type: NodePort
+  selector:
+    app: ray-head
+  ports:
+  - name: dashboard
+    port: 8265
+    targetPort: 8265
+    nodePort: 30265  # Ray dashboard
+  - name: redis
+    port: 6379
+    targetPort: 6379
+    nodePort: 30679  # Redis
+  - name: gcs
+    port: 10001
+    targetPort: 10001
+    nodePort: 31001  # GCS server
+  - name: serve
+    port: 8000
+    targetPort: 8000
+    nodePort: 30800  # Serve