DTS integration into tests

.github/workflows/ci.yaml

@@ -24,6 +24,10 @@ jobs:
         run: |
           docker compose up -d --build
 
+      - name: run docker compose dts
+        run: |
+          docker compose -f docker/dts/docker-compose.yaml up -d --build
+
       - name: instal pytest
         run: pip install pytest
 

docker-compose.yaml

@@ -53,53 +53,6 @@ services:
       - /opt/pysetup/files/staging/:/opt/pysetup/files/staging/
       - /var/run/docker.sock:/var/run/docker.sock
 
-  service-s3:
-    image: minio/minio
-    ports:
-      - "9000:9000"
-      - "9001:9001"
-    volumes:
-      - ./docker/dts/s3/data:/data
-    environment:
-      MINIO_ROOT_USER: root
-      MINIO_ROOT_PASSWORD: 123456789
-    command: server --console-address ":9001" /data
-
-  service-ftp:
-    build:
-      context: ./docker/dts/ftp/
-    image: ftp
-    ports:
-      - '2121-2130:2121-2130'
-    depends_on:
-      - service-s3
-
-  service-http:
-    build:
-      context: ./docker/dts/http/
-    image: http
-    ports:
-      - "5000:5000"
-    volumes:
-      - ./docker/dts/http/data:/data
-
-
-  tests-clients:
-    build:
-      context: ./docker/dts/clients/
-    image: clients
-    depends_on:
-      - tests-clients-init
-
-  tests-clients-init:
-    build:
-      context: ./docker/dts/clients/
-      dockerfile: Dockerfile.tests-init
-    depends_on:
-      - service-s3
-      - service-ftp
-      - service-http
-
 # Some cloud providers may require a lower MTU!
 networks:
   default:

docker/dts/docker-compose.yaml

@@ -30,7 +30,7 @@ services:
     ports:
       - "5000:5000"
     volumes:
-      - ./http/data:/data
+      - ../../tests/test_data:/data
 
   tests-clients:
     build:

tests/smoke_tests.py

@@ -85,6 +85,17 @@ def _test_activity(json, timeout_running, timeout_complete, state = 'COMPLETE'):
         return False
     return _wait_for_state(id, state, timeout_complete)
 
+    task_response = _wait_for_state(id, state, timeout_complete)
+
+    # Retrieve and print resources if they exist
+    resources = _gnv(task_response, "resources")
+    if resources:
+        print(f"Resources for task {id}: {resources}")
+    else:
+        print(f"No resources found for task {id}.")
+
+    return task_response
+
 def _test_sequence_simple(jsons, timeout, state = 'COMPLETE'):
     for file_name in jsons:
         json = f'{file_name}.json'
@@ -172,3 +183,22 @@ def test_task_cancel():
     assert task_id
     _post_request(f"/v1/tasks/{task_id}:cancel")
     assert _wait_for_state(task_id, "CANCELED", 60)
+
+def test_resource_check_with_limits():
+    # Load the JSON file
+    json_data = _open_json('resource_check.json')
+
+    # Extract CPU cores and RAM values
+    cpu_cores = _gnv(json_data, "resources.cpu_cores")
+    ram_gb = _gnv(json_data, "resources.ram_gb")
+
+    # Define the maximum limits
+    max_cpu_cores = 4
+    max_ram_gb = 3.8
+
+    # Perform the checks with limits
+    assert cpu_cores <= max_cpu_cores, f"CPU cores exceed the limit: {cpu_cores} > {max_cpu_cores}"
+    assert ram_gb <= max_ram_gb, f"RAM exceeds the limit: {ram_gb} > {max_ram_gb}"
+
+    print(f"Requested resources: CPU cores = {cpu_cores}, RAM = {ram_gb} GB (Limits: {max_cpu_cores} cores, {max_ram_gb} GB RAM)")
+

tests/test_data/env_test_1

@@ -0,0 +1 @@
+first upload succsessful

tests/test_data/env_test_2

@@ -0,0 +1 @@
+second upload succsessful

tests/test_data/test.txt

@@ -0,0 +1 @@
+this is a test file

tests/test_jsons/envs.json

@@ -1,22 +1,27 @@
 {
+    "outputs": [
+
+    {
+        "path": "/tmp/env_test_1",
+        "url": "http://172.17.0.1:5000/upload",
+        "type": "FILE"
+    },
+    {
+	"path": "/tmp/env_test_2",
+        "url": "http://172.17.0.1:5000/upload",
+        "type": "FILE"
+    }
+
+],	
     "executors":[
         {
             "image":"ubuntu",
             "command":[
-                "touch", "$TEST_FILE", "&&", "cat", "/tmp/test_file_42"
+                "echo", "$TEST_FILE", ">", "/tmp/env_test_1", "&&", "echo", "$TEST_FILE_2", ">", "/tmp/env_test_2"
             ],
             "env": {
-                "TEST_FILE": "/tmp/test_file_42"
-            }
-        },
-        {
-            "image":"ubuntu",
-            "command":[
-                "touch", "$TEST_FILE", "&&", "touch", "$TEST_FILE_2", "&&", "cat", "/tmp/test_file_7", "&&", "cat", "/tmp/test_file_2"
-            ],
-            "env": {
-                "TEST_FILE": "/tmp/test_file_7",
-                "TEST_FILE_2": "/tmp/test_file_2"
+                "TEST_FILE": "first upload succsessful",
+		"TEST_FILE_2": "second upload succsessful"
             }
         }
     ]

tests/test_jsons/inputs.json

@@ -7,7 +7,7 @@
         },
         {
             "path": "/data/file_http",
-            "url": "http://raw.githubusercontent.com/microsoft/vscode/main/README.md",
+            "url": "http://172.17.0.1:5000/download/test.txt",
             "type": "FILE"
         }
     ],

tests/test_jsons/resource_check.json

@@ -0,0 +1,17 @@
+{
+    "resources": {
+        "cpu_cores": 1,
+        "preemptible": false,
+        "ram_gb": 1,
+        "disk_gb": 1,
+        "zones": ["us-west-1"]
+    },
+    "executors":[
+        {
+            "image":"ubuntu",
+            "command":[
+                "sleep 20"
+            ]
+        }
+    ]
+}

tests/test_jsons/volumes.json

@@ -2,7 +2,7 @@
     "inputs": [
         {
             "path": "/data/file_http",
-            "url": "http://raw.githubusercontent.com/microsoft/vscode/main/README.md",
+            "url": "http://172.17.0.1:5000/download/test.txt",
             "type": "FILE"
         }
     ],