Add simulator dataset and test (#22)

tests/imaging_data/sim_dataset/labels.txt

@@ -0,0 +1,5 @@
+octagon,Y,96:160:80,68:223:187 -19,0,-44
+rectangle,O,101:210:28,37:104:143 -99,0,34
+rectangle,8,153:45:255,149:118:93 -64,0,59
+cross,T,118:73:48,16:64:37 -125,0,-65
+hexagon,O,22:160:151,99:7:45 -134,0,-25

tests/imaging_frontend_tests.py

@@ -2,7 +2,7 @@
 from torchvision.ops import box_iou
 import unittest
 from uavf_2024.imaging.image_processor import ImageProcessor
-from uavf_2024.imaging.imaging_types import FullPrediction
+from uavf_2024.imaging.imaging_types import FullPrediction, TargetDescription
 from uavf_2024.imaging.visualizations import visualize_predictions
 import numpy as np
 import cv2 as cv
@@ -26,10 +26,10 @@ def calc_metrics(predictions: list[FullPrediction], ground_truth: list[FullPredi
             x,y,x+w,y+h
         ]])
 
-        shape = np.argmax(truth.shape_confidences)
-        letter = np.argmax(truth.letter_confidences)
-        shape_col = np.argmax(truth.shape_color_confidences)
-        letter_col = np.argmax(truth.letter_color_confidences)
+        shape = np.argmax(truth.description.shape_probs)
+        letter = np.argmax(truth.description.letter_probs)
+        shape_col = np.argmax(truth.description.shape_col_probs)
+        letter_col = np.argmax(truth.description.letter_col_probs)
 
         this_target_was_detected = False
         for pred in predictions:
@@ -41,10 +41,10 @@ def calc_metrics(predictions: list[FullPrediction], ground_truth: list[FullPredi
             if iou>0.1:
                 true_positives+=1
                 this_target_was_detected = True
-                shape_top_1_accuracies.append(int(shape == np.argmax(pred.shape_confidences)))
-                letter_top_1_accuracies.append(int(letter == np.argmax(pred.letter_confidences)))
-                shape_color_top_1_accuracies.append(int(shape_col == np.argmax(pred.shape_color_confidences)))
-                letter_color_top_1_accuracies.append(int(letter_col == np.argmax(pred.letter_color_confidences)))
+                shape_top_1_accuracies.append(int(shape == np.argmax(pred.description.shape_probs)))
+                letter_top_1_accuracies.append(int(letter == np.argmax(pred.description.letter_probs)))
+                shape_color_top_1_accuracies.append(int(shape_col == np.argmax(pred.description.shape_col_probs)))
+                letter_color_top_1_accuracies.append(int(letter_col == np.argmax(pred.description.letter_col_probs)))
 
         if this_target_was_detected:
             targets_detected+=1
@@ -86,7 +86,10 @@ def parse_dataset(imgs_path, labels_path) -> tuple[list[np.ndarray], list[list[F
                 x,y,w,h = box.astype(int)
 
                 ground_truth.append(FullPrediction(
-                    x,y,w,h,np.eye(13)[shape], np.eye(36)[letter], np.eye(8)[shape_col], np.eye(8)[letter_col]
+                    x,y,w,h,
+                    TargetDescription(
+                        np.eye(13)[shape], np.eye(36)[letter], np.eye(8)[shape_col], np.eye(8)[letter_col]
+                    )
                 ))
         imgs.append(img)
         labels.append(ground_truth)
@@ -97,11 +100,11 @@ def setUp(self) -> None:
         self.image_processor = ImageProcessor()
 
     def test_runs_without_crashing(self):
-        sample_input = cv.imread(f"{CURRENT_FILE_PATH}/fullsize_dataset/images/image0.png")
+        sample_input = cv.imread(f"{CURRENT_FILE_PATH}/imaging_data/fullsize_dataset/images/image0.png")
         res = self.image_processor.process_image(sample_input)
 
     def test_metrics(self):
-        imgs, labels = parse_dataset(f"{CURRENT_FILE_PATH}/tile_dataset/images", f"{CURRENT_FILE_PATH}/tile_dataset/labels")
+        imgs, labels = parse_dataset(f"{CURRENT_FILE_PATH}/imaging_data/tile_dataset/images", f"{CURRENT_FILE_PATH}/imaging_data/tile_dataset/labels")
 
         recalls = []
         precisions = []

tests/imaging_integ_tests.py

@@ -0,0 +1,111 @@
+import unittest
+from uavf_2024.imaging.target_tracker import TargetTracker
+from uavf_2024.imaging.image_processor import ImageProcessor
+from uavf_2024.imaging.color_classification import ColorClassifier
+from uavf_2024.imaging.imaging_types import TargetDescription, Target3D
+import os
+import numpy as np
+import cv2 as cv
+
+CURRENT_FILE_PATH = os.path.dirname(os.path.realpath(__file__))
+
+SHAPES = [
+ "circle",
+ "cross",
+ "heptagon",
+ "hexagon",
+ "octagon",
+ "pentagon",
+ "quartercircle",
+ "rectangle",
+ "semicircle",
+ "square",
+ "star",
+ "trapezoid",
+ "triangle",
+ "person"
+]
+
+LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ123456789"
+
+def csv_to_np(csv_str: str, delim: str = ","):
+    '''
+    Parses strings like "1,2,3" or "1:2:3" into numpy array [1,2,3]
+    '''
+    return np.array(
+        [
+            int(x) for x in
+            csv_str.split(delim)
+        ]
+    )
+
+class TestPipeline(unittest.TestCase):
+    def test_with_sim_dataset(self):
+        def latlng_to_local(latlng):
+            return np.array([latlng[0], 0, latlng[1]])
+        def local_to_latlng(coords):
+            return coords[0], coords[2]
+        target_tracker = TargetTracker(
+            latlng_to_local,
+            local_to_latlng,
+            67,
+            5312
+        )
+        color_classifier = ColorClassifier()
+        image_processor = ImageProcessor()
+        ground_truth: list[Target3D] = []
+
+        with open(f"{CURRENT_FILE_PATH}/imaging_data/sim_dataset/labels.txt", "r") as f:
+            for line in f.readlines():
+                label, location_str = line.split(" ")
+                location = csv_to_np(location_str)
+
+                shape_name, alphanumeric, shape_col_rgb, letter_col_rgb = label.split(",")
+                shape_probs = np.eye(13)[SHAPES.index(shape_name)]
+                letter_probs = np.eye(35)[LETTERS.index(alphanumeric)]
+
+                shape_col_rgb = csv_to_np(shape_col_rgb, ":")
+                letter_col_rgb = csv_to_np(letter_col_rgb, ":")
+                shape_col_probs = color_classifier.predict(shape_col_rgb)                
+                letter_color_probs = color_classifier.predict(letter_col_rgb)                
+
+                ground_truth.append(
+                    Target3D(
+                        location[0],
+                        location[2],
+                        TargetDescription(
+                            shape_probs,
+                            letter_probs,
+                            shape_col_probs,
+                            letter_color_probs
+                        )
+                    )
+                )
+
+        images_dirname = f"{CURRENT_FILE_PATH}/imaging_data/sim_dataset/images"
+        for file_name in os.listdir(images_dirname):
+            img = cv.imread(f"{images_dirname}/{file_name}")
+            pose_strs = file_name.split(".")[0].split("_")[1:]
+            cam_position = csv_to_np(pose_strs[0])
+            cam_angles = csv_to_np(pose_strs[1])
+
+            predictions = image_processor.process_image(img)
+            for pred in predictions:
+               target_tracker.update_with_new_data(pred, np.concatenate([cam_position, cam_angles])) 
+
+        EPSILON = 1e-6
+        scores = []
+        for gt_target in ground_truth:
+            closest_match = target_tracker.closest_match(gt_target.description)
+            if abs(closest_match.lat-gt_target.lat)<EPSILON and abs(closest_match.lng-gt_target.lng)<EPSILON:
+                scores.append(1)
+            else:
+                scores.append(0)
+
+        print(f"Imaging Sim Score: {np.mean(scores)}") 
+
+
+
+
+
+

tests/target_tracker_tests.py → tests/imaging_target_tracker_tests.py

@@ -1,28 +1,32 @@
 import unittest
 from uavf_2024.imaging.target_tracker import TargetTracker
-from uavf_2024.imaging.imaging_types import FullPrediction, TargetDescription, Target3D
+from uavf_2024.imaging.imaging_types import TargetDescription, Target3D
 import numpy as np
 
-class TestImagingFrontend(unittest.TestCase):
+class TestTargetTracker(unittest.TestCase):
     def test_with_certain_match(self):
-        tracker = TargetTracker()
+        tracker = TargetTracker(None, None, None, None)
 
         target_1 = Target3D(
                 12.34,
                 69.420,
-                np.eye(13)[1],
-                np.eye(35)[3],
-                np.eye(8)[3],
-                np.eye(8)[7],
+                TargetDescription(
+                    np.eye(13)[1],
+                    np.eye(35)[3],
+                    np.eye(8)[3],
+                    np.eye(8)[7],
+                )
             )
 
         target_2 = Target3D(
                 12.34,
                 69.420,
-                np.eye(13)[2],
-                np.eye(35)[2],
-                np.eye(8)[2],
-                np.eye(8)[2],
+                TargetDescription(
+                    np.eye(13)[2],
+                    np.eye(35)[2],
+                    np.eye(8)[2],
+                    np.eye(8)[2],
+                )
             )
 
         # violating private-ness for testing
@@ -34,24 +38,28 @@ def test_with_certain_match(self):
         assert tracker.closest_match(desc) is target_1
 
     def test_with_near_match(self):
-        tracker = TargetTracker()
+        tracker = TargetTracker(None, None, None, None)
 
         target_1 = Target3D(
                 12.34,
                 69.420,
-                np.eye(13)[1],
-                np.eye(35)[3],
-                np.eye(8)[3],
-                np.eye(8)[7],
+                TargetDescription(
+                    np.eye(13)[1],
+                    np.eye(35)[3],
+                    np.eye(8)[3],
+                    np.eye(8)[7],
+                )
             )
 
         target_2 = Target3D(
                 12.34,
                 69.420,
-                np.eye(13)[2],
-                np.eye(35)[2],
-                np.eye(8)[2],
-                np.eye(8)[2],
+                TargetDescription(
+                    np.eye(13)[2],
+                    np.eye(35)[2],
+                    np.eye(8)[2],
+                    np.eye(8)[2],
+                )
             )
 
         # violating private-ness for testing

uavf_2024/imaging/image_processor.py

@@ -1,7 +1,7 @@
 import numpy as np
 
 from uavf_2024.imaging.utils import generate_tiles
-from .imaging_types import FullPrediction,InstanceSegmentationResult
+from .imaging_types import FullPrediction, InstanceSegmentationResult, TargetDescription
 from .letter_classification import LetterClassifier
 from .shape_detection import ShapeInstanceSegmenter
 from .color_segmentation import color_segmentation
@@ -60,10 +60,12 @@ def process_image(self, img: np.ndarray) -> "list[FullPrediction]":
                     res.y,
                     res.width,
                     res.height,
-                    shape_conf,
-                    letter_conf,
-                    shape_color_conf,
-                    letter_color_conf
+                    TargetDescription(
+                        shape_conf,
+                        letter_conf,
+                        shape_color_conf,
+                        letter_color_conf
+                    )
                 )
             )
 

uavf_2024/imaging/imaging_types.py

@@ -1,6 +1,13 @@
 from dataclasses import dataclass
 import numpy as np
 
+@dataclass
+class TargetDescription:
+    shape_probs: np.ndarray
+    letter_probs: np.ndarray
+    shape_col_probs: np.ndarray
+    letter_col_probs: np.ndarray
+
 @dataclass
 class Tile:
     img: np.ndarray
@@ -16,12 +23,7 @@ class FullPrediction:
     '''
     We can worry about typechecking these later, but the gist is that they're probability distributions over the possible classes.
     '''
-    shape_confidences: np.ndarray
-    letter_confidences: np.ndarray
-    shape_color_confidences: np.ndarray
-    letter_color_confidences: np.ndarray
-
-
+    description: TargetDescription
 
 @dataclass
 class InstanceSegmentationResult:
@@ -36,13 +38,6 @@ class InstanceSegmentationResult:
     mask: np.ndarray
     img: np.ndarray
 
-@dataclass
-class TargetDescription:
-    shape: int
-    letter: int
-    shape_color: int
-    letter_color: int
-
 @dataclass
 class Target3D:
     '''
@@ -51,7 +46,4 @@ class Target3D:
     '''
     lat: float
     lng: float
-    shape_probs: np.ndarray
-    letter_probs: np.ndarray
-    shape_col_probs: np.ndarray
-    letter_col_probs: np.ndarray
+    description: TargetDescription

uavf_2024/imaging/target_tracker.py

@@ -1,23 +1,29 @@
 from .imaging_types import FullPrediction, Target3D, TargetDescription
 from .utils import calc_match_score
 import numpy as np
+from typing import Callable, Any
 
 
 class TargetTracker:
-    def __init__(self):
+    def __init__(self, 
+                 latlng_to_local: Callable[[tuple[float,float]], np.ndarray], 
+                 local_to_latlng: Callable[[np.ndarray], tuple[float,float]],
+                 camera_fov: float,
+                 camera_resolution: int
+                 ):
         self._targets: list[Target3D] = []
+        self.local_to_latlng = local_to_latlng
+        self.latlng_to_global = latlng_to_local
+        self.camera_fov = camera_fov
+        self.camera_resolution = camera_resolution
 
-    def update_with_new_data(self, new_preds: list[tuple[FullPrediction,np.ndarray]]):
+    def update_with_new_data(self, pred: FullPrediction, camera_pose: np.ndarray):
         '''
-            new_preds should look like this:
-            [
-                (prediction1, camera_pose1),
-                (prediction2, camera_pose2)
-                ... etc ...
-            ]
-            TODO: figure out format for camera pose, and update the Target3D class with our final decision for how to define a global position
-        
+            camera_pose needs to be like [x,y,z,rot_x,rot_y,rot_z]
+            where x,y, and z are euclidian right-hand coordinates, and rot_x,rot_y,and rot_z
+            are right-handed rotation angles in degrees around their respective axes, applied in order x->y->z
         '''
+
         raise NotImplementedError()
 
     def closest_match(self, target_desc: TargetDescription) -> Target3D:
@@ -28,7 +34,7 @@ def closest_match(self, target_desc: TargetDescription) -> Target3D:
         best_match, best_score = self._targets[0], 0
 
         for target in self._targets:
-            new_score = calc_match_score(target_desc, target)
+            new_score = calc_match_score(target_desc, target.description)
             if new_score > best_score:
                 best_match = target
                 best_score = new_score