BREAKING CHANGE(timeline): provide server side image thumbnails for f…

…aster UI render times, close #128

config.yaml

@@ -46,7 +46,7 @@ ai_models:
     labels: ai_models/coco_labels.txt
   face_detection: &tfm_face_detection
     model:
-      tflite: ai_models/mobilenet_ssd_v2_coco_quant_postprocess.tflite
+      tflite: ai_models/mobilenet_ssd_v2_face_quant_postprocess.tflite
       edgetpu: ai_models/mobilenet_ssd_v2_face_quant_postprocess_edgetpu.tflite
     labels: ai_models/coco_labels.txt
     top_k: 2

src/ambianic/pipeline/ai/face_detect.py

@@ -56,17 +56,16 @@ def process_sample(self, **sample):
                               len(person_regions))
                     for box in person_regions:
                         person_image = self.crop_image(image, box)
-                        inference_result = self.detect(image=person_image)
+                        thumbnail, tensor_image, inference_result = \
+                            self.detect(image=person_image)
                         log.debug('Face detection inference_result: %r',
                                   inference_result)
                         inf_meta = {
-                            'display': 'Face Detection'
-                            # id
-                            # version
-                            # etc
+                            'display': 'Face Detection',
                         }
                         processed_sample = {
                             'image': person_image,
+                            'thumbnail': thumbnail,
                             'inference_result': inference_result,
                             'inference_meta': inf_meta
                             }

src/ambianic/pipeline/ai/image_detection.py

@@ -62,13 +62,10 @@ def load_labels(self, label_path=None):
             lines = (p.match(line).groups() for line in f.readlines())
             return {int(num): text.strip() for num, text in lines}
 
-    def resize(self, image=None, desired_size=None):
-        """Resizes original image to size expected by input tensor.
-
-        Preserves aspect ratio to avoid confusing the AI model with
-        unnatural distortions. Pads the resulting image
-        with solid black color pixels to fill the desired size.
+    def thumbnail(self, image=None, desired_size=None):
+        """Resizes original image as close as possible to desired size.
 
+        Preserves aspect ratio of original image.
         Does not modify the original image.
 
         :Parameters:
@@ -87,10 +84,42 @@ def resize(self, image=None, desired_size=None):
         """
         assert image
         assert desired_size
-        log.debug('current image size = %r', image.size)
+        log.debug('input image size = %r', image.size)
         thumb = image.copy()
         thumb.thumbnail(desired_size)
         log.debug('thmubnail image size = %r', thumb.size)
+        return thumb
+
+    def resize(self, image=None, desired_size=None):
+        """Pad original image to exact size expected by input tensor.
+
+        Preserve aspect ratio to avoid confusing the AI model with
+        unnatural distortions. Pad the resulting image
+        with solid black color pixels to fill the desired size.
+
+        Do not modify the original image.
+
+        :Parameters:
+        ----------
+        image : PIL.Image
+            Input Image sized to fit an input tensor but without padding.
+            Its possible that one size fits one tensor dimension exactly
+            but the other size is smaller than
+            the input tensor other dimension.
+
+        desired_size : (width, height)
+            Exact size expected by the AI model.
+
+        :Returns:
+        -------
+        PIL.Image
+            Resized image fitting exactly the AI model input tensor.
+
+        """
+        assert image
+        assert desired_size
+        log.debug('input image size = %r', image.size)
+        thumb = image.copy()
         delta_w = desired_size[0] - thumb.size[0]
         delta_h = desired_size[1] - thumb.size[1]
         padding = (0, 0, delta_w, delta_h)
@@ -119,7 +148,8 @@ def detect(self, image=None):
         :Parameters:
         ----------
         image : PIL.Image
-            Input image in raw RGB format.
+            Input image in raw RGB format
+            with the exact size of the input tensor.
 
         :Returns:
         -------
@@ -139,7 +169,17 @@ def detect(self, image=None):
         height = tfe.input_details[0]['shape'][1]
         width = tfe.input_details[0]['shape'][2]
 
-        new_im = self.resize(image=image, desired_size=(width, height))
+        # thumbnail is a proportionately resized image
+        thumbnail = self.thumbnail(image=image, desired_size=(width, height))
+
+        # convert thumbnail into an image with exact size as tensor
+        # preserving proportions by padding as needed
+        new_im = self.resize(image=thumbnail, desired_size=(width, height))
+
+        # calculate what fraction of the new image is the thumbnail size
+        # we will use these factors to adjust detection box coordinates
+        w_factor = thumbnail.size[0] / new_im.size[0]
+        h_factor = thumbnail.size[1] / new_im.size[1]
 
         # add N dim
         input_data = np.expand_dims(new_im, axis=0)
@@ -209,25 +249,22 @@ def detect(self, image=None):
                 if (li < len(self._labels)):
                     label = self._labels[li]
                     box = boxes[0, i, :]
-                    x0 = box[1]
-                    y0 = box[0]
-                    x1 = box[3]
-                    y1 = box[2]
+                    # refit detections into original image size
+                    # without overflowing outside image borders
+                    x0 = box[1] / w_factor
+                    y0 = box[0] / h_factor
+                    x1 = min(box[3] / w_factor, 1)
+                    y1 = min(box[2] / h_factor, 1)
+                    log.debug('thumbnail image size: %r , '
+                              'tensor image size: %r',
+                              thumbnail.size,
+                              new_im.size)
+                    log.debug('resizing detection box (x0, y0, x1, y1) '
+                              'from: %r to %r',
+                              (box[1], box[0], box[3], box[2]),
+                              (x0, y0, x1, y1))
                     inference_result.append((
                         label,
                         confidence,
                         (x0, y0, x1, y1)))
-        return inference_result
-#        objs = self.engine.DetectWithImage(
-#            image,
-#            threshold=self.confidence_threshold,
-#            keep_aspect_ratio=True,
-#            relative_coord=True,
-#            top_k=3)
-#
-#        for obj in objs:
-#            x0, y0, x1, y1 = obj.bounding_box.flatten().tolist()
-#            confidence = obj.score
-#            label = self.labels[obj.label_id]
-#            inference_result.append((label, confidence, (x0, y0, x1, y1)))
-#        return inference_result
+        return thumbnail, new_im, inference_result

src/ambianic/pipeline/ai/object_detect.py

@@ -19,13 +19,17 @@ def process_sample(self, **sample):
         else:
             try:
                 image = sample['image']
-                inference_result = self.detect(image=image)
+                thumbnail, tensor_image, inference_result = \
+                    self.detect(image=image)
+                log.debug('Object detection inference_result: %r',
+                          inference_result)
                 inf_meta = {
-                    'display': 'Object Detection'
+                    'display': 'Object Detection',
                 }
                 # pass on the results to the next connected pipe element
                 processed_sample = {
                     'image': image,
+                    'thumbnail': thumbnail,
                     'inference_result': inference_result,
                     'inference_meta': inf_meta
                     }

src/ambianic/pipeline/store.py

@@ -67,11 +67,18 @@ def __init__(self,
         self._idle_interval = datetime.timedelta(seconds=ii)
         self._time_latest_saved_idle = self._time_latest_saved_detection
 
-    def _save_sample(self, now, image, inference_result, inference_meta):
-        time_prefix = now.strftime("%Y%m%d-%H%M%S.%f%z.{fext}")
-        image_file = time_prefix.format(fext='jpg')
+    def _save_sample(self,
+                     inf_time=None,
+                     image=None,
+                     thumbnail=None,
+                     inference_result=None,
+                     inference_meta=None):
+        time_prefix = inf_time.strftime("%Y%m%d-%H%M%S.%f%z-{suffix}.{fext}")
+        image_file = time_prefix.format(suffix='image', fext='jpg')
         image_path = self._output_directory / image_file
-        json_file = time_prefix.format(fext='json')
+        thumbnail_file = time_prefix.format(suffix='thumbnail', fext='jpg')
+        thumbnail_path = self._output_directory / thumbnail_file
+        json_file = time_prefix.format(suffix='inference', fext='json')
         json_path = self._output_directory / json_file
         inf_json = []
         for label, confidence, box in inference_result:
@@ -92,8 +99,9 @@ def _save_sample(self, now, image, inference_result, inference_meta):
             inf_json.append(one_inf)
         save_json = {
             'id': uuid.uuid4().hex,
-            'datetime': now.isoformat(),
+            'datetime': inf_time.isoformat(),
             'image_file_name': image_file,
+            'thumbnail_file_name': thumbnail_file,
             'json_file_name': json_file,
             # rel_dir is relative to system data dir
             # this will be important when resloving REST API data
@@ -103,6 +111,7 @@ def _save_sample(self, now, image, inference_result, inference_meta):
             'inference_meta': inference_meta
         }
         image.save(image_path)
+        thumbnail.save(thumbnail_path)
         # save samples to local disk
         with open(json_path, 'w', encoding='utf-8') as f:
             json.dump(save_json, f, ensure_ascii=False, indent=4)
@@ -112,6 +121,7 @@ def _save_sample(self, now, image, inference_result, inference_meta):
 
     def process_sample(self,
                        image=None,
+                       thumbnail=None,
                        inference_result=None,
                        inference_meta=None,
                        **sample):
@@ -132,21 +142,23 @@ def process_sample(self,
                     # the user specified positive_interval
                     if now - self._time_latest_saved_detection >= \
                       self._positive_interval:
-                        self._save_sample(now,
-                                          image,
-                                          inference_result,
-                                          inference_meta)
+                        self._save_sample(inf_time=now,
+                                          image=image,
+                                          thumbnail=thumbnail,
+                                          inference_result=inference_result,
+                                          inference_meta=inference_meta)
                         self._time_latest_saved_detection = now
                 else:
                     # non-empty result, there is a detection
                     # let's save it if its been longer than
                     #  the user specified positive_interval
                     if now - self._time_latest_saved_idle >= \
                       self._idle_interval:
-                        self._save_sample(now,
-                                          image,
-                                          inference_result,
-                                          inference_meta)
+                        self._save_sample(inf_time=now,
+                                          image=image,
+                                          thumbnail=thumbnail,
+                                          inference_result=inference_result,
+                                          inference_meta=inference_meta)
                         self._time_latest_saved_idle = now
             except Exception as e:
                 log.exception('Error %r while saving sample %r',

tests/pipeline/ai/test_face_detect.py

@@ -47,7 +47,7 @@ def _face_detect_config():
             },
         'labels': _good_labels,
         'top_k': 2,
-        'confidence_threshold': 0.5,
+        'confidence_threshold': 0.6,
     }
     return config
 
@@ -229,8 +229,8 @@ def sample_callback(image=None, inference_result=None, **kwargs):
     assert not result
 
 
-def test_one_person_two_stage_pipe_low_person_confidence():
-    """Fail to detect person in 1st stage hence no face in 2nd stage."""
+def test_one_person_two_stage_pipe_high_face_confidence():
+    """Detect a person in 1st stage and a face in 2nd stage."""
     object_config = _object_detect_config()
     face_config = _face_detect_config()
     result = None
@@ -246,7 +246,13 @@ def sample_callback(image=None, inference_result=None, **kwargs):
     face_detector.connect_to_next_element(output)
     img = _get_image(file_name='person-face.jpg')
     object_detector.receive_next_sample(image=img)
-    assert not result
+    assert result
+    assert len(result) == 1
+    label, confidence, (x0, y0, x1, y1) = result[0]
+    assert label == 'person'
+    assert confidence > 0.9
+    assert x0 > 0 and x0 < x1
+    assert y0 > 0 and y0 < y1
 
 
 def test_two_person_high_confidence_one_face_high_confidence_two_stage_pipe():
@@ -287,7 +293,7 @@ def sample_callback(image=None, inference_result=None, **kwargs):
     assert len(result) == 1
     label, confidence, (x0, y0, x1, y1) = result[0]
     assert label == 'person'
-    assert confidence > 0.9
+    assert confidence > 0.8
     assert x0 > 0 and x0 < x1
     assert y0 > 0 and y0 < y1
 

tests/pipeline/test_store.py

@@ -25,11 +25,20 @@ class _TestSaveDetectionSamples(SaveDetectionSamples):
     _json_path = None
     _inf_result = None
 
-    def _save_sample(self, now, image, inference_result, inference_meta):
+    def _save_sample(self,
+                     inf_time=None,
+                     image=None,
+                     thumbnail=None,
+                     inference_result=None,
+                     inference_meta=None):
         self._save_sample_called = True
         self._inf_result = inference_result
         self._img_path, self._json_path = \
-            super()._save_sample(now, image, inference_result, inference_meta)
+            super()._save_sample(inf_time=inf_time,
+                                 image=image,
+                                 thumbnail=thumbnail,
+                                 inference_result=inference_result,
+                                 inference_meta=inference_meta)
 
 
 def test_store_positive_detection():
@@ -49,6 +58,7 @@ def test_store_positive_detection():
             ('person', 0.98, (0, 1, 2, 3))
     ]
     processed_samples = list(store.process_sample(image=img,
+                                                  thumbnail=img,
                                                   inference_result=detections))
     assert len(processed_samples) == 1
     print(processed_samples)
@@ -109,6 +119,7 @@ def test_store_negative_detection():
     img = Image.new('RGB', (60, 30), color='red')
     detections = []
     processed_samples = list(store.process_sample(image=img,
+                                                  thumbnail=img,
                                                   inference_result=detections))
     assert len(processed_samples) == 1
     print(processed_samples)
@@ -147,7 +158,12 @@ class _TestSaveDetectionSamples2(SaveDetectionSamples):
 
     _save_sample_called = False
 
-    def _save_sample(self, now, image, inference_result, inference_meta):
+    def _save_sample(self,
+                     inf_time=None,
+                     image=None,
+                     thumbnail=None,
+                     inference_result=None,
+                     inference_meta=None):
         self._save_sample_called = True
         raise RuntimeError()
 

tests/run-tests.sh

@@ -13,7 +13,7 @@ echo "Script location: ${BASEDIR}"
 # where codecov can find the generated reports
 cd $BASEDIR/../
 echo PWD=$PWD
-python3 -m pytest tests/ --cov=ambianic --cov-report=xml
+python3 -m pytest --cov=ambianic --cov-report=xml tests/
 # pytest --cov-report=xml --cov=ambianic tests
 # codecov
 # pylint --errors-only src/ambianic