From 1705cda4ce294d4850b904a7144fb16fd8815e53 Mon Sep 17 00:00:00 2001 From: Ivelin Ivanov Date: Tue, 12 Nov 2019 21:59:34 -0600 Subject: [PATCH] BREAKING CHANGE(timeline): provide server side image thumbnails for faster UI render times, close #128 --- config.yaml | 2 +- src/ambianic/pipeline/ai/face_detect.py | 9 +- src/ambianic/pipeline/ai/image_detection.py | 91 +++++++++++++++------ src/ambianic/pipeline/ai/object_detect.py | 8 +- src/ambianic/pipeline/store.py | 38 ++++++--- tests/pipeline/ai/test_face_detect.py | 16 ++-- tests/pipeline/test_store.py | 22 ++++- tests/run-tests.sh | 2 +- 8 files changed, 131 insertions(+), 57 deletions(-) diff --git a/config.yaml b/config.yaml index 9d276077..bfa2604a 100755 --- a/config.yaml +++ b/config.yaml @@ -46,7 +46,7 @@ ai_models: labels: ai_models/coco_labels.txt face_detection: &tfm_face_detection model: - tflite: ai_models/mobilenet_ssd_v2_coco_quant_postprocess.tflite + tflite: ai_models/mobilenet_ssd_v2_face_quant_postprocess.tflite edgetpu: ai_models/mobilenet_ssd_v2_face_quant_postprocess_edgetpu.tflite labels: ai_models/coco_labels.txt top_k: 2 diff --git a/src/ambianic/pipeline/ai/face_detect.py b/src/ambianic/pipeline/ai/face_detect.py index be4d865b..639ac5fd 100755 --- a/src/ambianic/pipeline/ai/face_detect.py +++ b/src/ambianic/pipeline/ai/face_detect.py @@ -56,17 +56,16 @@ def process_sample(self, **sample): len(person_regions)) for box in person_regions: person_image = self.crop_image(image, box) - inference_result = self.detect(image=person_image) + thumbnail, tensor_image, inference_result = \ + self.detect(image=person_image) log.debug('Face detection inference_result: %r', inference_result) inf_meta = { - 'display': 'Face Detection' - # id - # version - # etc + 'display': 'Face Detection', } processed_sample = { 'image': person_image, + 'thumbnail': thumbnail, 'inference_result': inference_result, 'inference_meta': inf_meta } diff --git a/src/ambianic/pipeline/ai/image_detection.py b/src/ambianic/pipeline/ai/image_detection.py index ebe182dd..e5849292 100644 --- a/src/ambianic/pipeline/ai/image_detection.py +++ b/src/ambianic/pipeline/ai/image_detection.py @@ -62,13 +62,10 @@ def load_labels(self, label_path=None): lines = (p.match(line).groups() for line in f.readlines()) return {int(num): text.strip() for num, text in lines} - def resize(self, image=None, desired_size=None): - """Resizes original image to size expected by input tensor. - - Preserves aspect ratio to avoid confusing the AI model with - unnatural distortions. Pads the resulting image - with solid black color pixels to fill the desired size. + def thumbnail(self, image=None, desired_size=None): + """Resizes original image as close as possible to desired size. + Preserves aspect ratio of original image. Does not modify the original image. :Parameters: @@ -87,10 +84,42 @@ def resize(self, image=None, desired_size=None): """ assert image assert desired_size - log.debug('current image size = %r', image.size) + log.debug('input image size = %r', image.size) thumb = image.copy() thumb.thumbnail(desired_size) log.debug('thmubnail image size = %r', thumb.size) + return thumb + + def resize(self, image=None, desired_size=None): + """Pad original image to exact size expected by input tensor. + + Preserve aspect ratio to avoid confusing the AI model with + unnatural distortions. Pad the resulting image + with solid black color pixels to fill the desired size. + + Do not modify the original image. + + :Parameters: + ---------- + image : PIL.Image + Input Image sized to fit an input tensor but without padding. + Its possible that one size fits one tensor dimension exactly + but the other size is smaller than + the input tensor other dimension. + + desired_size : (width, height) + Exact size expected by the AI model. + + :Returns: + ------- + PIL.Image + Resized image fitting exactly the AI model input tensor. + + """ + assert image + assert desired_size + log.debug('input image size = %r', image.size) + thumb = image.copy() delta_w = desired_size[0] - thumb.size[0] delta_h = desired_size[1] - thumb.size[1] padding = (0, 0, delta_w, delta_h) @@ -119,7 +148,8 @@ def detect(self, image=None): :Parameters: ---------- image : PIL.Image - Input image in raw RGB format. + Input image in raw RGB format + with the exact size of the input tensor. :Returns: ------- @@ -139,7 +169,17 @@ def detect(self, image=None): height = tfe.input_details[0]['shape'][1] width = tfe.input_details[0]['shape'][2] - new_im = self.resize(image=image, desired_size=(width, height)) + # thumbnail is a proportionately resized image + thumbnail = self.thumbnail(image=image, desired_size=(width, height)) + + # convert thumbnail into an image with exact size as tensor + # preserving proportions by padding as needed + new_im = self.resize(image=thumbnail, desired_size=(width, height)) + + # calculate what fraction of the new image is the thumbnail size + # we will use these factors to adjust detection box coordinates + w_factor = thumbnail.size[0] / new_im.size[0] + h_factor = thumbnail.size[1] / new_im.size[1] # add N dim input_data = np.expand_dims(new_im, axis=0) @@ -209,25 +249,22 @@ def detect(self, image=None): if (li < len(self._labels)): label = self._labels[li] box = boxes[0, i, :] - x0 = box[1] - y0 = box[0] - x1 = box[3] - y1 = box[2] + # refit detections into original image size + # without overflowing outside image borders + x0 = box[1] / w_factor + y0 = box[0] / h_factor + x1 = min(box[3] / w_factor, 1) + y1 = min(box[2] / h_factor, 1) + log.debug('thumbnail image size: %r , ' + 'tensor image size: %r', + thumbnail.size, + new_im.size) + log.debug('resizing detection box (x0, y0, x1, y1) ' + 'from: %r to %r', + (box[1], box[0], box[3], box[2]), + (x0, y0, x1, y1)) inference_result.append(( label, confidence, (x0, y0, x1, y1))) - return inference_result -# objs = self.engine.DetectWithImage( -# image, -# threshold=self.confidence_threshold, -# keep_aspect_ratio=True, -# relative_coord=True, -# top_k=3) -# -# for obj in objs: -# x0, y0, x1, y1 = obj.bounding_box.flatten().tolist() -# confidence = obj.score -# label = self.labels[obj.label_id] -# inference_result.append((label, confidence, (x0, y0, x1, y1))) -# return inference_result + return thumbnail, new_im, inference_result diff --git a/src/ambianic/pipeline/ai/object_detect.py b/src/ambianic/pipeline/ai/object_detect.py index cc706802..cae3df4c 100755 --- a/src/ambianic/pipeline/ai/object_detect.py +++ b/src/ambianic/pipeline/ai/object_detect.py @@ -19,13 +19,17 @@ def process_sample(self, **sample): else: try: image = sample['image'] - inference_result = self.detect(image=image) + thumbnail, tensor_image, inference_result = \ + self.detect(image=image) + log.debug('Object detection inference_result: %r', + inference_result) inf_meta = { - 'display': 'Object Detection' + 'display': 'Object Detection', } # pass on the results to the next connected pipe element processed_sample = { 'image': image, + 'thumbnail': thumbnail, 'inference_result': inference_result, 'inference_meta': inf_meta } diff --git a/src/ambianic/pipeline/store.py b/src/ambianic/pipeline/store.py index 3213c408..972c3901 100755 --- a/src/ambianic/pipeline/store.py +++ b/src/ambianic/pipeline/store.py @@ -67,11 +67,18 @@ def __init__(self, self._idle_interval = datetime.timedelta(seconds=ii) self._time_latest_saved_idle = self._time_latest_saved_detection - def _save_sample(self, now, image, inference_result, inference_meta): - time_prefix = now.strftime("%Y%m%d-%H%M%S.%f%z.{fext}") - image_file = time_prefix.format(fext='jpg') + def _save_sample(self, + inf_time=None, + image=None, + thumbnail=None, + inference_result=None, + inference_meta=None): + time_prefix = inf_time.strftime("%Y%m%d-%H%M%S.%f%z-{suffix}.{fext}") + image_file = time_prefix.format(suffix='image', fext='jpg') image_path = self._output_directory / image_file - json_file = time_prefix.format(fext='json') + thumbnail_file = time_prefix.format(suffix='thumbnail', fext='jpg') + thumbnail_path = self._output_directory / thumbnail_file + json_file = time_prefix.format(suffix='inference', fext='json') json_path = self._output_directory / json_file inf_json = [] for label, confidence, box in inference_result: @@ -92,8 +99,9 @@ def _save_sample(self, now, image, inference_result, inference_meta): inf_json.append(one_inf) save_json = { 'id': uuid.uuid4().hex, - 'datetime': now.isoformat(), + 'datetime': inf_time.isoformat(), 'image_file_name': image_file, + 'thumbnail_file_name': thumbnail_file, 'json_file_name': json_file, # rel_dir is relative to system data dir # this will be important when resloving REST API data @@ -103,6 +111,7 @@ def _save_sample(self, now, image, inference_result, inference_meta): 'inference_meta': inference_meta } image.save(image_path) + thumbnail.save(thumbnail_path) # save samples to local disk with open(json_path, 'w', encoding='utf-8') as f: json.dump(save_json, f, ensure_ascii=False, indent=4) @@ -112,6 +121,7 @@ def _save_sample(self, now, image, inference_result, inference_meta): def process_sample(self, image=None, + thumbnail=None, inference_result=None, inference_meta=None, **sample): @@ -132,10 +142,11 @@ def process_sample(self, # the user specified positive_interval if now - self._time_latest_saved_detection >= \ self._positive_interval: - self._save_sample(now, - image, - inference_result, - inference_meta) + self._save_sample(inf_time=now, + image=image, + thumbnail=thumbnail, + inference_result=inference_result, + inference_meta=inference_meta) self._time_latest_saved_detection = now else: # non-empty result, there is a detection @@ -143,10 +154,11 @@ def process_sample(self, # the user specified positive_interval if now - self._time_latest_saved_idle >= \ self._idle_interval: - self._save_sample(now, - image, - inference_result, - inference_meta) + self._save_sample(inf_time=now, + image=image, + thumbnail=thumbnail, + inference_result=inference_result, + inference_meta=inference_meta) self._time_latest_saved_idle = now except Exception as e: log.exception('Error %r while saving sample %r', diff --git a/tests/pipeline/ai/test_face_detect.py b/tests/pipeline/ai/test_face_detect.py index 577e22a5..19ad9aa5 100644 --- a/tests/pipeline/ai/test_face_detect.py +++ b/tests/pipeline/ai/test_face_detect.py @@ -47,7 +47,7 @@ def _face_detect_config(): }, 'labels': _good_labels, 'top_k': 2, - 'confidence_threshold': 0.5, + 'confidence_threshold': 0.6, } return config @@ -229,8 +229,8 @@ def sample_callback(image=None, inference_result=None, **kwargs): assert not result -def test_one_person_two_stage_pipe_low_person_confidence(): - """Fail to detect person in 1st stage hence no face in 2nd stage.""" +def test_one_person_two_stage_pipe_high_face_confidence(): + """Detect a person in 1st stage and a face in 2nd stage.""" object_config = _object_detect_config() face_config = _face_detect_config() result = None @@ -246,7 +246,13 @@ def sample_callback(image=None, inference_result=None, **kwargs): face_detector.connect_to_next_element(output) img = _get_image(file_name='person-face.jpg') object_detector.receive_next_sample(image=img) - assert not result + assert result + assert len(result) == 1 + label, confidence, (x0, y0, x1, y1) = result[0] + assert label == 'person' + assert confidence > 0.9 + assert x0 > 0 and x0 < x1 + assert y0 > 0 and y0 < y1 def test_two_person_high_confidence_one_face_high_confidence_two_stage_pipe(): @@ -287,7 +293,7 @@ def sample_callback(image=None, inference_result=None, **kwargs): assert len(result) == 1 label, confidence, (x0, y0, x1, y1) = result[0] assert label == 'person' - assert confidence > 0.9 + assert confidence > 0.8 assert x0 > 0 and x0 < x1 assert y0 > 0 and y0 < y1 diff --git a/tests/pipeline/test_store.py b/tests/pipeline/test_store.py index c70708aa..3820cfeb 100644 --- a/tests/pipeline/test_store.py +++ b/tests/pipeline/test_store.py @@ -25,11 +25,20 @@ class _TestSaveDetectionSamples(SaveDetectionSamples): _json_path = None _inf_result = None - def _save_sample(self, now, image, inference_result, inference_meta): + def _save_sample(self, + inf_time=None, + image=None, + thumbnail=None, + inference_result=None, + inference_meta=None): self._save_sample_called = True self._inf_result = inference_result self._img_path, self._json_path = \ - super()._save_sample(now, image, inference_result, inference_meta) + super()._save_sample(inf_time=inf_time, + image=image, + thumbnail=thumbnail, + inference_result=inference_result, + inference_meta=inference_meta) def test_store_positive_detection(): @@ -49,6 +58,7 @@ def test_store_positive_detection(): ('person', 0.98, (0, 1, 2, 3)) ] processed_samples = list(store.process_sample(image=img, + thumbnail=img, inference_result=detections)) assert len(processed_samples) == 1 print(processed_samples) @@ -109,6 +119,7 @@ def test_store_negative_detection(): img = Image.new('RGB', (60, 30), color='red') detections = [] processed_samples = list(store.process_sample(image=img, + thumbnail=img, inference_result=detections)) assert len(processed_samples) == 1 print(processed_samples) @@ -147,7 +158,12 @@ class _TestSaveDetectionSamples2(SaveDetectionSamples): _save_sample_called = False - def _save_sample(self, now, image, inference_result, inference_meta): + def _save_sample(self, + inf_time=None, + image=None, + thumbnail=None, + inference_result=None, + inference_meta=None): self._save_sample_called = True raise RuntimeError() diff --git a/tests/run-tests.sh b/tests/run-tests.sh index 626162a1..dc99328e 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -13,7 +13,7 @@ echo "Script location: ${BASEDIR}" # where codecov can find the generated reports cd $BASEDIR/../ echo PWD=$PWD -python3 -m pytest tests/ --cov=ambianic --cov-report=xml +python3 -m pytest --cov=ambianic --cov-report=xml tests/ # pytest --cov-report=xml --cov=ambianic tests # codecov # pylint --errors-only src/ambianic