From a2b7c99039d92d9ba622906c39b5219210d4ad62 Mon Sep 17 00:00:00 2001 From: ellenyuX Date: Tue, 15 Oct 2024 14:24:03 +0200 Subject: [PATCH 1/5] changes in storage.py --- src/storage.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/storage.py b/src/storage.py index 1da4a3e..5655600 100644 --- a/src/storage.py +++ b/src/storage.py @@ -4,8 +4,6 @@ import uuid import time -import logging -from src.singleplayer import api from threading import Thread from azure.storage.blob import BlobClient from azure.storage.blob import BlobServiceClient @@ -36,7 +34,7 @@ def save_image(image, label, certainty): blob_name=file_name, ) blob.upload_blob(image) - container_client = blob_connection() + container_client = blob_connection(setup.CONTAINER_NAME_NEW) # update metadata in blob try: image_count = int( @@ -49,9 +47,8 @@ def save_image(image, label, certainty): metadata = {"image_count": str(image_count + 1)} container_client.set_container_metadata(metadata=metadata) except Exception as e: - api.app.logger.error(e) + raise Exception("Could not save image from storage.py" + str(e)) url = base_url + "/" + container_name + "/" + file_name - logging.info(url) return url @@ -62,7 +59,7 @@ def clear_dataset(): NOTE: container is deleted by garbage collection, which does not happen instantly. A new blob cannot be initalized before old is collected. """ - container_client = blob_connection() + container_client = blob_connection(setup.CONTAINER_NAME_NEW) try: container_client.delete_container() except Exception as e: @@ -77,7 +74,7 @@ def create_container(): """ tries = setup.CREATE_CONTAINER_TRIES waiting_time = setup.CREATE_CONTAINER_WAITER - container_client = blob_connection() + container_client = blob_connection(setup.CONTAINER_NAME_NEW) success = False metadata = {"image_count": "0"} for i in range(tries): @@ -91,8 +88,7 @@ def create_container(): ) success = True except Exception as e: - api.app.logger.error(e) - + raise Exception("Could not create container") def image_count(): """ @@ -128,7 +124,7 @@ def get_n_random_images_from_label(n, label): """ Returns n random images from the blob storage container with the given label. """ - container_client = blob_connection(setup.CONTAINER_NAME_ORIGINAL) + container_client = blob_connection(Keys.get("CONTAINER_NAME")) blob_prefix = f"{label}/" blobs = list(container_client.list_blobs(name_starts_with=blob_prefix)) selected_blobs = random.sample(blobs, min(n, len(blobs))) From cc8688c1558ca21fdfff42a191903dcdf057c60c Mon Sep 17 00:00:00 2001 From: ellenyuX Date: Tue, 15 Oct 2024 14:34:42 +0200 Subject: [PATCH 2/5] only concatenate if str --- src/admin/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admin/api.py b/src/admin/api.py index 9a744ff..cf15244 100644 --- a/src/admin/api.py +++ b/src/admin/api.py @@ -135,7 +135,7 @@ def admin_page(action): "BLOB_image_count": new_blob_image_count, } except Exception as e: - current_app.logger.error("Something in admin/status failed: " + e) + current_app.logger.error("Something in admin/status failed: " + str(e)) return json.dumps(data), 200 elif action == "logging": From 80ba4fcc6514ec40a3cbbd06f259803b96e5f9a4 Mon Sep 17 00:00:00 2001 From: ellenyuX Date: Tue, 15 Oct 2024 14:43:50 +0200 Subject: [PATCH 3/5] literal int not base 10 failures --- src/admin/api.py | 3 +++ src/models.py | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/admin/api.py b/src/admin/api.py index cf15244..80bb28b 100644 --- a/src/admin/api.py +++ b/src/admin/api.py @@ -67,6 +67,9 @@ def get_available_years(): try: available_years = shared_models.get_available_years() + if available_years == "": + raise Exception("No years recorded") + return jsonify(available_years), 200 except Exception as e: return json.dumps({e}), 400 diff --git a/src/models.py b/src/models.py index 3937368..865c05d 100644 --- a/src/models.py +++ b/src/models.py @@ -6,7 +6,6 @@ import os import random import json -import sys class Iteration(db.Model): From a7cc89eb81492604f9b141461efa8020c3572e34 Mon Sep 17 00:00:00 2001 From: Tobias Hermansen Date: Tue, 15 Oct 2024 14:45:49 +0200 Subject: [PATCH 4/5] add error handling of calls to storage --- src/admin/api.py | 5 ++++- src/multiplayer/api.py | 19 ++++++++++++++----- src/singleplayer/api.py | 10 ++++++++-- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/admin/api.py b/src/admin/api.py index cf15244..e4829e5 100644 --- a/src/admin/api.py +++ b/src/admin/api.py @@ -120,7 +120,10 @@ def admin_page(action): elif action == "hardReset": # Delete all images in CV, upload all orignal images and retrain classifier.delete_all_images() - storage.clear_dataset() + try: + storage.clear_dataset() + except Exception as e: + current_app.logger.error(e) Thread(target=classifier.hard_reset_retrain).start() response = {"success": "All images deleted, model now training"} return json.dumps(response), 200 diff --git a/src/multiplayer/api.py b/src/multiplayer/api.py index becf618..6e486dd 100644 --- a/src/multiplayer/api.py +++ b/src/multiplayer/api.py @@ -200,9 +200,12 @@ def get_example_drawings(json_data, emitEndpoint="getExampleDrawings"): label, number_of_images ) - example_drawings = storage.get_images_from_relative_url( - example_drawing_urls - ) + try: + example_drawings = storage.get_images_from_relative_url( + example_drawing_urls + ) + except Exception as e: + current_app.logger.error(e) emit(emitEndpoint, json.dumps(example_drawings), room=game_id) @@ -257,7 +260,10 @@ def handle_classify(data, image, correct_label=None): if time_out: # to break race condition if both players timeout time.sleep(0.5 * random.random()) - storage.save_image(image, correct_label, best_certainty) + try: + storage.save_image(image, correct_label, best_certainty) + except Exception as e: + current_app.logger.error(e) player = shared_models.get_player(player_id) opponent = models.get_opponent(game_id, player_id) if opponent.state == "Done": @@ -292,7 +298,10 @@ def handle_classify(data, image, correct_label=None): emit("prediction", response) if has_won: - storage.save_image(image, correct_label, best_certainty) + try: + storage.save_image(image, correct_label, best_certainty) + except Exception as e: + current_app.logger.error(e) player = shared_models.get_player(player_id) opponent = models.get_opponent(game_id, player_id) if opponent.state == "Done": diff --git a/src/singleplayer/api.py b/src/singleplayer/api.py index cbea210..aeb95ac 100644 --- a/src/singleplayer/api.py +++ b/src/singleplayer/api.py @@ -150,7 +150,10 @@ def classify(): player.game_id, player_id, 1, "Done" ) # save image - storage.save_image(image, label, best_certainty) + try: + storage.save_image(image, label, best_certainty) + except Exception as e: + current_app.logger.error(e) # Update game state to be done game_state = "Done" # Insert statistic for label @@ -256,7 +259,10 @@ def get_n_drawings_by_label(): image_urls = shared_models.get_n_random_example_images( label, number_of_images ) - images = storage.get_images_from_relative_url(image_urls) + try: + images = storage.get_images_from_relative_url(image_urls) + except Exception as e: + current_app.logger.error(e) current_app.logger.info( "singleplayer /getExampleDrawings " + " label: " + str(label) ) From 26b615c8f7cb3818f32c9cb016dc49869b90cc1f Mon Sep 17 00:00:00 2001 From: ellenyuX Date: Tue, 15 Oct 2024 15:26:04 +0200 Subject: [PATCH 5/5] linting --- src/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/storage.py b/src/storage.py index 5655600..0326545 100644 --- a/src/storage.py +++ b/src/storage.py @@ -87,9 +87,10 @@ def create_container(): metadata=metadata, public_access="container" ) success = True - except Exception as e: + except Exception: raise Exception("Could not create container") + def image_count(): """ Returns number of images in relevant container.