From 8e78c50f30d3752a52e363bd49477ffcc2dd39f8 Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Thu, 28 Sep 2023 14:47:34 -0400 Subject: [PATCH 01/13] Change button and modal layout --- imagedephi/templates/HomePage.html.j2 | 124 ++++++++++++++---------- imagedephi/templates/InputStep.html.j2 | 35 ++++--- imagedephi/templates/OutputStep.html.j2 | 22 ++--- 3 files changed, 108 insertions(+), 73 deletions(-) diff --git a/imagedephi/templates/HomePage.html.j2 b/imagedephi/templates/HomePage.html.j2 index ad1e3ee5..3c1a8850 100644 --- a/imagedephi/templates/HomePage.html.j2 +++ b/imagedephi/templates/HomePage.html.j2 @@ -67,64 +67,86 @@ ], }, } - - - -
+ + + + +
- - - -
-
-
-
-
- -
-
- - - -
+ + + +
+
+
+
+
+ +
+
+ + +
- {% include "InputStep.html.j2" %} - {% include "OutputStep.html.j2" %} - {% include "RulesetStep.html.j2" %} -
+ {% include "SidebarInputStep.html.j2" %} + {% include "SidebarOutputStep.html.j2" %} + {% include "SidebarRulesetStep.html.j2" %} + +
- - {% include "InputSelectorWidget.html.j2" %} - - {% include "OutputSelectorWidget.html.j2" %} - - {# TODO make into a separate component #} - {% if redacted %} -
-
-
-
-

Redaction Successful

-

- We successfully performed a redaction on the images in {{ - input_directory_data.directory - }}. - The redacted images are now in a new folder created in {{ output_directory_data.directory }} -

-
-
+
+ + + + + + {# TODO make into a separate component #} + {% if redacted %} +
+
+
+
+

Redaction Successful

+

+ We successfully performed a redaction on the images in {{ + input_directory_data.directory + }}. + The redacted images are now in a new folder created in {{ output_directory_data.directory }} +

+
+
+
- + {% endif %} +
+ + diff --git a/imagedephi/templates/InputStep.html.j2 b/imagedephi/templates/InputStep.html.j2 index c066bd88..ac291feb 100644 --- a/imagedephi/templates/InputStep.html.j2 +++ b/imagedephi/templates/InputStep.html.j2 @@ -1,22 +1,35 @@ {% extends "SidebarSteps.html.j2" %} {% block step_number %} - 1 +1 {% endblock step_number %} {% block step_title %} - Input Directory +Input Directory {% endblock step_title %} {% block step_help_text %} - Location of the images you’d like to process. +Location of the images you’d like to process. {% endblock step_help_text %} {% block modal %} - + +{# + #} {% endblock modal %} {% block selected_path %} - {% if input_directory_data.directory.name %} -
{{ input_directory_data.directory }}
- {% else %} -
No directory selected
- {% endif %} +{% if input_directory_data.directory.name %} +
{{ input_directory_data.directory }}
+{% else %} +
No directory selected
+{% endif %} {% endblock selected_path %} diff --git a/imagedephi/templates/OutputStep.html.j2 b/imagedephi/templates/OutputStep.html.j2 index 08101cc1..242849ba 100644 --- a/imagedephi/templates/OutputStep.html.j2 +++ b/imagedephi/templates/OutputStep.html.j2 @@ -1,22 +1,22 @@ {% extends "SidebarSteps.html.j2" %} {% block step_number %} - 2 +2 {% endblock step_number %} {% block step_title %} - Output Directory +Output Directory {% endblock step_title %} {% block step_help_text %} - Location of the images after they are processed. +Location of the images after they are processed. {% endblock step_help_text %} {% block modal %} - + {% endblock modal %} {% block selected_path %} - {% if output_directory_data.directory.name %} -
{{ output_directory_data.directory }}
- {% else %} -
No directory selected
- {% endif %} +{% if output_directory_data.directory.name %} +
{{ output_directory_data.directory }}
+{% else %} +
No directory selected
+{% endif %} {% endblock selected_path %} From 6283bad5b4b23a6ff76118b4e205ec0705531e4d Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Fri, 29 Sep 2023 19:21:10 -0400 Subject: [PATCH 02/13] Refactor modal --- imagedephi/templates/HomePage.html.j2 | 93 +++++++++---------------- imagedephi/templates/InputStep.html.j2 | 35 +++------- imagedephi/templates/OutputStep.html.j2 | 22 +++--- 3 files changed, 56 insertions(+), 94 deletions(-) diff --git a/imagedephi/templates/HomePage.html.j2 b/imagedephi/templates/HomePage.html.j2 index 3c1a8850..90507577 100644 --- a/imagedephi/templates/HomePage.html.j2 +++ b/imagedephi/templates/HomePage.html.j2 @@ -83,70 +83,45 @@
+
+
+ + + +
-
- - - -
+ {% include "SidebarInputStep.html.j2" %} + {% include "SidebarOutputStep.html.j2" %} + {% include "SidebarRulesetStep.html.j2" %} +
- {% include "SidebarInputStep.html.j2" %} - {% include "SidebarOutputStep.html.j2" %} - {% include "SidebarRulesetStep.html.j2" %} - -
-
- - - - - - {# TODO make into a separate component #} - {% if redacted %} -
-
-
-
-

Redaction Successful

-

- We successfully performed a redaction on the images in {{ - input_directory_data.directory - }}. - The redacted images are now in a new folder created in {{ output_directory_data.directory }} -

+ + {% include "InputSelectorWidget.html.j2" %} + + {% include "OutputSelectorWidget.html.j2" %} + + {# TODO make into a separate component #} + {% if redacted %} +
+
+
+
+

Redaction Successful

+

+ We successfully performed a redaction on the images in {{ + input_directory_data.directory + }}. + The redacted images are now in a new folder created in {{ output_directory_data.directory }} +

+
+
-
+ {% endif %}
- {% endif %} -
- - + diff --git a/imagedephi/templates/InputStep.html.j2 b/imagedephi/templates/InputStep.html.j2 index ac291feb..c066bd88 100644 --- a/imagedephi/templates/InputStep.html.j2 +++ b/imagedephi/templates/InputStep.html.j2 @@ -1,35 +1,22 @@ {% extends "SidebarSteps.html.j2" %} {% block step_number %} -1 + 1 {% endblock step_number %} {% block step_title %} -Input Directory + Input Directory {% endblock step_title %} {% block step_help_text %} -Location of the images you’d like to process. + Location of the images you’d like to process. {% endblock step_help_text %} {% block modal %} - -{# - #} + {% endblock modal %} {% block selected_path %} -{% if input_directory_data.directory.name %} -
{{ input_directory_data.directory }}
-{% else %} -
No directory selected
-{% endif %} + {% if input_directory_data.directory.name %} +
{{ input_directory_data.directory }}
+ {% else %} +
No directory selected
+ {% endif %} {% endblock selected_path %} diff --git a/imagedephi/templates/OutputStep.html.j2 b/imagedephi/templates/OutputStep.html.j2 index 242849ba..08101cc1 100644 --- a/imagedephi/templates/OutputStep.html.j2 +++ b/imagedephi/templates/OutputStep.html.j2 @@ -1,22 +1,22 @@ {% extends "SidebarSteps.html.j2" %} {% block step_number %} -2 + 2 {% endblock step_number %} {% block step_title %} -Output Directory + Output Directory {% endblock step_title %} {% block step_help_text %} -Location of the images after they are processed. + Location of the images after they are processed. {% endblock step_help_text %} {% block modal %} - + {% endblock modal %} {% block selected_path %} -{% if output_directory_data.directory.name %} -
{{ output_directory_data.directory }}
-{% else %} -
No directory selected
-{% endif %} + {% if output_directory_data.directory.name %} +
{{ output_directory_data.directory }}
+ {% else %} +
No directory selected
+ {% endif %} {% endblock selected_path %} From 01955f305016eb565e6f67b9408c13620a73c2db Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Fri, 29 Sep 2023 19:50:57 -0400 Subject: [PATCH 03/13] Rename steps --- imagedephi/templates/HomePage.html.j2 | 58 ++++++++++++++------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/imagedephi/templates/HomePage.html.j2 b/imagedephi/templates/HomePage.html.j2 index 90507577..99b96902 100644 --- a/imagedephi/templates/HomePage.html.j2 +++ b/imagedephi/templates/HomePage.html.j2 @@ -90,38 +90,40 @@
- {% include "SidebarInputStep.html.j2" %} - {% include "SidebarOutputStep.html.j2" %} - {% include "SidebarRulesetStep.html.j2" %} -
+ {% include "InputStep.html.j2" %} + {% include "OutputStep.html.j2" %} + {% include "RulesetStep.html.j2" %} +
- - {% include "InputSelectorWidget.html.j2" %} - - {% include "OutputSelectorWidget.html.j2" %} - - {# TODO make into a separate component #} - {% if redacted %} -
-
-
-
-

Redaction Successful

-

- We successfully performed a redaction on the images in {{ - input_directory_data.directory - }}. - The redacted images are now in a new folder created in {{ output_directory_data.directory }} -

-
-
+
+ + {% include "InputSelectorWidget.html.j2" %} + + {% include "OutputSelectorWidget.html.j2" %} + + {# TODO make into a separate component #} + {% if redacted %} +
+
+
+
+

Redaction Successful

+

+ We successfully performed a redaction on the images in {{ + input_directory_data.directory + }}. + The redacted images are now in a new folder created in {{ output_directory_data.directory }} +

- {% endif %} +
- + {% endif %} +
+ + From f779bef3e60cbc97d512852702bbc5621a92fedf Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Sat, 30 Sep 2023 13:25:33 -0400 Subject: [PATCH 04/13] Add logging for end of redactions --- imagedephi/main.py | 2 +- imagedephi/redact/redact.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/imagedephi/main.py b/imagedephi/main.py index 9ce46946..41d381ea 100644 --- a/imagedephi/main.py +++ b/imagedephi/main.py @@ -114,9 +114,9 @@ def imagedephi( @click.pass_obj def run(obj: ImagedephiContext, input_path: Path, output_dir: Path, verbose, quiet, log_file): """Perform the redaction of images.""" - redact_images(input_path, output_dir, obj.override_rule_set) if verbose or quiet or log_file: set_logging_config(verbose, quiet, log_file) + redact_images(input_path, output_dir, obj.override_rule_set) @imagedephi.command diff --git a/imagedephi/redact/redact.py b/imagedephi/redact/redact.py index 64cbeb5b..7ad51af6 100644 --- a/imagedephi/redact/redact.py +++ b/imagedephi/redact/redact.py @@ -79,7 +79,7 @@ def redact_images( redact_dir = create_redact_dir(output_dir) show_redaction_plan(input_path) for image_file in images_to_redact: - logger.info(f"Redacting {image_file.name}...") + logger.info(f"Redacting {image_file.name}. Image {output_file_counter} of {output_file_max} images") if image_file.suffix in FILE_EXTENSION_MAP: redaction_plan = build_redaction_plan(image_file, base_rules, override_rules) if not redaction_plan.is_comprehensive(): @@ -95,6 +95,8 @@ def redact_images( output_file_max, ) redaction_plan.save(output_path, overwrite) + if output_file_counter == output_file_max: + logger.info("Redactions completed") output_file_counter += 1 From d45a572f06caa22a84bae0781047cff3742d6098 Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Tue, 3 Oct 2023 13:16:33 -0400 Subject: [PATCH 05/13] Add redaction progress bar to cli --- imagedephi/redact/redact.py | 42 +++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/imagedephi/redact/redact.py b/imagedephi/redact/redact.py index 7ad51af6..5ad66d87 100644 --- a/imagedephi/redact/redact.py +++ b/imagedephi/redact/redact.py @@ -5,6 +5,7 @@ import importlib.resources from pathlib import Path +import click import tifftools import tifftools.constants import yaml @@ -78,26 +79,27 @@ def redact_images( output_file_max = len(images_to_redact) redact_dir = create_redact_dir(output_dir) show_redaction_plan(input_path) - for image_file in images_to_redact: - logger.info(f"Redacting {image_file.name}. Image {output_file_counter} of {output_file_max} images") - if image_file.suffix in FILE_EXTENSION_MAP: - redaction_plan = build_redaction_plan(image_file, base_rules, override_rules) - if not redaction_plan.is_comprehensive(): - logger.info(f"Redaction could not be performed for {image_file.name}.") - redaction_plan.report_missing_rules() - else: - redaction_plan.execute_plan() - output_path = _get_output_path( - image_file, - redact_dir, - output_file_name_base, - output_file_counter, - output_file_max, - ) - redaction_plan.save(output_path, overwrite) - if output_file_counter == output_file_max: - logger.info("Redactions completed") - output_file_counter += 1 + with click.progressbar(images_to_redact, label="Redacting Images") as bar: + for image_file in bar: + logger.info(f"Redacting {image_file.name}. Image {output_file_counter} of {output_file_max} images") + if image_file.suffix in FILE_EXTENSION_MAP: + redaction_plan = build_redaction_plan(image_file, base_rules, override_rules) + if not redaction_plan.is_comprehensive(): + logger.info(f"Redaction could not be performed for {image_file.name}.") + redaction_plan.report_missing_rules() + else: + redaction_plan.execute_plan() + output_path = _get_output_path( + image_file, + redact_dir, + output_file_name_base, + output_file_counter, + output_file_max, + ) + redaction_plan.save(output_path, overwrite) + if output_file_counter == output_file_max: + logger.info("Redactions completed") + output_file_counter += 1 def show_redaction_plan(input_path: Path, override_rules: Ruleset | None = None) -> None: From 42a75a1f6e38822c570aebce7fe6bac9dafe67b9 Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Tue, 3 Oct 2023 13:17:36 -0400 Subject: [PATCH 06/13] Disable redaction button and add redaction message on submit --- imagedephi/gui.py | 1 + imagedephi/js/redaction.js | 11 +++ imagedephi/redact/redact.py | 5 +- imagedephi/templates/HomePage.html.j2 | 99 +++++++++++++++------------ 4 files changed, 71 insertions(+), 45 deletions(-) create mode 100644 imagedephi/js/redaction.js diff --git a/imagedephi/gui.py b/imagedephi/gui.py index 4d2870cb..982f4b21 100644 --- a/imagedephi/gui.py +++ b/imagedephi/gui.py @@ -67,6 +67,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: shutdown_event = asyncio.Event() app.mount("/assets", StaticFiles(directory="imagedephi/assets"), name="assets") +app.mount("/js", StaticFiles(directory="imagedephi/js"), name="js") class DirectoryData: diff --git a/imagedephi/js/redaction.js b/imagedephi/js/redaction.js new file mode 100644 index 00000000..71dee49b --- /dev/null +++ b/imagedephi/js/redaction.js @@ -0,0 +1,11 @@ +window.onload = (event) => { + +const redactBanner = document.getElementById("redacting"); +const redactButton = document.getElementById("dephi"); +const redactForm = document.getElementById('redact') +redactForm.addEventListener("submit", () => { + redactButton.setAttribute("disabled", "true") + redactBanner.classList.remove("hidden") +}) + +} diff --git a/imagedephi/redact/redact.py b/imagedephi/redact/redact.py index 5ad66d87..ca97fc01 100644 --- a/imagedephi/redact/redact.py +++ b/imagedephi/redact/redact.py @@ -81,7 +81,10 @@ def redact_images( show_redaction_plan(input_path) with click.progressbar(images_to_redact, label="Redacting Images") as bar: for image_file in bar: - logger.info(f"Redacting {image_file.name}. Image {output_file_counter} of {output_file_max} images") + logger.info( + f"""Redacting {image_file.name}. + Image {output_file_counter} of {output_file_max} images""" + ) if image_file.suffix in FILE_EXTENSION_MAP: redaction_plan = build_redaction_plan(image_file, base_rules, override_rules) if not redaction_plan.is_comprehensive(): diff --git a/imagedephi/templates/HomePage.html.j2 b/imagedephi/templates/HomePage.html.j2 index 99b96902..8c0d9724 100644 --- a/imagedephi/templates/HomePage.html.j2 +++ b/imagedephi/templates/HomePage.html.j2 @@ -67,63 +67,74 @@ ], }, } - - - - -
+ + + + +
- - - -
-
-
-
-
- -
+ + + +
+
+
+
+
+ +
+ {% include "InputStep.html.j2" %} + {% include "OutputStep.html.j2" %} + {% include "RulesetStep.html.j2" %} +
- {% include "InputStep.html.j2" %} - {% include "OutputStep.html.j2" %} - {% include "RulesetStep.html.j2" %} -
-
- - {% include "InputSelectorWidget.html.j2" %} - - {% include "OutputSelectorWidget.html.j2" %} - - {# TODO make into a separate component #} - {% if redacted %} -
-
-
-
-

Redaction Successful

-

- We successfully performed a redaction on the images in {{ - input_directory_data.directory - }}. - The redacted images are now in a new folder created in {{ output_directory_data.directory }} -

+ + {% include "InputSelectorWidget.html.j2" %} + + {% include "OutputSelectorWidget.html.j2" %} + + {# TODO make into a separate component #} + + {% if redacted %} +
+
+
+
+

Redaction Successful

+

+ We successfully performed a redaction on the images in {{ + input_directory_data.directory + }}. + The redacted images are now in a new folder created in {{ output_directory_data.directory }} +

+
+
+
+
+ {% endif %}
- {% endif %} -
- - + From 150947fb53439211c37a07eb803ede178394b2aa Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Wed, 11 Oct 2023 17:39:45 -0400 Subject: [PATCH 07/13] Change progress to click because of logging redirect difficulties --- imagedephi/redact/redact.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/imagedephi/redact/redact.py b/imagedephi/redact/redact.py index ca97fc01..f21bba28 100644 --- a/imagedephi/redact/redact.py +++ b/imagedephi/redact/redact.py @@ -79,9 +79,9 @@ def redact_images( output_file_max = len(images_to_redact) redact_dir = create_redact_dir(output_dir) show_redaction_plan(input_path) - with click.progressbar(images_to_redact, label="Redacting Images") as bar: + with click.progressbar(images_to_redact, label="Redacting Images", show_pos=True) as bar: for image_file in bar: - logger.info( + click.echo( f"""Redacting {image_file.name}. Image {output_file_counter} of {output_file_max} images""" ) @@ -101,7 +101,7 @@ def redact_images( ) redaction_plan.save(output_path, overwrite) if output_file_counter == output_file_max: - logger.info("Redactions completed") + click.echo("Redactions completed") output_file_counter += 1 From 58eb5939aee0df0a3e74122c10ca9bb79bd3624a Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Tue, 7 Nov 2023 12:01:17 -0500 Subject: [PATCH 08/13] WIP Queuehandler feeding to websocket --- imagedephi/gui.py | 67 +++++++++++++++++++++++++++++++++++-- imagedephi/redact/redact.py | 25 +++++++++++--- pyproject.toml | 1 + 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/imagedephi/gui.py b/imagedephi/gui.py index 982f4b21..659cc3b1 100644 --- a/imagedephi/gui.py +++ b/imagedephi/gui.py @@ -5,13 +5,14 @@ from contextlib import asynccontextmanager import importlib.resources from io import BytesIO +import logging import os from pathlib import Path -from typing import TYPE_CHECKING +from queue import Queue import urllib.parse from PIL import Image, UnidentifiedImageError -from fastapi import BackgroundTasks, FastAPI, Form, HTTPException, Request +from fastapi import BackgroundTasks, FastAPI, Form, HTTPException, Request, WebSocket from fastapi.responses import HTMLResponse, PlainTextResponse, StreamingResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates @@ -19,7 +20,9 @@ from starlette.background import BackgroundTask import tifftools -from imagedephi.redact import iter_image_files, redact_images +from imagedephi.redact import iter_image_files, redact as redact_module, redact_images + +# from imagedephi.redact.redact import output_file_counter from imagedephi.utils.tiff import get_associated_image_svs, get_ifd_for_thumbnail, get_is_svs if TYPE_CHECKING: @@ -251,6 +254,7 @@ def redact( input_directory: Path = Form(), # noqa: B008 output_directory: Path = Form(), # noqa: B008 ): + if not input_directory.is_dir(): raise HTTPException(status_code=404, detail="Input directory not found") if not output_directory.is_dir(): @@ -269,3 +273,60 @@ def redact( "redacted": True, }, ) + + +# The following is websockets 'hello world' testing DO NOT MERGE with this still here + +html = """ + + + + Chat + + +

WebSocket Chat

+
+ + +
+
    +
+ + + +""" + +# Not sure if this should be a generator +def progress_logging(log_queue: Queue) -> logging.LogRecord | None: + if log_queue.not_empty: + return log_queue.get().getMessage() + + +@app.get("/websocket") +async def get(): + return HTMLResponse(html) + + +@app.websocket('/ws') +async def websocket_endpoint(websocket: WebSocket): + redact_module.output_file_counter = 1 + await websocket.accept() + while True: + if redact_module.ql.not_empty: + await websocket.send_text(f"log: {progress_logging(redact_module.ql)}") + await asyncio.sleep(.25) diff --git a/imagedephi/redact/redact.py b/imagedephi/redact/redact.py index f21bba28..c9aae822 100644 --- a/imagedephi/redact/redact.py +++ b/imagedephi/redact/redact.py @@ -3,7 +3,11 @@ from collections.abc import Generator import datetime import importlib.resources +from io import StringIO +import logging +from logging.handlers import QueueHandler from pathlib import Path +import queue import click import tifftools @@ -17,6 +21,10 @@ from .svs import MalformedAperioFileError from .tiff import UnsupportedFileTypeError +ql = queue.Queue(-1) + +qh = QueueHandler(ql) + def _get_output_path( file_path: Path, @@ -75,17 +83,26 @@ def redact_images( ) # Convert to a list in order to get the length images_to_redact = list(iter_image_files(input_path) if input_path.is_dir() else [input_path]) + global output_file_counter output_file_counter = 1 output_file_max = len(images_to_redact) redact_dir = create_redact_dir(output_dir) show_redaction_plan(input_path) - with click.progressbar(images_to_redact, label="Redacting Images", show_pos=True) as bar: + f = open("demofile2.txt", "a") + + file = StringIO() + with click.progressbar(images_to_redact, label="Redacting Images", show_pos=True, file=file, show_percent=True) as bar: + f.seek(0) + f.write(bar.file.read()) + f.close() for image_file in bar: - click.echo( - f"""Redacting {image_file.name}. - Image {output_file_counter} of {output_file_max} images""" + test = logging.makeLogRecord( + dict(msg=f"""Redacting {image_file.name}. + Image {output_file_counter} of {output_file_max} images""") ) + qh.emit(test) if image_file.suffix in FILE_EXTENSION_MAP: + # it looks like build_redaction_plan gets called twice. Below and as part of show_redaction_plan above redaction_plan = build_redaction_plan(image_file, base_rules, override_rules) if not redaction_plan.is_comprehensive(): logger.info(f"Redaction could not be performed for {image_file.name}.") diff --git a/pyproject.toml b/pyproject.toml index 2e97c431..74d4d894 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "hypercorn", "pyyaml", "Pillow", + "websockets", ] dynamic = ["version"] From 163bcf1a204dafbc124be0cfd8ab090d2d5599f6 Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Thu, 9 Nov 2023 14:59:17 -0500 Subject: [PATCH 09/13] Update websocket to use progress API --- imagedephi/gui.py | 19 ++++++++++++------- imagedephi/js/redaction.js | 14 +++++++------- imagedephi/redact/redact.py | 15 +++++---------- imagedephi/templates/HomePage.html.j2 | 19 ++++++++++++++++++- imagedephi/utils/progress_log.py | 25 +++++++++++++++++++++++++ 5 files changed, 67 insertions(+), 25 deletions(-) create mode 100644 imagedephi/utils/progress_log.py diff --git a/imagedephi/gui.py b/imagedephi/gui.py index 659cc3b1..344302cd 100644 --- a/imagedephi/gui.py +++ b/imagedephi/gui.py @@ -8,7 +8,9 @@ import logging import os from pathlib import Path -from queue import Queue +from queue import Empty, Queue +from time import sleep +from typing import TYPE_CHECKING import urllib.parse from PIL import Image, UnidentifiedImageError @@ -21,6 +23,7 @@ import tifftools from imagedephi.redact import iter_image_files, redact as redact_module, redact_images +from imagedephi.utils.progress_log import get_next_progress_message, push_progress # from imagedephi.redact.redact import output_file_counter from imagedephi.utils.tiff import get_associated_image_svs, get_ifd_for_thumbnail, get_is_svs @@ -312,9 +315,9 @@ def redact( """ # Not sure if this should be a generator -def progress_logging(log_queue: Queue) -> logging.LogRecord | None: - if log_queue.not_empty: - return log_queue.get().getMessage() +# def progress_logging(log_queue: Queue) -> logging.LogRecord | None: +# if log_queue.not_empty: +# return log_queue.get().getMessage() @app.get("/websocket") @@ -327,6 +330,8 @@ async def websocket_endpoint(websocket: WebSocket): redact_module.output_file_counter = 1 await websocket.accept() while True: - if redact_module.ql.not_empty: - await websocket.send_text(f"log: {progress_logging(redact_module.ql)}") - await asyncio.sleep(.25) + message = get_next_progress_message() + if message is not None: + await websocket.send_text(f"{message[0]} of {message[1]} ") + else: + await asyncio.sleep(.01) diff --git a/imagedephi/js/redaction.js b/imagedephi/js/redaction.js index 71dee49b..4a970f6e 100644 --- a/imagedephi/js/redaction.js +++ b/imagedephi/js/redaction.js @@ -1,11 +1,11 @@ window.onload = (event) => { -const redactBanner = document.getElementById("redacting"); -const redactButton = document.getElementById("dephi"); -const redactForm = document.getElementById('redact') -redactForm.addEventListener("submit", () => { - redactButton.setAttribute("disabled", "true") - redactBanner.classList.remove("hidden") -}) + const redactBanner = document.getElementById("redacting"); + const redactButton = document.getElementById("dephi"); + const redactForm = document.getElementById('redact') + redactForm.addEventListener("submit", () => { + redactButton.setAttribute("disabled", "true") + redactBanner.classList.remove("hidden") + }) } diff --git a/imagedephi/redact/redact.py b/imagedephi/redact/redact.py index c9aae822..4ff4d933 100644 --- a/imagedephi/redact/redact.py +++ b/imagedephi/redact/redact.py @@ -4,7 +4,6 @@ import datetime import importlib.resources from io import StringIO -import logging from logging.handlers import QueueHandler from pathlib import Path import queue @@ -16,11 +15,13 @@ from imagedephi.rules import Ruleset from imagedephi.utils.logger import logger +from imagedephi.utils.progress_log import push_progress from .build_redaction_plan import FILE_EXTENSION_MAP, build_redaction_plan from .svs import MalformedAperioFileError from .tiff import UnsupportedFileTypeError +# create separate progress logger and remove 'emit' ql = queue.Queue(-1) qh = QueueHandler(ql) @@ -88,19 +89,13 @@ def redact_images( output_file_max = len(images_to_redact) redact_dir = create_redact_dir(output_dir) show_redaction_plan(input_path) - f = open("demofile2.txt", "a") file = StringIO() with click.progressbar(images_to_redact, label="Redacting Images", show_pos=True, file=file, show_percent=True) as bar: - f.seek(0) - f.write(bar.file.read()) - f.close() + for image_file in bar: - test = logging.makeLogRecord( - dict(msg=f"""Redacting {image_file.name}. - Image {output_file_counter} of {output_file_max} images""") - ) - qh.emit(test) + push_progress(image_file.name, output_file_counter, output_file_max) + if image_file.suffix in FILE_EXTENSION_MAP: # it looks like build_redaction_plan gets called twice. Below and as part of show_redaction_plan above redaction_plan = build_redaction_plan(image_file, base_rules, override_rules) diff --git a/imagedephi/templates/HomePage.html.j2 b/imagedephi/templates/HomePage.html.j2 index 8c0d9724..72ee2c56 100644 --- a/imagedephi/templates/HomePage.html.j2 +++ b/imagedephi/templates/HomePage.html.j2 @@ -68,12 +68,13 @@ }, } +
-
+ @@ -115,6 +116,22 @@

Redacting Images

+
+
    +
+ +
diff --git a/imagedephi/utils/progress_log.py b/imagedephi/utils/progress_log.py new file mode 100644 index 00000000..a1d48df3 --- /dev/null +++ b/imagedephi/utils/progress_log.py @@ -0,0 +1,25 @@ +# import logging +# import logging.handlers +import queue + +# _progress_logger = logging.getLogger('progress') +# _progress_queue: queue.Queue[logging.LogRecord] = queue.Queue(-1) +_progress_queue: queue.Queue[tuple] = queue.Queue(-1) + +# _queue_handler = logging.handlers.QueueHandler(_progress_queue) +# _progress_logger.addHandler(_queue_handler) + + +def push_progress(file_name: str, count: int, max: int) -> None: + # _progress_logger.info("Redacting %s. Image %d of %d", file_name, count, max) + _progress_queue.put_nowait((count, max)) + + +def get_next_progress_message() -> tuple | None: + try: + record = _progress_queue.get_nowait() + except queue.Empty: + return None + else: + # return record.message + return record From 854b492dc5706c4452ce174a071cd9fd07ec9890 Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Tue, 14 Nov 2023 10:19:11 -0500 Subject: [PATCH 10/13] Temporary progress update fix --- imagedephi/gui.py | 59 ++------------------------- imagedephi/js/redaction.js | 1 + imagedephi/redact/redact.py | 6 +-- imagedephi/templates/HomePage.html.j2 | 12 ++---- 4 files changed, 11 insertions(+), 67 deletions(-) diff --git a/imagedephi/gui.py b/imagedephi/gui.py index 344302cd..c73ca446 100644 --- a/imagedephi/gui.py +++ b/imagedephi/gui.py @@ -5,11 +5,8 @@ from contextlib import asynccontextmanager import importlib.resources from io import BytesIO -import logging import os from pathlib import Path -from queue import Empty, Queue -from time import sleep from typing import TYPE_CHECKING import urllib.parse @@ -23,7 +20,7 @@ import tifftools from imagedephi.redact import iter_image_files, redact as redact_module, redact_images -from imagedephi.utils.progress_log import get_next_progress_message, push_progress +from imagedephi.utils.progress_log import get_next_progress_message # from imagedephi.redact.redact import output_file_counter from imagedephi.utils.tiff import get_associated_image_svs, get_ifd_for_thumbnail, get_is_svs @@ -257,7 +254,6 @@ def redact( input_directory: Path = Form(), # noqa: B008 output_directory: Path = Form(), # noqa: B008 ): - if not input_directory.is_dir(): raise HTTPException(status_code=404, detail="Input directory not found") if not output_directory.is_dir(): @@ -278,60 +274,13 @@ def redact( ) -# The following is websockets 'hello world' testing DO NOT MERGE with this still here - -html = """ - - - - Chat - - -

WebSocket Chat

- - - - -
    -
- - - -""" - -# Not sure if this should be a generator -# def progress_logging(log_queue: Queue) -> logging.LogRecord | None: -# if log_queue.not_empty: -# return log_queue.get().getMessage() - - -@app.get("/websocket") -async def get(): - return HTMLResponse(html) - - -@app.websocket('/ws') +@app.websocket("/ws") async def websocket_endpoint(websocket: WebSocket): redact_module.output_file_counter = 1 await websocket.accept() while True: message = get_next_progress_message() if message is not None: - await websocket.send_text(f"{message[0]} of {message[1]} ") + await websocket.send_text(str({message[0]})) else: - await asyncio.sleep(.01) + await asyncio.sleep(0.01) diff --git a/imagedephi/js/redaction.js b/imagedephi/js/redaction.js index 4a970f6e..8717bd02 100644 --- a/imagedephi/js/redaction.js +++ b/imagedephi/js/redaction.js @@ -6,6 +6,7 @@ window.onload = (event) => { redactForm.addEventListener("submit", () => { redactButton.setAttribute("disabled", "true") redactBanner.classList.remove("hidden") + location.reload() }) } diff --git a/imagedephi/redact/redact.py b/imagedephi/redact/redact.py index 4ff4d933..e70c14a6 100644 --- a/imagedephi/redact/redact.py +++ b/imagedephi/redact/redact.py @@ -91,13 +91,13 @@ def redact_images( show_redaction_plan(input_path) file = StringIO() - with click.progressbar(images_to_redact, label="Redacting Images", show_pos=True, file=file, show_percent=True) as bar: - + with click.progressbar( + images_to_redact, label="Redacting Images", show_pos=True, file=file, show_percent=True + ) as bar: for image_file in bar: push_progress(image_file.name, output_file_counter, output_file_max) if image_file.suffix in FILE_EXTENSION_MAP: - # it looks like build_redaction_plan gets called twice. Below and as part of show_redaction_plan above redaction_plan = build_redaction_plan(image_file, base_rules, override_rules) if not redaction_plan.is_comprehensive(): logger.info(f"Redaction could not be performed for {image_file.name}.") diff --git a/imagedephi/templates/HomePage.html.j2 b/imagedephi/templates/HomePage.html.j2 index 72ee2c56..978aaadc 100644 --- a/imagedephi/templates/HomePage.html.j2 +++ b/imagedephi/templates/HomePage.html.j2 @@ -68,7 +68,6 @@ }, } - @@ -117,17 +116,12 @@

Redacting Images

-
    -
+ Redacting image out of {{ input_directory_data.child_images|length }} From fca3faa8012ed3f02bcac1e24e3225fdf1321ce9 Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Tue, 14 Nov 2023 10:29:52 -0500 Subject: [PATCH 11/13] Fix type failures --- imagedephi/gui.py | 1 - imagedephi/redact/redact.py | 8 -------- 2 files changed, 9 deletions(-) diff --git a/imagedephi/gui.py b/imagedephi/gui.py index c73ca446..e69e9417 100644 --- a/imagedephi/gui.py +++ b/imagedephi/gui.py @@ -276,7 +276,6 @@ def redact( @app.websocket("/ws") async def websocket_endpoint(websocket: WebSocket): - redact_module.output_file_counter = 1 await websocket.accept() while True: message = get_next_progress_message() diff --git a/imagedephi/redact/redact.py b/imagedephi/redact/redact.py index e70c14a6..d205a912 100644 --- a/imagedephi/redact/redact.py +++ b/imagedephi/redact/redact.py @@ -4,9 +4,7 @@ import datetime import importlib.resources from io import StringIO -from logging.handlers import QueueHandler from pathlib import Path -import queue import click import tifftools @@ -21,11 +19,6 @@ from .svs import MalformedAperioFileError from .tiff import UnsupportedFileTypeError -# create separate progress logger and remove 'emit' -ql = queue.Queue(-1) - -qh = QueueHandler(ql) - def _get_output_path( file_path: Path, @@ -84,7 +77,6 @@ def redact_images( ) # Convert to a list in order to get the length images_to_redact = list(iter_image_files(input_path) if input_path.is_dir() else [input_path]) - global output_file_counter output_file_counter = 1 output_file_max = len(images_to_redact) redact_dir = create_redact_dir(output_dir) From da393a472c8bc20de4f3e3672354de8edd01491f Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Tue, 14 Nov 2023 10:39:44 -0500 Subject: [PATCH 12/13] Lint fix --- imagedephi/gui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imagedephi/gui.py b/imagedephi/gui.py index e69e9417..d93f1bf6 100644 --- a/imagedephi/gui.py +++ b/imagedephi/gui.py @@ -19,7 +19,7 @@ from starlette.background import BackgroundTask import tifftools -from imagedephi.redact import iter_image_files, redact as redact_module, redact_images +from imagedephi.redact import iter_image_files, redact_images from imagedephi.utils.progress_log import get_next_progress_message # from imagedephi.redact.redact import output_file_counter From fae0172cbae16ef0af8f20af1e773fe178cb7aef Mon Sep 17 00:00:00 2001 From: Mary Salvi Date: Wed, 6 Dec 2023 16:11:34 -0500 Subject: [PATCH 13/13] Fix redaction progress for cli --- imagedephi/gui.py | 2 +- imagedephi/templates/HomePage.html.j2 | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/imagedephi/gui.py b/imagedephi/gui.py index d93f1bf6..28154cf6 100644 --- a/imagedephi/gui.py +++ b/imagedephi/gui.py @@ -282,4 +282,4 @@ async def websocket_endpoint(websocket: WebSocket): if message is not None: await websocket.send_text(str({message[0]})) else: - await asyncio.sleep(0.01) + await asyncio.sleep(0.001) diff --git a/imagedephi/templates/HomePage.html.j2 b/imagedephi/templates/HomePage.html.j2 index 978aaadc..1ab570c0 100644 --- a/imagedephi/templates/HomePage.html.j2 +++ b/imagedephi/templates/HomePage.html.j2 @@ -118,7 +118,8 @@
Redacting image out of {{ input_directory_data.child_images|length }}