From e348716d320886ef5d2f02206f4da2a9bc49801d Mon Sep 17 00:00:00 2001 From: YooSunyoung Date: Tue, 11 Jun 2024 03:02:43 +0200 Subject: [PATCH 1/5] Add graylog as a dependency. --- pyproject.toml | 3 ++- requirements/base.in | 1 + requirements/base.txt | 12 ++++++------ requirements/basetest.txt | 6 +++--- requirements/ci.txt | 10 +++++----- requirements/dev.txt | 18 ++++++++---------- requirements/mypy.txt | 4 ++-- requirements/nightly.in | 1 + requirements/nightly.txt | 12 ++++++------ requirements/static.txt | 15 ++++++--------- 10 files changed, 40 insertions(+), 42 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aed3426..2dda564 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,8 @@ dependencies = [ "confluent_kafka", "ess-streaming-data-types", "requests", - "rich" + "rich", + "graypy" ] dynamic = ["version"] diff --git a/requirements/base.in b/requirements/base.in index b18da2b..14b2fb9 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -7,3 +7,4 @@ confluent_kafka ess-streaming-data-types requests rich +graypy diff --git a/requirements/base.txt b/requirements/base.txt index e072a4b..d70ec2c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -5,17 +5,17 @@ # # pip-compile-multi # -certifi==2024.2.2 +certifi==2024.6.2 # via requests charset-normalizer==3.3.2 # via requests -confluent-kafka==2.3.0 +confluent-kafka==2.4.0 # via -r base.in -ess-streaming-data-types==0.23.1 +ess-streaming-data-types==0.26.1 # via -r base.in flatbuffers==24.3.25 # via ess-streaming-data-types -idna==3.6 +idna==3.7 # via requests kafka-python==2.0.2 # via -r base.in @@ -25,9 +25,9 @@ mdurl==0.1.2 # via markdown-it-py numpy==1.26.4 # via ess-streaming-data-types -pygments==2.17.2 +pygments==2.18.0 # via rich -requests==2.31.0 +requests==2.32.3 # via -r base.in rich==13.7.1 # via -r base.in diff --git a/requirements/basetest.txt b/requirements/basetest.txt index 6eabd7a..aad7976 100644 --- a/requirements/basetest.txt +++ b/requirements/basetest.txt @@ -5,15 +5,15 @@ # # pip-compile-multi # -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via pytest iniconfig==2.0.0 # via pytest packaging==24.0 # via pytest -pluggy==1.4.0 +pluggy==1.5.0 # via pytest -pytest==8.1.1 +pytest==8.2.1 # via -r basetest.in tomli==2.0.1 # via pytest diff --git a/requirements/ci.txt b/requirements/ci.txt index 497a73c..e0da602 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -13,7 +13,7 @@ colorama==0.4.6 # via tox distlib==0.3.8 # via virtualenv -filelock==3.13.3 +filelock==3.14.0 # via # tox # virtualenv @@ -26,11 +26,11 @@ packaging==24.0 # -r ci.in # pyproject-api # tox -platformdirs==4.2.0 +platformdirs==4.2.2 # via # tox # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via tox pyproject-api==1.6.1 # via tox @@ -40,7 +40,7 @@ tomli==2.0.1 # via # pyproject-api # tox -tox==4.14.2 +tox==4.15.0 # via -r ci.in -virtualenv==20.25.1 +virtualenv==20.26.2 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index ce99385..67fbe09 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,4 +1,4 @@ -# SHA1:6b332a6c3e0e3001d497bf6635c1625d7e509301 +# SHA1:12be7e8f03b4a9d10aef9113021d5b2636818446 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -10,7 +10,7 @@ -r mypy.txt -r static.txt -r test.txt -annotated-types==0.6.0 +annotated-types==0.7.0 # via pydantic build==1.2.1 # via pip-tools @@ -20,11 +20,11 @@ click==8.1.7 # pip-tools copier==9.2.0 # via -r dev.in -dunamai==1.19.2 +dunamai==1.21.1 # via copier funcy==2.0 # via copier -jinja2==3.1.3 +jinja2==3.1.4 # via # copier # jinja2-ansible-filters @@ -38,20 +38,18 @@ pip-compile-multi==2.6.3 # via -r dev.in pip-tools==7.4.1 # via pip-compile-multi -plumbum==1.8.2 +plumbum==1.8.3 # via copier prompt-toolkit==3.0.36 # via questionary -pydantic==2.6.4 +pydantic==2.7.2 # via copier -pydantic-core==2.16.3 +pydantic-core==2.18.3 # via pydantic -pyproject-hooks==1.0.0 +pyproject-hooks==1.1.0 # via # build # pip-tools -pyyaml-include==1.4.1 - # via -r dev.in questionary==2.0.1 # via copier toposort==1.10 diff --git a/requirements/mypy.txt b/requirements/mypy.txt index 8d29307..9732b15 100644 --- a/requirements/mypy.txt +++ b/requirements/mypy.txt @@ -6,9 +6,9 @@ # pip-compile-multi # -r test.txt -mypy==1.9.0 +mypy==1.10.0 # via -r mypy.in mypy-extensions==1.0.0 # via mypy -typing-extensions==4.11.0 +typing-extensions==4.12.1 # via mypy diff --git a/requirements/nightly.in b/requirements/nightly.in index 6a10cbf..f36bed9 100644 --- a/requirements/nightly.in +++ b/requirements/nightly.in @@ -6,3 +6,4 @@ confluent_kafka ess-streaming-data-types requests rich +graypy diff --git a/requirements/nightly.txt b/requirements/nightly.txt index 4048d6b..d20cc75 100644 --- a/requirements/nightly.txt +++ b/requirements/nightly.txt @@ -6,17 +6,17 @@ # pip-compile-multi # -r basetest.txt -certifi==2024.2.2 +certifi==2024.6.2 # via requests charset-normalizer==3.3.2 # via requests -confluent-kafka==2.3.0 +confluent-kafka==2.4.0 # via -r nightly.in -ess-streaming-data-types==0.23.1 +ess-streaming-data-types==0.26.1 # via -r nightly.in flatbuffers==24.3.25 # via ess-streaming-data-types -idna==3.6 +idna==3.7 # via requests kafka-python==2.0.2 # via -r nightly.in @@ -26,9 +26,9 @@ mdurl==0.1.2 # via markdown-it-py numpy==1.26.4 # via ess-streaming-data-types -pygments==2.17.2 +pygments==2.18.0 # via rich -requests==2.31.0 +requests==2.32.3 # via -r nightly.in rich==13.7.1 # via -r nightly.in diff --git a/requirements/static.txt b/requirements/static.txt index bc4d3df..ea619e4 100644 --- a/requirements/static.txt +++ b/requirements/static.txt @@ -9,20 +9,17 @@ cfgv==3.4.0 # via pre-commit distlib==0.3.8 # via virtualenv -filelock==3.13.3 +filelock==3.14.0 # via virtualenv -identify==2.5.35 +identify==2.5.36 # via pre-commit -nodeenv==1.8.0 +nodeenv==1.9.0 # via pre-commit -platformdirs==4.2.0 +platformdirs==4.2.2 # via virtualenv -pre-commit==3.7.0 +pre-commit==3.7.1 # via -r static.in pyyaml==6.0.1 # via pre-commit -virtualenv==20.25.1 +virtualenv==20.26.2 # via pre-commit - -# The following packages are considered to be unsafe in a requirements file: -# setuptools From 22a2654cb5076388dbe1e59d48ac7501bbede1a9 Mon Sep 17 00:00:00 2001 From: Max Novelli Date: Fri, 31 May 2024 15:18:04 +0200 Subject: [PATCH 2/5] adding comments for splitting in two scripts --- src/scicat_ingestor.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/scicat_ingestor.py b/src/scicat_ingestor.py index 72ff38f..524ea33 100644 --- a/src/scicat_ingestor.py +++ b/src/scicat_ingestor.py @@ -52,3 +52,21 @@ def main() -> None: # Receive messages for message in wrdn_messages(consumer, logger): logger.info("Processing message: %s", message) + + # check if we have received a WRDN message + # if message is not a WRDN, we get None back + if message: + + # extract nexus file name from message + + # extract job id from message + + # saves the WRDN message in a file + + # instantiate a new process and runs backeground ingestor + # on the nexus file + + # check if we need to commit the individual message + if config.kafka_options.individual_message_commit: + consumer.commit(message=message) + \ No newline at end of file From 9ab405217254310c2ec345702e298b12c9ff7cef Mon Sep 17 00:00:00 2001 From: Max Novelli Date: Fri, 31 May 2024 15:19:21 +0200 Subject: [PATCH 3/5] adding wrdn message selection --- src/scicat_kafka.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/scicat_kafka.py b/src/scicat_kafka.py index 26535aa..9c6bb35 100644 --- a/src/scicat_kafka.py +++ b/src/scicat_kafka.py @@ -125,5 +125,11 @@ def wrdn_messages( logger.error("Consumer error: %s", message.error()) yield None else: - logger.info("Received message.") - yield _deserialise_wrdn(message.value(), logger) + # retrieve type of message + message_value = message.value() + message_type = message_value[4:8] + logger.info("Received message. Type : %s", message_type) + if message_value == b"wrdn": + yield _deserialise_wrdn(message.value(), logger) + else: + yield None From a27bd52901c55ed4f5475805d4e3662e995ef4b4 Mon Sep 17 00:00:00 2001 From: Max Novelli Date: Mon, 3 Jun 2024 14:00:05 +0200 Subject: [PATCH 4/5] fixed linting and syntax --- src/scicat_ingestor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scicat_ingestor.py b/src/scicat_ingestor.py index 524ea33..64d1ba5 100644 --- a/src/scicat_ingestor.py +++ b/src/scicat_ingestor.py @@ -56,7 +56,6 @@ def main() -> None: # check if we have received a WRDN message # if message is not a WRDN, we get None back if message: - # extract nexus file name from message # extract job id from message @@ -66,7 +65,8 @@ def main() -> None: # instantiate a new process and runs backeground ingestor # on the nexus file + pass + # check if we need to commit the individual message if config.kafka_options.individual_message_commit: consumer.commit(message=message) - \ No newline at end of file From 83c68fa3744a4e4d673a5af20635a84f7255ca5e Mon Sep 17 00:00:00 2001 From: Sunyoung Yoo Date: Tue, 11 Jun 2024 11:07:39 +0200 Subject: [PATCH 5/5] Update src/scicat_kafka.py --- src/scicat_kafka.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scicat_kafka.py b/src/scicat_kafka.py index 9c6bb35..5a5c2a7 100644 --- a/src/scicat_kafka.py +++ b/src/scicat_kafka.py @@ -129,7 +129,7 @@ def wrdn_messages( message_value = message.value() message_type = message_value[4:8] logger.info("Received message. Type : %s", message_type) - if message_value == b"wrdn": - yield _deserialise_wrdn(message.value(), logger) + if message_type == b"wrdn": + yield _deserialise_wrdn(message_value, logger) else: yield None