Merge branch 'main' into metadata_configuration_files

pyproject.toml

@@ -32,7 +32,8 @@ dependencies = [
   "confluent_kafka",
   "ess-streaming-data-types",
   "requests",
-  "rich"
+  "rich",
+  "graypy"
 ]
 
 dynamic = ["version"]

requirements/base.in

@@ -7,3 +7,4 @@ confluent_kafka
 ess-streaming-data-types
 requests
 rich
+graypy

requirements/base.txt

@@ -5,17 +5,17 @@
 #
 #    pip-compile-multi
 #
-certifi==2024.2.2
+certifi==2024.6.2
     # via requests
 charset-normalizer==3.3.2
     # via requests
-confluent-kafka==2.3.0
+confluent-kafka==2.4.0
     # via -r base.in
-ess-streaming-data-types==0.23.1
+ess-streaming-data-types==0.26.1
     # via -r base.in
 flatbuffers==24.3.25
     # via ess-streaming-data-types
-idna==3.6
+idna==3.7
     # via requests
 kafka-python==2.0.2
     # via -r base.in
@@ -25,9 +25,9 @@ mdurl==0.1.2
     # via markdown-it-py
 numpy==1.26.4
     # via ess-streaming-data-types
-pygments==2.17.2
+pygments==2.18.0
     # via rich
-requests==2.31.0
+requests==2.32.3
     # via -r base.in
 rich==13.7.1
     # via -r base.in

requirements/basetest.txt

@@ -5,15 +5,15 @@
 #
 #    pip-compile-multi
 #
-exceptiongroup==1.2.0
+exceptiongroup==1.2.1
     # via pytest
 iniconfig==2.0.0
     # via pytest
 packaging==24.0
     # via pytest
-pluggy==1.4.0
+pluggy==1.5.0
     # via pytest
-pytest==8.1.1
+pytest==8.2.1
     # via -r basetest.in
 tomli==2.0.1
     # via pytest

requirements/ci.txt

@@ -13,7 +13,7 @@ colorama==0.4.6
     # via tox
 distlib==0.3.8
     # via virtualenv
-filelock==3.13.3
+filelock==3.14.0
     # via
     #   tox
     #   virtualenv
@@ -26,11 +26,11 @@ packaging==24.0
     #   -r ci.in
     #   pyproject-api
     #   tox
-platformdirs==4.2.0
+platformdirs==4.2.2
     # via
     #   tox
     #   virtualenv
-pluggy==1.4.0
+pluggy==1.5.0
     # via tox
 pyproject-api==1.6.1
     # via tox
@@ -40,7 +40,7 @@ tomli==2.0.1
     # via
     #   pyproject-api
     #   tox
-tox==4.14.2
+tox==4.15.0
     # via -r ci.in
-virtualenv==20.25.1
+virtualenv==20.26.2
     # via tox

requirements/dev.txt

@@ -1,4 +1,4 @@
-# SHA1:6b332a6c3e0e3001d497bf6635c1625d7e509301
+# SHA1:12be7e8f03b4a9d10aef9113021d5b2636818446
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -10,7 +10,7 @@
 -r mypy.txt
 -r static.txt
 -r test.txt
-annotated-types==0.6.0
+annotated-types==0.7.0
     # via pydantic
 build==1.2.1
     # via pip-tools
@@ -20,11 +20,11 @@ click==8.1.7
     #   pip-tools
 copier==9.2.0
     # via -r dev.in
-dunamai==1.19.2
+dunamai==1.21.1
     # via copier
 funcy==2.0
     # via copier
-jinja2==3.1.3
+jinja2==3.1.4
     # via
     #   copier
     #   jinja2-ansible-filters
@@ -38,20 +38,18 @@ pip-compile-multi==2.6.3
     # via -r dev.in
 pip-tools==7.4.1
     # via pip-compile-multi
-plumbum==1.8.2
+plumbum==1.8.3
     # via copier
 prompt-toolkit==3.0.36
     # via questionary
-pydantic==2.6.4
+pydantic==2.7.2
     # via copier
-pydantic-core==2.16.3
+pydantic-core==2.18.3
     # via pydantic
-pyproject-hooks==1.0.0
+pyproject-hooks==1.1.0
     # via
     #   build
     #   pip-tools
-pyyaml-include==1.4.1
-    # via -r dev.in
 questionary==2.0.1
     # via copier
 toposort==1.10

requirements/mypy.txt

@@ -6,9 +6,9 @@
 #    pip-compile-multi
 #
 -r test.txt
-mypy==1.9.0
+mypy==1.10.0
     # via -r mypy.in
 mypy-extensions==1.0.0
     # via mypy
-typing-extensions==4.11.0
+typing-extensions==4.12.1
     # via mypy

requirements/nightly.in

@@ -6,3 +6,4 @@ confluent_kafka
 ess-streaming-data-types
 requests
 rich
+graypy

requirements/nightly.txt

@@ -6,17 +6,17 @@
 #    pip-compile-multi
 #
 -r basetest.txt
-certifi==2024.2.2
+certifi==2024.6.2
     # via requests
 charset-normalizer==3.3.2
     # via requests
-confluent-kafka==2.3.0
+confluent-kafka==2.4.0
     # via -r nightly.in
-ess-streaming-data-types==0.23.1
+ess-streaming-data-types==0.26.1
     # via -r nightly.in
 flatbuffers==24.3.25
     # via ess-streaming-data-types
-idna==3.6
+idna==3.7
     # via requests
 kafka-python==2.0.2
     # via -r nightly.in
@@ -26,9 +26,9 @@ mdurl==0.1.2
     # via markdown-it-py
 numpy==1.26.4
     # via ess-streaming-data-types
-pygments==2.17.2
+pygments==2.18.0
     # via rich
-requests==2.31.0
+requests==2.32.3
     # via -r nightly.in
 rich==13.7.1
     # via -r nightly.in

requirements/static.txt

@@ -9,20 +9,17 @@ cfgv==3.4.0
     # via pre-commit
 distlib==0.3.8
     # via virtualenv
-filelock==3.13.3
+filelock==3.14.0
     # via virtualenv
-identify==2.5.35
+identify==2.5.36
     # via pre-commit
-nodeenv==1.8.0
+nodeenv==1.9.0
     # via pre-commit
-platformdirs==4.2.0
+platformdirs==4.2.2
     # via virtualenv
-pre-commit==3.7.0
+pre-commit==3.7.1
     # via -r static.in
 pyyaml==6.0.1
     # via pre-commit
-virtualenv==20.25.1
+virtualenv==20.26.2
     # via pre-commit
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

src/scicat_ingestor.py

@@ -56,7 +56,6 @@ def main() -> None:
             # check if we have received a WRDN message
             # if message is not a WRDN, we get None back
             if message:
-
                 # extract nexus file name from message
 
                 # extract job id from message
@@ -69,4 +68,3 @@ def main() -> None:
             # check if we need to commit the individual message
             if config.kafka_options.individual_message_commit:
                 consumer.commit(message=message)
-

src/scicat_kafka.py

@@ -129,7 +129,7 @@ def wrdn_messages(
             message_value = message.value()
             message_type = message_value[4:8]
             logger.info("Received message. Type : %s", message_type)
-            if message_value == b"wrdn":
-                yield _deserialise_wrdn(message.value(), logger)
+            if message_type == b"wrdn":
+                yield _deserialise_wrdn(message_value, logger)
             else:
                 yield None