adding comments for splitting in two scripts

src/scicat_ingestor.py

@@ -52,3 +52,21 @@ def main() -> None:
         # Receive messages
         for message in wrdn_messages(consumer, logger):
             logger.info("Processing message: %s", message)
+
+            # check if we have received a WRDN message
+            # if message is not a WRDN, we get None back
+            if message:
+
+                # extract nexus file name from message
+
+                # extract job id from message
+
+                # saves the WRDN message in a file
+
+                # instantiate a new process and runs backeground ingestor
+                # on the nexus file
+
+            # check if we need to commit the individual message
+            if config.kafka_options.individual_message_commit:
+                consumer.commit(message=message)
+