Archival package creation: add ability to log errors to a file.

cwrc · Jan 15, 2025 · bb3f045 · bb3f045
1 parent 3193c98
commit bb3f045
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 3 deletions.
diff --git a/README.md b/README.md
@@ -205,6 +205,22 @@ Note: if wanting to test [leaf-isle-bagger] and [isle-bagger] locally
 
 See the following as an alternative to specifying an OCI image registry and tag in the Dockerfile: <https://docs.docker.com/build/bake/reference/>. As an example, see [isle-buildkit] `docker-bake.hcl`.
 
+## FAQ
+
+### Debugging from the audit log:
+
+``` bash
+tmp=$(grep x /data/leaf-bagger/_leaf_bagger_audit_2025-01-03T_15-08-06.csv  | head -15 | cut -d ',' -f1 | tr '\n' ' ')
+for item in $tmp; do
+    ./venv/bin/python3 leaf-bagger.py \
+      --server ${BAGGER_DRUPAL_URL} \
+      --output /tmp/z.csv \
+      --force_single_node ${item} \
+      --container cwrc-test \
+      ;
+done
+```
+
 ---
 
 [isle-bagger]: https://github.com/cwrc/isle-bagger

diff --git a/rootfs/etc/s6-overlay/scripts/bagger-setup.sh b/rootfs/etc/s6-overlay/scripts/bagger-setup.sh
@@ -7,7 +7,7 @@ function setup_cron {
         cat <<EOF | crontab -u nginx -
 # min   hour    day     month   weekday command
 # ${BAGGER_CROND_SCHEDULE}        cd ${BAGGER_APP_DIR} && ./bin/console app:islandora_bagger:process_queue --queue=${BAGGER_QUEUE_PATH}
-${BAGGER_CROND_SCHEDULE}        cd ${LEAF_BAGGER_APP_DIR} && ./venv/bin/python3 leaf-bagger.py --server ${BAGGER_DRUPAL_URL} --output ${LEAF_BAGGER_OUTPUT_DIR}/_leaf_bagger_\$(date +"%Y-%m-%dT_%H-%M-%S").csv --container ${OS_CONTAINER}  --date \$(date -d "@\$((\$(date +%s) - ${LEAF_BAGGER_CROND_DATE_WINDOW}))" +"%Y-%m-%d") && ./venv/bin/python3 leaf-bagger-audit.py --server ${BAGGER_DRUPAL_URL} --output ${LEAF_BAGGER_AUDIT_OUTPUT_DIR}/_leaf_bagger_audit_\$(date +"%Y-%m-%dT_%H-%M-%S").csv --container ${OS_CONTAINER}
+${BAGGER_CROND_SCHEDULE}        cd ${LEAF_BAGGER_APP_DIR} && ./venv/bin/python3 leaf-bagger.py --server ${BAGGER_DRUPAL_URL} --output ${LEAF_BAGGER_OUTPUT_DIR}/_leaf_bagger_\$(date +"%Y-%m-%dT_%H-%M-%S").csv --container ${OS_CONTAINER}  --date \$(date -d "@\$((\$(date +%s) - ${LEAF_BAGGER_CROND_DATE_WINDOW}))" +"%Y-%m-%d") --error_log ${LEAF_BAGGER_OUTPUT_DIR}/_leaf_bagger_\$(date +"%Y-%m-%dT_%H-%M-%S")_error.log; ./venv/bin/python3 leaf-bagger-audit.py --server ${BAGGER_DRUPAL_URL} --output ${LEAF_BAGGER_AUDIT_OUTPUT_DIR}/_leaf_bagger_audit_\$(date +"%Y-%m-%dT_%H-%M-%S").csv --container ${OS_CONTAINER};
 EOF
     fi
 }

diff --git a/rootfs/var/www/leaf-isle-bagger/drupal/utilities.py b/rootfs/var/www/leaf-isle-bagger/drupal/utilities.py
@@ -125,6 +125,7 @@ def create_aip(node_list, bagger_app_path):
                 stderr=subprocess.STDOUT,
                 check=True,
                 cwd=bagger_app_path,
+                text=True,
             )
         except subprocess.CalledProcessError as e:
             logging.error(f"{e}")

diff --git a/rootfs/var/www/leaf-isle-bagger/leaf-bagger.py b/rootfs/var/www/leaf-isle-bagger/leaf-bagger.py
@@ -28,6 +28,12 @@ def parse_args():
     parser.add_argument(
         "--output", required=True, help="Location to store CSV output file."
     )
+    parser.add_argument(
+        "--error_log",
+        required=False,
+        default=None,
+        help="Location to store error logs.",
+    )
     parser.add_argument(
         "--date", required=False, help="Items changed after the given date."
     )
@@ -117,8 +123,14 @@ def main():
 
     args = parse_args()
 
-    logging.basicConfig(level=args.logging_level)
-    logging.getLogger('swiftclient').setLevel(logging.CRITICAL)
+    # Create logging handlers
+    logging_handlers = [logging.StreamHandler()]
+    if args.error_log is not None:
+        logging_handlers.append(logging.FileHandler(args.error_log))
+
+    # Config Logging
+    logging.basicConfig(level=args.logging_level, handers=logging_handlers)
+    logging.getLogger("swiftclient").setLevel(logging.CRITICAL)
 
     username, password = drupalUtilities.get_drupal_credentials()