Merge pull request #104 from OpenMined/aziz/error_report

persist logs and add `uv run syftbox client report` command to export them

pyproject.toml

@@ -23,6 +23,7 @@ dependencies = [
     "httpx>=0.27.2",
     "pyyaml>=6.0.2",
     "psutil>=6.0.0",
+    "loguru>=0.7.2",
 ]
 
 [project.optional-dependencies]

syftbox/app/manager.py

@@ -4,6 +4,8 @@
 from collections import namedtuple
 from pathlib import Path
 
+from loguru import logger
+
 from ..lib import DEFAULT_CONFIG_PATH, ClientConfig
 from .install import install
 
@@ -19,28 +21,28 @@ def list_app(client_config: ClientConfig, silent: bool = False) -> list[str]:
 
     if len(apps):
         if not silent:
-            print("\nInstalled apps:")
+            logger.info("\nInstalled apps:")
             for app in apps:
-                print(f"✅ {app}")
+                logger.info(f"✅ {app}")
     else:
         if not silent:
-            print(
+            logger.info(
                 "\nYou have no apps installed.\n\n"
                 f"Try:\nsyftbox app install OpenMined/github_app_updater\n\nor copy an app to: {apps_path}"
             )
     return apps
 
 
 def uninstall_app(client_config: ClientConfig) -> None:
-    print("Uninstalling Apps")
+    logger.info("Uninstalling Apps")
 
 
 def update_app(client_config: ClientConfig) -> None:
-    print("Updating Apps")
+    logger.info("Updating Apps")
 
 
 def upgrade_app(client_config: ClientConfig) -> None:
-    print("Upgrading Apps")
+    logger.info("Upgrading Apps")
 
 
 Commands = namedtuple("Commands", ["description", "execute"])
@@ -105,6 +107,6 @@ def main(parser, args_list) -> None:
             # we should make this a type
             if isinstance(result, tuple):
                 step, exception = result
-                print(f"Error during {step}: ", str(exception))
+                logger.info(f"Error during {step}: ", str(exception))
     else:
         parser.print_help()

syftbox/client/client.py

@@ -5,9 +5,9 @@
 import os
 import sys
 import time
-import traceback
 import types
 from dataclasses import dataclass
+from datetime import datetime
 from functools import partial
 from pathlib import Path
 from typing import Any
@@ -24,19 +24,22 @@
 from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
+from loguru import logger
 from pydantic import BaseModel
 
 from syftbox.client.fsevents import (
     AnyFileSystemEventHandler,
     FileSystemEvent,
     FSWatchdog,
 )
+from syftbox.client.utils.error_reporting import make_error_report
 from syftbox.lib import (
     DEFAULT_CONFIG_PATH,
     ClientConfig,
     SharedState,
     load_or_create_config,
 )
+from syftbox.lib.logger import zip_logs
 
 
 class CustomFastAPI(FastAPI):
@@ -116,7 +119,7 @@ def load_plugins(client_config: ClientConfig) -> dict[str, Plugin]:
                     )
                     loaded_plugins[plugin_name] = plugin
                 except Exception as e:
-                    print(e)
+                    logger.info(e)
 
     return loaded_plugins
 
@@ -166,11 +169,10 @@ def run_plugin(plugin_name, *args, **kwargs):
         module = app.loaded_plugins[plugin_name].module
         module.run(app.shared_state, *args, **kwargs)
     except Exception as e:
-        traceback.print_exc()
-        print("error", e)
+        logger.exception(e)
 
 
-def start_plugin(app: FastAPI, plugin_name: str):
+def start_plugin(app: CustomFastAPI, plugin_name: str):
     if plugin_name not in app.loaded_plugins:
         raise HTTPException(
             status_code=400,
@@ -202,7 +204,7 @@ def start_plugin(app: FastAPI, plugin_name: str):
             }
             return {"message": f"Plugin {plugin_name} started successfully"}
         else:
-            print(f"Job {existing_job}, already added")
+            logger.info(f"Job {existing_job}, already added")
             return {"message": f"Plugin {plugin_name} already started"}
     except Exception as e:
         raise HTTPException(
@@ -228,6 +230,15 @@ def parse_args():
         default="https://syftbox.openmined.org",
         help="Server",
     )
+    subparsers = parser.add_subparsers(dest="command", help="Sub-command help")
+    start_parser = subparsers.add_parser("report", help="Generate an error report")
+    start_parser.add_argument(
+        "--path",
+        type=str,
+        help="Path to the error report file",
+        default=f"./syftbox_logs_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}",
+    )
+
     return parser.parse_args()
 
 
@@ -250,7 +261,7 @@ def sync_on_event(event: FileSystemEvent):
 @contextlib.asynccontextmanager
 async def lifespan(app: CustomFastAPI, client_config: ClientConfig | None = None):
     # Startup
-    print("> Starting Client")
+    logger.info("> Starting Client")
 
     # client_config needs to be closed if it was created in this context
     # if it is passed as lifespan arg (eg for testing) it should be managed by the caller instead.
@@ -266,7 +277,7 @@ async def lifespan(app: CustomFastAPI, client_config: ClientConfig | None = None
     app.job_file = job_file
     if os.path.exists(job_file):
         os.remove(job_file)
-        print(f"> Cleared existing job file: {job_file}")
+        logger.info(f"> Cleared existing job file: {job_file}")
 
     # Start the scheduler
     jobstores = {"default": SQLAlchemyJobStore(url=f"sqlite:///{job_file}")}
@@ -277,16 +288,16 @@ async def lifespan(app: CustomFastAPI, client_config: ClientConfig | None = None
     app.scheduler = scheduler
     app.running_plugins = {}
     app.loaded_plugins = load_plugins(client_config)
-    print("> Loaded plugins:", sorted(list(app.loaded_plugins.keys())))
+    logger.info("> Loaded plugins:", sorted(list(app.loaded_plugins.keys())))
     app.watchdog = start_watchdog(app)
 
-    print("> Starting autorun plugins:", sorted(client_config.autorun_plugins))
+    logger.info("> Starting autorun plugins:", sorted(client_config.autorun_plugins))
     for plugin in client_config.autorun_plugins:
         start_plugin(app, plugin)
 
     yield  # This yields control to run the application
 
-    print("> Shutting down...")
+    logger.info("> Shutting down...")
     scheduler.shutdown()
     app.watchdog.stop()
     if close_client_config:
@@ -297,7 +308,7 @@ def stop_scheduler(app: FastAPI):
     # Remove the lock file if it exists
     if os.path.exists(app.job_file):
         os.remove(app.job_file)
-        print("> Scheduler stopped and lock file removed.")
+        logger.info("> Scheduler stopped and lock file removed.")
 
 
 app: CustomFastAPI = FastAPI(lifespan=lifespan)
@@ -463,12 +474,22 @@ def get_syftbox_src_path():
 def main() -> None:
     args = parse_args()
     client_config = load_or_create_config(args)
+    error_config = make_error_report(client_config)
+
+    if args.command == "report":
+        output_path = Path(args.path).resolve()
+        output_path_with_extension = zip_logs(output_path)
+        logger.info(f"Logs saved to: {output_path_with_extension}.")
+        logger.info("Please share your bug report together with the zipped logs")
+        return
+
+    logger.info(f"Client metadata: {error_config.model_dump_json(indent=2)}")
 
     os.environ["SYFTBOX_DATASITE"] = client_config.email
     os.environ["SYFTBOX_CLIENT_CONFIG_PATH"] = client_config.config_path
 
-    print("Dev Mode: ", os.environ.get("SYFTBOX_DEV"))
-    print("Wheel: ", os.environ.get("SYFTBOX_WHEEL"))
+    logger.info("Dev Mode: ", os.environ.get("SYFTBOX_DEV"))
+    logger.info("Wheel: ", os.environ.get("SYFTBOX_WHEEL"))
 
     debug = True
     port = client_config.port
@@ -488,9 +509,9 @@ def main() -> None:
         except SystemExit as e:
             if e.code != 1:  # If it's not the "Address already in use" error
                 raise
-            print(f"Failed to start server on port {port}. Trying next port.")
+            logger.info(f"Failed to start server on port {port}. Trying next port.")
             port = 0
-    print(f"Unable to find an available port after {max_attempts} attempts.")
+    logger.info(f"Unable to find an available port after {max_attempts} attempts.")
     sys.exit(1)
 
 

syftbox/client/plugins/apps.py

@@ -44,10 +44,10 @@ def find_and_run_script(task_path, extra_args):
                 env=env,
             )
 
-            # print("✅ Script run.sh executed successfully.")
+            # logger.info("✅ Script run.sh executed successfully.")
             return result
         except Exception as e:
-            print("Error running shell script", e)
+            logger.info("Error running shell script", e)
     else:
         raise FileNotFoundError(f"run.sh not found in {task_path}")
 
@@ -64,7 +64,7 @@ def find_and_run_script(task_path, extra_args):
 
 # def copy_default_apps(apps_path):
 #     if not os.path.exists(DEFAULT_APPS_PATH):
-#         print(f"Default apps directory not found: {DEFAULT_APPS_PATH}")
+#         logger.info(f"Default apps directory not found: {DEFAULT_APPS_PATH}")
 #         return
 
 #     for app in os.listdir(DEFAULT_APPS_PATH):
@@ -73,11 +73,11 @@ def find_and_run_script(task_path, extra_args):
 
 #         if os.path.isdir(src_app_path):
 #             if os.path.exists(dst_app_path):
-#                 print(f"App already installed at: {dst_app_path}")
+#                 logger.info(f"App already installed at: {dst_app_path}")
 #                 # shutil.rmtree(dst_app_path)
 #             else:
 #                 shutil.copytree(src_app_path, dst_app_path)
-#             print(f"Copied default app: {app}")
+#             logger.info(f"Copied default app: {app}")
 
 
 def dict_to_namespace(data) -> SimpleNamespace | list | Any:
@@ -113,12 +113,13 @@ def run_apps(client_config):
     if os.path.exists(file_path):
         perm_file = SyftPermission.load(file_path)
     else:
-        print(f"> {client_config.email} Creating Apps Permfile")
+        logger.info(f"> {client_config.email} Creating Apps Permfile")
         try:
             perm_file = SyftPermission.datasite_default(client_config.email)
             perm_file.save(file_path)
         except Exception as e:
-            print("Failed to create perm file", e)
+            logger.error("Failed to create perm file")
+            logger.exception(e)
 
     apps = os.listdir(apps_path)
     for app in apps:
@@ -128,7 +129,7 @@ def run_apps(client_config):
             if app_config is None:
                 run_app(client_config, app_path)
             elif RUNNING_APPS.get(app, None) is None:
-                print("⏱  Scheduling a  new app run.")
+                logger.info("⏱  Scheduling a  new app run.")
                 thread = threading.Thread(
                     target=run_custom_app_config,
                     args=(client_config, app_config, app_path),
@@ -157,7 +158,7 @@ def run_custom_app_config(client_config, app_config, path):
 
     env.update(app_envs)
     while True:
-        print(f"👟 Running {app_name}")
+        logger.info(f"👟 Running {app_name}")
         _ = subprocess.run(
             app_config.app.run.command,
             cwd=path,
@@ -174,22 +175,22 @@ def run_app(client_config, path):
 
     extra_args = []
     try:
-        print(f"👟 Running {app_name} app", end="")
+        logger.info(f"👟 Running {app_name} app", end="")
         result = find_and_run_script(path, extra_args)
         if hasattr(result, "returncode"):
             if "Already generated" not in str(result.stdout):
-                print("\n")
-                print(result.stdout)
+                logger.info("\n")
+                logger.info(result.stdout)
             else:
-                print(" - no change")
+                logger.info(" - no change")
             exit_code = result.returncode
             if exit_code != 0:
-                print(f"Error running: {app_name}", result.stdout, result.stderr)
+                logger.info(f"Error running: {app_name}", result.stdout, result.stderr)
     except Exception as e:
-        print(f"Failed to run. {e}")
+        logger.info(f"Failed to run. {e}")
 
 
 def run(shared_state):
-    # print("> Running Apps")
+    # logger.info("> Running Apps")
     client_config = shared_state.client_config
     run_apps(client_config)

syftbox/client/plugins/create_datasite.py

@@ -18,25 +18,27 @@ def claim_datasite(client_config):
     if os.path.exists(file_path):
         perm_file = SyftPermission.load(file_path)
     else:
-        print(f"> {client_config.email} Creating Datasite + Permfile")
+        logger.info(f"> {client_config.email} Creating Datasite + Permfile")
         try:
             perm_file = SyftPermission.datasite_default(client_config.email)
             perm_file.save(file_path)
         except Exception as e:
-            print("Failed to create perm file", e)
+            logger.error("Failed to create perm file")
+            logger.exception(e)
 
     public_path = client_config.datasite_path + "/" + "public"
     os.makedirs(public_path, exist_ok=True)
     public_file_path = perm_file_path(public_path)
     if os.path.exists(public_file_path):
         public_perm_file = SyftPermission.load(public_file_path)
     else:
-        print(f"> {client_config.email} Creating Public Permfile")
+        logger.info(f"> {client_config.email} Creating Public Permfile")
         try:
             public_perm_file = SyftPermission.mine_with_public_read(client_config.email)
             public_perm_file.save(public_file_path)
         except Exception as e:
-            print("Failed to create perm file", e)
+            logger.error("Failed to create perm file")
+            logger.exception(e)
 
 
 def run(shared_state):

syftbox/client/plugins/init.py

@@ -1,5 +1,7 @@
 from threading import Event
 
+from loguru import logger
+
 stop_event = Event()
 
 
@@ -21,4 +23,4 @@ def run(shared_state):
     if not stop_event.is_set():
         if not shared_state.client_config.token:
             register(shared_state.client_config)
-            print("> Register Complete")
+            logger.info("> Register Complete")