Refactor source files builder

src/writer/app_runner.py

@@ -1,6 +1,5 @@
 import asyncio
 import concurrent.futures
-import glob
 import importlib.util
 import logging
 import logging.handlers
@@ -106,7 +105,6 @@ def __init__(self,
         self.app_path = app_path
         self.mode = mode
         self.run_code = run_code
-        self.source_files: SourceFilesDirectory = {"children": {}, "type": "directory"}
         self.bmc_components = bmc_components
         self.is_app_process_server_ready = is_app_process_server_ready
         self.is_app_process_server_failed = is_app_process_server_failed 
@@ -656,7 +654,7 @@ def _start_wf_project_process_write_files(self):
 
     def load(self) -> None:
         self.run_code = self.load_persisted_script("main.py")
-        self.source_files = self._build_source_files()
+        self.source_files = wf_project.build_source_files(self.app_path)
         self.bmc_components = self._load_persisted_components()
 
         if self.mode == "edit":
@@ -672,53 +670,6 @@ def load(self) -> None:
         # parent pid and pid.
         self._subscribe_terminal_signal()
 
-    def _build_source_files(self) -> SourceFilesDirectory:
-        """
-        Build a file tree as `Dict` wherein the key represent the filename. The value is a `Dict` with a `type` as:
-
-        - `directory`, so it's a directory containing `children`
-        - `file`, so it's a file with `content` as string. We limit the file to the first X characters and set `"complete": False` if content is truncated
-
-        Example:
-
-        >>> {'type': 'directory', 'children': {'README.md': {'type': 'file', 'content': 'This app w', 'complete': False}}}
-        """
-        files = glob.glob(os.path.join(self.app_path, '**', "*"), recursive=True)
-        file_tree: SourceFilesDirectory = {
-            "type": "directory",
-            "children": {}
-        }
-
-        for file in files:
-            relative_path = os.path.relpath(file, self.app_path)
-            parts = relative_path.split(os.sep)
-            current_level: SourceFilesDirectory = file_tree
-
-            for part in parts:
-                if os.path.isdir(os.path.join(self.app_path, *parts[:parts.index(part) + 1])):
-                    if part not in current_level["children"]:
-                        current_level["children"][part] = { "type": "directory", "children": {} }
-
-                    next_level = current_level["children"][part]
-
-                    if next_level["type"] == "directory":
-                        current_level = next_level
-                else:
-                    try:
-                        content = self.load_persisted_script(relative_path)
-                        # limit only the first 100 characters to limit bandwidth usage, the rest will be lazy loaded
-                        exerpt = content[0:100]
-                        current_level["children"][part] = {
-                            "type": "file",
-                            "content": exerpt,
-                            "complete": exerpt == content,
-                        }
-                    except UnicodeDecodeError:
-                        print(relative_path)
-                        pass
-
-        return file_tree
-
     async def dispatch_message(self, session_id: Optional[str], request: AppProcessServerRequest) -> AppProcessServerResponse:
 
         """
@@ -761,7 +712,7 @@ def create_persisted_script(self, file = "main.py"):
         with open(path, "x", encoding='utf-8') as f:
             f.write('')
 
-        self.source_files = self._build_source_files()
+        self.source_files = wf_project.build_source_files(self.app_path)
 
     def rename_persisted_script(self, from_path: str, to_path: str):
         if from_path == 'main.py':
@@ -777,7 +728,7 @@ def rename_persisted_script(self, from_path: str, to_path: str):
 
         os.renames(from_path_abs, to_path_abs)
 
-        self.source_files = self._build_source_files()
+        self.source_files = wf_project.build_source_files(self.app_path)
 
     def delete_persisted_script(self, file: str):
         if file == 'main.py':
@@ -788,7 +739,7 @@ def delete_persisted_script(self, file: str):
 
         os.remove(path)
 
-        self.source_files = self._build_source_files()
+        self.source_files = wf_project.build_source_files(self.app_path)
 
     def load_persisted_script(self, file = "main.py") -> str:
         path = os.path.join(self.app_path, file)
@@ -895,7 +846,7 @@ def save_code(self, session_id: str, code: str, path: List[str] = ['main.py']) -
         with open(filepath, "w") as f:
             f.write(code)
 
-        self.source_files = self._build_source_files()
+        self.source_files = wf_project.build_source_files(self.app_path)
 
     def _clean_process(self) -> None:
         # Terminate the AppProcess server by sending an empty message
@@ -983,7 +934,7 @@ def update_code(self, session_id: Optional[str], run_code: str) -> None:
         if not self.is_app_process_server_ready.is_set():
             return
         self.run_code = run_code
-        self.source_files = self._build_source_files()
+        self.source_files = wf_project.build_source_files(self.app_path)
         self._clean_process()
         self._start_app_process()
         self.is_app_process_server_ready.wait()

src/writer/wf_project.py

@@ -6,6 +6,7 @@
 >>> metadata, components = wf_project.read_files('app/hello')
 """
 import dataclasses
+import glob
 import io
 import json
 import logging
@@ -19,7 +20,7 @@
 from typing import Any, Dict, List, Tuple
 
 from writer import core_ui
-from writer.ss_types import ComponentDefinition, MetadataDefinition
+from writer.ss_types import ComponentDefinition, MetadataDefinition, SourceFilesDirectory
 
 ROOTS = ['root', 'workflows_root']
 COMPONENT_ROOTS = ['page', 'workflows_workflow']
@@ -321,3 +322,55 @@ def can_create_project(path: str) -> bool:
         return True
 
     return False
+
+
+def build_source_files(app_path: str) -> SourceFilesDirectory:
+    """
+    Build a file tree as `Dict` wherein the key represent the filename. The value is a `Dict` with a `type` as:
+
+    - `directory`, so it's a directory containing `children`
+    - `file`, so it's a file with `content` as string. We limit the file to the first X characters and set `"complete": False` if content is truncated
+
+    Example:
+
+    >>> {'type': 'directory', 'children': {'README.md': {'type': 'file', 'content': 'This app w', 'complete': False}}}
+    """
+    files = glob.glob(os.path.join(app_path, '**', "*"), recursive=True)
+    file_tree: SourceFilesDirectory = {
+        "type": "directory",
+        "children": {}
+    }
+
+    def load_persisted_script(file: str) -> str:
+        path = os.path.join(app_path, file)
+        with open(path, "r", encoding='utf-8') as f:
+            return f.read()
+
+    for file in files:
+        relative_path = os.path.relpath(file, app_path)
+        parts = relative_path.split(os.sep)
+        current_level: SourceFilesDirectory = file_tree
+
+        for part in parts:
+            if os.path.isdir(os.path.join(app_path, *parts[:parts.index(part) + 1])):
+                if part not in current_level["children"]:
+                    current_level["children"][part] = { "type": "directory", "children": {} }
+
+                next_level = current_level["children"][part]
+
+                if next_level["type"] == "directory":
+                    current_level = next_level
+            else:
+                try:
+                    content = load_persisted_script(relative_path)
+                    # limit only the first 100 characters to limit bandwidth usage, the rest will be lazy loaded
+                    exerpt = content[0:100]
+                    current_level["children"][part] = {
+                        "type": "file",
+                        "content": exerpt,
+                        "complete": exerpt == content,
+                    }
+                except (UnicodeDecodeError, FileNotFoundError):
+                    pass
+
+    return file_tree