Merge remote-tracking branch 'origin/dev' into master

pyro/Application.py

@@ -30,7 +30,7 @@ def __init__(self, parser: argparse.ArgumentParser) -> None:
 
         self.args.input_path = self._try_fix_input_path(self.args.input_path or self.args.input_path_deprecated)
 
-        if not os.path.isfile(self.args.input_path):
+        if not self.args.create_project and not os.path.isfile(self.args.input_path):
             Application.log.error(f'Cannot load nonexistent PPJ at given path: "{self.args.input_path}"')
             sys.exit(1)
 
@@ -149,37 +149,41 @@ def run(self) -> int:
         build.try_compile()
 
         if ppj.options.anonymize:
-            if build.failed_count == 0 or ppj.options.ignore_errors:
+            if build.compile_data.failed_count == 0 or ppj.options.ignore_errors:
                 build.try_anonymize()
             else:
-                Application.log.error(f'Cannot anonymize scripts because {build.failed_count} scripts failed to compile')
-                sys.exit(build.failed_count)
+                Application.log.error(f'Cannot anonymize scripts because {build.compile_data.failed_count} scripts failed to compile')
+                sys.exit(build.compile_data.failed_count)
         else:
             Application.log.info('Cannot anonymize scripts because Anonymize is disabled in project')
 
         if ppj.options.package:
-            if build.failed_count == 0 or ppj.options.ignore_errors:
+            if build.compile_data.failed_count == 0 or ppj.options.ignore_errors:
                 build.try_pack()
             else:
-                Application.log.error(f'Cannot create Packages because {build.failed_count} scripts failed to compile')
-                sys.exit(build.failed_count)
+                Application.log.error(f'Cannot create Packages because {build.compile_data.failed_count} scripts failed to compile')
+                sys.exit(build.compile_data.failed_count)
         else:
             Application.log.info('Cannot create Packages because Package is disabled in project')
 
         if ppj.options.zip:
-            if build.failed_count == 0 or ppj.options.ignore_errors:
+            if build.compile_data.failed_count == 0 or ppj.options.ignore_errors:
                 build.try_zip()
             else:
-                Application.log.error(f'Cannot create ZipFile because {build.failed_count} scripts failed to compile')
-                sys.exit(build.failed_count)
+                Application.log.error(f'Cannot create ZipFile because {build.compile_data.failed_count} scripts failed to compile')
+                sys.exit(build.compile_data.failed_count)
         else:
             Application.log.info('Cannot create ZipFile because Zip is disabled in project')
 
-        Application.log.info(build.build_time if build.success_count > 0 else 'No scripts were compiled.')
+        Application.log.info(build.compile_data.to_string() if build.compile_data.success_count > 0 else 'No scripts were compiled.')
+
+        Application.log.info(build.package_data.to_string() if build.package_data.file_count > 0 else 'No files were packaged.')
+
+        Application.log.info(build.zipping_data.to_string() if build.zipping_data.file_count > 0 else 'No files were zipped.')
 
         Application.log.info('DONE!')
 
-        if ppj.use_post_build_event and build.failed_count == 0:
+        if ppj.use_post_build_event and build.compile_data.failed_count == 0:
             ppj.try_run_event(BuildEvent.POST)
 
-        return build.failed_count
+        return build.compile_data.failed_count

pyro/BuildFacade.py

@@ -11,39 +11,25 @@
 from pyro.PackageManager import PackageManager
 from pyro.PapyrusProject import PapyrusProject
 from pyro.PathHelper import PathHelper
+from pyro.Performance.CompileData import CompileData
+from pyro.Performance.PackageData import PackageData
+from pyro.Performance.ZippingData import ZippingData
 from pyro.PexReader import PexReader
 from pyro.ProcessManager import ProcessManager
 from pyro.Enums.ProcessState import ProcessState
-from pyro.TimeElapsed import TimeElapsed
 
-from pyro.Comparators import endswith, startswith
+from pyro.Comparators import (endswith,
+                              startswith)
 
 
 class BuildFacade:
     log: logging.Logger = logging.getLogger('pyro')
 
     ppj: PapyrusProject = None
 
-    time_elapsed: TimeElapsed = TimeElapsed()
-
-    scripts_count: int = 0
-    success_count: int = 0
-    command_count: int = 0
-
-    @property
-    def failed_count(self) -> int:
-        return self.command_count - self.success_count
-
-    @property
-    def build_time(self) -> str:
-        raw_time, avg_time = ('{0:.3f}s'.format(t)
-                              for t in (self.time_elapsed.value(), self.time_elapsed.average(self.success_count)))
-
-        return f'Compilation time: ' \
-               f'{raw_time} ({avg_time}/script) - ' \
-               f'{self.success_count} succeeded, ' \
-               f'{self.failed_count} failed ' \
-               f'({self.scripts_count} scripts)'
+    compile_data: CompileData = CompileData()
+    package_data: PackageData = PackageData()
+    zipping_data: ZippingData = ZippingData()
 
     def __init__(self, ppj: PapyrusProject) -> None:
         self.ppj = ppj
@@ -56,7 +42,7 @@ def __init__(self, ppj: PapyrusProject) -> None:
         for key in options:
             if key in ('args', 'input_path', 'anonymize', 'package', 'zip', 'zip_compression'):
                 continue
-            if startswith(key, ('ignore_', 'no_', 'force_', 'resolve_'), ignorecase=True):
+            if startswith(key, ('ignore_', 'no_', 'force_', 'create_', 'resolve_'), ignorecase=True):
                 continue
             if endswith(key, '_token', ignorecase=True):
                 continue
@@ -102,26 +88,26 @@ def try_compile(self) -> None:
         """Builds and passes commands to Papyrus Compiler"""
         commands: list = self.ppj.build_commands()
 
-        self.command_count = len(commands)
+        self.compile_data.command_count = len(commands)
 
-        self.time_elapsed.start_time = time.time()
+        self.compile_data.time.start_time = time.time()
 
-        if self.ppj.options.no_parallel or self.command_count == 1:
+        if self.ppj.options.no_parallel or self.compile_data.command_count == 1:
             for command in commands:
                 if ProcessManager.run_compiler(command) == ProcessState.SUCCESS:
-                    self.success_count += 1
-        elif self.command_count > 0:
+                    self.compile_data.success_count += 1
+        elif self.compile_data.command_count > 0:
             multiprocessing.freeze_support()
-            worker_limit = min(self.command_count, self.ppj.options.worker_limit)
+            worker_limit = min(self.compile_data.command_count, self.ppj.options.worker_limit)
             with multiprocessing.Pool(processes=worker_limit,
                                       initializer=BuildFacade._limit_priority) as pool:
                 for state in pool.imap(ProcessManager.run_compiler, commands):
                     if state == ProcessState.SUCCESS:
-                        self.success_count += 1
+                        self.compile_data.success_count += 1
                 pool.close()
                 pool.join()
 
-        self.time_elapsed.end_time = time.time()
+        self.compile_data.time.end_time = time.time()
 
     def try_anonymize(self) -> None:
         """Obfuscates identifying metadata in compiled scripts"""
@@ -140,10 +126,16 @@ def try_anonymize(self) -> None:
 
     def try_pack(self) -> None:
         """Generates BSA/BA2 packages for project"""
+        self.package_data.time.start_time = time.time()
         package_manager = PackageManager(self.ppj)
         package_manager.create_packages()
+        self.package_data.time.end_time = time.time()
+        self.package_data.file_count = package_manager.includes
 
     def try_zip(self) -> None:
         """Generates ZIP file for project"""
+        self.zipping_data.time.start_time = time.time()
         package_manager = PackageManager(self.ppj)
         package_manager.create_zip()
+        self.zipping_data.time.end_time = time.time()
+        self.zipping_data.file_count = package_manager.includes

pyro/Constants.py

@@ -1,5 +1,3 @@
-from dataclasses import dataclass
-
 from pyro.Constant import Constant
 
 
@@ -21,8 +19,7 @@ class GameType(Constant):
     TES5: str = 'tes5'
 
 
-@dataclass
-class XmlAttributeName:
+class XmlAttributeName(Constant):
     ANONYMIZE: str = 'Anonymize'
     COMPRESSION: str = 'Compression'
     DESCRIPTION: str = 'Description'
@@ -44,8 +41,7 @@ class XmlAttributeName:
     ZIP: str = 'Zip'
 
 
-@dataclass
-class XmlTagName:
+class XmlTagName(Constant):
     FOLDER: str = 'Folder'
     FOLDERS: str = 'Folders'
     IMPORT: str = 'Import'

pyro/PackageManager.py

@@ -36,13 +36,33 @@ class PackageManager:
     DEFAULT_GLFLAGS = glob.NODIR | glob.MATCHBASE | glob.SPLIT | glob.REALPATH | glob.FOLLOW | glob.IGNORECASE | glob.MINUSNEGATE
     DEFAULT_WCFLAGS = wcmatch.SYMLINKS | wcmatch.IGNORECASE | wcmatch.MINUSNEGATE
 
+    includes: int = 0
+
     def __init__(self, ppj: PapyrusProject) -> None:
         self.ppj = ppj
         self.options = ppj.options
 
         self.pak_extension = '.ba2' if self.options.game_type == GameType.FO4 else '.bsa'
         self.zip_extension = '.zip'
 
+    @staticmethod
+    def _can_compress_package(containing_folder: str):
+        flags = wcmatch.RECURSIVE | wcmatch.IGNORECASE
+
+        # voices bad because bethesda no likey
+        for _ in wcmatch.WcMatch(containing_folder, '*.fuz', flags=flags).imatch():
+            return False
+
+        # sounds bad because bethesda no likey
+        for _ in wcmatch.WcMatch(containing_folder, '*.wav|*.xwm', flags=flags).imatch():
+            return False
+
+        # strings bad because wrye bash no likey
+        for _ in wcmatch.WcMatch(containing_folder, '*.*strings', flags=flags).imatch():
+            return False
+
+        return True
+
     @staticmethod
     def _check_write_permission(file_path: str) -> None:
         if os.path.isfile(file_path):
@@ -129,6 +149,7 @@ def _generate_include_paths(includes_node: etree.ElementBase, root_path: str, zi
             attr_in: str = match_node.get(XmlAttributeName.IN).strip()
             attr_no_recurse: bool = match_node.get(XmlAttributeName.NO_RECURSE) == 'True'
             attr_exclude: str = match_node.get(XmlAttributeName.EXCLUDE).strip()
+            attr_path: str = match_node.get(XmlAttributeName.PATH).strip()
 
             in_path: str = os.path.normpath(attr_in)
 
@@ -155,6 +176,7 @@ def _generate_include_paths(includes_node: etree.ElementBase, root_path: str, zi
 
             yield from PackageManager._match(in_path, match_text,
                                              exclude_pattern=attr_exclude,
+                                             user_path=attr_path,
                                              no_recurse=attr_no_recurse)
 
     def _fix_package_extension(self, package_name: str) -> str:
@@ -186,20 +208,33 @@ def build_commands(self, containing_folder: str, output_path: str) -> str:
         arguments.append(containing_folder, enquote_value=True)
         arguments.append(output_path, enquote_value=True)
 
+        compressed_package = PackageManager._can_compress_package(containing_folder)
+
+        flags = wcmatch.RECURSIVE | wcmatch.IGNORECASE
+
         if self.options.game_type == GameType.FO4:
-            arguments.append('-fo4')
+            for _ in wcmatch.WcMatch(containing_folder, '!*.dds', flags=flags).imatch():
+                arguments.append('-fo4')
+                break
+            else:
+                arguments.append('-fo4dds')
         elif self.options.game_type == GameType.SSE:
             arguments.append('-sse')
 
-            # SSE has an ctd bug with uncompressed textures in a bsa that
-            # has an Embed Filenames flag on it, so force it to false.
-            for _ in wcmatch.WcMatch(containing_folder, '*.dds',
-                                     flags=wcmatch.RECURSIVE | wcmatch.IGNORECASE).imatch():
-                arguments.append('-af:0x3')
-                break
+            if not compressed_package:
+                # SSE crashes when uncompressed BSA has Embed Filenames flag and contains textures
+                for _ in wcmatch.WcMatch(containing_folder, '*.dds', flags=flags).imatch():
+                    arguments.append('-af:0x3')
+                    break
         else:
             arguments.append('-tes5')
 
+        # binary identical files share same data to preserve space
+        arguments.append('-share')
+
+        if compressed_package:
+            arguments.append('-z')
+
         return arguments.join()
 
     def create_packages(self) -> None:
@@ -256,6 +291,8 @@ def create_packages(self) -> None:
                 os.makedirs(os.path.dirname(target_path), exist_ok=True)
                 shutil.copy2(source_path, target_path)
 
+                self.includes += 1
+
             # run bsarch
             command: str = self.build_commands(self.options.temp_path, file_path)
             ProcessManager.run_bsarch(command)
@@ -318,6 +355,8 @@ def create_zip(self) -> None:
                             PackageManager.log.info('+ "{}"'.format(arcname))
                             z.write(include_path, arcname, compress_type=compress_type.value)
 
+                            self.includes += 1
+
                     PackageManager.log.info(f'Wrote ZIP file: "{file_path}"')
                 except PermissionError:
                     PackageManager.log.error(f'Cannot open ZIP file for writing: "{file_path}"')

pyro/PapyrusProject.py

@@ -60,6 +60,9 @@ def __init__(self, options: ProjectOptions) -> None:
 
         xml_parser: etree.XMLParser = etree.XMLParser(remove_blank_text=True, remove_comments=True)
 
+        if self.options.create_project:
+            sys.exit(1)
+
         # strip comments from raw text because lxml.etree.XMLParser does not remove XML-unsupported comments
         # e.g., '<PapyrusProject <!-- xmlns="PapyrusProject.xsd" -->>'
         xml_document: io.StringIO = XmlHelper.strip_xml_comments(self.options.input_path)
@@ -88,7 +91,7 @@ def __init__(self, options: ProjectOptions) -> None:
         # options can be overridden by arguments when the BuildFacade is initialized
         self._update_attributes(self.ppj_root.node)
 
-        if self.options.resolve_ppj:
+        if self.options.resolve_project:
             xml_output = etree.tostring(self.ppj_root.node, encoding='utf-8', xml_declaration=True, pretty_print=True)
             PapyrusProject.log.debug(f'Resolved PPJ. Text output:{os.linesep * 2}{xml_output.decode()}')
             sys.exit(1)
@@ -330,10 +333,10 @@ def _update_attributes(self, parent_node: etree.ElementBase) -> None:
             elif tag in (XmlTagName.FOLDER, XmlTagName.INCLUDE, XmlTagName.MATCH):
                 if XmlAttributeName.NO_RECURSE not in node.attrib:
                     node.set(XmlAttributeName.NO_RECURSE, 'False')
-                if tag == XmlTagName.INCLUDE:
+                if tag in (XmlTagName.INCLUDE, XmlTagName.MATCH):
                     if XmlAttributeName.PATH not in node.attrib:
                         node.set(XmlAttributeName.PATH, '')
-                elif tag == XmlTagName.MATCH:
+                if tag == XmlTagName.MATCH:
                     if XmlAttributeName.IN not in node.attrib:
                         node.set(XmlAttributeName.IN, os.curdir)
                     if XmlAttributeName.EXCLUDE not in node.attrib:

pyro/PapyrusProject.xsd

@@ -103,6 +103,7 @@
             <xs:extension base="recursablePath">
                 <xs:attribute name="In" type="xs:string" default=""/>
                 <xs:attribute name="Exclude" type="xs:string" default=""/>
+                <xs:attribute name="Path" type="xs:string" default=""/>
             </xs:extension>
         </xs:complexContent>
     </xs:complexType>

pyro/Performance/CompileData.py

@@ -0,0 +1,29 @@
+from dataclasses import (dataclass,
+                         field)
+
+from pyro.TimeElapsed import TimeElapsed
+
+
+@dataclass
+class CompileData:
+    time: TimeElapsed = field(init=False, default_factory=TimeElapsed)
+    scripts_count: int = field(init=False, default_factory=int)
+    success_count: int = field(init=False, default_factory=int)
+    command_count: int = field(init=False, default_factory=int)
+
+    def __post_init__(self):
+        self.time = TimeElapsed()
+
+    @property
+    def failed_count(self) -> int:
+        return self.command_count - self.success_count
+
+    def to_string(self):
+        raw_time, avg_time = ('{0:.3f}s'.format(t)
+                              for t in (self.time.value(), self.time.average(self.success_count)))
+
+        return f'Compile time: ' \
+               f'{raw_time} ({avg_time}/script) - ' \
+               f'{self.success_count} succeeded, ' \
+               f'{self.failed_count} failed ' \
+               f'({self.scripts_count} scripts)'

pyro/Performance/PackageData.py

@@ -0,0 +1,20 @@
+from dataclasses import (dataclass,
+                         field)
+
+from pyro.TimeElapsed import TimeElapsed
+
+
+@dataclass
+class PackageData:
+    time: TimeElapsed = field(init=False, default_factory=TimeElapsed)
+    file_count: int = field(init=False, default_factory=int)
+
+    def __post_init__(self):
+        self.time = TimeElapsed()
+
+    def to_string(self):
+        raw_time, avg_time = ('{0:.3f}s'.format(t)
+                              for t in (self.time.value(), self.time.average(self.file_count)))
+
+        return f'Package time: ' \
+               f'{raw_time} ({avg_time}/file, {self.file_count} files)'