From 3f4cef71e3ae9c0ed8775bfc9e671083bb571ea5 Mon Sep 17 00:00:00 2001 From: fireundubh Date: Sun, 13 Jun 2021 03:23:36 -0700 Subject: [PATCH 1/3] Implemented non-glob wildcard Match node Allowed files outside project root to be included in ZIP archives Reduced time to fail by reordering some project validations before remote downloading Lowercased log output for packaged files to match case in BSA/BA2 packages Fixed issue where some absolute paths in Include nodes were not handled correctly Fixed issue where some relative paths in Include nodes were not handled correctly Fixed issue where some info and error log messages were warnings Fixed issue where some errors did not cause application to exit with nonzero code Updated XSD --- pyro/Anonymizer.py | 20 ++--- pyro/Application.py | 56 ++++++++------ pyro/BuildFacade.py | 8 +- pyro/Comparators.py | 4 + pyro/Constants.py | 3 + pyro/PackageManager.py | 165 ++++++++++++++++++++++++---------------- pyro/PapyrusProject.py | 45 +++++++---- pyro/PapyrusProject.xsd | 25 ++++-- pyro/PathHelper.py | 21 ++--- pyro/ProjectBase.py | 12 +-- requirements.txt | 1 + 11 files changed, 219 insertions(+), 141 deletions(-) diff --git a/pyro/Anonymizer.py b/pyro/Anonymizer.py index 18685a03..259899b7 100644 --- a/pyro/Anonymizer.py +++ b/pyro/Anonymizer.py @@ -2,12 +2,14 @@ import os import random import string +import sys from pyro.PexHeader import PexHeader from pyro.PexReader import PexReader from pyro.Comparators import endswith + class Anonymizer: log: logging.Logger = logging.getLogger('pyro') @@ -25,7 +27,7 @@ def anonymize_script(path: str) -> None: header: PexHeader = PexReader.get_header(path) except ValueError: Anonymizer.log.error(f'Cannot anonymize script due to unknown file magic: "{path}"') - return + sys.exit(1) file_path: str = header.script_path.value user_name: str = header.user_name.value @@ -36,20 +38,20 @@ def anonymize_script(path: str) -> None: return if not endswith(file_path, '.psc', ignorecase=True): - Anonymizer.log.warning(f'Cannot anonymize script due to invalid file extension: "{path}"') - return + Anonymizer.log.error(f'Cannot anonymize script due to invalid file extension: "{path}"') + sys.exit(1) if not len(file_path) > 0: - Anonymizer.log.warning(f'Cannot anonymize script due to zero-length file path: "{path}"') - return + Anonymizer.log.error(f'Cannot anonymize script due to zero-length file path: "{path}"') + sys.exit(1) if not len(user_name) > 0: - Anonymizer.log.warning(f'Cannot anonymize script due to zero-length user name: "{path}"') - return + Anonymizer.log.error(f'Cannot anonymize script due to zero-length user name: "{path}"') + sys.exit(1) if not len(computer_name) > 0: - Anonymizer.log.warning(f'Cannot anonymize script due to zero-length computer name: "{path}"') - return + Anonymizer.log.error(f'Cannot anonymize script due to zero-length computer name: "{path}"') + sys.exit(1) with open(path, mode='r+b') as f: f.seek(header.script_path.offset, os.SEEK_SET) diff --git a/pyro/Application.py b/pyro/Application.py index 1f73eb66..b56a4b88 100644 --- a/pyro/Application.py +++ b/pyro/Application.py @@ -46,30 +46,16 @@ def _try_fix_input_path(input_path: str) -> str: if not os.path.isabs(input_path): cwd = os.getcwd() - Application.log.warning(f'Using working directory: "{cwd}"') + Application.log.info(f'Using working directory: "{cwd}"') input_path = os.path.join(cwd, input_path) - Application.log.warning(f'Using input path: "{input_path}"') + Application.log.info(f'Using input path: "{input_path}"') return input_path @staticmethod - def _validate_project(ppj: PapyrusProject) -> None: - compiler_path = ppj.get_compiler_path() - if not compiler_path or not os.path.isfile(compiler_path): - Application.log.error('Cannot proceed without compiler path') - sys.exit(1) - - flags_path = ppj.get_flags_path() - if not flags_path: - Application.log.error('Cannot proceed without flags path') - sys.exit(1) - - if not ppj.options.game_type: - Application.log.error('Cannot determine game type from arguments or Papyrus Project') - sys.exit(1) - + def _validate_project_file(ppj: PapyrusProject): if not ppj.has_imports_node: Application.log.error('Cannot proceed without imports defined in project') sys.exit(1) @@ -86,6 +72,22 @@ def _validate_project(ppj: PapyrusProject) -> None: Application.log.error('Cannot proceed with Zip enabled without ZipFile defined in project') sys.exit(1) + @staticmethod + def _validate_project_paths(ppj: PapyrusProject) -> None: + compiler_path = ppj.get_compiler_path() + if not compiler_path or not os.path.isfile(compiler_path): + Application.log.error('Cannot proceed without compiler path') + sys.exit(1) + + flags_path = ppj.get_flags_path() + if not flags_path: + Application.log.error('Cannot proceed without flags path') + sys.exit(1) + + if not ppj.options.game_type: + Application.log.error('Cannot determine game type from arguments or Papyrus Project') + sys.exit(1) + if not os.path.isabs(flags_path) and \ not any([os.path.isfile(os.path.join(import_path, flags_path)) for import_path in ppj.import_paths]): Application.log.error('Cannot proceed without flags file in any import folder') @@ -107,6 +109,9 @@ def run(self) -> int: options = ProjectOptions(self.args.__dict__) ppj = PapyrusProject(options) + + self._validate_project_file(ppj) + ppj.try_initialize_remotes() ppj.try_import_event(ImportEvent.PRE) @@ -117,7 +122,7 @@ def run(self) -> int: ppj.find_missing_scripts() ppj.try_set_game_path() - self._validate_project(ppj) + self._validate_project_paths(ppj) Application.log.info('Imports found:') for path in ppj.import_paths: @@ -142,25 +147,28 @@ def run(self) -> int: if build.failed_count == 0 or ppj.options.ignore_errors: build.try_anonymize() else: - Application.log.warning(f'Cannot anonymize scripts because {build.failed_count} scripts failed to compile') + Application.log.error(f'Cannot anonymize scripts because {build.failed_count} scripts failed to compile') + sys.exit(build.failed_count) else: - Application.log.warning('Cannot anonymize scripts because Anonymize is disabled in project') + Application.log.info('Cannot anonymize scripts because Anonymize is disabled in project') if ppj.options.package: if build.failed_count == 0 or ppj.options.ignore_errors: build.try_pack() else: - Application.log.warning(f'Cannot create Packages because {build.failed_count} scripts failed to compile') + Application.log.error(f'Cannot create Packages because {build.failed_count} scripts failed to compile') + sys.exit(build.failed_count) else: - Application.log.warning('Cannot create Packages because Package is disabled in project') + Application.log.info('Cannot create Packages because Package is disabled in project') if ppj.options.zip: if build.failed_count == 0 or ppj.options.ignore_errors: build.try_zip() else: - Application.log.warning(f'Cannot create ZipFile because {build.failed_count} scripts failed to compile') + Application.log.error(f'Cannot create ZipFile because {build.failed_count} scripts failed to compile') + sys.exit(build.failed_count) else: - Application.log.warning('Cannot create ZipFile because Zip is disabled in project') + Application.log.info('Cannot create ZipFile because Zip is disabled in project') Application.log.info(build.build_time if build.success_count > 0 else 'No scripts were compiled.') diff --git a/pyro/BuildFacade.py b/pyro/BuildFacade.py index 1dd6812a..638d1708 100644 --- a/pyro/BuildFacade.py +++ b/pyro/BuildFacade.py @@ -80,8 +80,8 @@ def _find_modified_scripts(self) -> list: try: header = PexReader.get_header(pex_path) except ValueError: - BuildFacade.log.warning(f'Cannot determine compilation time due to unknown magic: "{pex_path}"') - continue + BuildFacade.log.error(f'Cannot determine compilation time due to unknown magic: "{pex_path}"') + sys.exit(1) psc_last_modified: float = os.path.getmtime(script_path) pex_last_compiled: float = float(header.compilation_time.value) @@ -132,8 +132,8 @@ def try_anonymize(self) -> None: # these are absolute paths. there's no reason to manipulate them. for pex_path in self.ppj.pex_paths: if not os.path.isfile(pex_path): - BuildFacade.log.warning(f'Cannot locate file to anonymize: "{pex_path}"') - continue + BuildFacade.log.error(f'Cannot locate file to anonymize: "{pex_path}"') + sys.exit(1) Anonymizer.anonymize_script(pex_path) diff --git a/pyro/Comparators.py b/pyro/Comparators.py index 5b35ae92..6e145f6a 100644 --- a/pyro/Comparators.py +++ b/pyro/Comparators.py @@ -60,6 +60,10 @@ def is_include_node(node: etree.ElementBase) -> bool: return node is not None and endswith(node.tag, 'Include') and node.text is not None +def is_match_node(node: etree.ElementBase) -> bool: + return node is not None and endswith(node.tag, 'Match') and node.text is not None + + def is_package_node(node: etree.ElementBase) -> bool: return node is not None and endswith(node.tag, 'Package') diff --git a/pyro/Constants.py b/pyro/Constants.py index 85d9bdfd..abe62c71 100644 --- a/pyro/Constants.py +++ b/pyro/Constants.py @@ -26,9 +26,11 @@ class XmlAttributeName: ANONYMIZE: str = 'Anonymize' COMPRESSION: str = 'Compression' DESCRIPTION: str = 'Description' + EXCLUDE: str = 'Exclude' FINAL: str = 'Final' FLAGS: str = 'Flags' GAME: str = 'Game' + IN: str = 'In' NAME: str = 'Name' NO_RECURSE: str = 'NoRecurse' OPTIMIZE: str = 'Optimize' @@ -49,6 +51,7 @@ class XmlTagName: IMPORT: str = 'Import' IMPORTS: str = 'Imports' INCLUDE: str = 'Include' + MATCH: str = 'Match' PACKAGE: str = 'Package' PACKAGES: str = 'Packages' PAPYRUS_PROJECT: str = 'PapyrusProject' diff --git a/pyro/PackageManager.py b/pyro/PackageManager.py index 77a35caf..c5ad276a 100644 --- a/pyro/PackageManager.py +++ b/pyro/PackageManager.py @@ -1,5 +1,3 @@ -import fnmatch -import glob import logging import os import shutil @@ -8,10 +6,13 @@ import zipfile from lxml import etree +from wcmatch import (glob, + wcmatch) from pyro.CommandArguments import CommandArguments from pyro.Comparators import (endswith, is_include_node, + is_match_node, is_package_node, is_zipfile_node, startswith) @@ -20,7 +21,6 @@ XmlAttributeName) from pyro.Enums.ZipCompression import ZipCompression from pyro.PapyrusProject import PapyrusProject -from pyro.PathHelper import PathHelper from pyro.ProcessManager import ProcessManager from pyro.ProjectOptions import ProjectOptions @@ -33,6 +33,9 @@ class PackageManager: pak_extension: str = '' zip_extension: str = '' + DEFAULT_GLFLAGS = glob.NODIR | glob.MATCHBASE | glob.SPLIT | glob.REALPATH | glob.GLOBSTAR | glob.FOLLOW | glob.IGNORECASE | glob.MINUSNEGATE + DEFAULT_WCFLAGS = wcmatch.SYMLINKS | wcmatch.IGNORECASE | wcmatch.MINUSNEGATE + def __init__(self, ppj: PapyrusProject) -> None: self.ppj = ppj self.options = ppj.options @@ -50,59 +53,96 @@ def _check_write_permission(file_path: str) -> None: sys.exit(1) @staticmethod - def _generate_include_paths(includes_node: etree.ElementBase, root_path: str) -> typing.Generator: + def _generate_include_paths(includes_node: etree.ElementBase, root_path: str, zip_mode: bool = False) -> typing.Generator: for include_node in filter(is_include_node, includes_node): - no_recurse: bool = include_node.get(XmlAttributeName.NO_RECURSE) == 'True' - wildcard_pattern: str = '*' if no_recurse else r'**\*' + attr_no_recurse: bool = include_node.get(XmlAttributeName.NO_RECURSE) == 'True' + attr_path: str = (include_node.get(XmlAttributeName.PATH) or '').strip() + search_path: str = include_node.text.strip() - user_path: str = (include_node.get(XmlAttributeName.PATH) or '').strip() + if not search_path: + PackageManager.log.error(f'Include path at line {include_node.sourceline} in project file is empty') + sys.exit(1) - if startswith(include_node.text, os.pardir): - PackageManager.log.warning(f'Include paths cannot start with "{os.pardir}"') - continue + if not zip_mode and startswith(search_path, os.pardir): + PackageManager.log.error(f'Include paths cannot start with "{os.pardir}"') + sys.exit(1) - if startswith(include_node.text, os.curdir): - include_node.text = include_node.text.replace(os.curdir, root_path, 1) + if startswith(search_path, os.curdir): + search_path = search_path.replace(os.curdir, root_path, 1) - # normalize path - path_or_pattern = os.path.normpath(include_node.text) + # fix invalid pattern with leading separator + if not zip_mode and startswith(search_path, (os.path.sep, os.path.altsep)): + search_path = '**' + search_path - # populate files list using simple glob patterns - if '*' in path_or_pattern: - if not os.path.isabs(path_or_pattern): - search_path = os.path.join(root_path, wildcard_pattern) - elif root_path in path_or_pattern: - search_path = path_or_pattern - else: - PackageManager.log.warning(f'Cannot include path outside RootDir: "{path_or_pattern}"') - continue - - for include_path in glob.iglob(search_path, recursive=not no_recurse): - if os.path.isfile(include_path) and fnmatch.fnmatch(include_path, path_or_pattern): - yield include_path, user_path + # populate files list using glob patterns or relative paths + if '*' in search_path or not os.path.isabs(search_path): + for include_path in glob.iglob(search_path, + root_dir=root_path, + flags=PackageManager.DEFAULT_GLFLAGS): + yield include_path, attr_path # populate files list using absolute paths - elif os.path.isabs(path_or_pattern): - if root_path not in path_or_pattern: - PackageManager.log.warning(f'Cannot include path outside RootDir: "{path_or_pattern}"') - continue + else: + if not zip_mode and root_path not in search_path: + PackageManager.log.error(f'Cannot include path outside RootDir: "{search_path}"') + sys.exit(1) - if os.path.isfile(path_or_pattern): - yield path_or_pattern, user_path + search_path = os.path.abspath(os.path.normpath(search_path)) + + if os.path.isfile(search_path): + yield search_path, attr_path else: - search_path = os.path.join(path_or_pattern, wildcard_pattern) - yield from PathHelper.find_include_paths(search_path, no_recurse, user_path) + user_flags = wcmatch.RECURSIVE if not attr_no_recurse else 0x0 - else: - # populate files list using relative file path - test_path = os.path.join(root_path, path_or_pattern) - if not os.path.isdir(test_path): - yield test_path, user_path + matcher = wcmatch.WcMatch(search_path, '*.*', + flags=PackageManager.DEFAULT_WCFLAGS | user_flags) - # populate files list using relative folder path - else: - search_path = os.path.join(root_path, path_or_pattern, wildcard_pattern) - yield from PathHelper.find_include_paths(search_path, no_recurse, user_path) + matcher.on_reset() + matcher._skipped = 0 + for f in matcher._walk(): + yield f, attr_path + + for match_node in filter(is_match_node, includes_node): + attr_in: str = match_node.get(XmlAttributeName.IN).strip() + attr_no_recurse: bool = match_node.get(XmlAttributeName.NO_RECURSE) == 'True' + attr_exclude: str = (match_node.get(XmlAttributeName.EXCLUDE) or '').strip() + + if not attr_in: + PackageManager.log.error(f'Include path at line {match_node.sourceline} in project file is empty') + sys.exit(1) + + in_path: str = os.path.normpath(attr_in) + + if in_path == os.curdir: + in_path = in_path.replace(os.curdir, root_path, 1) + elif in_path == os.pardir: + in_path = in_path.replace(os.pardir, os.path.normpath(os.path.join(root_path, os.pardir)), 1) + elif os.path.sep in os.path.normpath(in_path): + if startswith(in_path, os.pardir): + in_path = in_path.replace(os.pardir, os.path.normpath(os.path.join(root_path, os.pardir)), 1) + elif startswith(in_path, os.curdir): + in_path = in_path.replace(os.curdir, root_path, 1) + + if not os.path.isabs(in_path): + in_path = os.path.join(root_path, in_path) + elif zip_mode and root_path not in in_path: + PackageManager.log.error(f'Cannot match path outside RootDir: "{in_path}"') + sys.exit(1) + + if not os.path.isdir(in_path): + PackageManager.log.error(f'Cannot match path that does not exist or is not a directory: "{in_path}"') + sys.exit(1) + + user_flags = wcmatch.RECURSIVE if not attr_no_recurse else 0x0 + + matcher = wcmatch.WcMatch(in_path, match_node.text, + exclude_pattern=attr_exclude, + flags=PackageManager.DEFAULT_WCFLAGS | user_flags) + + matcher.on_reset() + matcher._skipped = 0 + for f in matcher._walk(): + yield f, attr_in def _fix_package_extension(self, package_name: str) -> str: if not endswith(package_name, ('.ba2', '.bsa'), ignorecase=True): @@ -140,17 +180,10 @@ def build_commands(self, containing_folder: str, output_path: str) -> str: # SSE has an ctd bug with uncompressed textures in a bsa that # has an Embed Filenames flag on it, so force it to false. - has_textures = False - - for f in glob.iglob(os.path.join(containing_folder, r'**/*'), recursive=True): - if not os.path.isfile(f): - continue - if endswith(f, '.dds', ignorecase=True): - has_textures = True - break - - if has_textures: + for _ in wcmatch.WcMatch(containing_folder, '*.dds', + flags=wcmatch.RECURSIVE | wcmatch.IGNORECASE).imatch(): arguments.append('-af:0x3') + break else: arguments.append('-tes5') @@ -191,11 +224,15 @@ def create_packages(self) -> None: PackageManager.log.info(f'Creating "{file_name}"...') for source_path, _ in self._generate_include_paths(package_node, root_dir): - PackageManager.log.info(f'+ "{source_path}"') + if os.path.isabs(source_path): + relpath = os.path.relpath(source_path, root_dir) + else: + relpath = source_path - relpath = os.path.relpath(source_path, root_dir) target_path = os.path.join(self.options.temp_path, relpath) + PackageManager.log.info(f'+ "{relpath.casefold()}"') + # fix target path if user passes a deeper package root (RootDir) if endswith(source_path, '.pex', ignorecase=True) and not startswith(relpath, 'scripts', ignorecase=True): target_path = os.path.join(self.options.temp_path, 'Scripts', relpath) @@ -250,18 +287,18 @@ def create_zip(self) -> None: try: with zipfile.ZipFile(file_path, mode='w', compression=compress_type.value) as z: - for include_path, user_path in self._generate_include_paths(zip_node, zip_root_path): - if zip_root_path not in include_path: - PackageManager.log.warning(f'Cannot add file to ZIP outside RootDir: "{include_path}"') - continue - + for include_path, user_path in self._generate_include_paths(zip_node, zip_root_path, True): if not user_path: - arcname: str = os.path.relpath(include_path, zip_root_path) + if zip_root_path in include_path: + arcname = os.path.relpath(include_path, zip_root_path) + else: + # just add file to zip root + arcname = os.path.basename(include_path) else: _, file_name = os.path.split(include_path) - arcname: str = os.path.join(user_path, file_name) + arcname = file_name if user_path == os.curdir else os.path.join(user_path, file_name) - PackageManager.log.info(f'+ "{arcname}"') + PackageManager.log.info('+ "{}"'.format(arcname)) z.write(include_path, arcname, compress_type=compress_type.value) PackageManager.log.info(f'Wrote ZIP file: "{file_path}"') diff --git a/pyro/PapyrusProject.py b/pyro/PapyrusProject.py index f314b233..7cb95ab9 100644 --- a/pyro/PapyrusProject.py +++ b/pyro/PapyrusProject.py @@ -1,5 +1,4 @@ import configparser -import glob import hashlib import io import os @@ -8,6 +7,7 @@ from copy import deepcopy from lxml import etree +from wcmatch import wcmatch from pyro.Enums.BuildEvent import BuildEvent from pyro.Enums.ImportEvent import ImportEvent @@ -232,7 +232,7 @@ def try_set_game_type(self) -> None: self.options.game_type = GameType.get(game_type) if self.options.game_type: - PapyrusProject.log.warning(f'Using game type: {GameName.get(game_type)} (determined from Papyrus Project)') + PapyrusProject.log.info(f'Using game type: {GameName.get(game_type)} (determined from Papyrus Project)') if not self.options.game_type: self.options.game_type = self.get_game_type() @@ -344,11 +344,17 @@ def _update_attributes(self, parent_node: etree.ElementBase) -> None: if XmlAttributeName.ROOT_DIR not in node.attrib: node.set(XmlAttributeName.ROOT_DIR, self.project_path) - elif tag in (XmlTagName.FOLDER, XmlTagName.INCLUDE): + elif tag in (XmlTagName.FOLDER, XmlTagName.INCLUDE, XmlTagName.MATCH): if XmlAttributeName.NO_RECURSE not in node.attrib: node.set(XmlAttributeName.NO_RECURSE, 'False') - if tag == XmlTagName.INCLUDE and XmlAttributeName.PATH not in node.attrib: - node.set(XmlAttributeName.PATH, '') + if tag == XmlTagName.INCLUDE: + if XmlAttributeName.PATH not in node.attrib: + node.set(XmlAttributeName.PATH, '') + elif tag == XmlTagName.MATCH: + if XmlAttributeName.IN not in node.attrib: + node.set(XmlAttributeName.IN, os.curdir) + if XmlAttributeName.EXCLUDE not in node.attrib: + node.set(XmlAttributeName.EXCLUDE, '') elif tag == XmlTagName.ZIP_FILES: if XmlAttributeName.OUTPUT not in node.attrib: @@ -415,8 +421,8 @@ def _get_import_paths(self) -> list: import_path = os.path.normpath(import_node.text) if import_path == os.pardir: - self.log.warning(f'Import paths cannot be equal to "{os.pardir}"') - continue + PapyrusProject.log.error(f'Import paths cannot be equal to "{os.pardir}"') + sys.exit(1) if import_path == os.curdir: import_path = self.project_path @@ -427,7 +433,7 @@ def _get_import_paths(self) -> list: if os.path.isdir(import_path): results.append(import_path) else: - self.log.error(f'Import path does not exist: "{import_path}"') + PapyrusProject.log.error(f'Import path does not exist: "{import_path}"') sys.exit(1) return PathHelper.uniqify(results) @@ -553,7 +559,9 @@ def _get_remote_path(self, node: etree.ElementBase) -> str: local_path = os.path.join(temp_path, url_path) - for f in glob.iglob(os.path.join(local_path, r'**/*.psc'), recursive=True): + matcher = wcmatch.WcMatch(local_path, '*.psc', flags=wcmatch.IGNORECASE | wcmatch.RECURSIVE) + + for f in matcher.imatch(): return os.path.dirname(f) return local_path @@ -605,11 +613,16 @@ def _get_script_paths_from_folders_node(self) -> typing.Generator: if os.path.isdir(test_path): # count scripts to avoid issue where an errant `test_path` may exist and contain no sources # this can be a problem if that folder contains sources but user error is hard to fix - search_path: str = os.path.join(test_path, '*' if no_recurse else r'**\*') - script_count: int = sum(1 for f in glob.iglob(search_path, recursive=not no_recurse) - if endswith(f, '.psc', ignorecase=True)) - if script_count > 0: - yield from PathHelper.find_script_paths_from_folder(test_path, no_recurse) + test_passed = False + + user_flags = wcmatch.RECURSIVE if not no_recurse else 0x0 + matcher = wcmatch.WcMatch(test_path, '*.psc', flags=wcmatch.IGNORECASE | user_flags) + for _ in matcher.imatch(): + test_passed = True + break + + if test_passed: + yield from PathHelper.find_script_paths_from_folder(test_path, no_recurse, matcher) continue # try to add import-relative folder path @@ -652,8 +665,8 @@ def _try_exclude_unmodified_scripts(self) -> dict: try: header = PexReader.get_header(matching_path) except ValueError: - PapyrusProject.log.warning(f'Cannot determine compilation time due to unknown magic: "{matching_path}"') - continue + PapyrusProject.log.error(f'Cannot determine compilation time due to unknown magic: "{matching_path}"') + sys.exit(1) compiled_time: int = header.compilation_time.value if os.path.getmtime(script_path) < compiled_time: diff --git a/pyro/PapyrusProject.xsd b/pyro/PapyrusProject.xsd index 4314a00a..93a2f334 100644 --- a/pyro/PapyrusProject.xsd +++ b/pyro/PapyrusProject.xsd @@ -37,7 +37,8 @@ - + + @@ -90,13 +91,27 @@ - - - + + + + + + + + + + + + + + - + + + + diff --git a/pyro/PathHelper.py b/pyro/PathHelper.py index 06534e06..91890a98 100644 --- a/pyro/PathHelper.py +++ b/pyro/PathHelper.py @@ -1,9 +1,10 @@ -import glob import os from collections import OrderedDict from typing import Generator, Iterable from urllib.parse import unquote_plus, urlparse +from wcmatch import wcmatch + from pyro.Comparators import endswith, startswith @@ -30,19 +31,13 @@ def calculate_relative_object_name(script_path: str, import_paths: list) -> str: return file_name @staticmethod - def find_include_paths(search_path: str, no_recurse: bool, user_path: str = '') -> Generator: - """Yields existing file paths from absolute search path""" - for include_path in glob.iglob(search_path, recursive=not no_recurse): - if os.path.isfile(include_path): - yield include_path, user_path - - @staticmethod - def find_script_paths_from_folder(folder_path: str, no_recurse: bool) -> Generator: + def find_script_paths_from_folder(folder_path: str, no_recurse: bool, matcher: wcmatch.WcMatch = None) -> Generator: """Yields existing script paths starting from absolute folder path""" - search_path: str = os.path.join(folder_path, '*' if no_recurse else r'**\*') - for script_path in glob.iglob(search_path, recursive=not no_recurse): - if os.path.isfile(script_path) and endswith(script_path, '.psc', ignorecase=True): - yield script_path + if not matcher: + user_flags = wcmatch.RECURSIVE if not no_recurse else 0x0 + matcher = wcmatch.WcMatch(folder_path, '*.psc', flags=wcmatch.IGNORECASE | user_flags) + for script_path in matcher.imatch(): + yield script_path @staticmethod def uniqify(items: Iterable) -> list: diff --git a/pyro/ProjectBase.py b/pyro/ProjectBase.py index 046952e2..d3a34c49 100644 --- a/pyro/ProjectBase.py +++ b/pyro/ProjectBase.py @@ -251,35 +251,35 @@ def get_game_type(self) -> str: if self.options.game_path: for game_type, game_name in GameName.items(): if endswith(self.options.game_path, game_name, ignorecase=True): - ProjectBase.log.warning(f'Using game type: {game_name} (determined from game path)') + ProjectBase.log.info(f'Using game type: {game_name} (determined from game path)') return GameType.get(game_type) if self.options.registry_path: game_type = self._get_game_type_from_path(self.options.registry_path) if game_type: - ProjectBase.log.warning(f'Using game type: {GameName.get(game_type)} (determined from registry path)') + ProjectBase.log.info(f'Using game type: {GameName.get(game_type)} (determined from registry path)') return game_type if self.import_paths: for import_path in reversed(self.import_paths): game_type = self._get_game_type_from_path(import_path) if game_type: - ProjectBase.log.warning(f'Using game type: {GameName.get(game_type)} (determined from import paths)') + ProjectBase.log.info(f'Using game type: {GameName.get(game_type)} (determined from import paths)') return game_type if self.options.flags_path: if endswith(self.options.flags_path, FlagsName.FO4, ignorecase=True): - ProjectBase.log.warning(f'Using game type: {GameName.FO4} (determined from flags path)') + ProjectBase.log.info(f'Using game type: {GameName.FO4} (determined from flags path)') return GameType.FO4 if endswith(self.options.flags_path, FlagsName.TES5, ignorecase=True): try: self.get_game_path('sse') except FileNotFoundError: - ProjectBase.log.warning(f'Using game type: {GameName.TES5} (determined from flags path)') + ProjectBase.log.info(f'Using game type: {GameName.TES5} (determined from flags path)') return GameType.TES5 else: - ProjectBase.log.warning(f'Using game type: {GameName.SSE} (determined from flags path)') + ProjectBase.log.info(f'Using game type: {GameName.SSE} (determined from flags path)') return GameType.SSE raise AssertionError('Cannot determine game type from game path, registry path, import paths, or flags path') diff --git a/requirements.txt b/requirements.txt index 9ef0b62a..f2069d16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ lxml nuitka psutil +wcmatch From ac5f509ebcb3d7ad088bd0a83a1bb3a2c463e410 Mon Sep 17 00:00:00 2001 From: fireundubh Date: Sun, 13 Jun 2021 03:54:40 -0700 Subject: [PATCH 2/3] Fixed issue where package creation would fail on copying relative includes when application run outside project root --- pyro/PackageManager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyro/PackageManager.py b/pyro/PackageManager.py index c5ad276a..a770445b 100644 --- a/pyro/PackageManager.py +++ b/pyro/PackageManager.py @@ -228,6 +228,7 @@ def create_packages(self) -> None: relpath = os.path.relpath(source_path, root_dir) else: relpath = source_path + source_path = os.path.join(self.ppj.project_path, source_path) target_path = os.path.join(self.options.temp_path, relpath) From 5f031e6143c633cc9e42f2e0b38682ce61e70009 Mon Sep 17 00:00:00 2001 From: fireundubh Date: Sun, 13 Jun 2021 03:59:45 -0700 Subject: [PATCH 3/3] Added prefix to XML attribute variable names --- pyro/PackageManager.py | 44 +++++++++++++++++++++--------------------- pyro/PapyrusProject.py | 20 +++++++++---------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/pyro/PackageManager.py b/pyro/PackageManager.py index a770445b..8308fc0f 100644 --- a/pyro/PackageManager.py +++ b/pyro/PackageManager.py @@ -201,27 +201,27 @@ def create_packages(self) -> None: file_names = CaseInsensitiveList() for i, package_node in enumerate(filter(is_package_node, self.ppj.packages_node)): - file_name: str = package_node.get(XmlAttributeName.NAME) + attr_file_name: str = package_node.get(XmlAttributeName.NAME) # noinspection PyProtectedMember root_dir = self.ppj._get_path(package_node.get(XmlAttributeName.ROOT_DIR), relative_root_path=self.ppj.project_path, - fallback_path=[self.ppj.project_path, os.path.basename(file_name)]) + fallback_path=[self.ppj.project_path, os.path.basename(attr_file_name)]) # prevent clobbering files previously created in this session - if file_name in file_names: - file_name = f'{self.ppj.project_name} ({i})' + if attr_file_name in file_names: + attr_file_name = f'{self.ppj.project_name} ({i})' - if file_name not in file_names: - file_names.append(file_name) + if attr_file_name not in file_names: + file_names.append(attr_file_name) - file_name = self._fix_package_extension(file_name) + attr_file_name = self._fix_package_extension(attr_file_name) - file_path: str = os.path.join(self.options.package_path, file_name) + file_path: str = os.path.join(self.options.package_path, attr_file_name) self._check_write_permission(file_path) - PackageManager.log.info(f'Creating "{file_name}"...') + PackageManager.log.info(f'Creating "{attr_file_name}"...') for source_path, _ in self._generate_include_paths(package_node, root_dir): if os.path.isabs(source_path): @@ -257,18 +257,18 @@ def create_zip(self) -> None: file_names = CaseInsensitiveList() for i, zip_node in enumerate(filter(is_zipfile_node, self.ppj.zip_files_node)): - file_name: str = zip_node.get(XmlAttributeName.NAME) + attr_file_name: str = zip_node.get(XmlAttributeName.NAME) # prevent clobbering files previously created in this session - if file_name in file_names: - file_name = f'{file_name} ({i})' + if attr_file_name in file_names: + attr_file_name = f'{attr_file_name} ({i})' - if file_name not in file_names: - file_names.append(file_name) + if attr_file_name not in file_names: + file_names.append(attr_file_name) - file_name = self._fix_zip_extension(file_name) + attr_file_name = self._fix_zip_extension(attr_file_name) - file_path: str = os.path.join(self.options.zip_output_path, file_name) + file_path: str = os.path.join(self.options.zip_output_path, attr_file_name) self._check_write_permission(file_path) @@ -280,11 +280,11 @@ def create_zip(self) -> None: except KeyError: compress_type = ZipCompression.STORE - root_dir: str = zip_node.get(XmlAttributeName.ROOT_DIR) - zip_root_path: str = self._try_resolve_project_relative_path(root_dir) + attr_root_dir: str = zip_node.get(XmlAttributeName.ROOT_DIR) + zip_root_path: str = self._try_resolve_project_relative_path(attr_root_dir) if zip_root_path: - PackageManager.log.info(f'Creating "{file_name}"...') + PackageManager.log.info(f'Creating "{attr_file_name}"...') try: with zipfile.ZipFile(file_path, mode='w', compression=compress_type.value) as z: @@ -296,8 +296,8 @@ def create_zip(self) -> None: # just add file to zip root arcname = os.path.basename(include_path) else: - _, file_name = os.path.split(include_path) - arcname = file_name if user_path == os.curdir else os.path.join(user_path, file_name) + _, attr_file_name = os.path.split(include_path) + arcname = attr_file_name if user_path == os.curdir else os.path.join(user_path, attr_file_name) PackageManager.log.info('+ "{}"'.format(arcname)) z.write(include_path, arcname, compress_type=compress_type.value) @@ -307,5 +307,5 @@ def create_zip(self) -> None: PackageManager.log.error(f'Cannot open ZIP file for writing: "{file_path}"') sys.exit(1) else: - PackageManager.log.error(f'Cannot resolve RootDir path to existing folder: "{root_dir}"') + PackageManager.log.error(f'Cannot resolve RootDir path to existing folder: "{attr_root_dir}"') sys.exit(1) diff --git a/pyro/PapyrusProject.py b/pyro/PapyrusProject.py index 7cb95ab9..6c7e55cc 100644 --- a/pyro/PapyrusProject.py +++ b/pyro/PapyrusProject.py @@ -228,11 +228,11 @@ def try_set_game_type(self) -> None: # we need to set the game type after imports are populated but before pex paths are populated # allow xml to set game type but defer to passed argument if self.options.game_type not in GameType.values(): - game_type: str = self.ppj_root.get(XmlAttributeName.GAME, default='') - self.options.game_type = GameType.get(game_type) + attr_game_type: str = self.ppj_root.get(XmlAttributeName.GAME, default='') + self.options.game_type = GameType.get(attr_game_type) if self.options.game_type: - PapyrusProject.log.info(f'Using game type: {GameName.get(game_type)} (determined from Papyrus Project)') + PapyrusProject.log.info(f'Using game type: {GameName.get(attr_game_type)} (determined from Papyrus Project)') if not self.options.game_type: self.options.game_type = self.get_game_type() @@ -577,11 +577,11 @@ def _get_script_paths_from_folders_node(self) -> typing.Generator: for folder_node in filter(is_folder_node, self.folders_node): self.try_fix_namespace_path(folder_node) - no_recurse: bool = folder_node.get(XmlAttributeName.NO_RECURSE) == 'True' + attr_no_recurse: bool = folder_node.get(XmlAttributeName.NO_RECURSE) == 'True' # try to add project path if folder_node.text == os.curdir: - yield from PathHelper.find_script_paths_from_folder(self.project_path, no_recurse) + yield from PathHelper.find_script_paths_from_folder(self.project_path, attr_no_recurse) continue # handle . and .. in path @@ -598,14 +598,14 @@ def _get_script_paths_from_folders_node(self) -> typing.Generator: PapyrusProject.log.info(f'Adding import path from remote: "{local_path}"...') self.import_paths.insert(0, local_path) PapyrusProject.log.info(f'Adding folder path from remote: "{local_path}"...') - yield from PathHelper.find_script_paths_from_folder(local_path, no_recurse) + yield from PathHelper.find_script_paths_from_folder(local_path, attr_no_recurse) continue folder_path: str = os.path.normpath(folder_node.text) # try to add absolute path if os.path.isabs(folder_path) and os.path.isdir(folder_path): - yield from PathHelper.find_script_paths_from_folder(folder_path, no_recurse) + yield from PathHelper.find_script_paths_from_folder(folder_path, attr_no_recurse) continue # try to add project-relative folder path @@ -615,21 +615,21 @@ def _get_script_paths_from_folders_node(self) -> typing.Generator: # this can be a problem if that folder contains sources but user error is hard to fix test_passed = False - user_flags = wcmatch.RECURSIVE if not no_recurse else 0x0 + user_flags = wcmatch.RECURSIVE if not attr_no_recurse else 0x0 matcher = wcmatch.WcMatch(test_path, '*.psc', flags=wcmatch.IGNORECASE | user_flags) for _ in matcher.imatch(): test_passed = True break if test_passed: - yield from PathHelper.find_script_paths_from_folder(test_path, no_recurse, matcher) + yield from PathHelper.find_script_paths_from_folder(test_path, attr_no_recurse, matcher) continue # try to add import-relative folder path for import_path in self.import_paths: test_path = os.path.join(import_path, folder_path) if os.path.isdir(test_path): - yield from PathHelper.find_script_paths_from_folder(test_path, no_recurse) + yield from PathHelper.find_script_paths_from_folder(test_path, attr_no_recurse) # noinspection DuplicatedCode def _get_script_paths_from_scripts_node(self) -> typing.Generator: