diff --git a/README.md b/README.md index e1142fa..ae6563b 100644 --- a/README.md +++ b/README.md @@ -126,8 +126,9 @@ Module Docs - https://github.com/ClericPy/morebuiltins/blob/master/doc.md - [x] asyncio free-flying tasks(bg_task) - v0.0.3 - [x] named lock with timeout - v0.0.4 - [x] functools.FuncSchema (parse function to get the query-dict) - v0.0.4 -- [x] morebuiltins.download_python [standalone python](https://github.com/indygreg/python-build-standalone/releases/latest) downloader - v0.0.4 -- [ ] pip.pip_install +- [x] `python -m morebuiltins.download_python` [standalone python](https://github.com/indygreg/python-build-standalone/releases/latest) downloader - v0.0.4 +- [x] `from morebuiltins.zipapps import pip_install_target;pip_install_target("./mock_dir", ["six"], force=False, sys_path=0); import six;print(six.__file__)` - v0.0.5 +- [ ] changelog.md - [ ] progress_bar - [ ] http.server (upload) - [ ] time reach syntax diff --git a/morebuiltins/download_python/__init__.py b/morebuiltins/download_python/__init__.py index 720171d..e69de29 100644 --- a/morebuiltins/download_python/__init__.py +++ b/morebuiltins/download_python/__init__.py @@ -1,3 +0,0 @@ -from .main import download_python - -__all__ = ["download_python"] diff --git a/morebuiltins/download_python/__main__.py b/morebuiltins/download_python/__main__.py index 16b699e..1ec8a6a 100644 --- a/morebuiltins/download_python/__main__.py +++ b/morebuiltins/download_python/__main__.py @@ -1,5 +1,5 @@ -from .main import download_python if __name__ == "__main__": + from ..zipapps.download_python import download_python download_python() diff --git a/morebuiltins/zipapps/__init__.py b/morebuiltins/zipapps/__init__.py index 22266b9..87dc83f 100644 --- a/morebuiltins/zipapps/__init__.py +++ b/morebuiltins/zipapps/__init__.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- -from .main import create_app, __version__, ZipApp +from .main import create_app, __version__, ZipApp, pip_install_target from .activate_zipapps import activate -__all__ = ['create_app', 'activate', '__version__', 'ZipApp'] +__all__ = ["create_app", "activate", "__version__", "ZipApp", "pip_install_target"] +__doc__ = "Package your python code into one zip file, even a virtual environment." diff --git a/morebuiltins/zipapps/__main__.py b/morebuiltins/zipapps/__main__.py index 2f5887b..27d253d 100644 --- a/morebuiltins/zipapps/__main__.py +++ b/morebuiltins/zipapps/__main__.py @@ -2,12 +2,13 @@ import json import sys import time +import typing from pathlib import Path from . import __version__ from .main import ZipApp -USAGE = r''' +USAGE = r""" =========================================================================== 0. package your code without any requirements @@ -51,73 +52,77 @@ > python3 app.pyz -c "import lxml.html;print(lxml.html.__file__)" PS: all the unknown args will be used by "pip install". -===========================================================================''' +===========================================================================""" -PIP_PYZ_URL = 'https://bootstrap.pypa.io/pip/pip.pyz' +PIP_PYZ_URL = "https://bootstrap.pypa.io/pip/pip.pyz" +DOWNLOAD_PYTHON_URL = "https://www.github.com/indygreg/python-build-standalone" def _get_now(): - return time.strftime('%Y-%m-%d %H:%M:%S') + return time.strftime("%Y-%m-%d %H:%M:%S") def _get_pth_path(): py_exe_path = Path(sys.executable) - for _path in py_exe_path.parent.glob('*._pth'): + for _path in py_exe_path.parent.glob("*._pth"): _pth_path = _path break else: - fname = f'python{sys.version_info.major}{sys.version_info.minor}._pth' + fname = f"python{sys.version_info.major}{sys.version_info.minor}._pth" _pth_path = py_exe_path.parent / fname return _pth_path def _append_pth(): import re + _pth_path = _get_pth_path() if _pth_path.is_file(): - print('find _pth file:', - _pth_path.as_posix(), - flush=True, - file=sys.stderr) + print("find _pth file:", _pth_path.as_posix(), flush=True, file=sys.stderr) _path_bytes = _pth_path.read_bytes() else: - _path_bytes = b'' - if not re.search(b'^import site$', _path_bytes): - _path_bytes += b'\nimport site\n' - if not re.search(b'^pip\.pyz$', _path_bytes): - _path_bytes += b'\npip.pyz\n' + _path_bytes = b"" + if not re.search(b"^import site$", _path_bytes): + _path_bytes += b"\nimport site\n" + if not re.search(b"^pip\.pyz$", _path_bytes): + _path_bytes += b"\npip.pyz\n" _pth_path.write_bytes(_path_bytes) -def download_pip_pyz(target: Path = None, log=True): +def download_pip_pyz(target: typing.Optional[Path] = None, log=True): from urllib.request import urlretrieve - pip_pyz_path = Path(target or (Path(sys.executable).parent / 'pip.pyz')) + pip_pyz_path = Path(target or (Path(sys.executable).parent / "pip.pyz")) if log: - msg = f'Download {PIP_PYZ_URL} -> {pip_pyz_path.absolute().as_posix()}' + msg = f"Download {PIP_PYZ_URL} -> {pip_pyz_path.absolute().as_posix()}" print(_get_now(), msg, flush=True, file=sys.stderr) if pip_pyz_path.is_file(): pip_pyz_path.unlink() - urlretrieve(url=PIP_PYZ_URL, - filename=pip_pyz_path.absolute().as_posix(), - reporthook=lambda a, b, c: print( - _get_now(), - f'Downloading {int(100*(1+a)*b/c)}%, {(1 + a) * b} / {c}', - end='\r', - flush=True, - file=sys.stderr, - ) if log else None) + urlretrieve( + url=PIP_PYZ_URL, + filename=pip_pyz_path.absolute().as_posix(), + reporthook=lambda a, b, c: print( + _get_now(), + f"Downloading {int(100*(1+a)*b/c)}%, {(1 + a) * b} / {c}", + end="\r", + flush=True, + file=sys.stderr, + ) + if log + else None, + ) try: sys.path.append(pip_pyz_path.absolute().as_posix()) - import pip as _ + import pip + + assert pip + if log: - print(f'\n{_get_now()} install pip ok', flush=True, file=sys.stderr) + print(f"\n{_get_now()} install pip ok", flush=True, file=sys.stderr) return True except ImportError: if log: - print(f'\n{_get_now()} install pip failed', - flush=True, - file=sys.stderr) + print(f"\n{_get_now()} install pip failed", flush=True, file=sys.stderr) def handle_win32_embeded(): @@ -126,268 +131,310 @@ def handle_win32_embeded(): if not _pth_path.is_file(): return try: - import pip as _ + import pip + + assert pip + return except ImportError: need_install = ( - input(f'\n{"=" * 50}\npip module not found, try installing?(Y/n)' - ).lower().strip() or 'y') - if need_install != 'y': + input(f'\n{"=" * 50}\npip module not found, try installing?(Y/n)') + .lower() + .strip() + or "y" + ) + if need_install != "y": return - print('find _pth file:', _pth_path.as_posix(), flush=True, file=sys.stderr) + print("find _pth file:", _pth_path.as_posix(), flush=True, file=sys.stderr) if download_pip_pyz(): _append_pth() import os - os.system('%s -m pip -V' % Path(sys.executable).as_posix()) - os.system('pause') + os.system("%s -m pip -V" % Path(sys.executable).as_posix()) + os.system("pause") def main(): - parser = argparse.ArgumentParser(usage=USAGE, prog='Zipapps') - parser.add_argument('--version', action='version', version=__version__) - parser.add_argument('--output', - '-o', - default=ZipApp.DEFAULT_OUTPUT_PATH, - help='The path of the output file, defaults to' - f' "{ZipApp.DEFAULT_OUTPUT_PATH}".') - parser.add_argument( - '--python', - '-p', - dest='interpreter', + parser = argparse.ArgumentParser(usage=USAGE, prog="Zipapps") + parser.add_argument("--version", action="version", version=__version__) + parser.add_argument( + "--output", + "-o", + default=ZipApp.DEFAULT_OUTPUT_PATH, + help="The path of the output file, defaults to" + f' "{ZipApp.DEFAULT_OUTPUT_PATH}".', + ) + parser.add_argument( + "--python", + "-p", + dest="interpreter", default=None, - help='The path of the Python interpreter which will be ' - 'set as the `shebang line`, defaults to `None`. With shebang `/usr/bin/python3` you can run app with `./app.pyz` directly, no need for `python3 app.pyz`' - ) - parser.add_argument( - '--main', - '-m', - default='', - help='The entry point function of the application, ' - 'the format is: `package` | `package.module` | `package.module:function` | `module:function`' - ) - parser.add_argument('--compress', - '-c', - dest='compressed', - action='store_true', - help='compress files with the deflate method or not.') - parser.add_argument( - '--includes', - '--add', - '-a', - default='', - help='The given paths will be copied to `cache_path` while packaging, ' + help="The path of the Python interpreter which will be " + "set as the `shebang line`, defaults to `None`. With shebang `/usr/bin/python3` you can run app with `./app.pyz` directly, no need for `python3 app.pyz`", + ) + parser.add_argument( + "--main", + "-m", + default="", + help="The entry point function of the application, " + "the format is: `package` | `package.module` | `package.module:function` | `module:function`", + ) + parser.add_argument( + "--compress", + "-c", + dest="compressed", + action="store_true", + help="compress files with the deflate method or not.", + ) + parser.add_argument( + "--includes", + "--add", + "-a", + default="", + help="The given paths will be copied to `cache_path` while packaging, " 'which can be used while running. The path strings will be splited by ",". ' - 'such as `my_package_dir,my_module.py,my_config.json`, often used for libs not from `pypi` or some special config files' + "such as `my_package_dir,my_module.py,my_config.json`, often used for libs not from `pypi` or some special config files", ) parser.add_argument( - '--unzip', - '-u', - default='', + "--unzip", + "-u", + default="", help='The names which need to be unzipped while running, splited by "," ' - '`without ext`, such as `bottle,aiohttp`, or the complete path like `bin/bottle.py,temp.py`. For `.so/.pyd` files(which can not be loaded by zipimport), or packages with operations of static files. if unzip is set to "*", then will unzip all files and folders. if unzip is set to **AUTO**, then will add the `.pyd` and `.so` files automatically. Can be overwrite with environment variable `ZIPAPPS_UNZIP`' - ) - parser.add_argument( - '--unzip-exclude', - '-ue', - default='', - dest='unzip_exclude', - help='The opposite of `--unzip` / `-u` which will not be unzipped, ' - 'should be used with `--unzip` / `-u`. Can be overwrite with environment variable `ZIPAPPS_UNZIP_EXCLUDE`' - ) - parser.add_argument( - '--unzip-path', - '-up', - default='', - help='If `unzip` arg is not null, cache files will be unzipped to the ' - 'given path while running. Defaults to `zipapps_cache`, support some internal variables: `$TEMP/$HOME/$SELF/$PID/$CWD` as internal variables, for example `$HOME/zipapps_cache`. `$TEMP` means `tempfile.gettempdir()`, `$HOME` means `Path.home()`, `$SELF` means `.pyz` file path, `$PID` means `os.getpid()`, `$CWD` means `Path.cwd()`.' - ) - parser.add_argument( - '-cc', - '--pyc', - '--compiled', - action='store_true', - dest='compiled', - help='Compile .py to .pyc for fast import, but zipapp does not work ' - 'unless you unzip it.') - parser.add_argument('--cache-path', - '--source-dir', - '-cp', - default=None, - dest='cache_path', - help='The cache path of zipapps to store ' - 'site-packages and `includes` files, ' - 'which will be treat as PYTHONPATH. If not set, will ' - 'create and clean-up in TEMP dir automately.') - parser.add_argument('--shell', - '-s', - action='store_true', - help='Only while `main` is not set, used for shell=True' - ' in subprocess.Popen.') - parser.add_argument( - '--main-shell', - '-ss', - action='store_true', - dest='main_shell', - help='Only for `main` is not null, call `main` with subprocess.Popen: ' - '`python -c "import a.b;a.b.c()"`. This is used for `psutil` ImportError of DLL load.' - ) - parser.add_argument( - '--strict-python-path', - '-spp', - action='store_true', - dest='ignore_system_python_path', - help='Ignore global PYTHONPATH, only use zipapps_cache and app.pyz.') - parser.add_argument( - '-b', - '--build-id', - default='', - dest='build_id', - help='a string to skip duplicate builds,' - ' it can be the paths of files/folders which splited by ",", then the modify time will be used as build_id. If build_id contains `*`, will use `glob` function to get paths. For example, you can set requirements.txt as your build_id by `python3 -m zipapps -b requirements.txt -r requirements.txt` when you use pyz as venv.' - ) - parser.add_argument( - '--zipapps', - '--env-paths', - default='', - dest='env_paths', - help='Default --zipapps arg if it is not given while running.' - ' Also support $TEMP/$HOME/$SELF/$PID/$CWD prefix, separated by commas.' - ) - parser.add_argument( - '--delay', - '-d', - '--lazy-pip', - '--lazy-install', - '--lazy-pip-install', - action='store_true', - dest='lazy_install', - help='Install packages with pip while running, which means ' - 'requirements will not be install into pyz file. Default unzip path will be changed to `SELF/zipapps_cache`' - ) - parser.add_argument( - '-pva', - '--python-version-accuracy', - '--python-version-slice', + '`without ext`, such as `bottle,aiohttp`, or the complete path like `bin/bottle.py,temp.py`. For `.so/.pyd` files(which can not be loaded by zipimport), or packages with operations of static files. if unzip is set to "*", then will unzip all files and folders. if unzip is set to **AUTO**, then will add the `.pyd` and `.so` files automatically. Can be overwrite with environment variable `ZIPAPPS_UNZIP`', + ) + parser.add_argument( + "--unzip-exclude", + "-ue", + default="", + dest="unzip_exclude", + help="The opposite of `--unzip` / `-u` which will not be unzipped, " + "should be used with `--unzip` / `-u`. Can be overwrite with environment variable `ZIPAPPS_UNZIP_EXCLUDE`", + ) + parser.add_argument( + "--unzip-path", + "-up", + default="", + help="If `unzip` arg is not null, cache files will be unzipped to the " + "given path while running. Defaults to `zipapps_cache`, support some internal variables: `$TEMP/$HOME/$SELF/$PID/$CWD` as internal variables, for example `$HOME/zipapps_cache`. `$TEMP` means `tempfile.gettempdir()`, `$HOME` means `Path.home()`, `$SELF` means `.pyz` file path, `$PID` means `os.getpid()`, `$CWD` means `Path.cwd()`.", + ) + parser.add_argument( + "-cc", + "--pyc", + "--compiled", + action="store_true", + dest="compiled", + help="Compile .py to .pyc for fast import, but zipapp does not work " + "unless you unzip it.", + ) + parser.add_argument( + "--cache-path", + "--source-dir", + "-cp", + default=None, + dest="cache_path", + help="The cache path of zipapps to store " + "site-packages and `includes` files, " + "which will be treat as PYTHONPATH. If not set, will " + "create and clean-up in TEMP dir automately.", + ) + parser.add_argument( + "--shell", + "-s", + action="store_true", + help="Only while `main` is not set, used for shell=True" + " in subprocess.Popen.", + ) + parser.add_argument( + "--main-shell", + "-ss", + action="store_true", + dest="main_shell", + help="Only for `main` is not null, call `main` with subprocess.Popen: " + '`python -c "import a.b;a.b.c()"`. This is used for `psutil` ImportError of DLL load.', + ) + parser.add_argument( + "--strict-python-path", + "-spp", + action="store_true", + dest="ignore_system_python_path", + help="Ignore global PYTHONPATH, only use zipapps_cache and app.pyz.", + ) + parser.add_argument( + "-b", + "--build-id", + default="", + dest="build_id", + help="a string to skip duplicate builds," + ' it can be the paths of files/folders which splited by ",", then the modify time will be used as build_id. If build_id contains `*`, will use `glob` function to get paths. For example, you can set requirements.txt as your build_id by `python3 -m zipapps -b requirements.txt -r requirements.txt` when you use pyz as venv.', + ) + parser.add_argument( + "--zipapps", + "--env-paths", + default="", + dest="env_paths", + help="Default --zipapps arg if it is not given while running." + " Also support $TEMP/$HOME/$SELF/$PID/$CWD prefix, separated by commas.", + ) + parser.add_argument( + "--delay", + "-d", + "--lazy-pip", + "--lazy-install", + "--lazy-pip-install", + action="store_true", + dest="lazy_install", + help="Install packages with pip while running, which means " + "requirements will not be install into pyz file. Default unzip path will be changed to `SELF/zipapps_cache`", + ) + parser.add_argument( + "-pva", + "--python-version-accuracy", + "--python-version-slice", default=2, type=int, - dest='python_version_slice', - help='Only work for lazy-install mode, then `pip` target folders differ ' - 'according to sys.version_info[:_slice], defaults to 2, which means ' - '3.8.3 equals to 3.8.4 for same version accuracy 3.8') - parser.add_argument( - '--sys-paths', - '--sys-path', - '--py-path', - '--python-path', - default='', - dest='sys_paths', - help='Paths be insert to sys.path[-1] while running.' - ' Support $TEMP/$HOME/$SELF/$PID/$CWD prefix, separated by commas.') - parser.add_argument( - '--activate', - default='', - dest='activate', - help='Activate the given paths of zipapps app, ' - 'only activate them but not run them, separated by commas.') - parser.add_argument( - '--ensure-pip', - action='store_true', - dest='ensure_pip', - help='Add the ensurepip package to your pyz file, works for ' - 'embed-python(windows) or other python versions without `pip`' - ' installed but `lazy-install` mode is enabled. [EXPERIMENTAL]') - parser.add_argument( - '--layer-mode', - action='store_true', - dest='layer_mode', - help='Layer mode for the serverless use case, ' - '__main__.py / ensure_zipapps.py / activate_zipapps.py files will not be set in this mode.' - ) - parser.add_argument('--layer-mode-prefix', - default='python', - dest='layer_mode_prefix', - help='Only work while --layer-mode is set, ' - 'will move the files in the given prefix folder.') - parser.add_argument( - '-czc', - '--clear-zipapps-cache', - action='store_true', - dest='clear_zipapps_cache', - help='Clear the zipapps cache folder after running, ' - 'but maybe failed for .pyd/.so files.', - ) - parser.add_argument( - '-czs', - '--clear-zipapps-self', - action='store_true', - dest='clear_zipapps_self', - help='Clear the zipapps pyz file self after running.', - ) - parser.add_argument( - '--chmod', - default='', - dest='chmod', - help='os.chmod(int(chmod, 8)) for unzip files with `--chmod=777`,' - ' unix-like system only', - ) - parser.add_argument( - '--dump-config', - default='', - dest='dump_config', - help='Dump zipapps build args into JSON string.' - ' A file path needed and `-` means stdout.', - ) - parser.add_argument( - '--load-config', - default='', - dest='load_config', - help='Load zipapps build args from a JSON file.', - ) - parser.add_argument( - '--freeze-reqs', - default='', - dest='freeze', - help='Freeze package versions of pip args with venv,' - ' output to the given file path.', - ) - parser.add_argument( - '-q', - '--quite', + dest="python_version_slice", + help="Only work for lazy-install mode, then `pip` target folders differ " + "according to sys.version_info[:_slice], defaults to 2, which means " + "3.8.3 equals to 3.8.4 for same version accuracy 3.8", + ) + parser.add_argument( + "--sys-paths", + "--sys-path", + "--py-path", + "--python-path", + default="", + dest="sys_paths", + help="Paths be insert to sys.path[-1] while running." + " Support $TEMP/$HOME/$SELF/$PID/$CWD prefix, separated by commas.", + ) + parser.add_argument( + "--activate", + default="", + dest="activate", + help="Activate the given paths of zipapps app, " + "only activate them but not run them, separated by commas.", + ) + parser.add_argument( + "--ensure-pip", + action="store_true", + dest="ensure_pip", + help="Add the ensurepip package to your pyz file, works for " + "embed-python(windows) or other python versions without `pip`" + " installed but `lazy-install` mode is enabled. [EXPERIMENTAL]", + ) + parser.add_argument( + "--layer-mode", + action="store_true", + dest="layer_mode", + help="Layer mode for the serverless use case, " + "__main__.py / ensure_zipapps.py / activate_zipapps.py files will not be set in this mode.", + ) + parser.add_argument( + "--layer-mode-prefix", + default="python", + dest="layer_mode_prefix", + help="Only work while --layer-mode is set, " + "will move the files in the given prefix folder.", + ) + parser.add_argument( + "-czc", + "--clear-zipapps-cache", + action="store_true", + dest="clear_zipapps_cache", + help="Clear the zipapps cache folder after running, " + "but maybe failed for .pyd/.so files.", + ) + parser.add_argument( + "-czs", + "--clear-zipapps-self", + action="store_true", + dest="clear_zipapps_self", + help="Clear the zipapps pyz file self after running.", + ) + parser.add_argument( + "--chmod", + default="", + dest="chmod", + help="os.chmod(int(chmod, 8)) for unzip files with `--chmod=777`," + " unix-like system only", + ) + parser.add_argument( + "--dump-config", + default="", + dest="dump_config", + help="Dump zipapps build args into JSON string." + " A file path needed and `-` means stdout.", + ) + parser.add_argument( + "--load-config", + default="", + dest="load_config", + help="Load zipapps build args from a JSON file.", + ) + parser.add_argument( + "--freeze-reqs", + default="", + dest="freeze", + help="Freeze package versions of pip args with venv," + " output to the given file path.", + ) + parser.add_argument( + "-q", + "--quite", action="count", - dest='quite_mode', - help='mute logs.', + dest="quite_mode", + help="mute logs.", + ) + parser.add_argument( + "--download-pip-pyz", + default="", + dest="download_pip_pyz", + help=f'Download pip.pyz from "{PIP_PYZ_URL}"', + ) + parser.add_argument( + "--download-python", + action="store_true", + dest="download_python", + help=f'Download standalone python from "{DOWNLOAD_PYTHON_URL}"', + ) + parser.add_argument( + "--rm-patterns", + default="*.dist-info,__pycache__", + dest="rm_patterns", + help='Delete useless files or folders, splited by "," and defaults to `*.dist-info,__pycache__`. Recursively glob: **/*.pyc', ) - parser.add_argument('--download-pip-pyz', - default='', - dest='download_pip_pyz', - help=f'Download pip.pyz from "{PIP_PYZ_URL}"') - if len(sys.argv) == 1: parser.print_help() handle_win32_embeded() return args, pip_args = parser.parse_known_args() - if args.download_pip_pyz: + if args.download_python: + from .download_python import download_python + + return download_python() + elif args.download_pip_pyz: return download_pip_pyz(args.download_pip_pyz) if args.quite_mode: ZipApp.LOGGING = False - if '-q' not in pip_args and '--quiet' not in pip_args: + if "-q" not in pip_args and "--quiet" not in pip_args: pip_args.append(f'-{"q" * args.quite_mode}') - ZipApp._log(f'zipapps args: {args}, pip install args: {pip_args}') + ZipApp._log(f"zipapps args: {args}, pip install args: {pip_args}") if args.activate: from .activate_zipapps import activate - for path in args.activate.split(','): + + for path in args.activate.split(","): activate(path) return if args.freeze: from .freezing import FreezeTool + with FreezeTool(args.freeze, pip_args) as ft: ft.run() return if args.load_config: - with open(args.load_config, 'r', encoding='utf-8') as f: - app = ZipApp(**json.load(f)) + with open(args.load_config, "r", encoding="utf-8") as f: + kwargs = json.load(f) + app = ZipApp(**kwargs) else: app = ZipApp( includes=args.includes, @@ -415,13 +462,14 @@ def main(): unzip_exclude=args.unzip_exclude, chmod=args.chmod, clear_zipapps_self=args.clear_zipapps_self, + rm_patterns=args.rm_patterns, ) if args.dump_config: config_json = json.dumps(app.kwargs) - if args.dump_config == '-': + if args.dump_config == "-": print(config_json) else: - with open(args.dump_config, 'w', encoding='utf-8') as f: + with open(args.dump_config, "w", encoding="utf-8") as f: f.write(config_json) else: return app.build() diff --git a/morebuiltins/zipapps/activate_zipapps.py b/morebuiltins/zipapps/activate_zipapps.py index 8beef9d..cbb510a 100644 --- a/morebuiltins/zipapps/activate_zipapps.py +++ b/morebuiltins/zipapps/activate_zipapps.py @@ -14,5 +14,5 @@ def activate(path=None): del _tmp return True except ImportError as err: - stderr.write(f'WARNING: activate failed for {err!r}\n') + stderr.write(f"WARNING: activate failed for {err!r}\n") raise err diff --git a/morebuiltins/download_python/main.py b/morebuiltins/zipapps/download_python.py similarity index 95% rename from morebuiltins/download_python/main.py rename to morebuiltins/zipapps/download_python.py index d7ecee5..b5268cc 100644 --- a/morebuiltins/download_python/main.py +++ b/morebuiltins/zipapps/download_python.py @@ -89,8 +89,7 @@ def download_python(): https://github.com/indygreg/python-build-standalone/releases/download/20240415/cpython-3.12.3%2B20240415-x86_64-pc-windows-msvc-install_only.tar.gz D:\github\morebuiltins\morebuiltins\download_python\cpython-3.12.3+20240415-x86_64-pc-windows-msvc-install_only.tar.gz [10:56:44] Downloading: 39.12 / 39.12 MB | 100.00% | 11.3 MB/s | 0s - [10:56:44] Download complete. -""" + [10:56:44] Download complete.""" print( f"[{get_time()}] Checking https://api.github.com/repos/indygreg/python-build-standalone/releases/latest", flush=True, @@ -164,7 +163,10 @@ def sort_key(s): if target == "q": return target_path = Path(target) - target_path.unlink(missing_ok=True) + try: + target_path.unlink() + except FileNotFoundError: + pass print(f"[{get_time()}] Start downloading...") print(download_url) print(target_path.absolute(), flush=True) @@ -225,14 +227,20 @@ def reporthook(blocknum, blocksize, totalsize): except http.client.RemoteDisconnected: continue except KeyboardInterrupt: - temp_path.unlink(missing_ok=True) + try: + temp_path.unlink() + except FileNotFoundError: + pass print() print(f"\n[{get_time()}] Download canceled.", flush=True) return except Exception: print() traceback.print_exc() - temp_path.unlink(missing_ok=True) + try: + temp_path.unlink() + except FileNotFoundError: + pass break print("Press enter to exit.", flush=True) input() diff --git a/morebuiltins/zipapps/freezing.py b/morebuiltins/zipapps/freezing.py index 22a70fb..6de4c35 100644 --- a/morebuiltins/zipapps/freezing.py +++ b/morebuiltins/zipapps/freezing.py @@ -4,14 +4,14 @@ import sys import tempfile import time +import typing import venv from pathlib import Path -def ttime(timestamp=None, - tzone=int(-time.timezone / 3600), - fail="", - fmt="%Y-%m-%d %H:%M:%S"): +def ttime( + timestamp=None, tzone=int(-time.timezone / 3600), fail="", fmt="%Y-%m-%d %H:%M:%S" +): fix_tz = tzone * 3600 if timestamp is None: timestamp = time.time() @@ -28,29 +28,29 @@ def ttime(timestamp=None, class FreezeTool(object): - VENV_NAME = 'zipapps_venv' + VENV_NAME = "zipapps_venv" # not stable FASTER_PREPARE_PIP = False def __init__(self, output: str, pip_args: list): if not pip_args: - raise RuntimeError('pip args is null') - self.temp_dir: tempfile.TemporaryDirectory = None + raise RuntimeError("pip args is null") + self.temp_dir: typing.Optional[tempfile.TemporaryDirectory] = None self.pip_args = pip_args self.output_path = output def log(self, msg, flush=False): - _msg = f'{ttime()} | {msg}' + _msg = f"{ttime()} | {msg}" print(_msg, file=sys.stderr, flush=flush) def run(self): self.log( - 'All the logs will be redirected to stderr to ensure the output is stdout.' + "All the logs will be redirected to stderr to ensure the output is stdout." ) - self.temp_dir = tempfile.TemporaryDirectory(prefix='zipapps_') + self.temp_dir = tempfile.TemporaryDirectory(prefix="zipapps_") self.temp_path = Path(self.temp_dir.name) self.log( - f'Start mkdir temp folder: {self.temp_path.absolute()}, exist={self.temp_path.is_dir()}' + f"Start mkdir temp folder: {self.temp_path.absolute()}, exist={self.temp_path.is_dir()}" ) self.install_env() output = self.install_packages() @@ -59,18 +59,19 @@ def run(self): def install_env(self): venv_path = self.temp_path / self.VENV_NAME - self.log(f'Initial venv with pip: {venv_path.absolute()}') + self.log(f"Initial venv with pip: {venv_path.absolute()}") if self.FASTER_PREPARE_PIP: venv.create(env_dir=venv_path, system_site_packages=False, with_pip=False) import shutil import pip + pip_dir = Path(pip.__file__).parent - if os.name == 'nt': - target = venv_path / 'Lib' / 'site-packages' / 'pip' + if os.name == "nt": + target = venv_path / "Lib" / "site-packages" / "pip" else: - pyv = 'python%d.%d' % sys.version_info[:2] - target = venv_path / 'lib' / pyv / 'site-packages' / 'pip' + pyv = "python%d.%d" % sys.version_info[:2] + target = venv_path / "lib" / pyv / "site-packages" / "pip" shutil.copytree(pip_dir, target) else: venv.create(env_dir=venv_path, system_site_packages=False, with_pip=True) @@ -78,15 +79,15 @@ def install_env(self): raise FileNotFoundError(str(venv_path)) def install_packages(self): - if os.name == 'nt': - python_path = self.temp_path / self.VENV_NAME / 'Scripts' / 'python.exe' + if os.name == "nt": + python_path = self.temp_path / self.VENV_NAME / "Scripts" / "python.exe" else: - python_path = self.temp_path / self.VENV_NAME / 'bin' / 'python' + python_path = self.temp_path / self.VENV_NAME / "bin" / "python" args = [ str(python_path.absolute()), - '-m', - 'pip', - 'install', + "-m", + "pip", + "install", ] + self.pip_args self.log(f'Install packages in venv: {args}\n{"-" * 30}') with subprocess.Popen(args, stdout=subprocess.PIPE) as proc: @@ -94,32 +95,32 @@ def install_packages(self): try: line = line.decode() except ValueError: - line = line.decode('utf-8', 'ignore') + line = line.decode("utf-8", "ignore") print(line.rstrip(), file=sys.stderr, flush=True) - args = [str(python_path.absolute()), '-m', 'pip', 'freeze'] + args = [str(python_path.absolute()), "-m", "pip", "freeze"] print("-" * 30, file=sys.stderr) - self.log(f'Freeze packages in venv: {args}') + self.log(f"Freeze packages in venv: {args}") output = subprocess.check_output(args) try: - result = output.decode('utf-8') + result = output.decode("utf-8") except ValueError: result = output.decode() - result = re.sub('(\n|\r)+', '\n', result).strip() + result = re.sub("(\n|\r)+", "\n", result).strip() return result def freeze_requirements(self, output): - if self.output_path == '-': + if self.output_path == "-": print(output, flush=True) else: print(output, file=sys.stderr, flush=True) - with open(self.output_path, 'w', encoding='utf-8') as f: + with open(self.output_path, "w", encoding="utf-8") as f: print(output, file=f, flush=True) def remove_env(self): if self.temp_dir and self.temp_path.is_dir(): self.temp_dir.cleanup() self.log( - f'Delete temp folder: {self.temp_path.absolute()}, exist={self.temp_path.is_dir()}' + f"Delete temp folder: {self.temp_path.absolute()}, exist={self.temp_path.is_dir()}" ) def __del__(self): @@ -133,10 +134,10 @@ def __exit__(self, *e): def test(): - with FreezeTool('-', ['six==1.15.0']) as ft: + with FreezeTool("-", ["six==1.15.0"]) as ft: result = ft.run() # print(result) - assert result == 'six==1.15.0' + assert result == "six==1.15.0" if __name__ == "__main__": diff --git a/morebuiltins/zipapps/main.py b/morebuiltins/zipapps/main.py index 1aaaefc..fa7cae9 100644 --- a/morebuiltins/zipapps/main.py +++ b/morebuiltins/zipapps/main.py @@ -15,7 +15,7 @@ from pkgutil import get_data from zipfile import ZIP_DEFLATED, ZIP_STORED, BadZipFile, ZipFile -__version__ = '2024.04.22' +__version__ = "2024.06.04" def get_pip_main(ensurepip_root=None): @@ -23,25 +23,29 @@ def get_pip_main(ensurepip_root=None): import pip except ImportError: import ensurepip + assert ensurepip._bootstrap(root=ensurepip_root) == 0 if ensurepip_root: - for _path in Path(ensurepip_root).glob('**/pip/'): + for _path in Path(ensurepip_root).glob("**/pip/"): if _path.is_dir(): sys.path.append(str(_path.parent.absolute())) break import pip try: from pip._internal.cli.main import main + return main except ImportError: pass try: from pip import main + return main except ImportError: pass try: from pip._internal import main + return main except ImportError: pass @@ -49,54 +53,55 @@ def get_pip_main(ensurepip_root=None): class ZipApp(object): - DEFAULT_OUTPUT_PATH = 'app.pyz' - DEFAULT_UNZIP_CACHE_PATH = 'zipapps_cache' - AUTO_FIX_UNZIP_KEYS = {'AUTO_UNZIP', 'AUTO'} + DEFAULT_OUTPUT_PATH = "app.pyz" + DEFAULT_UNZIP_CACHE_PATH = "zipapps_cache" + AUTO_FIX_UNZIP_KEYS = {"AUTO_UNZIP", "AUTO"} COMPILE_KWARGS: typing.Dict[str, typing.Any] = {} - HANDLE_OTHER_ENVS_FLAG = '--zipapps' - LAZY_PIP_DIR_NAME = '_zipapps_lazy_pip' - PATH_SPLIT_TAG = ',' - HANDLE_ACTIVATE_ZIPAPPS = '--activate-zipapps' + HANDLE_OTHER_ENVS_FLAG = "--zipapps" + LAZY_PIP_DIR_NAME = "_zipapps_lazy_pip" + PATH_SPLIT_TAG = "," + HANDLE_ACTIVATE_ZIPAPPS = "--activate-zipapps" ENV_ALIAS = { - 'unzip': 'ZIPAPPS_UNZIP', - 'unzip_exclude': 'ZIPAPPS_UNZIP_EXCLUDE', - 'unzip_path': 'ZIPAPPS_CACHE', - 'ignore_system_python_path': 'STRICT_PYTHON_PATH', - 'python_version_slice': 'PYTHON_VERSION_SLICE', - 'clear_zipapps_cache': 'CLEAR_ZIPAPPS_CACHE', - 'clear_zipapps_self': 'CLEAR_ZIPAPPS_SELF', - 'chmod': 'UNZIP_CHMOD', + "unzip": "ZIPAPPS_UNZIP", + "unzip_exclude": "ZIPAPPS_UNZIP_EXCLUDE", + "unzip_path": "ZIPAPPS_CACHE", + "ignore_system_python_path": "STRICT_PYTHON_PATH", + "python_version_slice": "PYTHON_VERSION_SLICE", + "clear_zipapps_cache": "CLEAR_ZIPAPPS_CACHE", + "clear_zipapps_self": "CLEAR_ZIPAPPS_SELF", + "chmod": "UNZIP_CHMOD", } LOGGING = True def __init__( self, - includes: str = '', - cache_path: str = None, - main: str = '', - output: str = None, - interpreter: str = None, + includes: str = "", + cache_path: typing.Optional[str] = None, + main: str = "", + output: typing.Optional[str] = None, + interpreter: typing.Optional[str] = None, compressed: bool = False, shell: bool = False, - unzip: str = '', - unzip_path: str = '', + unzip: str = "", + unzip_path: str = "", ignore_system_python_path=False, main_shell=False, - pip_args: list = None, + pip_args: typing.Optional[list] = None, compiled: bool = False, - build_id: str = '', - env_paths: str = '', + build_id: str = "", + env_paths: str = "", lazy_install: bool = False, - sys_paths: str = '', + sys_paths: str = "", python_version_slice: int = 2, ensure_pip: bool = False, layer_mode: bool = False, - layer_mode_prefix: str = 'python', + layer_mode_prefix: str = "python", clear_zipapps_cache: bool = False, - unzip_exclude: str = '', - chmod: str = '', + unzip_exclude: str = "", + chmod: str = "", clear_zipapps_self: bool = False, + rm_patterns: str = "*.dist-info,__pycache__", ): """Zip your code. @@ -150,6 +155,8 @@ def __init__( :type chmod: str, optional :param clear_zipapps_self: Clear the zipapps pyz file after running. :type clear_zipapps_self: bool, optional + :param rm_patterns: Delete useless files or folders, splited by "," and defaults to `*.dist-info,__pycache__`. Recursively glob: **/*.pyc + :type rm_patterns: str """ self.includes = includes self.cache_path = cache_path @@ -177,16 +184,19 @@ def __init__( self.clear_zipapps_cache = clear_zipapps_cache self.clear_zipapps_self = clear_zipapps_self self.chmod = chmod + self.rm_patterns = rm_patterns - self._tmp_dir: tempfile.TemporaryDirectory = None + self._tmp_dir: typing.Optional[tempfile.TemporaryDirectory] = None self._build_success = False - self._is_greater_than_python_37 = sys.version_info.minor >= 7 and sys.version_info.major >= 3 + self._is_greater_than_python_37 = ( + sys.version_info.minor >= 7 and sys.version_info.major >= 3 + ) @property def kwargs(self): return dict( includes=self.includes, - cache_path=str(self.cache_path), + cache_path=str(self.cache_path or ""), main=self.main, output=self.output, interpreter=self.interpreter, @@ -216,44 +226,47 @@ def ensure_args(self): if not self.unzip: if self.unzip_exclude: self._log( - '[WARN]: The arg `unzip_exclude` should not be with `unzip` but `unzip` is null.' + "[WARN]: The arg `unzip_exclude` should not be with `unzip` but `unzip` is null." ) if self.compiled: self._log( - '[WARN]: The arg `compiled` should not be True while `unzip` is null, because .pyc files of __pycache__ folder may not work in zip file.' + "[WARN]: The arg `compiled` should not be True while `unzip` is null, because .pyc files of __pycache__ folder may not work in zip file." ) if self.lazy_install: self._log( '[WARN]: the `unzip` arg has been changed to "*" while `lazy_install` is True.' ) - self.unzip = '*' + self.unzip = "*" if self.cache_path: self._cache_path = Path(self.cache_path) else: - self._tmp_dir = tempfile.TemporaryDirectory(prefix='zipapps_') + self._tmp_dir = tempfile.TemporaryDirectory(prefix="zipapps_") self._cache_path = Path(self._tmp_dir.name) if not self.unzip_path: if self.lazy_install: self._log( - f'[WARN]: the arg `unzip_path` has been changed to `SELF/{self.DEFAULT_UNZIP_CACHE_PATH}` while `lazy_install` is True and `unzip_path` is null.' + f"[WARN]: the arg `unzip_path` has been changed to `SELF/{self.DEFAULT_UNZIP_CACHE_PATH}` while `lazy_install` is True and `unzip_path` is null." ) - self.unzip_path = f'SELF/{ZipApp.DEFAULT_UNZIP_CACHE_PATH}' + self.unzip_path = f"SELF/{ZipApp.DEFAULT_UNZIP_CACHE_PATH}" else: self._log( - f'[INFO]: the arg `unzip_path` has been changed to `{self.DEFAULT_UNZIP_CACHE_PATH}` by default.' + f"[INFO]: the arg `unzip_path` has been changed to `{self.DEFAULT_UNZIP_CACHE_PATH}` by default." ) self.unzip_path = self.DEFAULT_UNZIP_CACHE_PATH self.build_id_name = self.get_build_id_name() self._log( - f'[INFO]: output path is `{self._output_path}`, you can reset it with the arg `output`.' + f"[INFO]: output path is `{self._output_path}`, you can reset it with the arg `output`." ) def prepare_ensure_pip(self): if self.ensure_pip: import ensurepip + ensurepip_dir_path = Path(ensurepip.__file__).parent - shutil.copytree(str(ensurepip_dir_path.absolute()), - self._cache_path / ensurepip_dir_path.name) + shutil.copytree( + str(ensurepip_dir_path.absolute()), + self._cache_path / ensurepip_dir_path.name, + ) def build(self): self._log( @@ -272,6 +285,7 @@ def build(self): (self._cache_path / self.build_id_name).touch() if self.compiled: compileall.compile_dir(self._cache_path, **ZipApp.COMPILE_KWARGS) + self.clean_pip_pycache() if self.layer_mode: self.create_archive_layer() else: @@ -286,44 +300,46 @@ def create_archive_layer(self): else: compression = ZIP_DEFLATED compresslevel = 0 - _kwargs = dict(mode='w', compression=compression) + _kwargs = dict(mode="w", compression=compression) if self._is_greater_than_python_37: - _kwargs['compresslevel'] = compresslevel + _kwargs["compresslevel"] = compresslevel with ZipFile(str(self._output_path), **_kwargs) as zf: - for f in self._cache_path.glob('**/*'): + for f in self._cache_path.glob("**/*"): zf.write(f, str(f.relative_to(self._cache_path))) def create_archive(self): if self._is_greater_than_python_37: - zipapp.create_archive(source=self._cache_path, - target=str(self._output_path.absolute()), - interpreter=self.interpreter, - compressed=self.compressed) + zipapp.create_archive( + source=self._cache_path, + target=str(self._output_path.absolute()), + interpreter=self.interpreter, + compressed=self.compressed, + ) elif self.compressed: - raise RuntimeError('The arg `compressed` only support python3.7+') + raise RuntimeError("The arg `compressed` only support python3.7+") else: - zipapp.create_archive(source=self._cache_path, - target=str(self._output_path.absolute()), - interpreter=self.interpreter) + zipapp.create_archive( + source=self._cache_path, + target=str(self._output_path.absolute()), + interpreter=self.interpreter, + ) def prepare_entry_point(self): # reset unzip_names - unzip_names = set(self.unzip.split(',')) if self.unzip else set() + unzip_names = set(self.unzip.split(",")) if self.unzip else set() warning_names: typing.Dict[str, dict] = {} for path in self._cache_path.iterdir(): _name_not_included = path.name not in unzip_names if path.is_dir(): - pyd_counts = len(list(path.glob('**/*.pyd'))) - so_counts = len(list(path.glob('**/*.so'))) + pyd_counts = len(list(path.glob("**/*.pyd"))) + so_counts = len(list(path.glob("**/*.so"))) if (pyd_counts or so_counts) and _name_not_included: # warn which libs need to be unzipped if pyd_counts: - warning_names.setdefault(path.name, - {})['.pyd'] = pyd_counts + warning_names.setdefault(path.name, {})[".pyd"] = pyd_counts if so_counts: - warning_names.setdefault(path.name, - {})['.so'] = so_counts - elif path.is_file() and path.suffix in ('.pyd', '.so'): + warning_names.setdefault(path.name, {})[".so"] = so_counts + elif path.is_file() and path.suffix in (".pyd", ".so"): if _name_not_included and path.stem not in unzip_names: warning_names.setdefault(path.name, {})[path.suffix] = 1 # remove the special keys from unzip_names @@ -331,149 +347,177 @@ def prepare_entry_point(self): unzip_names -= auto_unzip_keys if warning_names: if self.clear_zipapps_cache: - msg = f'[WARN]: clear_zipapps_cache is True but .pyd/.so files were found {warning_names}' + msg = f"[WARN]: clear_zipapps_cache is True but .pyd/.so files were found {warning_names}" self._log(msg) if auto_unzip_keys: unzip_names |= warning_names.keys() else: _fix_unzip_names = ",".join(warning_names.keys()) - msg = f'[WARN]: .pyd/.so files may be imported incorrectly, set `--unzip={_fix_unzip_names}` or `--unzip=AUTO` or `--unzip=*` to fix it. {warning_names}' + msg = f"[WARN]: .pyd/.so files may be imported incorrectly, set `--unzip={_fix_unzip_names}` or `--unzip=AUTO` or `--unzip=*` to fix it. {warning_names}" self._log(msg) - new_unzip = ','.join(unzip_names) + new_unzip = ",".join(unzip_names) self.unzip = new_unzip if self.unzip: self._log( - f'[INFO]: these names will be unzipped while running: {self.unzip}' + f"[INFO]: these names will be unzipped while running: {self.unzip}" ) self.prepare_active_zipapps() def prepare_active_zipapps(self): output_name = Path(self._output_path).stem - if not re.match(r'^[0-9a-zA-Z_]+$', output_name): - raise ValueError( - 'The name of `output` should match regex: ^[0-9a-zA-Z_]+$') + if not re.match(r"^[0-9a-zA-Z_]+$", output_name): + raise ValueError("The name of `output` should match regex: ^[0-9a-zA-Z_]+$") def make_runner(): if self.main: - if re.match(r'^\w+(\.\w+)?(:\w+)?$', self.main): - module, _, function = self.main.partition(':') + pattern = r"^\w+(\.\w+)?(:\w+)?$" + if re.match(pattern, self.main): + module, _, function = self.main.partition(":") if module: # main may be: 'module.py:main' or 'module.submodule:main' # replace module.py to module module_path = self._cache_path / module if module_path.is_file(): module = module_path.stem - runner = f'import {module}' + runner = f"import {module}" if function: - runner += f'; {module}.{function}()' + runner += f"; {module}.{function}()" self._log( - f"[INFO]: -m: matches re.match(r'^\w+(\.\w+)?(:\w+)?$', self.main), add as `{runner}`." + f"[INFO]: -m: matches re.match(r'{pattern}', self.main), add as `{runner}`." ) return runner else: self._log( - f"[INFO]: -m: not matches re.match(r'^\w+(\.\w+)?(:\w+)?$', self.main), add as raw code `{self.main}`." + f"[INFO]: -m: not matches re.match(r'{pattern}', self.main), add as raw code `{self.main}`." ) return self.main - return '' + return "" kwargs = { - 'ts': self.setup_timestamp_file(), - 'shell': self.shell, - 'main_shell': self.main_shell, - 'unzip': repr(self.unzip), - 'unzip_exclude': repr(self.unzip_exclude), - 'output_name': output_name, - 'unzip_path': repr(self.unzip_path), - 'ignore_system_python_path': self.ignore_system_python_path, - 'has_main': bool(self.main), - 'run_main': make_runner(), - 'HANDLE_OTHER_ENVS_FLAG': self.HANDLE_OTHER_ENVS_FLAG, - 'env_paths': repr(self.env_paths), - 'LAZY_PIP_DIR_NAME': repr(self.LAZY_PIP_DIR_NAME), - 'pip_args_repr': repr(self.pip_args), - 'sys_paths': repr(self.sys_paths), - 'python_version_slice': repr(self.python_version_slice), - 'pip_args_md5': self.pip_args_md5, - 'clear_zipapps_cache': repr(self.clear_zipapps_cache), - 'HANDLE_ACTIVATE_ZIPAPPS': self.HANDLE_ACTIVATE_ZIPAPPS, - 'chmod': repr(self.chmod), - 'clear_zipapps_self': repr(self.clear_zipapps_self), + "ts": self.setup_timestamp_file(), + "shell": self.shell, + "main_shell": self.main_shell, + "unzip": repr(self.unzip), + "unzip_exclude": repr(self.unzip_exclude), + "output_name": output_name, + "unzip_path": repr(self.unzip_path), + "ignore_system_python_path": self.ignore_system_python_path, + "has_main": bool(self.main), + "run_main": make_runner(), + "HANDLE_OTHER_ENVS_FLAG": self.HANDLE_OTHER_ENVS_FLAG, + "env_paths": repr(self.env_paths), + "LAZY_PIP_DIR_NAME": repr(self.LAZY_PIP_DIR_NAME), + "pip_args_repr": repr(self.pip_args), + "sys_paths": repr(self.sys_paths), + "python_version_slice": repr(self.python_version_slice), + "pip_args_md5": self.pip_args_md5, + "clear_zipapps_cache": repr(self.clear_zipapps_cache), + "HANDLE_ACTIVATE_ZIPAPPS": self.HANDLE_ACTIVATE_ZIPAPPS, + "chmod": repr(self.chmod), + "clear_zipapps_self": repr(self.clear_zipapps_self), } for k, v in self.ENV_ALIAS.items(): - kwargs[f'{k}_env'] = repr(v) - code = get_data(__name__, 'entry_point.py.template').decode('u8') - (self._cache_path / '__main__.py').write_text(code.format(**kwargs)) + kwargs[f"{k}_env"] = repr(v) + code = get_data(__name__, "entry_point.py.template").decode("u8") + (self._cache_path / "__main__.py").write_text(code.format(**kwargs)) - code = get_data(__name__, 'ensure_zipapps.py.template').decode('u8') - (self._cache_path / 'ensure_zipapps.py').write_text( - code.format(**kwargs)) + code = get_data(__name__, "ensure_zipapps.py.template").decode("u8") + (self._cache_path / "ensure_zipapps.py").write_text(code.format(**kwargs)) - code = get_data(__name__, 'activate_zipapps.py').decode('u8') - (self._cache_path / 'activate_zipapps.py').write_text(code) - code += '\n\nactivate()' + code = get_data(__name__, "activate_zipapps.py").decode("u8") + (self._cache_path / "activate_zipapps.py").write_text(code) + code += "\n\nactivate()" - if output_name != 'zipapps': - (self._cache_path / f'ensure_{output_name}.py').write_text(code) - (self._cache_path / f'ensure_zipapps_{output_name}.py').write_text(code) - (self._cache_path / 'zipapps_config.json').write_text( - json.dumps(self.kwargs)) + if output_name != "zipapps": + (self._cache_path / f"ensure_{output_name}.py").write_text(code) + (self._cache_path / f"ensure_zipapps_{output_name}.py").write_text(code) + (self._cache_path / "zipapps_config.json").write_text(json.dumps(self.kwargs)) - def setup_timestamp_file(self,): + def setup_timestamp_file( + self, + ): ts = str(int(time.time() * 10000000)) - (self._cache_path / ('_zip_time_%s' % ts)).touch() + (self._cache_path / ("_zip_time_%s" % ts)).touch() return ts + @staticmethod + def get_md5(value: typing.Any): + if not isinstance(value, bytes): + value = str(value).encode("utf-8") + return md5(value).hexdigest() + def prepare_pip(self): - self.pip_args_md5 = '' + self.pip_args_md5 = "" if self.pip_args: - if '-t' in self.pip_args or '--target' in self.pip_args: + if "-t" in self.pip_args or "--target" in self.pip_args: raise RuntimeError( - '`-t` / `--target` arg can be set with `--cache-path` to rewrite the zipapps cache path.' + "`-t` / `--target` arg can be set with `--cache-path`/`cache_path` to rewrite the zipapps cache path." ) if self.lazy_install: # copy files to cache folder _temp_pip_path = self._cache_path / self.LAZY_PIP_DIR_NAME _temp_pip_path.mkdir(parents=True, exist_ok=True) - _md5_str = md5(str(self.pip_args).encode('utf-8')).hexdigest() + _md5_str = self.get_md5(self.pip_args) # overwrite path args to new path, such as requirements.txt or xxx.whl for index, arg in enumerate(self.pip_args): path = Path(arg) if path.is_file(): - _md5_str += md5(path.read_bytes()).hexdigest() + _md5_str += self.get_md5(path.read_bytes()) new_path = _temp_pip_path / path.name shutil.copyfile(path, new_path) _r_path = Path(self.LAZY_PIP_DIR_NAME) / path.name self.pip_args[index] = _r_path.as_posix() - self.pip_args_md5 = md5(_md5_str.encode('utf-8')).hexdigest() + self.pip_args_md5 = self.get_md5(_md5_str.encode("utf-8")) self._log( - f'[INFO]: pip_args_md5 has been generated: {self.pip_args_md5}' + f"[INFO]: pip_args_md5 has been generated: {self.pip_args_md5}" ) else: self.pip_install() - def pip_install(self): - if self.layer_mode: - _target_dir = self._cache_path.absolute() / self.layer_mode_prefix - target = str(_target_dir) - else: - target = str(self._cache_path.absolute()) - _pip_args = ['install', '--target', target] + self.pip_args + @classmethod + def _rm_with_patterns( + cls, + target_dir: Path, + patterns=("*.dist-info", "__pycache__"), + ): + target_dir = Path(target_dir) + for pattern in patterns: + if pattern: + for path in target_dir.glob(pattern): + if path.is_dir(): + shutil.rmtree(path, ignore_errors=True) + else: + try: + path.unlink() + except FileNotFoundError: + pass + + @classmethod + def _pip_install(cls, target_dir: Path, pip_args: list): + target_dir = Path(target_dir) + _pip_args = [ + "install", + "--target", + target_dir.absolute().as_posix(), + ] + pip_args pip_main = get_pip_main() result = pip_main(_pip_args) - assert result == 0, 'pip install failed %s' % result - self.clean_pip_pycache() + if result != 0: + raise RuntimeError("pip install failed: return code=%s" % result) def clean_pip_pycache(self): if self.layer_mode: - root = self._cache_path / self.layer_mode_prefix + target_dir = self._cache_path / self.layer_mode_prefix + else: + target_dir = self._cache_path + return self._rm_with_patterns(target_dir, patterns=self.rm_patterns.split(",")) + + def pip_install(self): + if self.layer_mode: + _target_dir = self._cache_path.absolute() / self.layer_mode_prefix else: - root = self._cache_path - for dist_path in root.glob('*.dist-info'): - shutil.rmtree(dist_path) - pycache = root / '__pycache__' - if pycache.is_dir(): - shutil.rmtree(pycache) + _target_dir = self._cache_path + return self._pip_install(target_dir=_target_dir, pip_args=self.pip_args) def prepare_includes(self): if not self.includes: @@ -490,7 +534,7 @@ def prepare_includes(self): elif include_path.is_file(): shutil.copyfile(include_path, _target_dir / include_path.name) else: - raise RuntimeError('%s is not exist' % include_path.absolute()) + raise RuntimeError("%s is not exist" % include_path.absolute()) def build_exists(self): if self.build_id_name and self._output_path.is_file(): @@ -505,12 +549,12 @@ def build_exists(self): def get_build_id_name(self): if not self.build_id: - return '' - build_id_str = '' - if '*' in self.build_id: + return "" + build_id_str = "" + if "*" in self.build_id: paths = glob(self.build_id) else: - paths = self.build_id.split(',') + paths = self.build_id.split(",") for p in paths: try: path = Path(p) @@ -518,37 +562,38 @@ def get_build_id_name(self): except FileNotFoundError: pass build_id_str = build_id_str or str(self.build_id) - md5_id = md5(build_id_str.encode('utf-8')).hexdigest() - return f'_build_id_{md5_id}' + md5_id = self.get_md5(build_id_str.encode("utf-8")) + return f"_build_id_{md5_id}" @classmethod def create_app( cls, - includes: str = '', - cache_path: str = None, - main: str = '', - output: str = None, - interpreter: str = None, + includes: str = "", + cache_path: typing.Optional[str] = None, + main: str = "", + output: typing.Optional[str] = None, + interpreter: typing.Optional[str] = None, compressed: bool = False, shell: bool = False, - unzip: str = '', - unzip_path: str = '', + unzip: str = "", + unzip_path: str = "", ignore_system_python_path=False, main_shell=False, - pip_args: list = None, + pip_args: typing.Optional[list] = None, compiled: bool = False, - build_id: str = '', - env_paths: str = '', + build_id: str = "", + env_paths: str = "", lazy_install: bool = False, - sys_paths: str = '', + sys_paths: str = "", python_version_slice: int = 2, ensure_pip: bool = False, layer_mode: bool = False, - layer_mode_prefix: str = 'python', + layer_mode_prefix: str = "python", clear_zipapps_cache: bool = False, - unzip_exclude: str = '', - chmod: str = '', + unzip_exclude: str = "", + chmod: str = "", clear_zipapps_self: bool = False, + rm_patterns: str = "*.dist-info,__pycache__", ): app = cls( includes=includes, @@ -576,6 +621,7 @@ def create_app( unzip_exclude=unzip_exclude, chmod=chmod, clear_zipapps_self=clear_zipapps_self, + rm_patterns=rm_patterns, ) return app.build() @@ -587,8 +633,7 @@ def _log(cls, text): def __del__(self): if self._tmp_dir: self._tmp_dir.cleanup() - self._log( - f'[INFO]: Temp cache has been cleaned. ({self._tmp_dir!r})') + self._log(f"[INFO]: Temp cache has been cleaned. ({self._tmp_dir!r})") if self._build_success: self._log( f'[INFO]: {"=" * 10} Successfully built `{self._output_path}` {"=" * 10}' @@ -597,4 +642,42 @@ def __del__(self): self._log(f'[ERROR]: {"=" * 10} Build failed {"=" * 10}') +def pip_install_target( + target: Path, + pip_args: list, + rm_patterns: str = "*.dist-info,__pycache__", + force=False, + clear_dir=False, + sys_path: typing.Optional[int] = None, +): + """ + Installs target dependencies using pip and cleans up the installed files afterwards. + + Args: + target (Path): The target directory for dependency installation. + pip_args (list): List of arguments for the pip install command. + rm_patterns (str, optional): Patterns of files to remove after installation, defaults to "*.dist-info,__pycache__". + force (bool, optional): Flag to force reinstallation, defaults to False. + clear_dir (bool, optional): will rmtree the target directory while clear_dir=True. + sys_path (typing.Optional[int], optional): If provided, inserts the target directory at the specified position in sys.path. + + Returns: + bool: Returns True if the installation is successful. + """ + target = Path(target) + md5_path = target / ZipApp.get_md5(pip_args) + if not force and md5_path.exists(): + return False + if clear_dir and target.is_dir(): + shutil.rmtree(target.as_posix(), ignore_errors=True) + ZipApp._pip_install(target_dir=target, pip_args=pip_args) + if rm_patterns: + ZipApp._rm_with_patterns(target, patterns=rm_patterns.split(",")) + + md5_path.touch() + if isinstance(sys_path, int): + sys.path.insert(sys_path, target.absolute().as_posix()) + return True + + create_app = ZipApp.create_app