Merge pull request #5 from hfg-gmuend/dev

Merge dev 0.8

README.md

@@ -8,14 +8,13 @@ Zoomaker is a command-line tool that helps install AI models, git repositories a
 - **single source of truth**: all resources are neatly defined in the `zoo.yaml` file
 - **freeze versions**: know exactly which revision of a resources is installed at any time
 - **only download once**: optimize bandwidth and cache your models locally
-- **optimize disk usage**: downloaded models are symlinked to the installation folder (small files <5MB are duplicate)
+- **optimize disk usage**: downloaded models are cached
 
 ## 😻 TL;DR
 
 1. Install Zoomaker `pip install zoomaker`
 2. Define your resources in the `zoo.yaml` file
 3. Run `zoomaker install` to install them
-(on Windows: `zoomaker install --no-symlinks`, see [hints](https://github.com/hfg-gmuend/zoomaker#%EF%B8%8F-limitations-on-windows) below)
 
 
 ## 📦 Installation
@@ -121,7 +120,7 @@ scripts:
   start_webui: |
     conda activate automatic1111
     cd /home/$(whoami)/stable-diffusion-webui/
-    ./webui.sh --theme dark --xformers --no-half
+    ./webui.sh --xformers --no-half
 ```
 </details>
 
@@ -138,7 +137,7 @@ resources:
       rename_to: analog-diffusion-v1.safetensors
 ```
 Please note:
-The resource `type: download` can be seen as the last resort. Currently there is no caching or symlinking of web downloads. Recommended to avoid it :)
+The resource `type: download` can be seen as the last resort. Existing web downloads are skipped, but no other caching. It is recommended to avoid web downloads :)
 </details>
 
 ## 🧮 zoo.yaml Structure
@@ -171,25 +170,23 @@ All commands are run from the root of the project, where also your `zoo.yaml` fi
 | `zoomaker run <script_name>`    | Run CLI scripts as defined in `zoo.yaml` |
 | `zoomaker --help` | Get help using the Zoomaker CLI                     |
 | `zoomaker --version` | Show current Zoomaker version                     |
-| `zoomaker --no-symlinks` | Do not use symlinks for installing resources  |
 
-## ⚠️ Limitations on Windows
-Symlinks are not widely supported on Windows, which limits the caching mechanism used by Zoomaker. To work around this limitation, you can disable symlinks by using the `--no-symlinks` flag with the install command:
 
-```bash
-zoomaker install --no-symlinks
-```
-
-This will still use the cache directory for checking if files are already cached, but if not, they will be downloaded and duplicated directly to the installation directory, saving bandwidth but increasing disk usage. Alternatively, you can use the [Windows Subsystem for Linux "WSL"](https://docs.microsoft.com/en-us/windows/wsl/install-win10) (don't forget to [enable developer mode](https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development)) or run Zoomaker as an administrator to enable symlink support on Windows.
-
-## 🤗 Hugging Face Access Token
+## 🤗 Hugging Face Access Token and Custom Cache Location
 
 You might be asked for a [Hugging Face Access Token](https://huggingface.co/docs/hub/security-tokens) during `zoomaker install`. Some resources on Hugging Face require accepting the terms of use of the model. You can set your access token by running this command in a terminal. The command `huggingface-cli` is automatically shipped alongside zoomaker.
 
 ```bash
 huggingface-cli login
 ```
 
+You can specify a custom cache location by setting the HF_HOME environment variable. The default cache location is `~/.cache/huggingface/`.
+
+```bash
+export HF_HOME=/path/to/your/cache
+zoomaker install
+```
+
 ## 🙏 Acknowledgements
 - Most of the internal heavy lifting is done be the [huggingface_hub library](https://huggingface.co/docs/huggingface_hub/guides/download) by Hugging Face. Thanks!
 - "Zoomaker Safari Hacker Cat" cover image by Alia Tasler, based on this [OpenMoji](https://openmoji.org/library/emoji-1F431-200D-1F4BB/). Thanks!

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "zoomaker"
-version = "0.7.0"
+version = "0.8.0"
 description = "Zoomaker - Friendly house keeping for your AI model zoo and related resources."
 authors = ["Benedikt Groß"]
 readme = "README.md"
@@ -10,8 +10,8 @@ repository = "https://github.com/hfg-gmuend/zoomaker"
 
 [tool.poetry.dependencies]
 python = "^3.9"
-huggingface-hub = "^0.14.0"
-GitPython = "^3.1.31"
+huggingface-hub = "^0.23.0"
+GitPython = "^3.1.43"
 
 [tool.poetry.scripts]
 zoomaker = 'zoomaker:main'

test/tests.py

@@ -6,6 +6,7 @@
 import io
 import shutil
 import sys
+from huggingface_hub import try_to_load_from_cache, _CACHED_NO_EXIST
 sys.path.append('..')
 from zoomaker import Zoomaker
 
@@ -67,12 +68,6 @@ def test_install_huggingface(self):
             """
             name: test
             resources:
-                models:
-                    - name: analog-diffusion-1.0
-                      src: wavymulder/Analog-Diffusion/analog-diffusion-1.0.safetensors
-                      type: huggingface
-                      install_to: ./tmp/models/Stable-diffusion/
-
                 embeddings:
                     - name: moebius
                       src: sd-concepts-library/moebius/learned_embeds.bin
@@ -83,39 +78,16 @@ def test_install_huggingface(self):
         )
         zoomaker = Zoomaker("zoo.yaml")
         zoomaker.install()
-        self.assertTrue(os.path.exists("./tmp/models/Stable-diffusion/analog-diffusion-1.0.safetensors"))
         self.assertTrue(os.path.exists("./tmp/embeddings/moebius.bin"))
-        # smybolik link
-        self.assertTrue(os.path.islink("./tmp/models/Stable-diffusion/analog-diffusion-1.0.safetensors"))
-        # no smybolik link, as smaller than 5MB
-        self.assertFalse(os.path.islink("./tmp/embeddings/moebius.bin"))
 
-    def test_install_no_symlinks(self):
-        create_zoo(
-            """
-            name: test
-            resources:
-                models:
-                    - name: analog-diffusion-1.0
-                      src: wavymulder/Analog-Diffusion/analog-diffusion-1.0.safetensors
-                      type: huggingface
-                      install_to: ./tmp/models/Stable-diffusion/
+    @unittest.skipIf(sys.platform.startswith("win"), "Skipping on Windows")
+    def test_install_huggingface_cached(self):
+        filepath = try_to_load_from_cache(
+            repo_id="sd-concepts-library/moebius", filename="learned_embeds.bin")
 
-                embeddings:
-                    - name: moebius
-                      src: sd-concepts-library/moebius/learned_embeds.bin
-                      type: huggingface
-                      install_to: ./tmp/embeddings/
-                      rename_to: moebius.bin
-            """
-        )
-        zoomaker = Zoomaker("zoo.yaml")
-        no_symlinks = True
-        zoomaker.install(no_symlinks)
-        self.assertTrue(os.path.exists("./tmp/models/Stable-diffusion/analog-diffusion-1.0.safetensors"))
-        self.assertTrue(os.path.exists("./tmp/embeddings/moebius.bin"))
-        self.assertFalse(os.path.islink("./tmp/models/Stable-diffusion/analog-diffusion-1.0.safetensors"))
-        self.assertFalse(os.path.islink("./tmp/embeddings/moebius.bin"))
+        self.assertTrue(isinstance(filepath, str))
+        self.assertTrue(os.path.exists(filepath))
+        self.assertFalse(filepath == _CACHED_NO_EXIST)
 
     def test_install_git(self):
         create_zoo(

zoomaker.py

@@ -35,12 +35,10 @@ def _check_yaml(self):
                 if type not in ["huggingface", "git", "download"]:
                     raise Exception(f"❌ Unknown resource type: {type}")
 
-    def install(self, no_symlinks: bool = False):
+    def install(self):
         print(f"👋 ===> {self.yaml_file} <===")
         print(f"name: {self.data.get('name', 'N/A')}")
         print(f"version: {self.data.get('version', 'N/A')}\n")
-        if no_symlinks:
-            print(f"⛔️ installing resources without symlinks ...")
         print(f"👇 installing resources ...")
         counter = 0;
         for group, resources in self.data["resources"].items():
@@ -61,7 +59,8 @@ def install(self, no_symlinks: bool = False):
                 if type == "huggingface":
                     repo_id = "/".join(src.split("/")[0:2])
                     repo_filepath = "/".join(src.split("/")[2:])
-                    downloaded = hf_hub_download(repo_id=repo_id, filename=repo_filepath, local_dir=install_to, revision=revision, local_dir_use_symlinks=False if no_symlinks else "auto")
+                    downloaded = hf_hub_download(repo_id=repo_id, filename=repo_filepath, local_dir=install_to, revision=revision)
+                    print(f"\t   size: {self._get_file_size(downloaded)}")
                     if rename_to:
                         self._rename_file(downloaded, os.path.join(install_to, rename_to))
                 # Git
@@ -90,11 +89,17 @@ def install(self, no_symlinks: bool = False):
                 # Download
                 else:
                     filename = self._slugify(os.path.basename(src))
-                    downloaded = self._download_file(src, os.path.join(install_to, filename))
-                    if rename_to:
-                        self._rename_file(downloaded, os.path.join(install_to, rename_to))
-                    if revision:
-                        print(f"\trevision is not supported for download. Ignoring revision: {revision}")
+                    destination = os.path.join(install_to, filename)
+                    destinationRenamed = os.path.join(install_to, rename_to)
+                    if os.path.exists(destination) or os.path.exists(destinationRenamed):
+                        print(f"\t   ℹ️ Skipping download: '{filename}' already exists")
+                    else:
+                        downloaded = self._download_file(src, destination)
+                        print(f"\t   size: {self._get_file_size(downloaded)}")
+                        if rename_to:
+                            self._rename_file(downloaded, destinationRenamed)
+                        if revision:
+                            print(f"\trevision is not supported for download. Ignoring revision: {revision}")
 
         print(f"\n✅ {counter} resources installed.")
 
@@ -123,6 +128,17 @@ def _rename_file(self, src, dest):
         else:
             os.rename(src, dest)
 
+    def _get_file_size(self, path):
+        size = os.stat(path).st_size
+        if size < 1024:
+            return f"{size} bytes"
+        elif size < pow(1024, 2):
+            return f"{round(size/1024, 2)} KB"
+        elif size < pow(1024, 3):
+            return f"{round(size/(pow(1024,2)), 2)} MB"
+        elif size < pow(1024, 4):
+            return f"{round(size/(pow(1024,3)), 2)} GB"
+
     def _download_file(self, url, filename):
         response = requests.get(url, stream=True)
         total_size_in_bytes = int(response.headers.get('content-length', 0))
@@ -160,12 +176,11 @@ def main():
     parser = argparse.ArgumentParser(description="Install models, git repos and run scripts defined in the zoo.yaml file.")
     parser.add_argument("command", nargs="?", choices=["install", "run"], help="The command to execute.")
     parser.add_argument("script", nargs="?", help="The script name to execute.")
-    parser.add_argument("--no-symlinks", action='store_true', help="Do not create symlinks for the installed resources.")
-    parser.add_argument("-v", "--version", action='version', help="The current version of the zoomaker.", version="0.7.0")
+    parser.add_argument("-v", "--version", action="version", help="The current version of the zoomaker.", version="0.8.0")
     args = parser.parse_args()
 
     if args.command == "install":
-        Zoomaker("zoo.yaml").install(args.no_symlinks)
+        Zoomaker("zoo.yaml").install()
     elif args.command == "run":
         Zoomaker("zoo.yaml").run(args.script)
     else: