Merge pull request #32 from gentlegiantJGC/improv-download

Improve resource pack downloading

minecraft_model_reader/api/mesh/block/block_mesh.py

@@ -206,7 +206,7 @@ def vert_tables(self) -> Dict[str, numpy.ndarray]:
                 key: numpy.hstack(
                     (
                         self._verts[key].reshape(-1, self._face_mode),
-                        self._texture_coords[key].reshape(-1, 2)
+                        self._texture_coords[key].reshape(-1, 2),
                         # TODO: add in face normals
                     )
                 ).ravel()

minecraft_model_reader/api/resource_pack/bedrock/resource_pack_manager.py

@@ -11,15 +11,13 @@
 from .blockshapes import BlockShapeClasses
 
 
-def _load_data() -> (
-    Tuple[
-        Dict[str, str],
-        Dict[
-            str,
-            Tuple[Tuple[Tuple[str, str], ...], Dict[Tuple[Union[str, int], ...], int]],
-        ],
-    ]
-):
+def _load_data() -> Tuple[
+    Dict[str, str],
+    Dict[
+        str,
+        Tuple[Tuple[Tuple[str, str], ...], Dict[Tuple[Union[str, int], ...], int]],
+    ],
+]:
     with open(os.path.join(os.path.dirname(__file__), "blockshapes.json")) as f:
         _block_shapes = comment_json.load(f)
 
@@ -48,9 +46,11 @@ def get_aux_value(block: Block) -> int:
         property_names, aux_map = AuxValues[name]
         properties = block.properties
         key = tuple(
-            properties[property_name].py_data
-            if property_name in properties
-            else default
+            (
+                properties[property_name].py_data
+                if property_name in properties
+                else default
+            )
             for property_name, default in property_names
         )
         return aux_map.get(key, 0)
@@ -69,12 +69,10 @@ def __init__(
     ):
         super().__init__()
         self._block_shapes: Dict[str, str] = {}  # block string to block shape
-        self._blocks: Dict[
-            str, Union[Dict[str, str], str, None]
-        ] = {}  # block string to short texture ids
-        self._terrain_texture: Dict[
-            str, Tuple[str, ...]
-        ] = (
+        self._blocks: Dict[str, Union[Dict[str, str], str, None]] = (
+            {}
+        )  # block string to short texture ids
+        self._terrain_texture: Dict[str, Tuple[str, ...]] = (
             {}
         )  # texture ids to list of relative paths. Each relates to a different data value.
         self._textures: Dict[str, str] = {}  # relative path to texture path

minecraft_model_reader/api/resource_pack/java/download_resources.py

@@ -2,7 +2,7 @@
 import shutil
 import zipfile
 import json
-from urllib.request import urlopen
+from urllib.request import urlopen, Request
 import io
 from typing import Generator, List
 import logging
@@ -114,18 +114,38 @@ def _remove_and_download_iter(path, version) -> Generator[float, None, None]:
     elif os.path.isdir(temp_path):
         shutil.rmtree(temp_path, ignore_errors=True)
 
-    try:
-        yield from download_resources_iter(temp_path, version)
-    except:
-        pass
-    else:
-        if os.path.isdir(path):
-            shutil.rmtree(path, ignore_errors=True)
+    yield from download_resources_iter(temp_path, version)
+    if os.path.isdir(path):
+        shutil.rmtree(path, ignore_errors=True)
 
-        shutil.move(temp_path, path)
+    shutil.move(temp_path, path)
 
-        with open(os.path.join(path, "version"), "w") as f:
-            f.write(version)
+    with open(os.path.join(path, "version"), "w") as f:
+        f.write(version)
+
+
+def download_with_retry(
+    url: str, chunk_size: int = 4096, attempts: int = 5
+) -> Generator[float, None, bytes]:
+    content_length_found = 0
+    content = []
+
+    for _ in range(attempts):
+        request = Request(url, headers={"Range": f"bytes={content_length_found}-"})
+        with urlopen(request, timeout=20) as response:
+            content_length = int(response.headers["content-length"].strip())
+            while content_length_found < content_length:
+                chunk = response.read(chunk_size)
+                if not chunk:
+                    break
+                content.append(chunk)
+                content_length_found += len(chunk)
+                yield min(1.0, content_length_found / content_length)
+        if content_length == content_length_found:
+            break
+    else:
+        raise RuntimeError(f"Failed to download")
+    return b"".join(content)
 
 
 def download_resources(path, version):
@@ -148,18 +168,14 @@ def download_resources_iter(
             version_manifest = json.load(vm)
         version_client_url = version_manifest["downloads"]["client"]["url"]
 
-        with urlopen(version_client_url, timeout=20) as response:
-            data = []
-            data_size = int(response.headers["content-length"].strip())
-            index = 0
-            chunk = b"hello"
-            while chunk:
-                chunk = response.read(chunk_size)
-                data.append(chunk)
-                index += 1
-                yield min(1.0, (index * chunk_size) / (data_size * 2))
+        downloader = download_with_retry(version_client_url)
+        try:
+            while True:
+                yield next(downloader) / 2
+        except StopIteration as e:
+            data = e.value
 
-        client = zipfile.ZipFile(io.BytesIO(b"".join(data)))
+        client = zipfile.ZipFile(io.BytesIO(data))
         paths: List[str] = [
             fpath for fpath in client.namelist() if fpath.startswith("assets/")
         ]

minecraft_model_reader/api/resource_pack/java/resource_pack_manager.py

@@ -145,9 +145,9 @@ def _load_iter(self) -> Generator[float, None, None]:
                     _, namespace, _, blockstate_file = os.path.normpath(
                         os.path.relpath(blockstate_path, pack.root_dir)
                     ).split(os.sep)
-                    blockstate_file_paths[
-                        (namespace, blockstate_file[:-5])
-                    ] = blockstate_path
+                    blockstate_file_paths[(namespace, blockstate_file[:-5])] = (
+                        blockstate_path
+                    )
                     yield sub_progress + (blockstate_index) / (
                         blockstate_count * pack_count * 3
                     )
@@ -170,9 +170,9 @@ def _load_iter(self) -> Generator[float, None, None]:
                         os.path.relpath(model_path, pack.root_dir)
                     ).split(os.sep)
                     rel_path = "/".join(rel_path_list)[:-5]
-                    model_file_paths[
-                        (namespace, rel_path.replace(os.sep, "/"))
-                    ] = model_path
+                    model_file_paths[(namespace, rel_path.replace(os.sep, "/"))] = (
+                        model_path
+                    )
                     yield sub_progress + (model_index) / (model_count * pack_count * 3)
 
         os.makedirs(os.path.dirname(transparency_cache_path), exist_ok=True)