diff --git a/minecraft_model_reader/api/mesh/block/block_mesh.py b/minecraft_model_reader/api/mesh/block/block_mesh.py index 70c08e9..990df5b 100644 --- a/minecraft_model_reader/api/mesh/block/block_mesh.py +++ b/minecraft_model_reader/api/mesh/block/block_mesh.py @@ -206,7 +206,7 @@ def vert_tables(self) -> Dict[str, numpy.ndarray]: key: numpy.hstack( ( self._verts[key].reshape(-1, self._face_mode), - self._texture_coords[key].reshape(-1, 2) + self._texture_coords[key].reshape(-1, 2), # TODO: add in face normals ) ).ravel() diff --git a/minecraft_model_reader/api/resource_pack/bedrock/resource_pack_manager.py b/minecraft_model_reader/api/resource_pack/bedrock/resource_pack_manager.py index 620c426..c597006 100644 --- a/minecraft_model_reader/api/resource_pack/bedrock/resource_pack_manager.py +++ b/minecraft_model_reader/api/resource_pack/bedrock/resource_pack_manager.py @@ -11,15 +11,13 @@ from .blockshapes import BlockShapeClasses -def _load_data() -> ( - Tuple[ - Dict[str, str], - Dict[ - str, - Tuple[Tuple[Tuple[str, str], ...], Dict[Tuple[Union[str, int], ...], int]], - ], - ] -): +def _load_data() -> Tuple[ + Dict[str, str], + Dict[ + str, + Tuple[Tuple[Tuple[str, str], ...], Dict[Tuple[Union[str, int], ...], int]], + ], +]: with open(os.path.join(os.path.dirname(__file__), "blockshapes.json")) as f: _block_shapes = comment_json.load(f) @@ -48,9 +46,11 @@ def get_aux_value(block: Block) -> int: property_names, aux_map = AuxValues[name] properties = block.properties key = tuple( - properties[property_name].py_data - if property_name in properties - else default + ( + properties[property_name].py_data + if property_name in properties + else default + ) for property_name, default in property_names ) return aux_map.get(key, 0) @@ -69,12 +69,10 @@ def __init__( ): super().__init__() self._block_shapes: Dict[str, str] = {} # block string to block shape - self._blocks: Dict[ - str, Union[Dict[str, str], str, None] - ] = {} # block string to short texture ids - self._terrain_texture: Dict[ - str, Tuple[str, ...] - ] = ( + self._blocks: Dict[str, Union[Dict[str, str], str, None]] = ( + {} + ) # block string to short texture ids + self._terrain_texture: Dict[str, Tuple[str, ...]] = ( {} ) # texture ids to list of relative paths. Each relates to a different data value. self._textures: Dict[str, str] = {} # relative path to texture path diff --git a/minecraft_model_reader/api/resource_pack/java/download_resources.py b/minecraft_model_reader/api/resource_pack/java/download_resources.py index a538d0f..bd56842 100644 --- a/minecraft_model_reader/api/resource_pack/java/download_resources.py +++ b/minecraft_model_reader/api/resource_pack/java/download_resources.py @@ -2,7 +2,7 @@ import shutil import zipfile import json -from urllib.request import urlopen +from urllib.request import urlopen, Request import io from typing import Generator, List import logging @@ -114,18 +114,38 @@ def _remove_and_download_iter(path, version) -> Generator[float, None, None]: elif os.path.isdir(temp_path): shutil.rmtree(temp_path, ignore_errors=True) - try: - yield from download_resources_iter(temp_path, version) - except: - pass - else: - if os.path.isdir(path): - shutil.rmtree(path, ignore_errors=True) + yield from download_resources_iter(temp_path, version) + if os.path.isdir(path): + shutil.rmtree(path, ignore_errors=True) - shutil.move(temp_path, path) + shutil.move(temp_path, path) - with open(os.path.join(path, "version"), "w") as f: - f.write(version) + with open(os.path.join(path, "version"), "w") as f: + f.write(version) + + +def download_with_retry( + url: str, chunk_size: int = 4096, attempts: int = 5 +) -> Generator[float, None, bytes]: + content_length_found = 0 + content = [] + + for _ in range(attempts): + request = Request(url, headers={"Range": f"bytes={content_length_found}-"}) + with urlopen(request, timeout=20) as response: + content_length = int(response.headers["content-length"].strip()) + while content_length_found < content_length: + chunk = response.read(chunk_size) + if not chunk: + break + content.append(chunk) + content_length_found += len(chunk) + yield min(1.0, content_length_found / content_length) + if content_length == content_length_found: + break + else: + raise RuntimeError(f"Failed to download") + return b"".join(content) def download_resources(path, version): @@ -148,18 +168,14 @@ def download_resources_iter( version_manifest = json.load(vm) version_client_url = version_manifest["downloads"]["client"]["url"] - with urlopen(version_client_url, timeout=20) as response: - data = [] - data_size = int(response.headers["content-length"].strip()) - index = 0 - chunk = b"hello" - while chunk: - chunk = response.read(chunk_size) - data.append(chunk) - index += 1 - yield min(1.0, (index * chunk_size) / (data_size * 2)) + downloader = download_with_retry(version_client_url) + try: + while True: + yield next(downloader) / 2 + except StopIteration as e: + data = e.value - client = zipfile.ZipFile(io.BytesIO(b"".join(data))) + client = zipfile.ZipFile(io.BytesIO(data)) paths: List[str] = [ fpath for fpath in client.namelist() if fpath.startswith("assets/") ] diff --git a/minecraft_model_reader/api/resource_pack/java/resource_pack_manager.py b/minecraft_model_reader/api/resource_pack/java/resource_pack_manager.py index cdad2b3..6988992 100644 --- a/minecraft_model_reader/api/resource_pack/java/resource_pack_manager.py +++ b/minecraft_model_reader/api/resource_pack/java/resource_pack_manager.py @@ -145,9 +145,9 @@ def _load_iter(self) -> Generator[float, None, None]: _, namespace, _, blockstate_file = os.path.normpath( os.path.relpath(blockstate_path, pack.root_dir) ).split(os.sep) - blockstate_file_paths[ - (namespace, blockstate_file[:-5]) - ] = blockstate_path + blockstate_file_paths[(namespace, blockstate_file[:-5])] = ( + blockstate_path + ) yield sub_progress + (blockstate_index) / ( blockstate_count * pack_count * 3 ) @@ -170,9 +170,9 @@ def _load_iter(self) -> Generator[float, None, None]: os.path.relpath(model_path, pack.root_dir) ).split(os.sep) rel_path = "/".join(rel_path_list)[:-5] - model_file_paths[ - (namespace, rel_path.replace(os.sep, "/")) - ] = model_path + model_file_paths[(namespace, rel_path.replace(os.sep, "/"))] = ( + model_path + ) yield sub_progress + (model_index) / (model_count * pack_count * 3) os.makedirs(os.path.dirname(transparency_cache_path), exist_ok=True)