Maintenance (#113)

* Update Progress badges to match other zelda games

* sync with dtk template

README.md

@@ -4,9 +4,9 @@ The Legend of Zelda: Skyward Sword
 
 [Build Status]: https://github.com/zeldaret/ss/actions/workflows/build.yml/badge.svg
 [actions]: https://github.com/zeldaret/ss/actions/workflows/build.yml
-[Progress]: https://img.shields.io/endpoint?label=Code&url=https%3A%2F%2Fprogress.decomp.club%2Fdata%2Fss%2FSOUE01%2Fall%2F%3Fmode%3Dshield%26measure%3Dcode
-[DOL Progress]: https://img.shields.io/endpoint?label=DOL&url=https%3A%2F%2Fprogress.decomp.club%2Fdata%2Fss%2FSOUE01%2Fdol%2F%3Fmode%3Dshield%26measure%3Dcode
-[RELs Progress]: https://img.shields.io/endpoint?label=RELs&url=https%3A%2F%2Fprogress.decomp.club%2Fdata%2Fss%2FSOUE01%2Fmodules%2F%3Fmode%3Dshield%26measure%3Dcode
+[Progress]: https://decomp.dev/zeldaret/ss.svg?mode=shield&measure=code&label=Code
+[DOL Progress]: https://decomp.dev/zeldaret/ss.svg?mode=shield&measure=code&category=dol&label=DOL
+[RELs Progress]: https://decomp.dev/zeldaret/ss.svg?mode=shield&measure=code&category=modules&label=RELs
 [Discord Badge]: https://img.shields.io/discord/688807550715560050?color=%237289DA&logo=discord&logoColor=%23FFFFFF
 [discord]: https://discord.zelda64.dev
 

configure.py

@@ -15,6 +15,7 @@
 import argparse
 import sys
 from pathlib import Path
+from typing import Any, Dict, List
 
 from tools.project import (
     Object,
@@ -124,6 +125,8 @@
 config = ProjectConfig()
 config.version = str(args.version)
 version_num = VERSIONS.index(config.version)
+
+# Apply arguments
 config.build_dir = args.build_dir
 config.dtk_path = args.dtk
 config.objdiff_path = args.objdiff
@@ -156,6 +159,8 @@
     "-I include",
     f"-I build/{config.version}/include",
     f"--defsym version={version_num}",
+    f"--defsym BUILD_VERSION={version_num}",
+    f"--defsym VERSION_{config.version}",
 ]
 config.linker_version = "Wii/1.5"
 config.ldflags = [
@@ -198,14 +203,15 @@
     "-enc SJIS",
     "-i include",
     f"-i build/{config.version}/include",
+    f"-DBUILD_VERSION={version_num}",
+    f"-DVERSION_{config.version}",
     "-i src",
     "-i src/PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Include",
     "-i src/PowerPC_EABI_Support/MSL/MSL_C/MSL_Common_Embedded/Math/Include",
     "-i src/PowerPC_EABI_Support/MSL/MSL_C/PPC_EABI/Include",
     "-i src/PowerPC_EABI_Support/MSL/MSL_C++/MSL_Common/Include",
     "-i src/PowerPC_EABI_Support/Runtime/Inc",
     "-i src/PowerPC_EABI_Support/MetroTRK",
-    f"-DGAME_VERSION={version_num}",
 ]
 
 # Debug flags
@@ -1661,9 +1667,7 @@ def MatchingFor(*versions):
     Rel(NonMatching, "d_a_obj_tornado", "REL/d/a/obj/d_a_obj_tornado.cpp"),
     Rel(NonMatching, "d_a_obj_tower_bomb", "REL/d/a/obj/d_a_obj_tower_bomb.cpp"),
     Rel(NonMatching, "d_a_obj_tower_D101", "REL/d/a/obj/d_a_obj_tower_D101.cpp"),
-    Rel(
-        Matching, "d_a_obj_tower_gearD101", "REL/d/a/obj/d_a_obj_tower_gearD101.cpp"
-    ),
+    Rel(Matching, "d_a_obj_tower_gearD101", "REL/d/a/obj/d_a_obj_tower_gearD101.cpp"),
     Rel(
         NonMatching,
         "d_a_obj_tower_hand_D101",
@@ -1877,6 +1881,24 @@ def MatchingFor(*versions):
     Rel(NonMatching, "d_t_tumble_weed", "REL/d/t/d_t_tumble_weed.cpp"),
 ]
 
+
+# Optional callback to adjust link order. This can be used to add, remove, or reorder objects.
+# This is called once per module, with the module ID and the current link order.
+#
+# For example, this adds "dummy.c" to the end of the DOL link order if configured with --non-matching.
+# "dummy.c" *must* be configured as a Matching (or Equivalent) object in order to be linked.
+def link_order_callback(module_id: int, objects: List[str]) -> List[str]:
+    # Don't modify the link order for matching builds
+    if not config.non_matching:
+        return objects
+    if module_id == 0:  # DOL
+        return objects + ["dummy.c"]
+    return objects
+
+
+# Uncomment to enable the link order callback.
+# config.link_order_callback = link_order_callback
+
 # Optional extra categories for progress tracking
 config.progress_categories = [
     ProgressCategory("core", "Core Game Engine"),

tools/download_tool.py

@@ -92,6 +92,23 @@ def wibo_url(tag: str) -> str:
 }
 
 
+def download(url, response, output) -> None:
+    if url.endswith(".zip"):
+        data = io.BytesIO(response.read())
+        with zipfile.ZipFile(data) as f:
+            f.extractall(output)
+        # Make all files executable
+        for root, _, files in os.walk(output):
+            for name in files:
+                os.chmod(os.path.join(root, name), 0o755)
+        output.touch(mode=0o755)  # Update dir modtime
+    else:
+        with open(output, "wb") as f:
+            shutil.copyfileobj(response, f)
+        st = os.stat(output)
+        os.chmod(output, st.st_mode | stat.S_IEXEC)
+
+
 def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("tool", help="Tool name")
@@ -104,21 +121,25 @@ def main() -> None:
 
     print(f"Downloading {url} to {output}")
     req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
-    with urllib.request.urlopen(req) as response:
-        if url.endswith(".zip"):
-            data = io.BytesIO(response.read())
-            with zipfile.ZipFile(data) as f:
-                f.extractall(output)
-            # Make all files executable
-            for root, _, files in os.walk(output):
-                for name in files:
-                    os.chmod(os.path.join(root, name), 0o755)
-            output.touch(mode=0o755)  # Update dir modtime
-        else:
-            with open(output, "wb") as f:
-                shutil.copyfileobj(response, f)
-            st = os.stat(output)
-            os.chmod(output, st.st_mode | stat.S_IEXEC)
+    try:
+        with urllib.request.urlopen(req) as response:
+            download(url, response, output)
+    except urllib.error.URLError as e:
+        if str(e).find("CERTIFICATE_VERIFY_FAILED") == -1:
+            raise e
+        try:
+            import certifi
+            import ssl
+        except:
+            print(
+                '"certifi" module not found. Please install it using "python -m pip install certifi".'
+            )
+            return
+
+        with urllib.request.urlopen(
+            req, context=ssl.create_default_context(cafile=certifi.where())
+        ) as response:
+            download(url, response, output)
 
 
 if __name__ == "__main__":

tools/project.py

@@ -17,7 +17,20 @@
 import platform
 import sys
 from pathlib import Path
-from typing import IO, Any, Dict, Iterable, List, Optional, Set, Tuple, Union, cast
+from typing import (
+    Any,
+    Callable,
+    cast,
+    Dict,
+    IO,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    TypedDict,
+    Union,
+)
 
 from . import ninja_syntax
 from .ninja_syntax import serialize_path
@@ -179,6 +192,9 @@ def __init__(self) -> None:
         self.scratch_preset_id: Optional[int] = (
             None  # Default decomp.me preset ID for scratches
         )
+        self.link_order_callback: Optional[Callable[[int, List[str]], List[str]]] = (
+            None  # Callback to add/remove/reorder units within a module
+        )
 
         # Progress output, progress.json and report.json config
         self.progress = True  # Enable report.json generation and CLI progress output
@@ -294,18 +310,46 @@ def make_flags_str(flags: Optional[List[str]]) -> str:
     return " ".join(flags)
 
 
+# Unit configuration
+class BuildConfigUnit(TypedDict):
+    object: Optional[str]
+    name: str
+    autogenerated: bool
+
+
+# Module configuration
+class BuildConfigModule(TypedDict):
+    name: str
+    module_id: int
+    ldscript: str
+    entry: str
+    units: List[BuildConfigUnit]
+
+
+# Module link configuration
+class BuildConfigLink(TypedDict):
+    modules: List[str]
+
+
+# Build configuration generated by decomp-toolkit
+class BuildConfig(BuildConfigModule):
+    version: str
+    modules: List[BuildConfigModule]
+    links: List[BuildConfigLink]
+
+
 # Load decomp-toolkit generated config.json
 def load_build_config(
     config: ProjectConfig, build_config_path: Path
-) -> Optional[Dict[str, Any]]:
+) -> Optional[BuildConfig]:
     if not build_config_path.is_file():
         return None
 
     def versiontuple(v: str) -> Tuple[int, ...]:
         return tuple(map(int, (v.split("."))))
 
     f = open(build_config_path, "r", encoding="utf-8")
-    build_config: Dict[str, Any] = json.load(f)
+    build_config: BuildConfig = json.load(f)
     config_version = build_config.get("version")
     if config_version is None:
         print("Invalid config.json, regenerating...")
@@ -321,6 +365,24 @@ def versiontuple(v: str) -> Tuple[int, ...]:
         return None
 
     f.close()
+
+    # Apply link order callback
+    if config.link_order_callback:
+        modules: List[BuildConfigModule] = [build_config, *build_config["modules"]]
+        for module in modules:
+            unit_names = list(map(lambda u: u["name"], module["units"]))
+            unit_names = config.link_order_callback(module["module_id"], unit_names)
+            units: List[BuildConfigUnit] = []
+            for unit_name in unit_names:
+                units.append(
+                    # Find existing unit or create a new one
+                    next(
+                        (u for u in module["units"] if u["name"] == unit_name),
+                        {"object": None, "name": unit_name, "autogenerated": False},
+                    )
+                )
+            module["units"] = units
+
     return build_config
 
 
@@ -338,7 +400,7 @@ def generate_build(config: ProjectConfig) -> None:
 def generate_build_ninja(
     config: ProjectConfig,
     objects: Dict[str, Object],
-    build_config: Optional[Dict[str, Any]],
+    build_config: Optional[BuildConfig],
 ) -> None:
     out = io.StringIO()
     n = ninja_syntax.Writer(out)
@@ -699,9 +761,9 @@ def map_path(path: Path) -> Path:
         return path.parent / (path.name + ".MAP")
 
     class LinkStep:
-        def __init__(self, config: Dict[str, Any]) -> None:
-            self.name: str = config["name"]
-            self.module_id: int = config["module_id"]
+        def __init__(self, config: BuildConfigModule) -> None:
+            self.name = config["name"]
+            self.module_id = config["module_id"]
             self.ldscript: Optional[Path] = Path(config["ldscript"])
             self.entry = config["entry"]
             self.inputs: List[str] = []
@@ -907,13 +969,14 @@ def asm_build(
 
             return obj_path
 
-        def add_unit(build_obj, link_step: LinkStep):
+        def add_unit(build_obj: BuildConfigUnit, link_step: LinkStep):
             obj_path, obj_name = build_obj["object"], build_obj["name"]
             obj = objects.get(obj_name)
             if obj is None:
                 if config.warn_missing_config and not build_obj["autogenerated"]:
                     print(f"Missing configuration for {obj_name}")
-                link_step.add(obj_path)
+                if obj_path is not None:
+                    link_step.add(Path(obj_path))
                 return
 
             link_built_obj = obj.completed
@@ -942,12 +1005,7 @@ def add_unit(build_obj, link_step: LinkStep):
                 link_step.add(built_obj_path)
             elif obj_path is not None:
                 # Use the original (extracted) object
-                link_step.add(obj_path)
-            else:
-                lib_name = obj.options["lib"]
-                sys.exit(
-                    f"Missing object for {obj_name}: {obj.src_path} {lib_name} {obj}"
-                )
+                link_step.add(Path(obj_path))
 
         # Add DOL link step
         link_step = LinkStep(build_config)
@@ -1268,7 +1326,7 @@ def add_unit(build_obj, link_step: LinkStep):
 def generate_objdiff_config(
     config: ProjectConfig,
     objects: Dict[str, Object],
-    build_config: Optional[Dict[str, Any]],
+    build_config: Optional[BuildConfig],
 ) -> None:
     if build_config is None:
         return
@@ -1333,7 +1391,7 @@ def generate_objdiff_config(
     }
 
     def add_unit(
-        build_obj: Dict[str, Any], module_name: str, progress_categories: List[str]
+        build_obj: BuildConfigUnit, module_name: str, progress_categories: List[str]
     ) -> None:
         obj_path, obj_name = build_obj["object"], build_obj["name"]
         base_object = Path(obj_name).with_suffix("")
@@ -1481,7 +1539,7 @@ def unix_path(input: Any) -> str:
 def generate_compile_commands(
     config: ProjectConfig,
     objects: Dict[str, Object],
-    build_config: Optional[Dict[str, Any]],
+    build_config: Optional[BuildConfig],
 ) -> None:
     if build_config is None or not config.generate_compile_commands:
         return
@@ -1570,7 +1628,7 @@ def generate_compile_commands(
 
     clangd_config = []
 
-    def add_unit(build_obj: Dict[str, Any]) -> None:
+    def add_unit(build_obj: BuildConfigUnit) -> None:
         obj = objects.get(build_obj["name"])
         if obj is None:
             return