Merge pull request #8 from Decompollaborate/develop

2.1.0

README.md

@@ -33,6 +33,7 @@ The list can be checked in runtime with `python3 -m mapfile_parser --help`.
 
 Each one of them can be executed with `python3 -m mapfile_parser utilityname`, for example `python3 -m mapfile_parser pj64_syms`.
 
+- `bss_check`: Check that globally visible bss has not been reordered.
 - `first_diff`: Find the first difference(s) between the built ROM and the base ROM.
 - `jsonify`: Converts a mapfile into a json format.
 - `pj64_syms`: Produce a PJ64 compatible symbol map.

pyproject.toml

@@ -3,7 +3,7 @@
 
 [project]
 name = "mapfile_parser"
-version = "2.0.1"
+version = "2.1.0"
 description = "Map file parser library focusing decompilation projects"
 readme = "README.md"
 requires-python = ">=3.7"

src/mapfile_parser/__init__.py

@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-__version_info__ = (2, 0, 1)
+__version_info__ = (2, 1, 0)
 __version__ = ".".join(map(str, __version_info__))
 __author__ = "Decompollaborate"
 
@@ -16,6 +16,8 @@
 from .mapfile import File as File
 from .mapfile import Segment as Segment
 from .mapfile import FoundSymbolInfo as FoundSymbolInfo
+from .mapfile import SymbolComparisonInfo as SymbolComparisonInfo
+from .mapfile import MapsComparisonInfo as MapsComparisonInfo
 
 from .progress_stats import ProgressStats as ProgressStats
 

src/mapfile_parser/__main__.py

@@ -11,10 +11,13 @@
 
 
 def mapfileParserMain():
-    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(description="Interface to call any of the mapfile_parser's CLI utilities", prog="mapfile_parser")
+
+    parser.add_argument("-V", "--version", action="version", version=f"%(prog)s {mapfile_parser.__version__}")
 
     subparsers = parser.add_subparsers(description="action", help="the action to perform", required=True)
 
+    mapfile_parser.frontends.bss_check.addSubparser(subparsers)
     mapfile_parser.frontends.first_diff.addSubparser(subparsers)
     mapfile_parser.frontends.jsonify.addSubparser(subparsers)
     mapfile_parser.frontends.pj64_syms.addSubparser(subparsers)

src/mapfile_parser/frontends/__init__.py

@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 
+from . import bss_check as bss_check
 from . import first_diff as first_diff
 from . import jsonify as jsonify
 from . import pj64_syms as pj64_syms

src/mapfile_parser/frontends/bss_check.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: © 2023 Decompollaborate
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import sys
+
+from .. import mapfile
+from .. import utils
+
+
+def getComparison(mapPath: Path, expectedMapPath: Path, *, reverseCheck: bool=True) -> mapfile.MapsComparisonInfo:
+    buildMap = mapfile.MapFile()
+    buildMap.readMapFile(mapPath)
+    buildMap = buildMap.filterBySectionType(".bss")
+
+    expectedMap = mapfile.MapFile()
+    expectedMap.readMapFile(expectedMapPath)
+    expectedMap = expectedMap.filterBySectionType(".bss")
+
+    return buildMap.compareFilesAndSymbols(expectedMap, checkOtherOnSelf=reverseCheck)
+
+def printSymbolComparison(comparisonInfo: mapfile.MapsComparisonInfo, printAll: bool=False):
+    print("Symbol Name,Build Address,Build File,Expected Address,Expected File,Difference,GOOD/BAD/MISSING")
+
+    # If it's bad or missing, don't need to do anything special.
+    # If it's good, check for if it's in a file with bad or missing stuff, and check if print all is on. If none of these, print it.
+
+    for symbolInfo in comparisonInfo.comparedList:
+        buildFile = symbolInfo.buildFile
+        expectedFile = symbolInfo.expectedFile
+        buildFilePath = buildFile.filepath if buildFile is not None else None
+        expectedFilePath = expectedFile.filepath if expectedFile is not None else None
+
+        if symbolInfo.diff is None:
+            print(f"{symbolInfo.symbol.name},{symbolInfo.buildAddress:X},{buildFilePath},{symbolInfo.expectedAddress:X},{expectedFilePath},{symbolInfo.diff},MISSING")
+            continue
+
+        symbolState = "BAD"
+        if symbolInfo.diff == 0:
+            symbolState = "GOOD"
+            if not buildFile in comparisonInfo.badFiles and not expectedFile in comparisonInfo.badFiles:
+                if not buildFile in comparisonInfo.badFiles and not expectedFile in comparisonInfo.badFiles:
+                    if not printAll:
+                        continue
+
+        if buildFile != expectedFile:
+            symbolState += " MOVED"
+        print(f"{symbolInfo.symbol.name},{symbolInfo.buildAddress:X},{buildFilePath},{symbolInfo.expectedAddress:X},{expectedFilePath},{symbolInfo.diff:X},{symbolState}")
+
+def printFileComparison(comparisonInfo: mapfile.MapsComparisonInfo):
+    utils.eprint("")
+
+    if len(comparisonInfo.badFiles) != 0:
+        utils.eprint("  BAD")
+
+        for file in comparisonInfo.badFiles:
+            utils.eprint(f"bss reordering in {file.filepath}")
+        utils.eprint("")
+
+    if len(comparisonInfo.missingFiles) != 0:
+        utils.eprint("  MISSING")
+
+        for file in comparisonInfo.missingFiles:
+            utils.eprint(f"Symbols missing from {file.filepath}")
+        utils.eprint("")
+
+        utils.eprint("Some files appear to be missing symbols. Have they been renamed or declared as static? You may need to remake 'expected'")
+
+
+def doBssCheck(mapPath: Path, expectedMapPath: Path, *, printAll: bool=False, reverseCheck: bool=True) -> int:
+    if not mapPath.exists():
+        utils.eprint(f"{mapPath} must exist")
+        return 1
+    if not expectedMapPath.exists():
+        utils.eprint(f"{expectedMapPath} must exist")
+        return 1
+
+    comparisonInfo = getComparison(mapPath, expectedMapPath, reverseCheck=reverseCheck)
+    printSymbolComparison(comparisonInfo, printAll)
+
+    if len(comparisonInfo.badFiles) + len(comparisonInfo.missingFiles) != 0:
+        printFileComparison(comparisonInfo)
+        return 1
+
+    utils.eprint("")
+    utils.eprint("  GOOD")
+
+    return 0
+
+
+def processArguments(args: argparse.Namespace):
+    mapPath: Path = args.mapfile
+    expectedMapPath: Path = args.expectedmap
+
+    printAll: bool = args.print_all
+    reverseCheck: bool = not args.no_reverse_check
+
+    exit(doBssCheck(mapPath, expectedMapPath, printAll=printAll, reverseCheck=reverseCheck))
+
+
+def addSubparser(subparser: argparse._SubParsersAction[argparse.ArgumentParser]):
+    parser = subparser.add_parser("bss_check", help="Check that globally visible bss has not been reordered.")
+
+    parser.add_argument("mapfile", help="Path to a map file", type=Path)
+    parser.add_argument("expectedmap", help="Path to the map file in the expected dir", type=Path)
+
+    parser.add_argument("-a", "--print-all", help="Print all bss, not just non-matching.", action="store_true")
+    parser.add_argument("--no-reverse-check", help="Disable looking for symbols on the expected map that are missing on the built map file.", action="store_true")
+
+    parser.set_defaults(func=processArguments)

src/mapfile_parser/frontends/jsonify.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# SPDX-FileCopyrightText: © 2022-2023 Decompollaborate
+# SPDX-FileCopyrightText: © 2023 Decompollaborate
 # SPDX-License-Identifier: MIT
 
 from __future__ import annotations

src/mapfile_parser/mapfile.py

@@ -41,6 +41,22 @@ def getAsStrPlusOffset(self, symName: str|None=None) -> str:
             message = "Symbol"
         return f"{message} {self.getAsStr()}"
 
+@dataclasses.dataclass
+class SymbolComparisonInfo:
+    symbol: Symbol
+    buildAddress: int
+    buildFile: File|None
+    expectedAddress: int
+    expectedFile: File|None
+    diff: int|None
+
+class MapsComparisonInfo:
+    def __init__(self):
+        self.badFiles: set[File] = set()
+        self.missingFiles: set[File] = set()
+        self.comparedList: list[SymbolComparisonInfo] = []
+
+
 @dataclasses.dataclass
 class Symbol:
     name: str
@@ -376,24 +392,27 @@ def __init__(self):
         self.debugging: bool = False
 
     def readMapFile(self, mapPath: Path):
-        tempSegmentsList: list[Segment] = list()
-        tempFilesListList: list[list[File]] = list()
-
         with mapPath.open("r") as f:
             mapData = f.read()
 
-            # Skip the stuff we don't care about
             startIndex = 0
             auxVar = 0
-            while auxVar != -1:
-                startIndex = auxVar
-                auxVar = mapData.find("\nLOAD ", startIndex+1)
-            auxVar = mapData.find("\n", startIndex+1)
+
+            # Skip the stuff we don't care about
+            # Looking for this string will only work on English machines (or C locales)
+            # but it doesn't matter much, because if this string is not found then the
+            # parsing should still work, but just a bit slower because of the extra crap
+            auxVar = mapData.find("\nLinker script and memory map", startIndex+1)
+            if auxVar != -1:
+                auxVar = mapData.find("\n", auxVar+1)
             if auxVar != -1:
                 startIndex = auxVar
             mapData = mapData[startIndex:]
         # print(len(mapData))
 
+        tempSegmentsList: list[Segment] = [Segment("$$dummysegment$$", 0, 0, 0)]
+        tempFilesListList: list[list[File]] = [[]]
+
         inFile = False
 
         prevLine = ""
@@ -420,7 +439,7 @@ def readMapFile(self, mapPath: Path):
             if not inFile:
                 fillMatch = regex_fill.search(line)
                 entryMatch = regex_fileDataEntry.search(line)
-                loadAddressMatch = regex_segmentEntry.search(line)
+                segmentEntryMatch = regex_segmentEntry.search(line)
 
                 if fillMatch is not None:
                     # Add *fill* size to last file
@@ -436,13 +455,14 @@ def readMapFile(self, mapPath: Path):
                     if size > 0:
                         inFile = True
                         tempFile = File(filepath, vram, size, sectionType)
+                        assert len(tempFilesListList) > 0, line
                         tempFilesListList[-1].append(tempFile)
 
-                elif loadAddressMatch is not None:
-                    name = loadAddressMatch["name"]
-                    vram = int(loadAddressMatch["vram"], 0)
-                    size = int(loadAddressMatch["size"], 0)
-                    vrom = int(loadAddressMatch["vrom"], 0)
+                elif segmentEntryMatch is not None:
+                    name = segmentEntryMatch["name"]
+                    vram = int(segmentEntryMatch["vram"], 0)
+                    size = int(segmentEntryMatch["size"], 0)
+                    vrom = int(segmentEntryMatch["vrom"], 0)
 
                     if name == "":
                         # If the segment name is too long then this line gets break in two lines
@@ -454,7 +474,8 @@ def readMapFile(self, mapPath: Path):
 
             prevLine = line
 
-        for i, segment in enumerate(tempSegmentsList):
+        # Skip dummy segment
+        for i, segment in enumerate(tempSegmentsList[1:]):
             filesList = tempFilesListList[i]
 
             vromOffset = segment.vrom
@@ -636,6 +657,34 @@ def getProgress(self, asmPath: Path, nonmatchings: Path, aliases: dict[str, str]
 
         return totalStats, progressPerFolder
 
+    # Useful for finding bss reorders
+    def compareFilesAndSymbols(self, otherMapFile: MapFile, *, checkOtherOnSelf: bool=True) -> MapsComparisonInfo:
+        compInfo = MapsComparisonInfo()
+
+        for segment in self:
+            for file in segment:
+                for symbol in file:
+                    foundSymInfo = otherMapFile.findSymbolByName(symbol.name)
+                    if foundSymInfo is not None:
+                        comp = SymbolComparisonInfo(symbol, symbol.vram, file, symbol.vram, foundSymInfo.file, symbol.vram - foundSymInfo.symbol.vram)
+                        compInfo.comparedList.append(comp)
+                        if comp.diff != 0:
+                            compInfo.badFiles.add(file)
+                    else:
+                        compInfo.missingFiles.add(file)
+                        compInfo.comparedList.append(SymbolComparisonInfo(symbol, symbol.vram, file, -1, None, None))
+
+        if checkOtherOnSelf:
+            for segment in otherMapFile:
+                for file in segment:
+                    for symbol in file:
+                        foundSymInfo = self.findSymbolByName(symbol.name)
+                        if foundSymInfo is None:
+                            compInfo.missingFiles.add(file)
+                            compInfo.comparedList.append(SymbolComparisonInfo(symbol, -1, None, symbol.vram, file, None))
+
+        return compInfo
+
     def printAsCsv(self, printVram: bool=True, skipWithoutSymbols: bool=True):
         File.printCsvHeader(printVram)
         for segment in self._segmentsList: