Fix compatibility with IDA 7.x and Python 3

gcdsp.py

@@ -10,50 +10,18 @@
 repository.
 """
 from idaapi import *
-import os.path
+from pathlib import Path
+
+from gcdsp_opcodes import OpType
+from gcdsp_generated import *
 
 GREETINGS_STRING = """\
 GC/Wii DSP processor for IDA (C) 2011 [email protected] - \
 licensed under the GPLv2 license\
 """
 
-class OpType:
-    """Enumeration of the different operand encoding types which can be found
-    in the GC DSP ISA. From DSPTables.h in Dolphin source code."""
-
-    NONE = 0
-    VAL = 1
-    IMM = 2
-    MEM = 3
-    STR = 4
-    ADDR_I = 5
-    ADDR_D = 6
-
-    REG = 0x8000
-    REG04 = REG | 0x0400
-    REG08 = REG | 0x0800
-    REG18 = REG | 0x1800
-    REGM18 = REG18
-    REG19 = REG | 0x1900
-    REGM19 = REG19
-    REG1A = REG | 0x1a80
-    REG1C = REG | 0x1c00
-    ACCL = REG | 0x1c00
-    ACCM = REG | 0x1e00
-    ACCM_D = REG | 0x1e80
-    ACC = REG | 0x2000
-    ACC_D = REG | 0x2080
-    AX = REG | 0x2200
-    REGS_MASK = 0x3f80
-
-    REF = REG | 0x4000
-    PRG = REF | REG
-
-
-# Get autogenerated parts from another file
-execfile(os.path.join(os.path.dirname(__file__), 'gcdsp_generated.py'))
-
-class Operand(object):
+
+class Operand:
     def __init__(self, type, size, loc, rshift, mask):
         self.type = type
         self.size = size
@@ -86,36 +54,36 @@ def parse(self, res, byte1, byte2):
 
         if type == OpType.REG:
             res.type = o_reg
-            res.dtyp = dt_byte  # TODO: fix (ACCs are 40-bit for example)
+            res.dtype = dt_byte  # TODO: fix (ACCs are 40-bit for example)
             res.reg = val
         elif type == OpType.PRG:
             res.type = o_phrase
-            res.dtyp = dt_byte
+            res.dtype = dt_byte
             res.phrase = val
         elif type == OpType.ADDR_I:
             res.type = o_near
-            res.dtyp = dt_byte
+            res.dtype = dt_byte
             res.addr = val
         elif type == OpType.ADDR_D:
             res.type = o_mem
-            res.dtyp = dt_byte
+            res.dtype = dt_byte
             res.addr = val
         elif type == OpType.IMM:
             res.type = o_imm
-            res.dtyp = dt_byte
+            res.dtype = dt_byte
             res.value = val
         elif type == OpType.MEM:
             if self.size != 2:
                 b = val & 0x80
                 if b:
                     val |= 0xFF00
             res.type = o_mem
-            res.dtyp = dt_byte
-            res.addr = val
+            res.dtype = dt_byte
+            res.addr = 0x10000 | val
         else:
             raise ValueError("unhandled type: %04X" % type)
 
-class Instr(object):
+class Instr:
     def __init__(self, name, opcode, mask, size, operands=[], ext_operands=[],
                  stops=False, calls=False, jumps=False, shifts=False,
                  hll=False):
@@ -170,7 +138,7 @@ class GCDSPProcessor(processor_t):
     instruc_start = 0
 
     assembler = {
-        "flag" : ASH_HEXF3 | ASD_DECF0 | ASO_OCTF1 | ASB_BINF3 | AS_NOTAB
+        "flag" : ASH_HEXF3 | ASD_DECF0 | ASO_OCTF1 | ASB_BINF3
                | AS_ASCIIC | AS_ASCIIZ,
         "uflag": 0,
         "name": "GNU assembler",
@@ -291,40 +259,41 @@ def _init_registers(self):
             self.reg_ids[reg] = i
 
         # Simulate fake segment registers
-        self.regFirstSreg = self.regCodeSreg = self.reg_ids["$CS"]
-        self.regLastSreg = self.regDataSreg = self.reg_ids["$DS"]
+        self.reg_first_sreg = self.reg_code_sreg = self.reg_ids["$CS"]
+        self.reg_last_sreg = self.reg_data_sreg = self.reg_ids["$DS"]
+
 
     def notify_init(self, idp_file):
         """Called at module initialization."""
-        cvar.inf.mf = True  # set to big endian... wtf
-        cvar.inf.wide_high_byte_first = True  # big endian for 16b bytes too
+        cvar.inf.set_be(True)
+        cvar.inf.lflags |= LFLG_WIDE_HBF  # big endian for 16b bytes too
         return True
 
     def notify_endbinary(self, ok):
         """Called when the binary finished loading."""
         if ok:
-            print GREETINGS_STRING
+            print(GREETINGS_STRING)
 
-    def _read_cmd_byte(self):
-        ea = self.cmd.ea + self.cmd.size
-        byte = get_full_byte(ea)
-        self.cmd.size += 1
+    def _read_cmd_byte(self, cmd):
+        ea = cmd.ea + cmd.size
+        byte = get_wide_byte(ea)
+        cmd.size += 1
         return byte
 
-    def ana(self):
-        """Analyze one instruction and fill the "cmd" instance member."""
-        cmd = self.cmd
-        byte1 = self._read_cmd_byte()
+    def notify_ana(self, cmd):
+        """Analyze one instruction and fill "cmd"."""
+        self.cmd = cmd
+        byte1 = self._read_cmd_byte(cmd)
         instr = self.instrs_opcode[byte1]
         if instr is None:
             return 0
 
         if instr.size == 2:
-            byte2 = self._read_cmd_byte()
+            byte2 = self._read_cmd_byte(cmd)
         else:
             byte2 = 0
 
-        operands = [cmd[i] for i in xrange(6)]
+        operands = [cmd[i] for i in range(6)]
         for to_fill in operands:
             to_fill.type = o_void
 
@@ -334,80 +303,78 @@ def ana(self):
         cmd.itype = instr.id
         return cmd.size
 
-    def _emu_operand(self, op):
+    def _emu_operand(self, cmd, op):
         """Emulated using one operand from the instruction."""
         if op.type == o_mem:
-            ua_dodata2(0, op.addr, op.dtyp)
-            ua_add_dref(0, op.addr, dr_R)  # TODO: dr_W ?
+            cmd.create_op_data(op.addr, 0, op.dtype)
+            cmd.add_dref(op.addr, 0, dr_R)
         elif op.type == o_near:
-            if self.cmd.get_canon_feature() & CF_CALL:
+            if cmd.get_canon_feature() & CF_CALL:
                 fl = fl_CN
             else:
                 fl = fl_JN
-            ua_add_cref(0, op.addr, fl)
+            cmd.add_cref(op.addr, 0, fl)
 
-    def emu(self):
+    def notify_emu(self, cmd):
         """Emulate instruction behavior and create x-refs, interpret operand
         values, etc."""
-        instr = self.instrs_list[self.cmd.itype]
+        instr = self.instrs_list[cmd.itype]
 
-        for i in xrange(len(instr.all_operands)):
-            self._emu_operand(self.cmd[i])
+        for i in range(len(instr.all_operands)):
+            self._emu_operand(cmd, cmd[i])
 
         if not instr.stops:  # add a link to next instr if code continues
-            ua_add_cref(0, self.cmd.ea + self.cmd.size, fl_F)
+            cmd.add_cref(cmd.ea + cmd.size, 0, fl_F)
 
         return True
 
-    def outop(self, op):
+    def notify_out_operand(self, ctx, op):
         """Generates text representation of an instruction operand."""
         if op.type == o_reg:
-            out_register(self.reg_names[op.reg])
+            ctx.out_register(self.reg_names[op.reg])
         elif op.type == o_phrase:
-            out_symbol('@')
-            out_register(self.reg_names[op.reg])
+            ctx.out_symbol('@')
+            ctx.out_register(self.reg_names[op.reg])
         elif op.type == o_imm:
-            OutValue(op, OOFW_IMM)
+            ctx.out_value(op, OOFW_IMM)
         elif op.type in [o_near, o_mem]:
-            ok = out_name_expr(op, op.addr, BADADDR)
+            ok = ctx.out_name_expr(op, op.addr, BADADDR)
             if not ok:
-                out_tagon(COLOR_ERROR)
-                OutLong(op.addr, 16)
-                out_tagoff(COLOR_ERROR)
-                QueueMark(Q_noName, self.cmd.ea)
+                ctx.out_tagon(COLOR_ERROR)
+                ctx.out_long(op.addr, 16)
+                ctx.out_tagoff(COLOR_ERROR)
+                remember_problem(PR_NONAME, self.cmd.ea)
         else:
             return False
         return True
 
-    def out(self):
+    def notify_out_insn(self, ctx):
         """Generates text representation of an instruction in the "cmd" inst
         member."""
         cmd = self.cmd
 
-        buf = init_output_buffer(1024)
-        OutMnem(15)  # max width = 15
+        ctx.out_mnem(15)  # max width = 15
 
         instr = self.instrs_list[cmd.itype]
 
         in_extended = False
-        for i in xrange(0, 6):
+        for i in range(0, 6):
             if cmd[i].type == o_void:
                 break
 
             if i != 0:
                 if not in_extended and i >= len(instr.operands):
                     in_extended = True
-                    OutChar(' ')
-                    out_symbol(':')
+                    ctx.out_char(' ')
+                    ctx.out_symbol(':')
                 else:
-                    out_symbol(',')
-                OutChar(' ')
+                    ctx.out_symbol(',')
+                ctx.out_char(' ')
 
-            out_one_operand(i)
+            ctx.out_one_operand(i)
 
-        term_output_buffer()
         cvar.gl_comm = 1  # allow comments at end of line
-        MakeLine(buf)
+        ctx.flush_outbuf()
 
 def PROCESSOR_ENTRY():
     return GCDSPProcessor()

gcdsp_generated.py

@@ -1,4 +1,6 @@
 #### THIS CODE WAS AUTO-GENERATED BY gen_from_tables.py ####
+from gcdsp_opcodes import OpType
+
 opcodes = [
     ["NOP",0x0000,0xfffc,1,0,[],False,False],
     ["DAR",0x0004,0xfffc,1,1,[[OpType.REG,1,0,0,0x0003]],False,False],

gcdsp_opcodes.py

@@ -0,0 +1,30 @@
+class OpType:
+    """Enumeration of the different operand encoding types which can be found
+    in the GC DSP ISA. From DSPTables.h in Dolphin source code."""
+
+    NONE = 0
+    VAL = 1
+    IMM = 2
+    MEM = 3
+    STR = 4
+    ADDR_I = 5
+    ADDR_D = 6
+
+    REG = 0x8000
+    REG04 = REG | 0x0400
+    REG08 = REG | 0x0800
+    REG18 = REG | 0x1800
+    REGM18 = REG18
+    REG19 = REG | 0x1900
+    REGM19 = REG19
+    REG1A = REG | 0x1a80
+    REG1C = REG | 0x1c00
+    ACCL = REG | 0x1c00
+    ACCM = REG | 0x1e00
+    ACCM_D = REG | 0x1e80
+    ACC = REG | 0x2000
+    ACC_D = REG | 0x2080
+    AX = REG | 0x2200
+    REGS_MASK = 0x3f80
+
+    PRG = REG | 0x4000

gen_from_tables.py

@@ -50,7 +50,7 @@ def gen_from_text(text):
             else:
                 fields = line.replace('{', '[').replace('}', ']').split(',')
                 fields = fields[0:3] + fields[5:-5] + [fields[-4]]
-                fields = map(str.strip, fields)
+                fields = list(map(str.strip, fields))
                 fields = ','.join(fields) + ']'
                 fields = fields.replace('true', 'True')
                 fields = fields.replace('false', 'False')
@@ -60,21 +60,22 @@ def gen_from_text(text):
                 else:
                     opcodes_ext.append(fields)
 
-    print 'opcodes = ['
-    print '    ' + ',\n    '.join(opcodes)
-    print ']'
-    print
-    print 'opcodes_ext = ['
-    print '    ' + ',\n    '.join(opcodes_ext)
-    print ']'
+    print('opcodes = [')
+    print('    ' + ',\n    '.join(opcodes))
+    print(']')
+    print()
+    print('opcodes_ext = [')
+    print('    ' + ',\n    '.join(opcodes_ext))
+    print(']')
 
 if __name__ == '__main__':
     if len(sys.argv) != 2:
-        print 'usage: %s /path/to/DSPTables.cpp' % sys.argv[0]
+        print('usage: %s /path/to/DSPTables.cpp' % sys.argv[0])
         sys.exit(1)
 
     text = open(sys.argv[1]).read()
 
-    print "#### THIS CODE WAS AUTO-GENERATED BY gen_from_tables.py ####"
+    print("#### THIS CODE WAS AUTO-GENERATED BY gen_from_tables.py ####")
+    print("from gcdsp_opcodes import OpType")
     gen_from_text(text)
-    print "#### END AUTO-GENERATED CODE ####"
+    print("#### END AUTO-GENERATED CODE ####")