Merge pull request #161 from mparisi20/pragma_proc

Add pragma processor tool and use it in code_801DD8C0.cpp

5 files changed, 351 insertions, 68 deletions

diff --git a/Makefile b/Makefile
index 69b7c42..b897334 100644
--- a/Makefile
+++ b/Makefile
@@ -84,12 +84,13 @@ SHA1SUM     := sha1sum
 PYTHON      := python3
 
 POSTPROC := tools/postprocess/postprocess.py
+PRAGMAPROC := tools/pragma/pragma.py
 
 # Options
 INCLUDES := -i . -I- -i include -i include/SDK -i include/MSL_C -include include/types.h
 ASFLAGS := -mgekko -I include
 LDFLAGS := -map $(MAP) -fp hard -nodefaults
-CFLAGS  := -Cpp_exceptions off -proc gekko -fp hard -O4,p -nodefaults -msgstyle gcc -ipa file $(INCLUDES) -W all
+CFLAGS  := -Cpp_exceptions off -proc gekko -fp hard -O4,p -nodefaults -msgstyle gcc -ipa file $(INCLUDES) -W all -w nopragmas
 
 # for postprocess.py
 PROCFLAGS := -fsymbol-fixup
@@ -101,7 +102,7 @@ SBSS_PDHR := 10
 infoshell = $(foreach line, $(shell $1 | sed "s/ /__SPACE__/g"), $(info $(subst __SPACE__, ,$(line))))
 
 TOOLS_DIR = tools
-TOOLDIRS = $(filter-out $(TOOLS_DIR)/mwcc_compiler $(TOOLS_DIR)/postprocess,$(wildcard $(TOOLS_DIR)/*))
+TOOLDIRS = $(filter-out $(TOOLS_DIR)/mwcc_compiler $(TOOLS_DIR)/postprocess $(TOOLS_DIR)/pragma,$(wildcard $(TOOLS_DIR)/*))
 TOOLBASE = $(TOOLDIRS:$(TOOLS_DIR)/%=%)
 TOOLS = $(foreach tool,$(TOOLBASE),$(TOOLS_DIR)/$(tool)/$(tool)$(EXE))
 
@@ -154,7 +155,7 @@ $(BUILD_DIR)/%.o: %.s
 	$(PYTHON) $(POSTPROC) $(PROCFLAGS) $@
 
 $(BUILD_DIR)/%.o: %.cpp
-	$(CC) $(CFLAGS) -lang c++ -c -o $@ $<
+	$(PYTHON) $(PRAGMAPROC) "$(CC)" "$(CFLAGS) -lang c++ -c" $@ $< -fix-regswaps
 
 $(BUILD_DIR)/%.o: %.c
-	$(CC) $(CFLAGS) -lang c99 -c -o $@ $<
+	$(PYTHON) $(PRAGMAPROC) "$(CC)" "$(CFLAGS) -lang c99 -c" $@ $< -fix-regswaps
diff --git a/src/code_801DD8C0.cpp b/src/code_801DD8C0.cpp
index 89c315c..bc86aa9 100644
--- a/src/code_801DD8C0.cpp
+++ b/src/code_801DD8C0.cpp
@@ -44,8 +44,9 @@ static inline void inline_ClearFunction(gUnkClass7* ptr)
         ptr->unk6 &= ~0x1;

 }

 

-#ifdef NONMATCHING

-// r4/r6 regswap

+#ifndef NONMATCHING

+#pragma regswap 801DDA28 801DDA80 r4 r6 801DD8C0

+#endif

 void GSanimationObject::func_801DD9C8(float p2)

 {

     if (unk4) {

@@ -70,68 +71,6 @@ void GSanimationObject::func_801DD9C8(float p2)
         }

     }

 }

-#else

-asm void GSanimationObject::func_801DD9C8(float p2)

-{

-    nofralloc

-/* 801DD9C8 001D9628  94 21 FF F0 */	stwu r1, -0x10(r1)

-/* 801DD9CC 001D962C  7C 08 02 A6 */	mflr r0

-/* 801DD9D0 001D9630  90 01 00 14 */	stw r0, 0x14(r1)

-/* 801DD9D4 001D9634  93 E1 00 0C */	stw r31, 0xc(r1)

-/* 801DD9D8 001D9638  7C 7F 1B 78 */	mr r31, r3

-/* 801DD9DC 001D963C  80 83 00 04 */	lwz r4, 4(r3)

-/* 801DD9E0 001D9640  2C 04 00 00 */	cmpwi r4, 0

-/* 801DD9E4 001D9644  41 82 00 9C */	beq lbl_801DDA80

-/* 801DD9E8 001D9648  80 04 00 00 */	lwz r0, 0(r4)

-/* 801DD9EC 001D964C  38 60 00 00 */	li r3, 0

-/* 801DD9F0 001D9650  2C 00 00 00 */	cmpwi r0, 0

-/* 801DD9F4 001D9654  41 82 00 14 */	beq lbl_801DDA08

-/* 801DD9F8 001D9658  A0 04 00 06 */	lhz r0, 6(r4)

-/* 801DD9FC 001D965C  54 00 07 FF */	clrlwi. r0, r0, 0x1f

-/* 801DDA00 001D9660  41 82 00 08 */	beq lbl_801DDA08

-/* 801DDA04 001D9664  38 60 00 01 */	li r3, 1

-lbl_801DDA08:

-/* 801DDA08 001D9668  2C 03 00 00 */	cmpwi r3, 0

-/* 801DDA0C 001D966C  41 82 00 74 */	beq lbl_801DDA80

-/* 801DDA10 001D9670  7C 83 23 78 */	mr r3, r4

-/* 801DDA14 001D9674  4B FF FB B5 */	bl func_801DD5C8

-/* 801DDA18 001D9678  80 7F 00 04 */	lwz r3, 4(r31)

-/* 801DDA1C 001D967C  7F E4 FB 78 */	mr r4, r31

-/* 801DDA20 001D9680  38 A0 00 00 */	li r5, 0

-/* 801DDA24 001D9684  4B FF FD D9 */	bl func_801DD7FC

-/* 801DDA28 001D9688  80 DF 00 04 */	lwz r6, 4(r31)

-/* 801DDA2C 001D968C  38 80 00 01 */	li r4, 1

-/* 801DDA30 001D9690  A0 A6 00 06 */	lhz r5, 6(r6)

-/* 801DDA34 001D9694  54 A0 07 39 */	rlwinm. r0, r5, 0, 0x1c, 0x1c

-/* 801DDA38 001D9698  40 82 00 2C */	bne lbl_801DDA64

-/* 801DDA3C 001D969C  80 06 00 00 */	lwz r0, 0(r6)

-/* 801DDA40 001D96A0  38 60 00 00 */	li r3, 0

-/* 801DDA44 001D96A4  2C 00 00 00 */	cmpwi r0, 0

-/* 801DDA48 001D96A8  41 82 00 10 */	beq lbl_801DDA58

-/* 801DDA4C 001D96AC  54 A0 07 FF */	clrlwi. r0, r5, 0x1f

-/* 801DDA50 001D96B0  41 82 00 08 */	beq lbl_801DDA58

-/* 801DDA54 001D96B4  38 60 00 01 */	li r3, 1

-lbl_801DDA58:

-/* 801DDA58 001D96B8  2C 03 00 00 */	cmpwi r3, 0

-/* 801DDA5C 001D96BC  41 82 00 08 */	beq lbl_801DDA64

-/* 801DDA60 001D96C0  38 80 00 00 */	li r4, 0

-lbl_801DDA64:

-/* 801DDA64 001D96C4  2C 04 00 00 */	cmpwi r4, 0

-/* 801DDA68 001D96C8  41 82 00 18 */	beq lbl_801DDA80

-/* 801DDA6C 001D96CC  A0 66 00 06 */	lhz r3, 6(r6)

-/* 801DDA70 001D96D0  54 60 06 F7 */	rlwinm. r0, r3, 0, 0x1b, 0x1b

-/* 801DDA74 001D96D4  40 82 00 0C */	bne lbl_801DDA80

-/* 801DDA78 001D96D8  54 60 04 3C */	rlwinm r0, r3, 0, 0x10, 0x1e

-/* 801DDA7C 001D96DC  B0 06 00 06 */	sth r0, 6(r6)

-lbl_801DDA80:

-/* 801DDA80 001D96E0  80 01 00 14 */	lwz r0, 0x14(r1)

-/* 801DDA84 001D96E4  83 E1 00 0C */	lwz r31, 0xc(r1)

-/* 801DDA88 001D96E8  7C 08 03 A6 */	mtlr r0

-/* 801DDA8C 001D96EC  38 21 00 10 */	addi r1, r1, 0x10

-/* 801DDA90 001D96F0  4E 80 00 20 */	blr

-}

-#pragma peephole on

-#endif

 

 // Search the linked list referenced by unk0 for a node with the specified id

 gUnkClass8* GSanimationObject::func_801DDA94(u16 id)

diff --git a/tools/pragma/LICENSE b/tools/pragma/LICENSE
new file mode 100644
index 0000000..3b8d049
--- /dev/null
+++ b/tools/pragma/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Max Parisi https://github.com/mparisi20
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/tools/pragma/README.md b/tools/pragma/README.md
new file mode 100644
index 0000000..2f9074c
--- /dev/null
+++ b/tools/pragma/README.md
@@ -0,0 +1,35 @@
+# pragma.py
+#### Pragma processor for GameCube/Wii decompilation projects
+###### by Max Parisi
+
+This is a custom pragma processor meant for use with matching decompilation projects for GameCube and Wii games. It currently adds support for `#pragma regswap`.
+
+If some reconstructed C/C++ source code compiles to object binary whose only differences with a target binary are misallocated registers, you can use `#pragma regswap` 
+to correct the register allocation. This is an alternative to the strategy of inlining the desired assembly that aims to be more compact, self-documenting, and convenient. 
+
+`pragma.py` will compile your source code, then fix the regswaps by patching the range of the object file specified in each pragma invocation.
+
+## Usage
+Invoke `#pragma regswap` one or more times in a C or C++ source file with the following parameters:
+```
+#pragma regswap start end regA regB startFile
+```
+
+where, 
+* `start` - absolute address of start of affected region in hexadecimal format
+* `end` - absolute address of end of affected region (in hex)
+* `regA` - register to be swapped with `regB` (r0-r31 or f0-f31)
+* `regB` - register to swapped with `regA` (r0-r31 or f0-f31)
+* `startFile` - absolute address of the first function provided by this file (in hex)
+
+
+
+Then, invoke the pragma processor with the `-fix-regswaps` option:
+```
+python3 pragma.py [-h] [-fix-regswaps] cc cflags output source
+```
+
+
+## Future Work
+* Add support for an instruction-swap pragma (`#pragma iswap ...`)
+* Support for more architectures besides Gekko/Broadway PowerPC
diff --git a/tools/pragma/pragma.py b/tools/pragma/pragma.py
new file mode 100644
index 0000000..75e0450
--- /dev/null
+++ b/tools/pragma/pragma.py
@@ -0,0 +1,287 @@
+# pragma.py 
+# By mparisi20
+# github.com/mparisi20/pragma_processor
+
+# #pragma regswap usage:
+# #pragma regswap start end regA regB startFile
+
+# start: absolute address of start of affected region (hex)
+# end: absolute address of end of affected region (hex)
+# regA: register to swap (r0-r31 or f0-f31)
+# regB: register to swap (r0-r31 or f0-f31)
+# startFile: absolute address of the first function provided by this file (hex)
+
+# TODO: add support for an instruction swap pragma
+# TODO: add "#pragma startaddr <hex-addr>" to avoid rewriting the start address in each regswap pragma?
+
+import os
+import sys
+import argparse
+import subprocess
+import tempfile
+import re
+
+# 10-bit extension field for instructions with opcode 31
+op31_map = {
+             'mask': 0x3ff,
+             'data':
+             {
+             frozenset([0, 32, 4, 86, 470, 54, 278, 246, 1014, 982]): (11, 16),
+
+             frozenset([28, 60, 284, 476, 124, 444, 412, 316, 24, 792, 
+              536, 119, 87, 375, 343, 311, 279, 55, 23, 247, 
+              215, 439, 407, 183, 151, 790, 534, 918, 662, 533, 
+              661, 20, 150, 631, 599, 567, 535, 759, 727, 983, 
+              695, 663, 310, 438]): (6, 11, 16), 
+
+             frozenset([26, 954, 922, 824, 597, 725]): (6, 11),
+
+             frozenset([19, 83, 339, 371, 144, 146, 467, 595, 210]): (6,),
+
+             frozenset([659, 242]): (6, 16),
+
+             frozenset([306]): (16,)
+             }
+           }
+
+# lower 9 bits
+op31_mask9_map = {
+                   'mask': 0x1ff,
+                   'data':
+                   {
+                   frozenset([266, 10, 138, 491, 459, 75, 11, 235, 40, 8, 136]): (6, 11, 16),
+                   frozenset([234, 202, 104, 232, 200]): (6, 11)
+                   }
+                 }
+
+# 10-bit extension field for instructions with opcode 63
+op63_map = { 
+             'mask': 0x3ff,
+             'data':
+             {
+             frozenset([14, 15, 12, 264, 72, 136, 40]): (6, 16),
+             frozenset([32, 0]): (11, 16),
+             frozenset([583, 711]): (6,)
+             }
+           }
+
+# lower 5 bits
+op63_mask5_map = { 
+                   'mask': 0x1f,
+                   'data':
+                   {
+                   frozenset([21, 18, 20]): (6, 11, 16),
+                   frozenset([25]): (6, 11, 21),
+                   frozenset([26]): (6, 16),
+                   frozenset([23, 29, 28, 31, 30]): (6, 11, 16, 21)
+                   }
+                 }
+
+# lower 5 bits of the 10-bit extension field for instructions with opcode 59
+op59_mask5_map = { 
+                   'mask': 0x1f,
+                   'data':
+                   {
+                   frozenset([21, 18, 20]): (6, 11, 16),
+                   frozenset([25]): (6, 11, 21),
+                   frozenset([24]): (6, 16),
+                   frozenset([29, 28, 31, 30]): (6, 11, 16, 21)
+                   }
+                 }
+
+# 10-bit extension field for instructions with opcode 4
+op4_map = {
+            'mask': 0x3ff,
+            'data':
+            {
+            frozenset([40, 72, 136, 264]): (6, 16),
+            frozenset([0, 32, 64, 96, 1014]): (11, 16),
+            frozenset([528, 560, 592, 624]): (6, 11, 16)
+            }
+          }
+
+# lower 6 bits
+op4_mask6_map = {
+                  'mask': 0x3f,
+                  'data':
+                  {
+                  frozenset([6, 7, 38, 39]): (6, 11, 16)
+                  }
+                }
+
+# lower 5 bits
+op4_mask5_map = {
+                  'mask': 0x1f,
+                  'data':
+                  {
+                  frozenset([18, 20, 21]): (6, 11, 16),
+                  frozenset([23, 28, 29, 30, 31, 10, 11, 14, 15]): (6, 11, 16, 21),
+                  frozenset([24, 26]): (6, 16),
+                  frozenset([25, 12, 13]): (6, 11, 21)
+                  }
+                }
+
+# 6-bit opcode field for miscellaneous opcodes
+misc_opcode_map = {
+                    'mask': 0x3f,
+                    'data':
+                    {
+                    frozenset([14, 12, 13, 15, 7, 8, 28, 29, 24, 25, 
+                     26, 27, 20, 21, 34, 35, 42, 43, 40, 41, 
+                     32, 33, 38, 39, 44, 45, 36, 37, 46, 47, 
+                     50, 51, 48, 49, 54, 55, 52, 53, 56, 57, 
+                     60, 61]): (6, 11),
+
+                    frozenset([11, 10, 3]): (11,),
+
+                    frozenset([23]): (6, 11, 16)
+                    }
+                  }
+
+class PPCInstr:
+
+    INSTR_SIZE = 32
+    REG_FIELD_SIZE = 5
+
+    def __init__(self, val):
+        self.v = val
+    
+    def get_field(self, left, right):
+        return (self.v >> (self.INSTR_SIZE - right - 1)) & ((1 << (right - left + 1)) - 1)
+    
+    def set_field(self, left, right, val):
+        width = right - left + 1
+        mask = (1 << width) - 1
+        shift = self.INSTR_SIZE - width - left
+        self.v = self.v & ~(mask << shift) | ((val & mask) << shift)
+    
+    def get_opcode(self):
+        return self.get_field(0, 5)
+    
+    def get_ext_opcode(self):
+        return self.get_field(21, 30)
+    
+    def search_opcode_maps(self, opcode, *maps):
+        for map in maps:
+            masked_opcode = opcode & map['mask']
+            for k in map['data'].keys():
+                if masked_opcode in k:
+                    return map['data'][k]
+    
+    # returns a tuple containing the bit position of each register field
+    # or None if the instruction does not use registers
+    def get_reg_fields(self):
+        opcode = self.get_opcode()
+        ext_opcode = self.get_ext_opcode()
+        if opcode == 31:
+            return self.search_opcode_maps(ext_opcode, op31_map, op31_mask9_map)
+        elif opcode == 59:
+            return self.search_opcode_maps(ext_opcode, op59_mask5_map)
+        elif opcode == 63:
+            return self.search_opcode_maps(ext_opcode, op63_map, op63_mask5_map)
+        elif opcode == 4:
+            return self.search_opcode_maps(ext_opcode, op4_map, op4_mask6_map, op4_mask5_map)
+        else:
+            return self.search_opcode_maps(opcode, misc_opcode_map)
+    
+    # edit the PPC instruction to swap the registers
+    def swap_registers(self, regA, regB):
+        # DEBUG_v = hex(self.v)
+        reg_fields = self.get_reg_fields()
+        # print(str(reg_fields) + ", " + DEBUG_v)
+        if reg_fields is None:
+            return
+        for left in reg_fields:
+            right = left + self.REG_FIELD_SIZE - 1
+            currReg = self.get_field(left, right)
+            if currReg == regA:
+                self.set_field(left, right, regB)
+            elif currReg == regB:
+                self.set_field(left, right, regA)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("cc", 
+                    help="path to a C/C++ compiler")
+parser.add_argument("cflags", 
+                    help="all flags and options to be invoked with cc")
+parser.add_argument("output", 
+                    help="path to the outputted object file")
+parser.add_argument("source", 
+                    help="path to the C/C++ source file")
+parser.add_argument("-fix-regswaps", 
+                    help="execute #pragma regswap", action="store_true")
+args = parser.parse_args()
+
+def parse_reg(str):
+    if str[0] == 'r' or str[0] == 'f':
+        reg = int(str[1:])
+        if reg >= 0 and reg <= 31:
+            return reg
+    raise ValueError("Failed to parse register argument (can be r0...r31 or f0...f31)")
+
+class RegswapTask:
+    def __init__(self, start, end, regA, regB):
+        self.start = start # .text section byte offset 
+        self.end = end     # .text section byte offset
+        self.regA = regA
+        self.regB = regB
+
+regswap_tasks = []
+with open(args.source, "r") as src:
+    regswap_pattern = re.compile("[ \t]*#pragma[ \t]+regswap[ \t]+")
+    for line in src:
+        if regswap_pattern.match(line):
+            if args.fix_regswaps:
+                params = line.split()[2:]
+                if len(params) != 5:
+                    raise ValueError("ERROR: " + len(params) + " arguments passed to #pragma regswap (expected 5)")
+                start = int(params[0], base=16)
+                end = int(params[1], base=16)
+                regA = parse_reg(params[2])
+                regB = parse_reg(params[3])
+                start_file = int(params[4], base=16)
+                if not (start % 4 == 0 and end % 4 == 0 and start_file % 4 == 0):
+                    raise ValueError("Invalid start, end, or start_file arguments (should have 4 byte aligment)")
+                if not (start >= start_file and end > start):
+                    raise ValueError("Invalid start, end, or start_file arguments (end must be > start, and start >= start_file)")
+                regswap_tasks.append(RegswapTask(start-start_file, end-start_file, regA, regB))
+    subprocess.run([*args.cc.strip().split(' '), *args.cflags.split(' '), "-o", args.output, args.source])
+
+instrs = []
+TEXT_INDEX = 1 # NOTE: assumes that mwcceppc always places the .text section header at index 1
+SHDR_32_SIZE = 40 # size of an Elf32_Shdr object
+
+if args.fix_regswaps and len(regswap_tasks) != 0:
+    with open(args.output, "rb") as f:
+        if f.read(7) != b'\x7FELF\x01\x02\x01':
+            raise ValueError("compiler output is not an current version ELF file for a 32-bit big endian architecture")
+        f.seek(0x20)
+        e_shoff = int.from_bytes(f.read(4), byteorder='big')
+        f.seek(0x30)
+        e_shnum = int.from_bytes(f.read(2), byteorder='big')
+        if e_shoff == 0 or e_shnum < 2:
+            raise ValueError("ELF file must contain at least two sections")
+        
+        # get .text section sh_offset and sh_size members
+        f.seek(e_shoff + TEXT_INDEX*SHDR_32_SIZE + 0x10)
+        text_offset = int.from_bytes(f.read(4), byteorder='big')
+        text_size = int.from_bytes(f.read(4), byteorder='big')
+        
+        # read .text section contents into buffer
+        f.seek(text_offset)
+        for i in range(text_size // 4):
+            instrs.append(PPCInstr(int.from_bytes(f.read(4), byteorder='big')))
+        
+        # perform regswap tasks
+        for task in regswap_tasks:
+            if task.end > text_size:
+                raise ValueError("End address " + (task.end + start_file) + " is past the end of the ELF file's .text section")
+            for i in range(task.start // 4, task.end // 4):
+                instrs[i].swap_registers(task.regA, task.regB)
+    
+    # write patched .text section back to the ELF
+    with open(args.output, "rb+") as f:
+        f.seek(text_offset)
+        for instr in instrs:
+            f.write(instr.v.to_bytes(4, byteorder='big'))