Add CI script to compare recompiled assembly with original code (#24)

* add test to compare assembly between functions

* ci: use abs path of wget

* ci: fix shell disambiguity

* ci: ensure capstone is installed

* ci: ensure correct filenames

* use better source for lego island files

* give me an idea of what the dir structure looks like

* make wine path function

* improved script and project

* fixed script on windows

* print debug info because now it literally only doesn't work on fucking github actions

* better source path resolving

For some reason, nmake compiles produce different symbols. I wonder if this affects the accuracy of the decomp.

.github/workflows/build.yml

@@ -54,6 +54,15 @@ jobs:
         call .\msvc420\bin\VCVARS32.BAT x86
         mkdir Release
         .\msvc420\bin\NMAKE.EXE /f isle.mak CFG="ISLE - Win32 Release"
+
+    - name: Summarize Accuracy
+      shell: cmd
+      run: |
+        C:\msys64\usr\bin\wget.exe https://legoisland.org/download/ISLE.EXE
+        C:\msys64\usr\bin\wget.exe https://legoisland.org/download/LEGO1.DLL
+        pip install capstone
+        python3 tools/reccomp/reccomp.py ISLE.EXE Release/ISLE.EXE Release/ISLE.PDB ISLE
+        python3 tools/reccomp/reccomp.py LEGO1.DLL Release/LEGO1.DLL Release/LEGO1.PDB LEGO1
       
     - name: Upload Artifact
       uses: actions/upload-artifact@master

ISLE/isle.cpp

@@ -9,6 +9,7 @@
 
 RECT windowRect = {0, 0, 640, 480};
 
+// OFFSET: ISLE 0x401000
 Isle::Isle()
 {
   m_hdPath = NULL;
@@ -51,6 +52,7 @@ Isle::Isle()
   LegoOmni::CreateInstance();
 }
 
+// OFFSET: ISLE 0x4011a0
 Isle::~Isle()
 {
   if (LegoOmni::GetInstance()) {
@@ -75,6 +77,7 @@ Isle::~Isle()
   }
 }
 
+// OFFSET: ISLE 0x401260
 void Isle::close()
 {
   MxDSAction ds;
@@ -109,6 +112,7 @@ void Isle::close()
   }
 }
 
+// OFFSET: ISLE 0x402740
 BOOL readReg(LPCSTR name, LPSTR outValue, DWORD outSize)
 {
   HKEY hKey;
@@ -127,6 +131,7 @@ BOOL readReg(LPCSTR name, LPSTR outValue, DWORD outSize)
   return out;
 }
 
+// OFFSET: ISLE 0x4027b0
 int readRegBool(LPCSTR name, BOOL *out)
 {
   char buffer[256];
@@ -146,6 +151,7 @@ int readRegBool(LPCSTR name, BOOL *out)
   return FALSE;
 }
 
+// OFFSET: ISLE 0x402880
 int readRegInt(LPCSTR name, int *out)
 {
   char buffer[256];
@@ -158,6 +164,7 @@ int readRegInt(LPCSTR name, int *out)
   return FALSE;
 }
 
+// OFFSET: ISLE 0x4028d0
 void Isle::loadConfig()
 {
   #define BUFFER_SIZE 1024
@@ -224,6 +231,7 @@ void Isle::loadConfig()
   }
 }
 
+// OFFSET: ISLE 0x401560
 void Isle::setupVideoFlags(BOOL fullScreen, BOOL flipSurfaces, BOOL backBuffers,
                            BOOL using8bit, BOOL m_using16bit, BOOL param_6, BOOL param_7,
                            BOOL wideViewAngle, char *deviceId)
@@ -244,6 +252,7 @@ void Isle::setupVideoFlags(BOOL fullScreen, BOOL flipSurfaces, BOOL backBuffers,
   }
 }
 
+// OFFSET: ISLE 0x4013b0
 BOOL Isle::setupLegoOmni()
 {
   char mediaPath[256];
@@ -258,6 +267,7 @@ BOOL Isle::setupLegoOmni()
   return FALSE;
 }
 
+// OFFSET: ISLE 0x402e80
 void Isle::setupCursor(WPARAM wParam)
 {
   switch (wParam) {
@@ -278,6 +288,7 @@ void Isle::setupCursor(WPARAM wParam)
   SetCursor(m_cursorCurrent);
 }
 
+// OFFSET: ISLE 0x401d20
 LRESULT WINAPI WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
 {
   if (!g_isle) {
@@ -447,6 +458,7 @@ LRESULT WINAPI WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
   return DefWindowProcA(hWnd,uMsg,wParam,lParam);
 }
 
+// OFFSET: ISLE 0x4023e0
 MxResult Isle::setupWindow(HINSTANCE hInstance)
 {
   WNDCLASSA wndclass;
@@ -554,6 +566,7 @@ MxResult Isle::setupWindow(HINSTANCE hInstance)
   return SUCCESS;
 }
 
+// OFFSET: ISLE 0x402c20
 void Isle::tick(BOOL sleepIfNotNextFrame)
 {
   if (this->m_windowActive) {

ISLE/main.cpp

@@ -5,6 +5,7 @@
 #include "isle.h"
 #include "legoomni.h"
 
+// OFFSET: ISLE 0x401ca0
 BOOL findExistingInstance(void)
 {
   HWND hWnd = FindWindowA(WNDCLASS_NAME, WINDOW_TITLE);
@@ -17,6 +18,7 @@ BOOL findExistingInstance(void)
   return 1;
 }
 
+// OFFSET: ISLE 0x401ce0
 BOOL startDirectSound(void)
 {
   LPDIRECTSOUND lpDS = 0;
@@ -29,6 +31,7 @@ BOOL startDirectSound(void)
   return FALSE;
 }
 
+// OFFSET: ISLE 0x401610
 int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nShowCmd)
 {
   // Look for another instance, if we find one, bring it to the foreground instead

isle.mak

@@ -68,19 +68,21 @@ CLEAN :
 	-@erase "$(INTDIR)\mxtimer.obj"
 	-@erase "$(INTDIR)\mxvideoparam.obj"
 	-@erase "$(INTDIR)\mxvideoparamflags.obj"
+	-@erase "$(INTDIR)\vc40.pdb"
 	-@erase ".\Release\LEGO1.DLL"
 	-@erase ".\Release\LEGO1.EXP"
 	-@erase ".\Release\LEGO1.LIB"
 	-@erase ".\Release\LEGO1.MAP"
+	-@erase ".\Release\LEGO1.PDB"
 
 "$(OUTDIR)" :
     if not exist "$(OUTDIR)/$(NULL)" mkdir "$(OUTDIR)"
 
 CPP=cl.exe
 # ADD BASE CPP /nologo /MT /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
-# ADD CPP /nologo /MT /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
+# ADD CPP /nologo /MT /W3 /GX /Zi /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
-CPP_PROJ=/nologo /MT /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS"\
+CPP_PROJ=/nologo /MT /W3 /GX /Zi /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS"\
- /Fp"$(INTDIR)/LEGO1.pch" /YX /Fo"$(INTDIR)/" /c 
+ /Fp"$(INTDIR)/LEGO1.pch" /YX /Fo"$(INTDIR)/" /Fd"$(INTDIR)/" /c 
 CPP_OBJS=.\LEGO1\Release/
 CPP_SBRS=.\.
 
@@ -117,12 +119,12 @@ BSC32_SBRS= \
 	
 LINK32=link.exe
 # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /dll /machine:I386
-# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib winmm.lib /nologo /subsystem:windows /dll /pdb:"Release/LEGO1.PDB" /map:"Release/LEGO1.MAP" /machine:I386 /out:"Release/LEGO1.DLL" /implib:"Release/LEGO1.LIB"
+# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib winmm.lib /nologo /subsystem:windows /dll /pdb:"Release/LEGO1.PDB" /map:"Release/LEGO1.MAP" /debug /machine:I386 /out:"Release/LEGO1.DLL" /implib:"Release/LEGO1.LIB"
 # SUBTRACT LINK32 /pdb:none
 LINK32_FLAGS=kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib\
  advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib\
  odbccp32.lib winmm.lib /nologo /subsystem:windows /dll /incremental:no\
- /pdb:"Release/LEGO1.PDB" /map:"Release/LEGO1.MAP" /machine:I386\
+ /pdb:"Release/LEGO1.PDB" /map:"Release/LEGO1.MAP" /debug /machine:I386\
  /out:"Release/LEGO1.DLL" /implib:"Release/LEGO1.LIB" 
 LINK32_OBJS= \
 	"$(INTDIR)\dllmain.obj" \
@@ -281,16 +283,18 @@ CLEAN :
 	-@erase "$(INTDIR)\isle.res"
 	-@erase "$(INTDIR)\main.obj"
 	-@erase "$(INTDIR)\mxomnicreateparambase.obj"
+	-@erase "$(INTDIR)\vc40.pdb"
 	-@erase ".\Release\ISLE.EXE"
+	-@erase ".\Release\ISLE.PDB"
 
 "$(OUTDIR)" :
     if not exist "$(OUTDIR)/$(NULL)" mkdir "$(OUTDIR)"
 
 CPP=cl.exe
 # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
-# ADD CPP /nologo /W3 /GX /O2 /I "LEGO1" /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
+# ADD CPP /nologo /W3 /GX /Zi /O2 /I "LEGO1" /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
-CPP_PROJ=/nologo /ML /W3 /GX /O2 /I "LEGO1" /D "WIN32" /D "NDEBUG" /D\
+CPP_PROJ=/nologo /ML /W3 /GX /Zi /O2 /I "LEGO1" /D "WIN32" /D "NDEBUG" /D\
- "_WINDOWS" /Fp"$(INTDIR)/ISLE.pch" /YX /Fo"$(INTDIR)/" /c 
+ "_WINDOWS" /Fp"$(INTDIR)/ISLE.pch" /YX /Fo"$(INTDIR)/" /Fd"$(INTDIR)/" /c 
 CPP_OBJS=.\ISLE\Release/
 CPP_SBRS=.\.
 
@@ -328,13 +332,13 @@ BSC32_SBRS= \
 	
 LINK32=link.exe
 # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /machine:I386
-# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib winmm.lib lego1.lib dsound.lib /nologo /subsystem:windows /pdb:"Release/ISLE.PDB" /machine:I386 /out:"Release/ISLE.EXE" /LIBPATH:"ISLE/ext"
+# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib winmm.lib lego1.lib dsound.lib /nologo /subsystem:windows /pdb:"Release/ISLE.PDB" /debug /machine:I386 /out:"Release/ISLE.EXE" /LIBPATH:"ISLE/ext"
 # SUBTRACT LINK32 /pdb:none
 LINK32_FLAGS=kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib\
  advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib\
  odbccp32.lib winmm.lib lego1.lib dsound.lib /nologo /subsystem:windows\
- /incremental:no /pdb:"Release/ISLE.PDB" /machine:I386 /out:"Release/ISLE.EXE"\
+ /incremental:no /pdb:"Release/ISLE.PDB" /debug /machine:I386\
- /LIBPATH:"ISLE/ext" 
+ /out:"Release/ISLE.EXE" /LIBPATH:"ISLE/ext" 
 LINK32_OBJS= \
 	"$(INTDIR)\define.obj" \
 	"$(INTDIR)\isle.obj" \
@@ -799,6 +803,8 @@ SOURCE=.\ISLE\main.cpp
 DEP_CPP_MAIN_=\
 	".\ISLE\define.h"\
 	".\ISLE\isle.h"\
+	".\LEGO1\lego3dmanager.h"\
+	".\LEGO1\lego3dview.h"\
 	".\LEGO1\legoanimationmanager.h"\
 	".\LEGO1\legobuildingmanager.h"\
 	".\LEGO1\legoentity.h"\
@@ -832,6 +838,7 @@ DEP_CPP_MAIN_=\
 	".\LEGO1\mxvariabletable.h"\
 	".\LEGO1\mxvideoparam.h"\
 	".\LEGO1\mxvideoparamflags.h"\
+	".\LEGO1\viewmanager.h"\
 	
 
 "$(INTDIR)\main.obj" : $(SOURCE) $(DEP_CPP_MAIN_) "$(INTDIR)"

tools/reccomp/reccomp.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+
+from capstone import *
+import difflib
+import struct
+import subprocess
+import os
+import sys
+
+def print_usage():
+  print('Usage: %s [options] <original-binary> <recompiled-binary> <recompiled-pdb> <decomp-dir>\n' % sys.argv[0])
+  print('\t-v, --verbose <offset>\t\t\tPrint assembly diff for specific function (original file\'s offset)')
+  sys.exit(1)
+
+positional_args = []
+verbose = None
+skip = False
+
+for i, arg in enumerate(sys.argv):
+  if skip:
+    skip = False
+    continue
+
+  if arg.startswith('-'):
+    # A flag rather than a positional arg
+    flag = arg[1:]
+
+    if flag == 'v' or flag == '-verbose':
+      verbose = int(sys.argv[i + 1], 16)
+      skip = True
+    else:
+      print('Unknown flag: %s' % arg)
+      print_usage()
+  else:
+    positional_args.append(arg)
+
+if len(positional_args) != 5:
+  print_usage()
+
+original = positional_args[1]
+if not os.path.isfile(original):
+  print('Invalid input: Original binary does not exist')
+  sys.exit(1)
+
+recomp = positional_args[2]
+if not os.path.isfile(recomp):
+  print('Invalid input: Recompiled binary does not exist')
+  sys.exit(1)
+
+syms = positional_args[3]
+if not os.path.isfile(syms):
+  print('Invalid input: Symbols PDB does not exist')
+  sys.exit(1)
+
+source = positional_args[4]
+if not os.path.isdir(source):
+  print('Invalid input: Source directory does not exist')
+  sys.exit(1)
+
+# Declare a class that can automatically convert virtual executable addresses
+# to file addresses
+class Bin:
+  def __init__(self, filename):
+    self.file = open(filename, 'rb')
+
+    #HACK: Strictly, we should be parsing the header, but we know where
+    #      everything is in these two files so we just jump straight there
+
+    # Read ImageBase
+    self.file.seek(0xB4)
+    self.imagebase = struct.unpack('i', self.file.read(4))[0]
+
+    # Read .text VirtualAddress
+    self.file.seek(0x184)
+    self.textvirt = struct.unpack('i', self.file.read(4))[0]
+
+    # Read .text PointerToRawData
+    self.file.seek(0x18C)
+    self.textraw = struct.unpack('i', self.file.read(4))[0]
+
+  def __del__(self):
+    if self.file:
+      self.file.close()
+
+  def get_addr(self, virt):
+    return virt - self.imagebase - self.textvirt + self.textraw
+
+  def read(self, offset, size):
+    self.file.seek(self.get_addr(offset))
+    return self.file.read(size)
+
+line_dump = None
+
+origfile = Bin(original)
+recompfile = Bin(recomp)
+
+class RecompiledInfo:
+  addr = None
+  size = None
+  name = None
+
+print()
+
+def get_recompiled_address(filename, line):
+  global line_dump, sym_dump
+
+  def get_wine_path(fn):
+    return subprocess.check_output(['winepath', '-w', fn]).decode('utf-8').strip()
+
+  # Load source lines from PDB
+  if not line_dump:
+    call = [os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), 'cvdump'), '-l', '-s']
+
+    if os.name != 'nt':
+      # Run cvdump through wine and convert path to Windows-friendly wine path
+      call.insert(0, 'wine')
+      call.append(get_wine_path(syms))
+    else:
+      call.append(syms)
+
+    line_dump = subprocess.check_output(call).decode('utf-8').split('\r\n')
+
+  # Find requested filename/line in PDB
+  if os.name != 'nt':
+    # Convert filename to Wine path
+    filename = get_wine_path(filename)
+
+  #print('Looking for ' + filename + ' line ' + str(line))
+
+  addr = None
+  found = False
+
+  for i, s in enumerate(line_dump):
+    try:
+      sourcepath = s.split()[0]
+      if os.path.isfile(sourcepath) and os.path.samefile(sourcepath, filename):
+        lines = line_dump[i + 2].split()
+        if line == int(lines[0]):
+          # Found address
+          addr = int(lines[1], 16)
+          found = True
+          break
+    except IndexError:
+      pass
+
+  if found:
+    # Find size of function
+    for i, s in enumerate(line_dump):
+      if 'S_GPROC32' in s:
+        if int(s[26:34], 16) == addr:
+          obj = RecompiledInfo()
+          obj.addr = addr + recompfile.imagebase + recompfile.textvirt
+          obj.size = int(s[41:49], 16)
+          obj.name = s[77:]
+
+          return obj
+
+md = Cs(CS_ARCH_X86, CS_MODE_32)
+
+def parse_asm(file, addr, size):
+  asm = []
+  data = file.read(addr, size)
+  for i in md.disasm(data, 0):
+    if i.mnemonic == 'call':
+      # Filter out "calls" because the offsets we're not currently trying to
+      # match offsets. As long as there's a call in the right place, it's
+      # probably accurate.
+      asm.append(i.mnemonic)
+    else:
+      asm.append("%s %s" % (i.mnemonic, i.op_str))
+  return asm
+
+function_count = 0
+total_accuracy = 0
+
+for subdir, dirs, files in os.walk(source):
+  for file in files:
+    srcfilename = os.path.join(os.path.abspath(subdir), file)
+    srcfile = open(srcfilename, 'r')
+    line_no = 0
+
+    while True:
+      try:
+        line = srcfile.readline()
+        line_no += 1
+
+        if not line:
+          break
+
+        if line.startswith('// OFFSET:'):
+          par = line[10:].strip().split()
+          module = par[0]
+          addr = int(par[1], 16)
+
+          find_open_bracket = line
+          while '{' not in find_open_bracket:
+            find_open_bracket = srcfile.readline()
+            line_no += 1
+
+          recinfo = get_recompiled_address(srcfilename, line_no)
+          if not recinfo:
+            print('Failed to find recompiled address of ' + hex(addr))
+            continue
+
+          origasm = parse_asm(origfile, addr, recinfo.size)
+          recompasm = parse_asm(recompfile, recinfo.addr, recinfo.size)
+
+          diff = difflib.SequenceMatcher(None, origasm, recompasm)
+          ratio = diff.ratio()
+          print('%s (%s / %s) is %.2f%% similar to the original' % (recinfo.name, hex(addr), hex(recinfo.addr), ratio * 100))
+
+          function_count += 1
+          total_accuracy += ratio
+
+          if verbose == addr:
+            udiff = difflib.unified_diff(origasm, recompasm)
+            for line in udiff:
+              print(line)
+            print()
+            print()
+
+      except UnicodeDecodeError:
+        break
+
+if function_count > 0:
+  print('\nTotal accuracy %.2f%% across %i functions' % (total_accuracy / function_count * 100, function_count))