Add CI script to compare recompiled assembly with original code (#24)

* add test to compare assembly between functions * ci: use abs path of wget * ci: fix shell disambiguity * ci: ensure capstone is installed * ci: ensure correct filenames * use better source for lego island files * give me an idea of what the dir structure looks like * make wine path function * improved script and project * fixed script on windows * print debug info because now it literally only doesn't work on fucking github actions * better source path resolving For some reason, nmake compiles produce different symbols. I wonder if this affects the accuracy of the decomp.
2024-11-26 01:17:55 -05:00 · 2023-06-18 20:28:18 -07:00 · 2023-06-18 20:28:18 -07:00 · 5aa7921e90
commit 5aa7921e90
parent f9a93406a8
7 changed files with 269 additions and 11 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -54,6 +54,15 @@ jobs:
        call .\msvc420\bin\VCVARS32.BAT x86
        mkdir Release
        .\msvc420\bin\NMAKE.EXE /f isle.mak CFG="ISLE - Win32 Release"
+
+    - name: Summarize Accuracy
+      shell: cmd
+      run: |
+        C:\msys64\usr\bin\wget.exe https://legoisland.org/download/ISLE.EXE
+        C:\msys64\usr\bin\wget.exe https://legoisland.org/download/LEGO1.DLL
+        pip install capstone
+        python3 tools/reccomp/reccomp.py ISLE.EXE Release/ISLE.EXE Release/ISLE.PDB ISLE
+        python3 tools/reccomp/reccomp.py LEGO1.DLL Release/LEGO1.DLL Release/LEGO1.PDB LEGO1
      
    - name: Upload Artifact
      uses: actions/upload-artifact@master
--- a/ISLE/isle.cpp
+++ b/ISLE/isle.cpp
@ -9,6 +9,7 @@

 RECT windowRect = {0, 0, 640, 480};

+// OFFSET: ISLE 0x401000
 Isle::Isle()
 {
  m_hdPath = NULL;
@ -51,6 +52,7 @@ Isle::Isle()
  LegoOmni::CreateInstance();
 }

+// OFFSET: ISLE 0x4011a0
 Isle::~Isle()
 {
  if (LegoOmni::GetInstance()) {
@ -75,6 +77,7 @@ Isle::~Isle()
  }
 }

+// OFFSET: ISLE 0x401260
 void Isle::close()
 {
  MxDSAction ds;
@ -109,6 +112,7 @@ void Isle::close()
  }
 }

+// OFFSET: ISLE 0x402740
 BOOL readReg(LPCSTR name, LPSTR outValue, DWORD outSize)
 {
  HKEY hKey;
@ -127,6 +131,7 @@ BOOL readReg(LPCSTR name, LPSTR outValue, DWORD outSize)
  return out;
 }

+// OFFSET: ISLE 0x4027b0
 int readRegBool(LPCSTR name, BOOL *out)
 {
  char buffer[256];
@ -146,6 +151,7 @@ int readRegBool(LPCSTR name, BOOL *out)
  return FALSE;
 }

+// OFFSET: ISLE 0x402880
 int readRegInt(LPCSTR name, int *out)
 {
  char buffer[256];
@ -158,6 +164,7 @@ int readRegInt(LPCSTR name, int *out)
  return FALSE;
 }

+// OFFSET: ISLE 0x4028d0
 void Isle::loadConfig()
 {
  #define BUFFER_SIZE 1024
@ -224,6 +231,7 @@ void Isle::loadConfig()
  }
 }

+// OFFSET: ISLE 0x401560
 void Isle::setupVideoFlags(BOOL fullScreen, BOOL flipSurfaces, BOOL backBuffers,
                           BOOL using8bit, BOOL m_using16bit, BOOL param_6, BOOL param_7,
                           BOOL wideViewAngle, char *deviceId)
@ -244,6 +252,7 @@ void Isle::setupVideoFlags(BOOL fullScreen, BOOL flipSurfaces, BOOL backBuffers,
  }
 }

+// OFFSET: ISLE 0x4013b0
 BOOL Isle::setupLegoOmni()
 {
  char mediaPath[256];
@ -258,6 +267,7 @@ BOOL Isle::setupLegoOmni()
  return FALSE;
 }

+// OFFSET: ISLE 0x402e80
 void Isle::setupCursor(WPARAM wParam)
 {
  switch (wParam) {
@ -278,6 +288,7 @@ void Isle::setupCursor(WPARAM wParam)
  SetCursor(m_cursorCurrent);
 }

+// OFFSET: ISLE 0x401d20
 LRESULT WINAPI WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
 {
  if (!g_isle) {
@ -447,6 +458,7 @@ LRESULT WINAPI WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
  return DefWindowProcA(hWnd,uMsg,wParam,lParam);
 }

+// OFFSET: ISLE 0x4023e0
 MxResult Isle::setupWindow(HINSTANCE hInstance)
 {
  WNDCLASSA wndclass;
@ -554,6 +566,7 @@ MxResult Isle::setupWindow(HINSTANCE hInstance)
  return SUCCESS;
 }

+// OFFSET: ISLE 0x402c20
 void Isle::tick(BOOL sleepIfNotNextFrame)
 {
  if (this->m_windowActive) {
--- a/ISLE/main.cpp
+++ b/ISLE/main.cpp
@ -5,6 +5,7 @@
 #include "isle.h"
 #include "legoomni.h"

+// OFFSET: ISLE 0x401ca0
 BOOL findExistingInstance(void)
 {
  HWND hWnd = FindWindowA(WNDCLASS_NAME, WINDOW_TITLE);
@ -17,6 +18,7 @@ BOOL findExistingInstance(void)
  return 1;
 }

+// OFFSET: ISLE 0x401ce0
 BOOL startDirectSound(void)
 {
  LPDIRECTSOUND lpDS = 0;
@ -29,6 +31,7 @@ BOOL startDirectSound(void)
  return FALSE;
 }

+// OFFSET: ISLE 0x401610
 int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nShowCmd)
 {
  // Look for another instance, if we find one, bring it to the foreground instead
--- a/isle.mak
+++ b/isle.mak
@ -68,19 +68,21 @@ CLEAN :
 	-@erase "$(INTDIR)\mxtimer.obj"
 	-@erase "$(INTDIR)\mxvideoparam.obj"
 	-@erase "$(INTDIR)\mxvideoparamflags.obj"
+	-@erase "$(INTDIR)\vc40.pdb"
 	-@erase ".\Release\LEGO1.DLL"
 	-@erase ".\Release\LEGO1.EXP"
 	-@erase ".\Release\LEGO1.LIB"
 	-@erase ".\Release\LEGO1.MAP"
+	-@erase ".\Release\LEGO1.PDB"

 "$(OUTDIR)" :
    if not exist "$(OUTDIR)/$(NULL)" mkdir "$(OUTDIR)"

 CPP=cl.exe
 # ADD BASE CPP /nologo /MT /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
-# ADD CPP /nologo /MT /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
-CPP_PROJ=/nologo /MT /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS"\
- /Fp"$(INTDIR)/LEGO1.pch" /YX /Fo"$(INTDIR)/" /c 
+# ADD CPP /nologo /MT /W3 /GX /Zi /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
+CPP_PROJ=/nologo /MT /W3 /GX /Zi /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS"\
+ /Fp"$(INTDIR)/LEGO1.pch" /YX /Fo"$(INTDIR)/" /Fd"$(INTDIR)/" /c 
 CPP_OBJS=.\LEGO1\Release/
 CPP_SBRS=.\.

@ -117,12 +119,12 @@ BSC32_SBRS= \
 	
 LINK32=link.exe
 # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /dll /machine:I386
-# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib winmm.lib /nologo /subsystem:windows /dll /pdb:"Release/LEGO1.PDB" /map:"Release/LEGO1.MAP" /machine:I386 /out:"Release/LEGO1.DLL" /implib:"Release/LEGO1.LIB"
+# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib winmm.lib /nologo /subsystem:windows /dll /pdb:"Release/LEGO1.PDB" /map:"Release/LEGO1.MAP" /debug /machine:I386 /out:"Release/LEGO1.DLL" /implib:"Release/LEGO1.LIB"
 # SUBTRACT LINK32 /pdb:none
 LINK32_FLAGS=kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib\
 advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib\
 odbccp32.lib winmm.lib /nologo /subsystem:windows /dll /incremental:no\
- /pdb:"Release/LEGO1.PDB" /map:"Release/LEGO1.MAP" /machine:I386\
+ /pdb:"Release/LEGO1.PDB" /map:"Release/LEGO1.MAP" /debug /machine:I386\
 /out:"Release/LEGO1.DLL" /implib:"Release/LEGO1.LIB" 
 LINK32_OBJS= \
 	"$(INTDIR)\dllmain.obj" \
@ -281,16 +283,18 @@ CLEAN :
 	-@erase "$(INTDIR)\isle.res"
 	-@erase "$(INTDIR)\main.obj"
 	-@erase "$(INTDIR)\mxomnicreateparambase.obj"
+	-@erase "$(INTDIR)\vc40.pdb"
 	-@erase ".\Release\ISLE.EXE"
+	-@erase ".\Release\ISLE.PDB"

 "$(OUTDIR)" :
    if not exist "$(OUTDIR)/$(NULL)" mkdir "$(OUTDIR)"

 CPP=cl.exe
 # ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
-# ADD CPP /nologo /W3 /GX /O2 /I "LEGO1" /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
-CPP_PROJ=/nologo /ML /W3 /GX /O2 /I "LEGO1" /D "WIN32" /D "NDEBUG" /D\
- "_WINDOWS" /Fp"$(INTDIR)/ISLE.pch" /YX /Fo"$(INTDIR)/" /c 
+# ADD CPP /nologo /W3 /GX /Zi /O2 /I "LEGO1" /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /YX /c
+CPP_PROJ=/nologo /ML /W3 /GX /Zi /O2 /I "LEGO1" /D "WIN32" /D "NDEBUG" /D\
+ "_WINDOWS" /Fp"$(INTDIR)/ISLE.pch" /YX /Fo"$(INTDIR)/" /Fd"$(INTDIR)/" /c 
 CPP_OBJS=.\ISLE\Release/
 CPP_SBRS=.\.

@ -328,13 +332,13 @@ BSC32_SBRS= \
 	
 LINK32=link.exe
 # ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:windows /machine:I386
-# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib winmm.lib lego1.lib dsound.lib /nologo /subsystem:windows /pdb:"Release/ISLE.PDB" /machine:I386 /out:"Release/ISLE.EXE" /LIBPATH:"ISLE/ext"
+# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib winmm.lib lego1.lib dsound.lib /nologo /subsystem:windows /pdb:"Release/ISLE.PDB" /debug /machine:I386 /out:"Release/ISLE.EXE" /LIBPATH:"ISLE/ext"
 # SUBTRACT LINK32 /pdb:none
 LINK32_FLAGS=kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib\
 advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib\
 odbccp32.lib winmm.lib lego1.lib dsound.lib /nologo /subsystem:windows\
- /incremental:no /pdb:"Release/ISLE.PDB" /machine:I386 /out:"Release/ISLE.EXE"\
- /LIBPATH:"ISLE/ext" 
+ /incremental:no /pdb:"Release/ISLE.PDB" /debug /machine:I386\
+ /out:"Release/ISLE.EXE" /LIBPATH:"ISLE/ext" 
 LINK32_OBJS= \
 	"$(INTDIR)\define.obj" \
 	"$(INTDIR)\isle.obj" \
@ -799,6 +803,8 @@ SOURCE=.\ISLE\main.cpp
 DEP_CPP_MAIN_=\
 	".\ISLE\define.h"\
 	".\ISLE\isle.h"\
+	".\LEGO1\lego3dmanager.h"\
+	".\LEGO1\lego3dview.h"\
 	".\LEGO1\legoanimationmanager.h"\
 	".\LEGO1\legobuildingmanager.h"\
 	".\LEGO1\legoentity.h"\
@ -832,6 +838,7 @@ DEP_CPP_MAIN_=\
 	".\LEGO1\mxvariabletable.h"\
 	".\LEGO1\mxvideoparam.h"\
 	".\LEGO1\mxvideoparamflags.h"\
+	".\LEGO1\viewmanager.h"\
 	

 "$(INTDIR)\main.obj" : $(SOURCE) $(DEP_CPP_MAIN_) "$(INTDIR)"
--- a/isle.mdp
+++ b/isle.mdp
--- a/tools/reccomp/cvdump.exe
+++ b/tools/reccomp/cvdump.exe
--- a/tools/reccomp/reccomp.py
+++ b/tools/reccomp/reccomp.py
@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+
+from capstone import *
+import difflib
+import struct
+import subprocess
+import os
+import sys
+
+def print_usage():
+  print('Usage: %s [options] <original-binary> <recompiled-binary> <recompiled-pdb> <decomp-dir>\n' % sys.argv[0])
+  print('\t-v, --verbose <offset>\t\t\tPrint assembly diff for specific function (original file\'s offset)')
+  sys.exit(1)
+
+positional_args = []
+verbose = None
+skip = False
+
+for i, arg in enumerate(sys.argv):
+  if skip:
+    skip = False
+    continue
+
+  if arg.startswith('-'):
+    # A flag rather than a positional arg
+    flag = arg[1:]
+
+    if flag == 'v' or flag == '-verbose':
+      verbose = int(sys.argv[i + 1], 16)
+      skip = True
+    else:
+      print('Unknown flag: %s' % arg)
+      print_usage()
+  else:
+    positional_args.append(arg)
+
+if len(positional_args) != 5:
+  print_usage()
+
+original = positional_args[1]
+if not os.path.isfile(original):
+  print('Invalid input: Original binary does not exist')
+  sys.exit(1)
+
+recomp = positional_args[2]
+if not os.path.isfile(recomp):
+  print('Invalid input: Recompiled binary does not exist')
+  sys.exit(1)
+
+syms = positional_args[3]
+if not os.path.isfile(syms):
+  print('Invalid input: Symbols PDB does not exist')
+  sys.exit(1)
+
+source = positional_args[4]
+if not os.path.isdir(source):
+  print('Invalid input: Source directory does not exist')
+  sys.exit(1)
+
+# Declare a class that can automatically convert virtual executable addresses
+# to file addresses
+class Bin:
+  def __init__(self, filename):
+    self.file = open(filename, 'rb')
+
+    #HACK: Strictly, we should be parsing the header, but we know where
+    #      everything is in these two files so we just jump straight there
+
+    # Read ImageBase
+    self.file.seek(0xB4)
+    self.imagebase = struct.unpack('i', self.file.read(4))[0]
+
+    # Read .text VirtualAddress
+    self.file.seek(0x184)
+    self.textvirt = struct.unpack('i', self.file.read(4))[0]
+
+    # Read .text PointerToRawData
+    self.file.seek(0x18C)
+    self.textraw = struct.unpack('i', self.file.read(4))[0]
+
+  def __del__(self):
+    if self.file:
+      self.file.close()
+
+  def get_addr(self, virt):
+    return virt - self.imagebase - self.textvirt + self.textraw
+
+  def read(self, offset, size):
+    self.file.seek(self.get_addr(offset))
+    return self.file.read(size)
+
+line_dump = None
+
+origfile = Bin(original)
+recompfile = Bin(recomp)
+
+class RecompiledInfo:
+  addr = None
+  size = None
+  name = None
+
+print()
+
+def get_recompiled_address(filename, line):
+  global line_dump, sym_dump
+
+  def get_wine_path(fn):
+    return subprocess.check_output(['winepath', '-w', fn]).decode('utf-8').strip()
+
+  # Load source lines from PDB
+  if not line_dump:
+    call = [os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), 'cvdump'), '-l', '-s']
+
+    if os.name != 'nt':
+      # Run cvdump through wine and convert path to Windows-friendly wine path
+      call.insert(0, 'wine')
+      call.append(get_wine_path(syms))
+    else:
+      call.append(syms)
+
+    line_dump = subprocess.check_output(call).decode('utf-8').split('\r\n')
+
+  # Find requested filename/line in PDB
+  if os.name != 'nt':
+    # Convert filename to Wine path
+    filename = get_wine_path(filename)
+
+  #print('Looking for ' + filename + ' line ' + str(line))
+
+  addr = None
+  found = False
+
+  for i, s in enumerate(line_dump):
+    try:
+      sourcepath = s.split()[0]
+      if os.path.isfile(sourcepath) and os.path.samefile(sourcepath, filename):
+        lines = line_dump[i + 2].split()
+        if line == int(lines[0]):
+          # Found address
+          addr = int(lines[1], 16)
+          found = True
+          break
+    except IndexError:
+      pass
+
+  if found:
+    # Find size of function
+    for i, s in enumerate(line_dump):
+      if 'S_GPROC32' in s:
+        if int(s[26:34], 16) == addr:
+          obj = RecompiledInfo()
+          obj.addr = addr + recompfile.imagebase + recompfile.textvirt
+          obj.size = int(s[41:49], 16)
+          obj.name = s[77:]
+
+          return obj
+
+md = Cs(CS_ARCH_X86, CS_MODE_32)
+
+def parse_asm(file, addr, size):
+  asm = []
+  data = file.read(addr, size)
+  for i in md.disasm(data, 0):
+    if i.mnemonic == 'call':
+      # Filter out "calls" because the offsets we're not currently trying to
+      # match offsets. As long as there's a call in the right place, it's
+      # probably accurate.
+      asm.append(i.mnemonic)
+    else:
+      asm.append("%s %s" % (i.mnemonic, i.op_str))
+  return asm
+
+function_count = 0
+total_accuracy = 0
+
+for subdir, dirs, files in os.walk(source):
+  for file in files:
+    srcfilename = os.path.join(os.path.abspath(subdir), file)
+    srcfile = open(srcfilename, 'r')
+    line_no = 0
+
+    while True:
+      try:
+        line = srcfile.readline()
+        line_no += 1
+
+        if not line:
+          break
+
+        if line.startswith('// OFFSET:'):
+          par = line[10:].strip().split()
+          module = par[0]
+          addr = int(par[1], 16)
+
+          find_open_bracket = line
+          while '{' not in find_open_bracket:
+            find_open_bracket = srcfile.readline()
+            line_no += 1
+
+          recinfo = get_recompiled_address(srcfilename, line_no)
+          if not recinfo:
+            print('Failed to find recompiled address of ' + hex(addr))
+            continue
+
+          origasm = parse_asm(origfile, addr, recinfo.size)
+          recompasm = parse_asm(recompfile, recinfo.addr, recinfo.size)
+
+          diff = difflib.SequenceMatcher(None, origasm, recompasm)
+          ratio = diff.ratio()
+          print('%s (%s / %s) is %.2f%% similar to the original' % (recinfo.name, hex(addr), hex(recinfo.addr), ratio * 100))
+
+          function_count += 1
+          total_accuracy += ratio
+
+          if verbose == addr:
+            udiff = difflib.unified_diff(origasm, recompasm)
+            for line in udiff:
+              print(line)
+            print()
+            print()
+
+      except UnicodeDecodeError:
+        break
+
+if function_count > 0:
+  print('\nTotal accuracy %.2f%% across %i functions' % (total_accuracy / function_count * 100, function_count))