mirror of
https://github.com/isledecomp/isle.git
synced 2024-11-22 23:57:54 -05:00
Faster reccmp.py on linux (#62)
* reccmp: avoid repeated execution of winepath Executing winepath many times is slow, so try we like to avoid it as much as possible. When the path start with a known prefix, replace it with a cached prefix and do some string manipulation. This change reduces execution time of reccmp.py from 90s to 2s. Which is nice. m * reccmp: continue looking when source cannot be found Most often, the reasons is mismatched sources. * reccmp: add basic logging + optional debug * Read the addresses in the exe headers as little endian
This commit is contained in:
parent
904640e028
commit
40dd0a93d4
1 changed files with 66 additions and 27 deletions
|
@ -6,6 +6,7 @@
|
||||||
import difflib
|
import difflib
|
||||||
import struct
|
import struct
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import colorama
|
import colorama
|
||||||
|
@ -24,8 +25,14 @@
|
||||||
parser.add_argument('--svg-icon', metavar='icon', help='Icon to use in SVG (PNG)')
|
parser.add_argument('--svg-icon', metavar='icon', help='Icon to use in SVG (PNG)')
|
||||||
parser.add_argument('--print-rec-addr', action='store_true', help='Print addresses of recompiled functions too')
|
parser.add_argument('--print-rec-addr', action='store_true', help='Print addresses of recompiled functions too')
|
||||||
|
|
||||||
|
parser.set_defaults(loglevel=logging.INFO)
|
||||||
|
parser.add_argument('--debug', action='store_const', const=logging.DEBUG, dest='loglevel', help='Print script debug information')
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
logging.basicConfig(level=args.loglevel, format='[%(levelname)s] %(message)s')
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
colorama.init()
|
colorama.init()
|
||||||
|
|
||||||
verbose = None
|
verbose = None
|
||||||
|
@ -61,6 +68,7 @@
|
||||||
# to file addresses
|
# to file addresses
|
||||||
class Bin:
|
class Bin:
|
||||||
def __init__(self, filename):
|
def __init__(self, filename):
|
||||||
|
logger.debug('Parsing headers of "%s"... ', filename)
|
||||||
self.file = open(filename, 'rb')
|
self.file = open(filename, 'rb')
|
||||||
|
|
||||||
#HACK: Strictly, we should be parsing the header, but we know where
|
#HACK: Strictly, we should be parsing the header, but we know where
|
||||||
|
@ -68,15 +76,16 @@ def __init__(self, filename):
|
||||||
|
|
||||||
# Read ImageBase
|
# Read ImageBase
|
||||||
self.file.seek(0xB4)
|
self.file.seek(0xB4)
|
||||||
self.imagebase = struct.unpack('i', self.file.read(4))[0]
|
self.imagebase, = struct.unpack('<i', self.file.read(4))
|
||||||
|
|
||||||
# Read .text VirtualAddress
|
# Read .text VirtualAddress
|
||||||
self.file.seek(0x184)
|
self.file.seek(0x184)
|
||||||
self.textvirt = struct.unpack('i', self.file.read(4))[0]
|
self.textvirt, = struct.unpack('<i', self.file.read(4))
|
||||||
|
|
||||||
# Read .text PointerToRawData
|
# Read .text PointerToRawData
|
||||||
self.file.seek(0x18C)
|
self.file.seek(0x18C)
|
||||||
self.textraw = struct.unpack('i', self.file.read(4))[0]
|
self.textraw, = struct.unpack('<i', self.file.read(4))
|
||||||
|
logger.debug('... Parsing finished')
|
||||||
|
|
||||||
def __del__(self):
|
def __del__(self):
|
||||||
if self.file:
|
if self.file:
|
||||||
|
@ -90,16 +99,38 @@ def read(self, offset, size):
|
||||||
return self.file.read(size)
|
return self.file.read(size)
|
||||||
|
|
||||||
class RecompiledInfo:
|
class RecompiledInfo:
|
||||||
addr = None
|
def __init__(self):
|
||||||
size = None
|
self.addr = None
|
||||||
name = None
|
self.size = None
|
||||||
start = None
|
self.name = None
|
||||||
|
self.start = None
|
||||||
|
|
||||||
def get_wine_path(fn):
|
class WinePathConverter:
|
||||||
return subprocess.check_output(['winepath', '-w', fn]).decode('utf-8').strip()
|
def __init__(self, unix_cwd):
|
||||||
|
self.unix_cwd = unix_cwd
|
||||||
|
self.win_cwd = self._call_winepath_unix2win(self.unix_cwd)
|
||||||
|
|
||||||
def get_unix_path(fn):
|
def get_wine_path(self, unix_fn: str) -> str:
|
||||||
return subprocess.check_output(['winepath', fn]).decode('utf-8').strip()
|
if unix_fn.startswith('./'):
|
||||||
|
return self.win_cmd + '\\' + unix_fn[2:].replace('/', '\\')
|
||||||
|
if unix_fn.startswith(self.unix_cwd):
|
||||||
|
return self.win_cwd + '\\' + unix_fn.removeprefix(self.unix_cwd).replace('/', '\\').lstrip('\\')
|
||||||
|
return self._call_winepath_unix2win(unix_fn)
|
||||||
|
|
||||||
|
def get_unix_path(self, win_fn: str) -> str:
|
||||||
|
if win_fn.startswith('.\\') or win_fn.startswith('./'):
|
||||||
|
return self.unix_cwd + '/' + win_fn[2:].replace('\\', '/')
|
||||||
|
if win_fn.startswith(self.win_cwd):
|
||||||
|
return self.unix_cwd + '/' + win_fn.removeprefix(self.win_cwd).replace('\\', '/')
|
||||||
|
return self._call_winepath_win2unix(win_fn)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _call_winepath_unix2win(fn: str) -> str:
|
||||||
|
return subprocess.check_output(['winepath', '-w', fn], text=True).strip()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _call_winepath_win2unix(fn: str) -> str:
|
||||||
|
return subprocess.check_output(['winepath', fn], text=True).strip()
|
||||||
|
|
||||||
def get_file_in_script_dir(fn):
|
def get_file_in_script_dir(fn):
|
||||||
return os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), fn)
|
return os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), fn)
|
||||||
|
@ -109,22 +140,24 @@ class SymInfo:
|
||||||
funcs = {}
|
funcs = {}
|
||||||
lines = {}
|
lines = {}
|
||||||
|
|
||||||
def __init__(self, pdb, file):
|
def __init__(self, pdb, file, wine_path_converter):
|
||||||
call = [get_file_in_script_dir('cvdump.exe'), '-l', '-s']
|
call = [get_file_in_script_dir('cvdump.exe'), '-l', '-s']
|
||||||
|
|
||||||
if os.name != 'nt':
|
if wine_path_converter:
|
||||||
# Run cvdump through wine and convert path to Windows-friendly wine path
|
# Run cvdump through wine and convert path to Windows-friendly wine path
|
||||||
call.insert(0, 'wine')
|
call.insert(0, 'wine')
|
||||||
call.append(get_wine_path(pdb))
|
call.append(wine_path_converter.get_wine_path(pdb))
|
||||||
else:
|
else:
|
||||||
call.append(pdb)
|
call.append(pdb)
|
||||||
|
|
||||||
print('Parsing %s...' % pdb)
|
logger.info('Parsing %s ...', pdb)
|
||||||
|
logger.debug('Command = %r', call)
|
||||||
line_dump = subprocess.check_output(call).decode('utf-8').split('\r\n')
|
line_dump = subprocess.check_output(call).decode('utf-8').split('\r\n')
|
||||||
|
|
||||||
current_section = None
|
current_section = None
|
||||||
|
|
||||||
|
logger.debug('Parsing output of cvdump.exe ...')
|
||||||
|
|
||||||
for i, line in enumerate(line_dump):
|
for i, line in enumerate(line_dump):
|
||||||
if line.startswith('***'):
|
if line.startswith('***'):
|
||||||
current_section = line[4:]
|
current_section = line[4:]
|
||||||
|
@ -132,8 +165,6 @@ def __init__(self, pdb, file):
|
||||||
if current_section == 'SYMBOLS' and 'S_GPROC32' in line:
|
if current_section == 'SYMBOLS' and 'S_GPROC32' in line:
|
||||||
addr = int(line[26:34], 16)
|
addr = int(line[26:34], 16)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
info = RecompiledInfo()
|
info = RecompiledInfo()
|
||||||
info.addr = addr + recompfile.imagebase + recompfile.textvirt
|
info.addr = addr + recompfile.imagebase + recompfile.textvirt
|
||||||
|
|
||||||
|
@ -155,9 +186,9 @@ def __init__(self, pdb, file):
|
||||||
elif current_section == 'LINES' and line.startswith(' ') and not line.startswith(' '):
|
elif current_section == 'LINES' and line.startswith(' ') and not line.startswith(' '):
|
||||||
sourcepath = line.split()[0]
|
sourcepath = line.split()[0]
|
||||||
|
|
||||||
if os.name != 'nt':
|
if wine_path_converter:
|
||||||
# Convert filename to Unix path for file compare
|
# Convert filename to Unix path for file compare
|
||||||
sourcepath = get_unix_path(sourcepath)
|
sourcepath = wine_path_converter.get_unix_path(sourcepath)
|
||||||
|
|
||||||
if sourcepath not in self.lines:
|
if sourcepath not in self.lines:
|
||||||
self.lines[sourcepath] = {}
|
self.lines[sourcepath] = {}
|
||||||
|
@ -178,18 +209,23 @@ def __init__(self, pdb, file):
|
||||||
|
|
||||||
j += 1
|
j += 1
|
||||||
|
|
||||||
|
logger.debug('... Parsing output of cvdump.exe finished')
|
||||||
|
|
||||||
def get_recompiled_address(self, filename, line):
|
def get_recompiled_address(self, filename, line):
|
||||||
addr = None
|
addr = None
|
||||||
found = False
|
found = False
|
||||||
|
|
||||||
#print('Looking for ' + filename + ' line ' + str(line))
|
logger.debug('Looking for %s:%d', filename, line)
|
||||||
|
|
||||||
for fn in self.lines:
|
for fn in self.lines:
|
||||||
# Sometimes a PDB is compiled with a relative path while we always have
|
# Sometimes a PDB is compiled with a relative path while we always have
|
||||||
# an absolute path. Therefore we must
|
# an absolute path. Therefore we must
|
||||||
|
try:
|
||||||
if os.path.samefile(fn, filename):
|
if os.path.samefile(fn, filename):
|
||||||
filename = fn
|
filename = fn
|
||||||
break
|
break
|
||||||
|
except FileNotFoundError as e:
|
||||||
|
continue
|
||||||
|
|
||||||
if filename in self.lines and line in self.lines[fn]:
|
if filename in self.lines and line in self.lines[fn]:
|
||||||
addr = self.lines[fn][line]
|
addr = self.lines[fn][line]
|
||||||
|
@ -197,13 +233,16 @@ def get_recompiled_address(self, filename, line):
|
||||||
if addr in self.funcs:
|
if addr in self.funcs:
|
||||||
return self.funcs[addr]
|
return self.funcs[addr]
|
||||||
else:
|
else:
|
||||||
print('Failed to find function symbol with address: %s' % hex(addr))
|
logger.error('Failed to find function symbol with address: 0x%x', addr)
|
||||||
else:
|
else:
|
||||||
print('Failed to find function symbol with filename and line: %s:%s' % (filename, str(line)))
|
logger.error('Failed to find function symbol with filename and line: %s:%d', filename, line)
|
||||||
|
|
||||||
|
wine_path_converter = None
|
||||||
|
if os.name != 'nt':
|
||||||
|
wine_path_converter = WinePathConverter(source)
|
||||||
origfile = Bin(original)
|
origfile = Bin(original)
|
||||||
recompfile = Bin(recomp)
|
recompfile = Bin(recomp)
|
||||||
syminfo = SymInfo(syms, recompfile)
|
syminfo = SymInfo(syms, recompfile, wine_path_converter)
|
||||||
|
|
||||||
print()
|
print()
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue