feat(extractor): runner_map axis from CMC_RunnerFactory

Script dispatch is getRunner(obj.getType())->run(id), so map type ids to
runners instead of guessing runner names from C++ classes. Type ids come
from the base-constructor argument, the map from emulating getRunner, and
runner vtables from RunnerFactory::initialize; vtable slots are read off
the labelled CMC_Runner vftable because they move between releases.

This recovers runners without symbols (VECTOR, FILTER, STATICFILTER,
PATH, FIFO/LIFO, MOVIE and the @ built-ins of type 0) and lets several
script types share one runner (INTEGER/STRING/BOOL/DOUBLE/EXPRESSION).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Patryk Gensch
2026-10-24 18:37:17 +02:00
co-authored by Claude Opus 5.5
parent 3132a5222a
commit 5c597a3a52
+379 -13
View File
@@ -268,13 +268,16 @@ def _is_method_name(s):
return bool(s) and all(c.isalnum() or c == "_" for c in s) return bool(s) and all(c.isalnum() or c == "_" for c in s)
def _extract_methods_from(program, runner_func): def _extract_methods_from(program, runner_func, runner=None):
"""Walk one CMC_*_Runner::prepareMthHashSet, recovering its directly-registered methods. """Walk one CMC_*_Runner::prepareMthHashSet, recovering its directly-registered methods.
Per method the engine emits: new CInteger(ID) ; new CStringHashCode("NAME") ; CHashtable::put. Per method the engine emits: new CInteger(ID) ; new CStringHashCode("NAME") ; CHashtable::put.
The method name is *loaded* (MOV), not pushed, so we track the most recent string literal The method name is *loaded* (MOV), not pushed, so we track the most recent string literal
referenced by any instruction rather than scanning PUSH operands. The leading call to the referenced by any instruction rather than scanning PUSH operands. The leading call to the
base class's prepareMthHashSet gives the inheritance link (inherited/global methods).""" base class's prepareMthHashSet gives the inheritance link (inherited/global methods).
`runner` names a registration reached through a vtable rather than a symbol (see
`extract_runner_map`); otherwise the function's namespace is the runner class."""
listing = program.getListing() listing = program.getListing()
methods = [] methods = []
base_runner = None base_runner = None
@@ -283,6 +286,7 @@ def _extract_methods_from(program, runner_func):
pending_name = None pending_name = None
recent = [] recent = []
if runner is None:
ns = runner_func.getParentNamespace() ns = runner_func.getParentNamespace()
runner = ns.getName() if ns is not None else "?" runner = ns.getName() if ns is not None else "?"
owner = _owner_from_runner(runner) owner = _owner_from_runner(runner)
@@ -316,23 +320,34 @@ def _extract_methods_from(program, runner_func):
return methods, runner, base_runner return methods, runner, base_runner
def extract_methods(program): def extract_methods(program, runner_map=None):
"""Returns (methods, inheritance). `methods` lists each Runner's directly-registered """Returns (methods, inheritance). `methods` lists each Runner's directly-registered
methods; `inheritance` maps each runner to the base runner it chains to, so the consumer methods; `inheritance` maps each runner to the base runner it chains to, so the consumer
can compose the full (incl. global) method set per type. Method id -> vtable address can compose the full (incl. global) method set per type. Runners without a symbol are
correlation is a later step.""" reached through `runner_map` vtables."""
fm = program.getFunctionManager() fm = program.getFunctionManager()
methods = [] methods = []
inheritance = [] inheritance = []
seen = set()
it = fm.getFunctions(True) it = fm.getFunctions(True)
while it.hasNext(): while it.hasNext():
f = it.next() f = it.next()
if f.getName() != "prepareMthHashSet": if f.getName() != "prepareMthHashSet":
continue continue
seen.add(resolve_thunk(f).getEntryPoint().getOffset())
own, runner, base = _extract_methods_from(program, f) own, runner, base = _extract_methods_from(program, f)
methods.extend(own) methods.extend(own)
if base is not None: if base is not None:
inheritance.append({"runner": runner, "base_runner": base}) inheritance.append({"runner": runner, "base_runner": base})
for r in (runner_map or {}).get("runners", []):
f = _function_at(program, r.get("prepare_addr"))
if f is None or resolve_thunk(f).getEntryPoint().getOffset() in seen:
continue
seen.add(resolve_thunk(f).getEntryPoint().getOffset())
own, runner, base = _extract_methods_from(program, f, r["runner"])
methods.extend(own)
if base is not None:
inheritance.append({"runner": runner, "base_runner": base})
if not methods: if not methods:
# Piklib 6.1 predates prepareMthHashSet: methods are dispatched by NAME inside # Piklib 6.1 predates prepareMthHashSet: methods are dispatched by NAME inside
# <Runner>::run, so recover the names from there instead (no numeric ids). # <Runner>::run, so recover the names from there instead (no numeric ids).
@@ -575,16 +590,18 @@ def _parse_switch(program, func):
"byte_table": byte_table} "byte_table": byte_table}
def extract_method_dispatch(program): def extract_method_dispatch(program, runner_map=None):
"""For each CMC_*_Runner::run, recover how method ids map to their implementation. """For each CMC_*_Runner::run, recover how method ids map to their implementation.
`run(int id, ...)` is a `switch(id)` (vtable slot 17, overridden per runner) whose every `run(int id, ...)` is a `switch(id)` (overridden per runner) whose every
`case id:` is the method body - either a tail-call to a named submethod (BlooMoo/MSVC8 `case id:` is the method body - either a tail-call to a named submethod (BlooMoo/MSVC8
keeps show()/load()/... as separate functions) or inline code whose leaves are virtual keeps show()/load()/... as separate functions) or inline code whose leaves are virtual
calls on the wrapped object (Piklib/MSVC6). We fingerprint each case by its ordered CALL calls on the wrapped object (Piklib/MSVC6). We fingerprint each case by its ordered CALL
anchors, so a later pass can diff method *bodies* by (owner, id). Join names via `methods`.""" anchors, so a later pass can diff method *bodies* by (owner, id). Join names via `methods`."""
fm = program.getFunctionManager() fm = program.getFunctionManager()
out = [] out = []
seen = set()
runs = []
it = fm.getFunctions(True) it = fm.getFunctions(True)
while it.hasNext(): while it.hasNext():
f = it.next() f = it.next()
@@ -592,8 +609,17 @@ def extract_method_dispatch(program):
continue continue
ns = f.getParentNamespace() ns = f.getParentNamespace()
runner = ns.getName() if ns is not None else "?" runner = ns.getName() if ns is not None else "?"
if not runner.endswith("_Runner"): if runner.endswith("_Runner"):
runs.append((f, runner))
for r in (runner_map or {}).get("runners", []):
f = _function_at(program, r.get("run_addr"))
if f is not None:
runs.append((f, r["runner"]))
for f, runner in runs:
entry = resolve_thunk(f).getEntryPoint().getOffset()
if entry in seen:
continue continue
seen.add(entry)
try: try:
out.extend(_dispatch_from_run(program, f, _owner_from_runner(runner), runner)) out.extend(_dispatch_from_run(program, f, _owner_from_runner(runner), runner))
except Exception as e: # one malformed runner shouldn't sink the whole axis except Exception as e: # one malformed runner shouldn't sink the whole axis
@@ -872,6 +898,343 @@ def extract_struct_layout(program):
ifc.dispose() ifc.dispose()
# --------------------------------------------------------------------------- runner map
# CMC_Runner virtuals the script ABI hangs on. Their vtable offsets moved between releases
# (prepareMthHashSet is +0x20 in Piklib 7.1, +0x28 from 7.2 on), so they are read off the
# labelled base vftable, never hard-coded.
_RUNNER_SLOTS = ("prepareMthHashSet", "run", "getParameter", "getParameterBool",
"getParameterDouble", "getParameterInteger", "getParameterString")
_MAX_VTABLE_SLOTS = 64
# How the demangler spells `??_7<class>@@6B@` differs between Ghidra versions/compilers.
_VFTABLE_LABELS = ("vftable", "_vftable_", "`vftable'")
# Scratch addresses for emulating CMC_RunnerFactory::getRunner; outside every image.
_EMU_STACK = 0x7ff00000
_EMU_RETURN = 0x7fe00000
def _addr(program, value):
return program.getAddressFactory().getDefaultAddressSpace().getAddress(value)
def _hex(addr):
return "0x%x" % addr.getOffset() if addr is not None else None
def _function_at(program, hex_addr):
if not hex_addr:
return None
return program.getFunctionManager().getFunctionAt(_addr(program, int(hex_addr, 16)))
def _read_ptr(program, addr):
try:
return _addr(program, program.getMemory().getInt(addr) & 0xffffffff)
except: # noqa: E722 - MemoryAccessException / AddressOutOfBounds
return None
def _labels(program, addr):
"""(namespace, name) of every label at `addr` and at the body an ILT stub there jumps to.
Identical-COMDAT folding gives one body many names (Piklib 8's getType is labelled
CMC_DLLExModuleInterface::getModuleInterface too), so the primary symbol is an arbitrary
alias: test membership, never trust the primary name alone."""
addrs = [addr]
f = program.getFunctionManager().getFunctionAt(addr)
body = resolve_thunk(f) if f is not None else None
if body is not None and body.getEntryPoint() != addr:
addrs.append(body.getEntryPoint())
elif f is None: # an ILT `JMP body` stub the analyzer never made a function
instr = program.getListing().getInstructionAt(addr)
if instr is not None and instr.getMnemonicString() == "JMP" and len(instr.getFlows()) == 1:
addrs.append(instr.getFlows()[0])
out = set()
for a in addrs:
for s in program.getSymbolTable().getSymbols(a):
ns = s.getParentNamespace()
out.add((ns.getName() if ns is not None else "", s.getName()))
return out
def _vftable_class(program, addr):
"""The class whose labelled vftable (`??_7<class>@@6B@`) sits at `addr`, or None."""
for ns, name in _labels(program, addr):
if name in _VFTABLE_LABELS:
return ns
return None
def _labelled_vftable(program, class_name):
st = program.getSymbolTable()
for label in _VFTABLE_LABELS:
it = st.getSymbols(label)
while it.hasNext():
s = it.next()
ns = s.getParentNamespace()
if ns is not None and ns.getName() == class_name:
return s.getAddress()
return None
def _ensure_function(program, addr):
"""The function at `addr`, created when auto-analysis missed it: bodies referenced only
from a vtable are often left undefined (Piklib 7.2's unnamed runners)."""
if addr is None:
return None
fm = program.getFunctionManager()
f = fm.getFunctionAt(addr)
if f is None and _executable(program, addr):
from ghidra.app.cmd.disassemble import DisassembleCommand
from ghidra.app.cmd.function import CreateFunctionCmd
tx = program.startTransaction("ams: define vtable target")
try:
DisassembleCommand(addr, None, True).applyTo(program)
CreateFunctionCmd(addr).applyTo(program)
finally:
program.endTransaction(tx, True)
f = fm.getFunctionAt(addr)
return f
def _runner_slots(program):
"""{CMC_Runner virtual: vtable offset}. `run(int, ...)` precedes `run(CXString, ...)` in
every observed build, so the first labelled slot of a name wins."""
vft = _labelled_vftable(program, "CMC_Runner")
slots = {}
if vft is None:
return slots
for i in range(_MAX_VTABLE_SLOTS):
target = _read_ptr(program, vft.add(i * 4))
if target is None or not _executable(program, target):
break
for ns, name in _labels(program, target):
if ns == "CMC_Runner" and name in _RUNNER_SLOTS and name not in slots:
slots[name] = i * 4
return slots
def _vftable_store(program, instr):
"""The vftable installed by `MOV dword ptr [reg], imm`, or None: a constructor (called or
inlined) sets the object's first dword to a non-code table that starts with code."""
if instr.getMnemonicString() != "MOV" or instr.getNumOperands() != 2:
return None
if not re.match(r"dword ptr \[E[A-Z]{2}\]$", instr.getDefaultOperandRepresentation(0)):
return None
sc = instr.getScalar(1)
if sc is None:
return None
try:
table = _addr(program, sc.getUnsignedValue())
except: # noqa: E722
return None
blk = program.getMemory().getBlock(table)
if blk is None or blk.isExecute():
return None
first = _read_ptr(program, table)
return table if first is not None and _executable(program, first) else None
def _last_vftable_store(program, func, depth=0):
"""The most-derived vftable a constructor installs: base constructors run first, so the
last store wins. Descends into callees when the body only delegates."""
found = None
callees = []
it = program.getListing().getInstructions(resolve_thunk(func).getBody(), True)
while it.hasNext():
instr = it.next()
table = _vftable_store(program, instr)
if table is not None:
found = table
elif instr.getMnemonicString() == "CALL":
cf = call_target(program, instr)
if cf is not None:
callees.append(cf)
if found is None and depth < 2:
for cf in callees:
found = _last_vftable_store(program, cf, depth + 1) or found
return found
def _runner_globals(program, init):
"""[(global, vftable)] from CMC_RunnerFactory::initialize, which builds every runner as
`p<X>Runner = new <X>_Runner`. The vftable installed since the previous global write
(inline constructor) or by the last constructor called belongs to the next write; the
NULL branch of `new` writes too and is skipped until the constructed store follows."""
out = []
done = set()
pending = None
calls = []
st = program.getSymbolTable()
it = program.getListing().getInstructions(init.getBody(), True)
while it.hasNext():
instr = it.next()
table = _vftable_store(program, instr)
if table is not None:
pending = table
if instr.getMnemonicString() == "CALL":
cf = call_target(program, instr)
if cf is not None and cf.getName() != OP_NEW:
calls.append(cf)
for ref in instr.getReferencesFrom():
if not ref.getReferenceType().isWrite():
continue
sym = st.getPrimarySymbol(ref.getToAddress())
name = sym.getName() if sym is not None else ""
if not (name.startswith("p") and name.endswith("Runner")) or name in done:
continue
table = pending
for cf in reversed(calls):
if table is not None:
break
table = _last_vftable_store(program, cf)
if table is None:
continue
out.append((name, ref.getToAddress(), table))
done.add(name)
pending = None
calls = []
return out
def _is_cmc_ctor_at(program, addr):
for ns, name in _labels(program, addr):
if name == ns and name.startswith(CMC_PREFIX):
return True
return False
def _is_base_ctor_call(program, addr):
"""A CMC_* constructor, or an unlabelled one that chains to it (BlooMoo's FIFO/LIFO share
an unexported queue base that calls CMC_Object)."""
if _is_cmc_ctor_at(program, addr):
return True
f = program.getFunctionManager().getFunctionAt(addr)
if f is None:
return False
it = program.getListing().getInstructions(resolve_thunk(f).getBody(), True)
while it.hasNext():
instr = it.next()
if instr.getMnemonicString() == "CALL":
flows = instr.getFlows()
return len(flows) == 1 and _is_cmc_ctor_at(program, flows[0])
return False
def _ctor_type_id(program, ctor_addr):
"""The type id a script-type constructor hands to its base. Every CMC_* base constructor
takes it as the *last* argument (`CMC_Object(container, element, int)`), which is pushed
first - so it is the first plain immediate pushed before the base-constructor call. The
SEH frame's `PUSH -1` and pushed addresses are not plain small immediates."""
func = _function_at(program, ctor_addr)
if func is None:
return None
first = None
it = program.getListing().getInstructions(resolve_thunk(func).getBody(), True)
while it.hasNext():
instr = it.next()
mn = instr.getMnemonicString()
if mn == "PUSH" and first is None and not instr.getReferencesFrom():
sc = instr.getScalar(0)
if sc is not None and 0 <= _s32(sc.getValue()) < 0x10000:
first = _s32(sc.getValue())
elif mn == "CALL":
flows = instr.getFlows()
if len(flows) == 1 and _is_base_ctor_call(program, flows[0]):
return first
return None
def _compared_ids(program, func):
"""Every small immediate `func` compares against - the case labels of its switch."""
ids = set()
it = program.getListing().getInstructions(func.getBody(), True)
while it.hasNext():
instr = it.next()
if instr.getMnemonicString() in ("CMP", "SUB", "ADD", "LEA") and instr.getNumOperands() == 2:
sc = instr.getScalar(1)
if sc is not None:
v = abs(_s32(sc.getValue()))
if v < 0x10000:
ids.add(v)
return ids
def _emulate_get_runner(program, get_runner, runner_globals, extra_ids):
"""{type id: runner global} by executing CMC_RunnerFactory::getRunner(int) for every id up
to the largest one it compares against. Its compiled switch (jump tables, binary search)
differs per compiler; running it doesn't care. Each global is seeded with its own address,
so the returned pointer names the slot the id selects."""
from ghidra.app.emulator import EmulatorHelper
entry = get_runner.getEntryPoint()
by_marker = {}
emu = EmulatorHelper(program)
out = {}
try:
for name, addr, _ in runner_globals:
emu.writeMemoryValue(addr, 4, addr.getOffset())
by_marker[addr.getOffset()] = name
emu.setBreakpoint(_addr(program, _EMU_RETURN))
top = max(_compared_ids(program, resolve_thunk(get_runner)) | set(extra_ids) | set([0]))
for tid in range(top + 2):
emu.writeRegister("ESP", _EMU_STACK)
emu.writeMemoryValue(_addr(program, _EMU_STACK), 4, _EMU_RETURN)
emu.writeMemoryValue(_addr(program, _EMU_STACK + 4), 4, tid)
emu.writeRegister("EAX", 0)
emu.writeRegister(emu.getPCRegister(), entry.getOffset())
if not emu.run(monitor):
continue
name = by_marker.get(int(emu.readRegister("EAX").longValue()) & 0xffffffff)
if name is not None:
out[tid] = name
finally:
emu.dispose()
return out
def _runner_from_global(name):
"""Runner class for a factory slot whose class has no symbol: BlooMoo's RTTI names the same
runners that Piklib exports only as slots (pVectorRunner -> CMC_Vector_Runner)."""
return "CMC_%s_Runner" % name[1:-len("Runner")]
def extract_runner_map(program, types):
"""Which runner executes each script type, from CMC_RunnerFactory (Piklib 7.1+, BlooMoo).
The engine dispatches `obj^METHOD` as getRunner(obj.getType())->run(id, ...), so the
authoritative link is type id -> runner, not a class name: INTEGER/STRING/BOOL/DOUBLE/
EXPRESSION share one runner, and VECTOR/FILTER/PATH/FIFO/LIFO/MOVIE plus the `@` built-ins
(type 0) run on runners that have no symbol at all. Annotates `types` with `type_id`.
None when the engine predates the factory (Piklib 6.1)."""
init = find_function_by_qualified(program, "CMC_RunnerFactory", "initialize")
get_runner = find_function_by_qualified(program, "CMC_RunnerFactory", "getRunner")
slots = _runner_slots(program)
if init is None or get_runner is None or "prepareMthHashSet" not in slots or "run" not in slots:
return None
for t in types:
t["type_id"] = _ctor_type_id(program, t.get("ctor_addr"))
runners = []
key_of = {}
globals_ = _runner_globals(program, resolve_thunk(init))
for name, _, table in globals_:
cls = _vftable_class(program, table)
key = cls if cls and cls != "CMC_Runner" else _runner_from_global(name)
key_of[name] = key
prepare = _ensure_function(program, _read_ptr(program, table.add(slots["prepareMthHashSet"])))
run_ = _ensure_function(program, _read_ptr(program, table.add(slots["run"])))
runners.append({"runner": key, "global": name, "class": cls, "vftable": _hex(table),
"prepare_addr": _hex(prepare.getEntryPoint()) if prepare else None,
"run_addr": _hex(run_.getEntryPoint()) if run_ else None})
ids = [t["type_id"] for t in types if t.get("type_id") is not None]
by_id = _emulate_get_runner(program, get_runner, globals_, ids)
return {
"slots": dict((k, "0x%x" % v) for k, v in slots.items()),
"runners": runners,
"by_type_id": [{"type_id": tid, "runner": key_of[g], "global": g}
for tid, g in sorted(by_id.items())],
}
# --------------------------------------------------------------------------- metadata # --------------------------------------------------------------------------- metadata
def detect_engine(factory): def detect_engine(factory):
@@ -936,14 +1299,15 @@ def run():
engine, compiler = detect_engine(factory) # namespace lives on the symbol/stub engine, compiler = detect_engine(factory) # namespace lives on the symbol/stub
factory = resolve_thunk(factory) # MSVC8 exposes the symbol as a JMP stub factory = resolve_thunk(factory) # MSVC8 exposes the symbol as a JMP stub
types = extract_types(program, factory) types = extract_types(program, factory)
methods, method_inheritance = extract_methods(program) runner_map = extract_runner_map(program, types)
methods, method_inheritance = extract_methods(program, runner_map)
events = extract_events(program) events = extract_events(program)
fields = extract_script_fields(program) fields = extract_script_fields(program)
struct_layout, field_inheritance = extract_struct_layout(program) struct_layout, field_inheritance = extract_struct_layout(program)
method_dispatch = extract_method_dispatch(program) method_dispatch = extract_method_dispatch(program, runner_map)
snapshot = { snapshot = {
"schema_version": 4, "schema_version": 5,
"binary": { "binary": {
"name": program.getName(), "name": program.getName(),
"sha256": sha256_of(program), "sha256": sha256_of(program),
@@ -959,6 +1323,7 @@ def run():
"field_inheritance": field_inheritance, "field_inheritance": field_inheritance,
"struct_layout": struct_layout, "struct_layout": struct_layout,
"method_dispatch": method_dispatch, "method_dispatch": method_dispatch,
"runner_map": runner_map,
} }
args = getScriptArgs() args = getScriptArgs()
@@ -969,9 +1334,10 @@ def run():
finally: finally:
fh.close() fh.close()
print("[+] %s [%s/%s]: %d types, %d methods, %d events, %d fields (%d layout, %d dispatch) -> %s" % ( print("[+] %s [%s/%s]: %d types, %d methods, %d events, %d fields (%d layout, %d dispatch, %d runners) -> %s" % (
program.getName(), engine, compiler, len(types), len(methods), program.getName(), engine, compiler, len(types), len(methods),
len(events), len(fields), len(struct_layout), len(method_dispatch), out_path)) len(events), len(fields), len(struct_layout), len(method_dispatch),
len(runner_map["runners"]) if runner_map else 0, out_path))
run() run()