From 5c597a3a521bb37572f9ceb8f18044c87e3dca49 Mon Sep 17 00:00:00 2001 From: Patryk Gensch <43010113+patryk025@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:37:17 +0200 Subject: [PATCH] feat(extractor): runner_map axis from CMC_RunnerFactory Script dispatch is getRunner(obj.getType())->run(id), so map type ids to runners instead of guessing runner names from C++ classes. Type ids come from the base-constructor argument, the map from emulating getRunner, and runner vtables from RunnerFactory::initialize; vtable slots are read off the labelled CMC_Runner vftable because they move between releases. This recovers runners without symbols (VECTOR, FILTER, STATICFILTER, PATH, FIFO/LIFO, MOVIE and the @ built-ins of type 0) and lets several script types share one runner (INTEGER/STRING/BOOL/DOUBLE/EXPRESSION). Co-Authored-By: Claude Opus 5.5 --- ghidra_scripts/extract_engine_surface.py | 396 ++++++++++++++++++++++- 1 file changed, 381 insertions(+), 15 deletions(-) diff --git a/ghidra_scripts/extract_engine_surface.py b/ghidra_scripts/extract_engine_surface.py index 0206d66..03da7ec 100644 --- a/ghidra_scripts/extract_engine_surface.py +++ b/ghidra_scripts/extract_engine_surface.py @@ -268,13 +268,16 @@ def _is_method_name(s): return bool(s) and all(c.isalnum() or c == "_" for c in s) -def _extract_methods_from(program, runner_func): +def _extract_methods_from(program, runner_func, runner=None): """Walk one CMC_*_Runner::prepareMthHashSet, recovering its directly-registered methods. Per method the engine emits: new CInteger(ID) ; new CStringHashCode("NAME") ; CHashtable::put. The method name is *loaded* (MOV), not pushed, so we track the most recent string literal referenced by any instruction rather than scanning PUSH operands. The leading call to the - base class's prepareMthHashSet gives the inheritance link (inherited/global methods).""" + base class's prepareMthHashSet gives the inheritance link (inherited/global methods). + + `runner` names a registration reached through a vtable rather than a symbol (see + `extract_runner_map`); otherwise the function's namespace is the runner class.""" listing = program.getListing() methods = [] base_runner = None @@ -283,8 +286,9 @@ def _extract_methods_from(program, runner_func): pending_name = None recent = [] - ns = runner_func.getParentNamespace() - runner = ns.getName() if ns is not None else "?" + if runner is None: + ns = runner_func.getParentNamespace() + runner = ns.getName() if ns is not None else "?" owner = _owner_from_runner(runner) it = listing.getInstructions(resolve_thunk(runner_func).getBody(), True) @@ -316,23 +320,34 @@ def _extract_methods_from(program, runner_func): return methods, runner, base_runner -def extract_methods(program): +def extract_methods(program, runner_map=None): """Returns (methods, inheritance). `methods` lists each Runner's directly-registered methods; `inheritance` maps each runner to the base runner it chains to, so the consumer - can compose the full (incl. global) method set per type. Method id -> vtable address - correlation is a later step.""" + can compose the full (incl. global) method set per type. Runners without a symbol are + reached through `runner_map` vtables.""" fm = program.getFunctionManager() methods = [] inheritance = [] + seen = set() it = fm.getFunctions(True) while it.hasNext(): f = it.next() if f.getName() != "prepareMthHashSet": continue + seen.add(resolve_thunk(f).getEntryPoint().getOffset()) own, runner, base = _extract_methods_from(program, f) methods.extend(own) if base is not None: inheritance.append({"runner": runner, "base_runner": base}) + for r in (runner_map or {}).get("runners", []): + f = _function_at(program, r.get("prepare_addr")) + if f is None or resolve_thunk(f).getEntryPoint().getOffset() in seen: + continue + seen.add(resolve_thunk(f).getEntryPoint().getOffset()) + own, runner, base = _extract_methods_from(program, f, r["runner"]) + methods.extend(own) + if base is not None: + inheritance.append({"runner": runner, "base_runner": base}) if not methods: # Piklib 6.1 predates prepareMthHashSet: methods are dispatched by NAME inside # ::run, so recover the names from there instead (no numeric ids). @@ -575,16 +590,18 @@ def _parse_switch(program, func): "byte_table": byte_table} -def extract_method_dispatch(program): +def extract_method_dispatch(program, runner_map=None): """For each CMC_*_Runner::run, recover how method ids map to their implementation. - `run(int id, ...)` is a `switch(id)` (vtable slot 17, overridden per runner) whose every + `run(int id, ...)` is a `switch(id)` (overridden per runner) whose every `case id:` is the method body - either a tail-call to a named submethod (BlooMoo/MSVC8 keeps show()/load()/... as separate functions) or inline code whose leaves are virtual calls on the wrapped object (Piklib/MSVC6). We fingerprint each case by its ordered CALL anchors, so a later pass can diff method *bodies* by (owner, id). Join names via `methods`.""" fm = program.getFunctionManager() out = [] + seen = set() + runs = [] it = fm.getFunctions(True) while it.hasNext(): f = it.next() @@ -592,8 +609,17 @@ def extract_method_dispatch(program): continue ns = f.getParentNamespace() runner = ns.getName() if ns is not None else "?" - if not runner.endswith("_Runner"): + if runner.endswith("_Runner"): + runs.append((f, runner)) + for r in (runner_map or {}).get("runners", []): + f = _function_at(program, r.get("run_addr")) + if f is not None: + runs.append((f, r["runner"])) + for f, runner in runs: + entry = resolve_thunk(f).getEntryPoint().getOffset() + if entry in seen: continue + seen.add(entry) try: out.extend(_dispatch_from_run(program, f, _owner_from_runner(runner), runner)) except Exception as e: # one malformed runner shouldn't sink the whole axis @@ -872,6 +898,343 @@ def extract_struct_layout(program): ifc.dispose() +# --------------------------------------------------------------------------- runner map + +# CMC_Runner virtuals the script ABI hangs on. Their vtable offsets moved between releases +# (prepareMthHashSet is +0x20 in Piklib 7.1, +0x28 from 7.2 on), so they are read off the +# labelled base vftable, never hard-coded. +_RUNNER_SLOTS = ("prepareMthHashSet", "run", "getParameter", "getParameterBool", + "getParameterDouble", "getParameterInteger", "getParameterString") +_MAX_VTABLE_SLOTS = 64 +# How the demangler spells `??_7@@6B@` differs between Ghidra versions/compilers. +_VFTABLE_LABELS = ("vftable", "_vftable_", "`vftable'") +# Scratch addresses for emulating CMC_RunnerFactory::getRunner; outside every image. +_EMU_STACK = 0x7ff00000 +_EMU_RETURN = 0x7fe00000 + + +def _addr(program, value): + return program.getAddressFactory().getDefaultAddressSpace().getAddress(value) + + +def _hex(addr): + return "0x%x" % addr.getOffset() if addr is not None else None + + +def _function_at(program, hex_addr): + if not hex_addr: + return None + return program.getFunctionManager().getFunctionAt(_addr(program, int(hex_addr, 16))) + + +def _read_ptr(program, addr): + try: + return _addr(program, program.getMemory().getInt(addr) & 0xffffffff) + except: # noqa: E722 - MemoryAccessException / AddressOutOfBounds + return None + + +def _labels(program, addr): + """(namespace, name) of every label at `addr` and at the body an ILT stub there jumps to. + + Identical-COMDAT folding gives one body many names (Piklib 8's getType is labelled + CMC_DLLExModuleInterface::getModuleInterface too), so the primary symbol is an arbitrary + alias: test membership, never trust the primary name alone.""" + addrs = [addr] + f = program.getFunctionManager().getFunctionAt(addr) + body = resolve_thunk(f) if f is not None else None + if body is not None and body.getEntryPoint() != addr: + addrs.append(body.getEntryPoint()) + elif f is None: # an ILT `JMP body` stub the analyzer never made a function + instr = program.getListing().getInstructionAt(addr) + if instr is not None and instr.getMnemonicString() == "JMP" and len(instr.getFlows()) == 1: + addrs.append(instr.getFlows()[0]) + out = set() + for a in addrs: + for s in program.getSymbolTable().getSymbols(a): + ns = s.getParentNamespace() + out.add((ns.getName() if ns is not None else "", s.getName())) + return out + + +def _vftable_class(program, addr): + """The class whose labelled vftable (`??_7@@6B@`) sits at `addr`, or None.""" + for ns, name in _labels(program, addr): + if name in _VFTABLE_LABELS: + return ns + return None + + +def _labelled_vftable(program, class_name): + st = program.getSymbolTable() + for label in _VFTABLE_LABELS: + it = st.getSymbols(label) + while it.hasNext(): + s = it.next() + ns = s.getParentNamespace() + if ns is not None and ns.getName() == class_name: + return s.getAddress() + return None + + +def _ensure_function(program, addr): + """The function at `addr`, created when auto-analysis missed it: bodies referenced only + from a vtable are often left undefined (Piklib 7.2's unnamed runners).""" + if addr is None: + return None + fm = program.getFunctionManager() + f = fm.getFunctionAt(addr) + if f is None and _executable(program, addr): + from ghidra.app.cmd.disassemble import DisassembleCommand + from ghidra.app.cmd.function import CreateFunctionCmd + tx = program.startTransaction("ams: define vtable target") + try: + DisassembleCommand(addr, None, True).applyTo(program) + CreateFunctionCmd(addr).applyTo(program) + finally: + program.endTransaction(tx, True) + f = fm.getFunctionAt(addr) + return f + + +def _runner_slots(program): + """{CMC_Runner virtual: vtable offset}. `run(int, ...)` precedes `run(CXString, ...)` in + every observed build, so the first labelled slot of a name wins.""" + vft = _labelled_vftable(program, "CMC_Runner") + slots = {} + if vft is None: + return slots + for i in range(_MAX_VTABLE_SLOTS): + target = _read_ptr(program, vft.add(i * 4)) + if target is None or not _executable(program, target): + break + for ns, name in _labels(program, target): + if ns == "CMC_Runner" and name in _RUNNER_SLOTS and name not in slots: + slots[name] = i * 4 + return slots + + +def _vftable_store(program, instr): + """The vftable installed by `MOV dword ptr [reg], imm`, or None: a constructor (called or + inlined) sets the object's first dword to a non-code table that starts with code.""" + if instr.getMnemonicString() != "MOV" or instr.getNumOperands() != 2: + return None + if not re.match(r"dword ptr \[E[A-Z]{2}\]$", instr.getDefaultOperandRepresentation(0)): + return None + sc = instr.getScalar(1) + if sc is None: + return None + try: + table = _addr(program, sc.getUnsignedValue()) + except: # noqa: E722 + return None + blk = program.getMemory().getBlock(table) + if blk is None or blk.isExecute(): + return None + first = _read_ptr(program, table) + return table if first is not None and _executable(program, first) else None + + +def _last_vftable_store(program, func, depth=0): + """The most-derived vftable a constructor installs: base constructors run first, so the + last store wins. Descends into callees when the body only delegates.""" + found = None + callees = [] + it = program.getListing().getInstructions(resolve_thunk(func).getBody(), True) + while it.hasNext(): + instr = it.next() + table = _vftable_store(program, instr) + if table is not None: + found = table + elif instr.getMnemonicString() == "CALL": + cf = call_target(program, instr) + if cf is not None: + callees.append(cf) + if found is None and depth < 2: + for cf in callees: + found = _last_vftable_store(program, cf, depth + 1) or found + return found + + +def _runner_globals(program, init): + """[(global, vftable)] from CMC_RunnerFactory::initialize, which builds every runner as + `pRunner = new _Runner`. The vftable installed since the previous global write + (inline constructor) or by the last constructor called belongs to the next write; the + NULL branch of `new` writes too and is skipped until the constructed store follows.""" + out = [] + done = set() + pending = None + calls = [] + st = program.getSymbolTable() + it = program.getListing().getInstructions(init.getBody(), True) + while it.hasNext(): + instr = it.next() + table = _vftable_store(program, instr) + if table is not None: + pending = table + if instr.getMnemonicString() == "CALL": + cf = call_target(program, instr) + if cf is not None and cf.getName() != OP_NEW: + calls.append(cf) + for ref in instr.getReferencesFrom(): + if not ref.getReferenceType().isWrite(): + continue + sym = st.getPrimarySymbol(ref.getToAddress()) + name = sym.getName() if sym is not None else "" + if not (name.startswith("p") and name.endswith("Runner")) or name in done: + continue + table = pending + for cf in reversed(calls): + if table is not None: + break + table = _last_vftable_store(program, cf) + if table is None: + continue + out.append((name, ref.getToAddress(), table)) + done.add(name) + pending = None + calls = [] + return out + + +def _is_cmc_ctor_at(program, addr): + for ns, name in _labels(program, addr): + if name == ns and name.startswith(CMC_PREFIX): + return True + return False + + +def _is_base_ctor_call(program, addr): + """A CMC_* constructor, or an unlabelled one that chains to it (BlooMoo's FIFO/LIFO share + an unexported queue base that calls CMC_Object).""" + if _is_cmc_ctor_at(program, addr): + return True + f = program.getFunctionManager().getFunctionAt(addr) + if f is None: + return False + it = program.getListing().getInstructions(resolve_thunk(f).getBody(), True) + while it.hasNext(): + instr = it.next() + if instr.getMnemonicString() == "CALL": + flows = instr.getFlows() + return len(flows) == 1 and _is_cmc_ctor_at(program, flows[0]) + return False + + +def _ctor_type_id(program, ctor_addr): + """The type id a script-type constructor hands to its base. Every CMC_* base constructor + takes it as the *last* argument (`CMC_Object(container, element, int)`), which is pushed + first - so it is the first plain immediate pushed before the base-constructor call. The + SEH frame's `PUSH -1` and pushed addresses are not plain small immediates.""" + func = _function_at(program, ctor_addr) + if func is None: + return None + first = None + it = program.getListing().getInstructions(resolve_thunk(func).getBody(), True) + while it.hasNext(): + instr = it.next() + mn = instr.getMnemonicString() + if mn == "PUSH" and first is None and not instr.getReferencesFrom(): + sc = instr.getScalar(0) + if sc is not None and 0 <= _s32(sc.getValue()) < 0x10000: + first = _s32(sc.getValue()) + elif mn == "CALL": + flows = instr.getFlows() + if len(flows) == 1 and _is_base_ctor_call(program, flows[0]): + return first + return None + + +def _compared_ids(program, func): + """Every small immediate `func` compares against - the case labels of its switch.""" + ids = set() + it = program.getListing().getInstructions(func.getBody(), True) + while it.hasNext(): + instr = it.next() + if instr.getMnemonicString() in ("CMP", "SUB", "ADD", "LEA") and instr.getNumOperands() == 2: + sc = instr.getScalar(1) + if sc is not None: + v = abs(_s32(sc.getValue())) + if v < 0x10000: + ids.add(v) + return ids + + +def _emulate_get_runner(program, get_runner, runner_globals, extra_ids): + """{type id: runner global} by executing CMC_RunnerFactory::getRunner(int) for every id up + to the largest one it compares against. Its compiled switch (jump tables, binary search) + differs per compiler; running it doesn't care. Each global is seeded with its own address, + so the returned pointer names the slot the id selects.""" + from ghidra.app.emulator import EmulatorHelper + entry = get_runner.getEntryPoint() + by_marker = {} + emu = EmulatorHelper(program) + out = {} + try: + for name, addr, _ in runner_globals: + emu.writeMemoryValue(addr, 4, addr.getOffset()) + by_marker[addr.getOffset()] = name + emu.setBreakpoint(_addr(program, _EMU_RETURN)) + top = max(_compared_ids(program, resolve_thunk(get_runner)) | set(extra_ids) | set([0])) + for tid in range(top + 2): + emu.writeRegister("ESP", _EMU_STACK) + emu.writeMemoryValue(_addr(program, _EMU_STACK), 4, _EMU_RETURN) + emu.writeMemoryValue(_addr(program, _EMU_STACK + 4), 4, tid) + emu.writeRegister("EAX", 0) + emu.writeRegister(emu.getPCRegister(), entry.getOffset()) + if not emu.run(monitor): + continue + name = by_marker.get(int(emu.readRegister("EAX").longValue()) & 0xffffffff) + if name is not None: + out[tid] = name + finally: + emu.dispose() + return out + + +def _runner_from_global(name): + """Runner class for a factory slot whose class has no symbol: BlooMoo's RTTI names the same + runners that Piklib exports only as slots (pVectorRunner -> CMC_Vector_Runner).""" + return "CMC_%s_Runner" % name[1:-len("Runner")] + + +def extract_runner_map(program, types): + """Which runner executes each script type, from CMC_RunnerFactory (Piklib 7.1+, BlooMoo). + + The engine dispatches `obj^METHOD` as getRunner(obj.getType())->run(id, ...), so the + authoritative link is type id -> runner, not a class name: INTEGER/STRING/BOOL/DOUBLE/ + EXPRESSION share one runner, and VECTOR/FILTER/PATH/FIFO/LIFO/MOVIE plus the `@` built-ins + (type 0) run on runners that have no symbol at all. Annotates `types` with `type_id`. + None when the engine predates the factory (Piklib 6.1).""" + init = find_function_by_qualified(program, "CMC_RunnerFactory", "initialize") + get_runner = find_function_by_qualified(program, "CMC_RunnerFactory", "getRunner") + slots = _runner_slots(program) + if init is None or get_runner is None or "prepareMthHashSet" not in slots or "run" not in slots: + return None + for t in types: + t["type_id"] = _ctor_type_id(program, t.get("ctor_addr")) + runners = [] + key_of = {} + globals_ = _runner_globals(program, resolve_thunk(init)) + for name, _, table in globals_: + cls = _vftable_class(program, table) + key = cls if cls and cls != "CMC_Runner" else _runner_from_global(name) + key_of[name] = key + prepare = _ensure_function(program, _read_ptr(program, table.add(slots["prepareMthHashSet"]))) + run_ = _ensure_function(program, _read_ptr(program, table.add(slots["run"]))) + runners.append({"runner": key, "global": name, "class": cls, "vftable": _hex(table), + "prepare_addr": _hex(prepare.getEntryPoint()) if prepare else None, + "run_addr": _hex(run_.getEntryPoint()) if run_ else None}) + ids = [t["type_id"] for t in types if t.get("type_id") is not None] + by_id = _emulate_get_runner(program, get_runner, globals_, ids) + return { + "slots": dict((k, "0x%x" % v) for k, v in slots.items()), + "runners": runners, + "by_type_id": [{"type_id": tid, "runner": key_of[g], "global": g} + for tid, g in sorted(by_id.items())], + } + + # --------------------------------------------------------------------------- metadata def detect_engine(factory): @@ -936,14 +1299,15 @@ def run(): engine, compiler = detect_engine(factory) # namespace lives on the symbol/stub factory = resolve_thunk(factory) # MSVC8 exposes the symbol as a JMP stub types = extract_types(program, factory) - methods, method_inheritance = extract_methods(program) + runner_map = extract_runner_map(program, types) + methods, method_inheritance = extract_methods(program, runner_map) events = extract_events(program) fields = extract_script_fields(program) struct_layout, field_inheritance = extract_struct_layout(program) - method_dispatch = extract_method_dispatch(program) + method_dispatch = extract_method_dispatch(program, runner_map) snapshot = { - "schema_version": 4, + "schema_version": 5, "binary": { "name": program.getName(), "sha256": sha256_of(program), @@ -959,6 +1323,7 @@ def run(): "field_inheritance": field_inheritance, "struct_layout": struct_layout, "method_dispatch": method_dispatch, + "runner_map": runner_map, } args = getScriptArgs() @@ -969,9 +1334,10 @@ def run(): finally: fh.close() - print("[+] %s [%s/%s]: %d types, %d methods, %d events, %d fields (%d layout, %d dispatch) -> %s" % ( + print("[+] %s [%s/%s]: %d types, %d methods, %d events, %d fields (%d layout, %d dispatch, %d runners) -> %s" % ( program.getName(), engine, compiler, len(types), len(methods), - len(events), len(fields), len(struct_layout), len(method_dispatch), out_path)) + len(events), len(fields), len(struct_layout), len(method_dispatch), + len(runner_map["runners"]) if runner_map else 0, out_path)) run()