diff --git a/ghidra_scripts/extract_engine_surface.py b/ghidra_scripts/extract_engine_surface.py index 03da7ec..9fe145d 100644 --- a/ghidra_scripts/extract_engine_surface.py +++ b/ghidra_scripts/extract_engine_surface.py @@ -597,7 +597,8 @@ def extract_method_dispatch(program, runner_map=None): `case id:` is the method body - either a tail-call to a named submethod (BlooMoo/MSVC8 keeps show()/load()/... as separate functions) or inline code whose leaves are virtual calls on the wrapped object (Piklib/MSVC6). We fingerprint each case by its ordered CALL - anchors, so a later pass can diff method *bodies* by (owner, id). Join names via `methods`.""" + anchors, so a later pass can diff method *bodies* by (owner, id). Join names via `methods`. + Returns (rows, {runner: run function address}).""" fm = program.getFunctionManager() out = [] seen = set() @@ -615,16 +616,18 @@ def extract_method_dispatch(program, runner_map=None): f = _function_at(program, r.get("run_addr")) if f is not None: runs.append((f, r["runner"])) + run_addrs = {} for f, runner in runs: entry = resolve_thunk(f).getEntryPoint().getOffset() if entry in seen: continue seen.add(entry) + run_addrs[runner] = "0x%x" % entry try: out.extend(_dispatch_from_run(program, f, _owner_from_runner(runner), runner)) except Exception as e: # one malformed runner shouldn't sink the whole axis print("[!] method_dispatch %s: %s" % (runner, e)) - return out + return out, run_addrs def _dispatch_from_run(program, run_func, owner, runner): @@ -954,9 +957,32 @@ def _labels(program, addr): for s in program.getSymbolTable().getSymbols(a): ns = s.getParentNamespace() out.add((ns.getName() if ns is not None else "", s.getName())) + pair = _mangled_pair(s.getName()) + if pair is not None: + out.add(pair) return out +# Folded aliases often stay as raw MSVC labels; class and member are enough to match them. +_MANGLED_CTOR = re.compile(r"^\?\?0(\w+)@") +_MANGLED_VFTABLE = re.compile(r"^\?\?_7(\w+)@") +_MANGLED_MEMBER = re.compile(r"^\?(\w+)@(\w+)@") + + +def _mangled_pair(label): + """(class, member) of an undemangled MSVC label, e.g. ?getType@CMC_Object@@QAEHXZ.""" + m = _MANGLED_CTOR.match(label) + if m: + return (m.group(1), m.group(1)) + m = _MANGLED_VFTABLE.match(label) + if m: + return (m.group(1), "vftable") + m = _MANGLED_MEMBER.match(label) + if m: + return (m.group(2), m.group(1)) + return None + + def _vftable_class(program, addr): """The class whose labelled vftable (`??_7@@6B@`) sits at `addr`, or None.""" for ns, name in _labels(program, addr): @@ -1028,11 +1054,7 @@ def _vftable_store(program, instr): table = _addr(program, sc.getUnsignedValue()) except: # noqa: E722 return None - blk = program.getMemory().getBlock(table) - if blk is None or blk.isExecute(): - return None - first = _read_ptr(program, table) - return table if first is not None and _executable(program, first) else None + return table if _is_vftable_address(program, table) else None def _last_vftable_store(program, func, depth=0): @@ -1056,6 +1078,49 @@ def _last_vftable_store(program, func, depth=0): return found +def _object_vftable(program, dec, ctor): + """The most-derived vftable a constructor installs into `this`: the last P-code STORE of + a vftable-shaped constant at this+0. P-code follows `this` through stack spills that a + register walk loses (MSVC6 reloads it around inline strlen).""" + high = dec.high(resolve_thunk(ctor)) + if high is None: + return None + this_vns = set(vn for vn, role in _storage_roles(program, high, {"ECX": "this"}).items()) + found = None + ops = high.getPcodeOps() + while ops.hasNext(): + op = ops.next() + if op.getOpcode() != PcodeOp.STORE or _trace_this_offset(op.getInput(1), this_vns) != 0: + continue + table = _constant_address(program, op.getInput(2)) + if table is not None and _is_vftable_address(program, table): + found = table + return found + + +def _constant_address(program, vn): + """The address a P-code value denotes when it is constant: a plain constant, or the + decompiler's `PTRSUB(0, addr)` spelling of `&global`.""" + vn = _strip(vn) + if vn is None: + return None + if vn.isConstant(): + return _addr(program, vn.getOffset()) + d = vn.getDef() + if d is not None and d.getOpcode() == PcodeOp.PTRSUB and d.getInput(0).isConstant() \ + and d.getInput(0).getOffset() == 0 and d.getInput(1).isConstant(): + return _addr(program, d.getInput(1).getOffset()) + return None + + +def _is_vftable_address(program, table): + blk = program.getMemory().getBlock(table) + if blk is None or blk.isExecute(): + return False + first = _read_ptr(program, table) + return first is not None and _executable(program, first) + + def _runner_globals(program, init): """[(global, vftable)] from CMC_RunnerFactory::initialize, which builds every runner as `pRunner = new _Runner`. The vftable installed since the previous global write @@ -1235,6 +1300,446 @@ def extract_runner_map(program, types): } +# --------------------------------------------------------------------------- method facts + +# Runner getter reading script argument N -> the type the method converts it to. +_PARAM_GETTERS = {"getParameter": "any", "getParameterBool": "bool", "getParameterDouble": "double", + "getParameterInteger": "int", "getParameterString": "string"} +# CMC_VariableFactory call building the result -> result type ("copy": same type as the object). +_RESULT_ALLOCATORS = {"allocBool": "bool", "allocDouble": "double", "allocInteger": "int", + "allocString": "string", "copyVariable": "copy"} +# __thiscall run(int id, CMC_Object* obj, container, CXVector* params): storage -> role. +# Keyed by storage, not parameter index: an optimised body without a signature drops an +# unused `this` and renumbers what is left. +_RUN_ROLES = {"ECX": "runner", 8: "object", 16: "params"} +# A case forwards to a runner member (MSVC8 keeps every method out of line), which may forward again. +_HELPER_DEPTH = 2 +_DECOMPILE_TIMEOUT = 60 + + +class _Decompiler(object): + """HighFunction cache: runner helpers are shared by several cases.""" + + def __init__(self, program): + self.ifc = DecompInterface() + self.ifc.openProgram(program) + self.cache = {} + + def high(self, func): + key = func.getEntryPoint().getOffset() + if key not in self.cache: + res = self.ifc.decompileFunction(func, _DECOMPILE_TIMEOUT, monitor) + ok = res is not None and res.decompileCompleted() + self.cache[key] = res.getHighFunction() if ok else None + return self.cache[key] + + def dispose(self): + self.ifc.dispose() + + +class _Facts(object): + """What one method body does with its arguments, its object and its result.""" + + def __init__(self): + self.params = {} # argument index (None: read in a loop) -> set of types + self.returns = set() + self.object_slots = set() + self.type_checks = set() # type ids the body branches on via obj.getType() + + def as_dict(self): + indexes = sorted(i for i in self.params if i is not None) + return { + "params": [{"index": i, "types": sorted(self.params[i])} for i in indexes], + "variadic": None in self.params, + "returns": sorted(self.returns), + "object_slots": ["0x%x" % off for off in sorted(self.object_slots)], + "type_checks": sorted(self.type_checks), + } + + +def _strip(vn): + """The value behind copies, casts and call side-effect placeholders.""" + while vn is not None: + d = vn.getDef() + if d is None or d.getOpcode() not in (PcodeOp.COPY, PcodeOp.CAST, PcodeOp.INDIRECT): + return vn + vn = d.getInput(0) + return vn + + +def _storage_key(storage): + if storage.isRegisterStorage(): + return storage.getRegister().getName() + if storage.isStackStorage(): + return storage.getStackOffset() + return None + + +def _storage_roles(program, high, by_storage): + """{varnode: role} for the parameters stored where `by_storage` says, plus `this` when the + body reads it as an untyped incoming ECX.""" + out = {} + syms = high.getLocalSymbolMap().getSymbols() + while syms.hasNext(): + s = syms.next() + role = by_storage.get(_storage_key(s.getStorage())) if s.isParameter() else None + if role and s.getHighVariable() is not None: + for vn in s.getHighVariable().getInstances(): + out[vn] = role + if by_storage.get("ECX"): + ops = high.getPcodeOps() + while ops.hasNext(): + op = ops.next() + for i in range(op.getNumInputs()): + vn = op.getInput(i) + if vn.isInput() and vn.isRegister(): + reg = program.getRegister(vn.getAddress(), vn.getSize()) + if reg is not None and reg.getName() == "ECX": + out[vn] = by_storage["ECX"] + return out + + +def _call_storage(program, op, roles): + """storage -> role of a call's arguments: the caller's values (ECX first for __thiscall, + then stack slots in order) and, where the call site lost them, the callee's demangled + parameter types.""" + target = program.getFunctionManager().getFunctionAt(op.getInput(0).getAddress()) + out = {} + if target is None: + return out + for p in target.getParameters(): + role = _role_of_type(p.getDataType().getName()) + key = _storage_key(p.getVariableStorage()) + if role and key is not None: + out[key] = role + slot = 4 + for i in range(1, op.getNumInputs()): + if i == 1 and target.getCallingConventionName() == "__thiscall": + key = "ECX" + else: + key, slot = slot, slot + 4 + role = _role(op.getInput(i), roles) + if role is not None: + out[key] = role + return out + + +def _role_of_type(name): + if "CXVector" in name: + return "params" + if "_Runner" in name: + return "runner" + if name.startswith(CMC_PREFIX): + return "object" + return None + + +def _is_runner_member(program, addr): + for ns, _ in _labels(program, addr): + if ns.endswith("_Runner"): + return True + return False + + +def _role(vn, roles): + vn = _strip(vn) + return roles.get(vn) if vn is not None else None + + +def _vcall(op, roles): + """(role, offset) of a virtual call `CALLIND [[x] + offset]` on the object `x` holds.""" + target = _strip(op.getInput(0)) + load = target.getDef() if target is not None else None + if load is None or load.getOpcode() != PcodeOp.LOAD: + return None + ptr = _strip(load.getInput(1)) + off = 0 + arith = ptr.getDef() if ptr is not None else None + if arith is not None and arith.getOpcode() in (PcodeOp.INT_ADD, PcodeOp.PTRSUB) and arith.getInput(1).isConstant(): + off = int(arith.getInput(1).getOffset()) + ptr = _strip(arith.getInput(0)) + vtable = ptr.getDef() if ptr is not None else None + if vtable is None or vtable.getOpcode() != PcodeOp.LOAD: + return None + role = _role(vtable.getInput(1), roles) + return (role, off) if role is not None else None + + +def _member(labels, cls, names): + for ns, name in labels: + if ns == cls and name in names: + return name + return None + + +def _reachable(starts, allowed): + seen = set() + order = [] + stack = list(starts) + while stack: + b = stack.pop() + if b in seen or not allowed(b): + continue + seen.add(b) + order.append(b) + for k in range(b.getOutSize()): + stack.append(b.getOut(k)) + return order + + +def _case_regions(high, entries): + """{case address: blocks reachable from it without entering another case}. Shared + epilogues belong to every case that reaches them.""" + blocks = high.getBasicBlocks() + by_start = {} + for i in range(blocks.size()): + b = blocks.get(i) + by_start[b.getStart().getOffset()] = b + heads = dict((a, by_start[a]) for a in entries if a in by_start) + head_set = set(heads.values()) + out = {} + for a, head in heads.items(): + out[a] = _reachable([head], lambda b, head=head: b not in head_set or b is head) + return out + + +def _passes(op, roles, wanted): + """Whether a call hands one of the `wanted` roles to its callee.""" + for i in range(1, op.getNumInputs()): + if _role(op.getInput(i), roles) in wanted: + return True + return False + + +def _getter_index(program, op, roles): + """Argument index of `runner->getParameter*(params, index)`; None when computed (a loop). + The decompiler may or may not pass `this`, or any argument when it misses the callee's + signature - then read the pushes: `index` is pushed right before `params`.""" + for i in range(1, op.getNumInputs() - 1): + if _role(op.getInput(i), roles) == "params": + index = _strip(op.getInput(i + 1)) + return int(index.getOffset()) if index is not None and index.isConstant() else None + return _pushed_constant(program, op.getSeqnum().getTarget(), 1) + + +def _pushed_constant(program, call_addr, arg): + """The constant pushed as argument `arg` (0-based) of the CALL at `call_addr`, following + `PUSH reg` back to `XOR reg,reg` / `MOV reg,imm` in the same straight line; else None.""" + instr = program.getListing().getInstructionAt(call_addr) + pushes = [] + known = {} + trail = [] + for _ in range(24): + instr = instr.getPrevious() if instr is not None else None + if instr is None or instr.getFlowType().isCall() or instr.getFlowType().isJump(): + break + trail.append(instr) + for instr in reversed(trail): # forward: register constants, then the pushes in order + mn = instr.getMnemonicString() + if mn == "XOR" and instr.getNumOperands() == 2 and \ + instr.getDefaultOperandRepresentation(0) == instr.getDefaultOperandRepresentation(1): + known[instr.getDefaultOperandRepresentation(0)] = 0 + elif mn == "MOV" and instr.getNumOperands() == 2 and instr.getScalar(1) is not None \ + and not instr.getReferencesFrom(): + known[instr.getDefaultOperandRepresentation(0)] = int(instr.getScalar(1).getValue()) + elif mn == "PUSH": + sc = instr.getScalar(0) + if sc is not None and not instr.getReferencesFrom(): + pushes.append(int(sc.getValue())) + else: + pushes.append(known.get(instr.getDefaultOperandRepresentation(0))) + elif instr.getNumOperands() > 0: + known.pop(instr.getDefaultOperandRepresentation(0), None) + pushes.reverse() # arguments are pushed right to left + return pushes[arg] if arg < len(pushes) else None + + +def _type_test(block, type_results): + """(opcode, K) when `block` ends branching on obj.getType() ==/!= K, else None.""" + ops = list(block.getIterator()) + if not ops or ops[-1].getOpcode() != PcodeOp.CBRANCH or block.getOutSize() != 2: + return None + cond = _strip(ops[-1].getInput(1)) + test = cond.getDef() if cond is not None else None + if test is None or test.getOpcode() not in (PcodeOp.INT_EQUAL, PcodeOp.INT_NOTEQUAL): + return None + if not test.getInput(1).isConstant() or _strip(test.getInput(0)) not in type_results: + return None + return (test.getOpcode(), int(test.getInput(1).getOffset())) + + +def _path_for_type(head, inside, tests, type_id): + """Blocks reachable from `head` when obj.getType() == type_id: type branches follow only + the side that type takes, every other branch both sides.""" + seen = set() + stack = [head] + while stack: + b = stack.pop() + if b in seen or b not in inside: + continue + seen.add(b) + test = tests.get(b) + if test is None: + for k in range(b.getOutSize()): + stack.append(b.getOut(k)) + continue + opcode, k = test + holds = (type_id == k) if opcode == PcodeOp.INT_EQUAL else (type_id != k) + stack.append(b.getTrueOut() if holds else b.getFalseOut()) + return seen + + +def _collect(program, dec, blocks, roles, getters, facts, depth, type_ids): + """Accumulate `facts` over `blocks` (head first) and return {type id: does work}: whether + the path that type takes reaches a virtual call, a runner helper doing work for it, or a + result allocation - an excluded type only reaches the error/skip exit.""" + type_results = set() + work = {} # block -> [True | {type id: bool} from a helper] + for b in blocks: + it = b.getIterator() + while it.hasNext(): + op = it.next() + code = op.getOpcode() + if code == PcodeOp.CALLIND: + vc = _vcall(op, roles) + if vc is None: + continue + role, off = vc + if role == "runner" and off in getters: + facts.params.setdefault(_getter_index(program, op, roles), set()).add(getters[off]) + elif role == "object": + facts.object_slots.add(off) + work.setdefault(b, []).append(True) + elif code == PcodeOp.CALL: + target = op.getInput(0).getAddress() + labels = _labels(program, target) + this_role = _role(op.getInput(1), roles) if op.getNumInputs() > 1 else None + alloc = _member(labels, "CMC_VariableFactory", _RESULT_ALLOCATORS) + helper = this_role == "runner" or (op.getNumInputs() == 1 and _is_runner_member(program, target)) + body = program.getFunctionManager().getFunctionAt(target) + if alloc: + facts.returns.add(_RESULT_ALLOCATORS[alloc]) + work.setdefault(b, []).append(True) + elif this_role == "object" and _member(labels, "CMC_Object", ("getType",)): + if op.getOutput() is not None: + type_results.add(op.getOutput()) + elif helper and body is not None and depth < _HELPER_DEPTH: + work.setdefault(b, []).append(_collect_function( + program, dec, resolve_thunk(body), _call_storage(program, op, roles), + getters, facts, depth + 1, type_ids)) + elif this_role == "object" or _passes(op, roles, ("object", "params")): + work.setdefault(b, []).append(True) + tests = {} + for b in blocks: + test = _type_test(b, type_results) + if test is not None: + tests[b] = test + facts.type_checks.add(test[1]) + inside = set(blocks) + applies = {} + for type_id in type_ids: + path = _path_for_type(blocks[0], inside, tests, type_id) if blocks else set() + applies[type_id] = any(w is True or w.get(type_id) for b in path for w in work.get(b, [])) + return applies + + +def _collect_function(program, dec, func, by_storage, getters, facts, depth, type_ids): + high = dec.high(func) + if high is None: + return dict((t, True) for t in type_ids) # unknown body: assume it does its job + blocks = high.getBasicBlocks() + return _collect(program, dec, [blocks.get(i) for i in range(blocks.size())], + _storage_roles(program, high, by_storage), getters, facts, depth, type_ids) + + +def annotate_method_facts(program, method_dispatch, run_addrs, runner_map, types): + """Per dispatch row: argument types read through CMC_Runner::getParameter*, the result + allocator, virtual calls on the object and branches on obj.getType() - the script + signature and per-type validity. Read from P-code because MSVC6 inlines bodies into + run() while MSVC8 moves them out of line. `run_addrs` maps runner -> run function.""" + slots = dict((k, int(v, 16)) for k, v in ((runner_map or {}).get("slots") or {}).items()) + getters = dict((slots[name], ty) for name, ty in _PARAM_GETTERS.items() if name in slots) + if not getters: + return + by_runner = {} + for row in method_dispatch: + by_runner.setdefault(row["runner"], []).append(row) + runner_of = dict((e["type_id"], e["runner"]) for e in runner_map["by_type_id"]) + shared = {} # runner -> {type id: script name}, for runners several types dispatch to + for t in types: + if t.get("type_id") in runner_of: + shared.setdefault(runner_of[t["type_id"]], {})[t["type_id"]] = t["script_name"] + dec = _Decompiler(program) + try: + for runner, rows in by_runner.items(): + run_func = _function_at(program, run_addrs.get(runner)) + high = dec.high(run_func) if run_func is not None else None + if high is None: + continue + roles = _storage_roles(program, high, _RUN_ROLES) + regions = _case_regions(high, set(int(r["case_addr"], 16) for r in rows)) + for row in rows: + region = regions.get(int(row["case_addr"], 16)) + if region is None: + continue + facts = _Facts() + group = shared.get(runner, {}) if len(shared.get(runner, {})) > 1 else {} + applies = _collect(program, dec, region, roles, getters, facts, 0, sorted(group)) + row.update(facts.as_dict()) + if group: + # Only an explicit obj.getType() branch can exclude a type. + row["per_type"] = dict((group[t], "applies" if ok or not facts.type_checks + else "excluded_by_type_check") for t, ok in applies.items()) + finally: + dec.dispose() + + +def _trivial(program, addr): + """A virtual a type leaves as `_purecall`, a bare RET or `return 0`: a no-op for it.""" + if addr is None: + return True + for _, name in _labels(program, addr): + if "purecall" in name: + return True + f = program.getFunctionManager().getFunctionAt(addr) + instr = program.getListing().getInstructionAt(resolve_thunk(f).getEntryPoint() if f else addr) + for _ in range(3): + if instr is None: + return False + mn = instr.getMnemonicString() + if mn == "RET": + return True + text = instr.toString() + if not (mn == "XOR" and text in ("XOR EAX,EAX", "XOR AL,AL")) and \ + not (mn == "MOV" and re.match(r"MOV (EAX|AL),0x0$", text)): + return False + instr = instr.getNext() + return False + + +def annotate_type_vftables(program, types, method_dispatch): + """Record each type's vftable and demote a shared-runner method to "trivial_virtual" for + a type whose vftable leaves every virtual the method calls as a no-op.""" + tables = {} + dec = _Decompiler(program) + try: + for t in types: + ctor = _function_at(program, t.get("ctor_addr")) + table = _object_vftable(program, dec, ctor) if ctor is not None else None + t["vftable"] = _hex(table) + tables[t["script_name"]] = table + finally: + dec.dispose() + for row in method_dispatch: + for name, verdict in (row.get("per_type") or {}).items(): + table = tables.get(name) + if verdict != "applies" or not row["object_slots"] or table is None: + continue + targets = [_read_ptr(program, table.add(int(off, 16))) for off in row["object_slots"]] + if all(_trivial(program, a) for a in targets): + row["per_type"][name] = "trivial_virtual" + + # --------------------------------------------------------------------------- metadata def detect_engine(factory): @@ -1304,7 +1809,10 @@ def run(): events = extract_events(program) fields = extract_script_fields(program) struct_layout, field_inheritance = extract_struct_layout(program) - method_dispatch = extract_method_dispatch(program, runner_map) + method_dispatch, run_addrs = extract_method_dispatch(program, runner_map) + if runner_map: + annotate_method_facts(program, method_dispatch, run_addrs, runner_map, types) + annotate_type_vftables(program, types, method_dispatch) snapshot = { "schema_version": 5,