From 4e65268a1ecafb54ce58918aa32b8728a84ae659 Mon Sep 17 00:00:00 2001 From: Patryk Gensch <43010113+patryk025@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:58:40 +0100 Subject: [PATCH] feat(extractor): extract scene modules (SCENE:DLLS=) like engines A module DLL has no engine factory: its CMC_ExternObject::resolve compares TYPE (CXString::equals or operator==), allocates and constructs. The runner of each type is the vftable its constructor or a virtual installs whose prepare slot chains to CMC_Runner::prepareMthHashSet (defined when analysis left it out); registrations, dispatch, signatures, per-type events and fields then come from the engine passes, limited to the module's own runners. Snapshots record module_of (the host engine DLL). An unoptimised run() keeps the switch index in a stack slot; its bounds are followed through that slot only when the register match finds none. The batch queue extracts plugins too, the inventory knows MATRIX, QMAN, INERTIA and BOARD, and the tracker marks module-mapped types. Co-Authored-By: Claude Opus 5.5 --- ams/acquire/batch.py | 8 +- ams/api/static/tracker.js | 2 +- ams/inventory.py | 6 +- ams/tracker.py | 10 +- ghidra_scripts/extract_engine_surface.py | 310 ++++++++++++++++++++--- tests/test_batch.py | 2 +- tests/test_tracker.py | 12 + 7 files changed, 301 insertions(+), 49 deletions(-) diff --git a/ams/acquire/batch.py b/ams/acquire/batch.py index afcb1d3..ee64aec 100644 --- a/ams/acquire/batch.py +++ b/ams/acquire/batch.py @@ -1,4 +1,4 @@ -"""Resumable, SHA-addressed engine extraction queue (POSIX).""" +"""Resumable, SHA-addressed extraction queue for engines and scene modules (POSIX).""" from __future__ import annotations import argparse @@ -115,6 +115,10 @@ def execute_worker(source, output, log, timeout, env): raise RuntimeError('Extractor exited with code ' + str(code)) +# Engines and the scene modules (SCENE:DLLS=) that add script types; the extractor handles both. +EXTRACTED_SCOPES = ('engine', 'plugin') + + def run_queue(manifest, out_dir, *, ghidra_home=None, timeout=1800, limit=None, retry_failed=False, worker=execute_worker): if timeout <= 0 or (limit is not None and limit < 0): @@ -146,7 +150,7 @@ def _run(builds, directory, ghidra_home, timeout, limit, retry_failed, worker): report = {'schema': 'extraction-batch/1', 'attempted': 0, 'succeeded': 0, 'pending': 0, 'skipped_success': 0, 'skipped_failed': 0, 'skipped_scope': 0, 'failures': []} for build in builds: - if build.get('scope') != 'engine': + if build.get('scope') not in EXTRACTED_SCOPES: report['skipped_scope'] += 1 continue sha = build['sha256'] diff --git a/ams/api/static/tracker.js b/ams/api/static/tracker.js index 95c1add..b930814 100644 --- a/ams/api/static/tracker.js +++ b/ams/api/static/tracker.js @@ -1,7 +1,7 @@ 'use strict'; const EVENT_STATUS={emitted:'emitowany w klasie typu',emitted_base:'emitowany w Variable',engine_literal:'literał w kodzie silnika',not_found:'brak literału w emulatorze'}; const FIELD_STATUS={read:'czytane w obsłudze typu',read_base:'czytane w Variable',engine_literal:'literał w innym kodzie',not_found:'brak literału w emulatorze'}; -const MAPPING={explicit:'Jawne (klasa C++ z fabryki)',runner_factory:'CMC_RunnerFactory (typ → runner)',class_constructor:'Runner tworzony w konstruktorze klasy',inferred:'Wywnioskowane z nazwy'}; +const MAPPING={explicit:'Jawne (klasa C++ z fabryki)',runner_factory:'CMC_RunnerFactory (typ → runner)',module:'Moduł sceny DLLS (typ → runner obiektu)',class_constructor:'Runner tworzony w konstruktorze klasy',inferred:'Wywnioskowane z nazwy'}; const $ = id => document.getElementById(id); const labels = {missing:'Brak rejestracji',registered:'Zarejestrowana',stub_candidate:'Podejrzenie stubu',partial:'Częściowa',implemented:'Zaimplementowana',unverified:'Niezweryfikowana',asm_reviewed:'Przegląd ASM',runtime_tested:'Test zachowania',stale:'Dowód wymaga aktualizacji'}; let data, selected = null; diff --git a/ams/inventory.py b/ams/inventory.py index b26d46b..13e9df4 100644 --- a/ams/inventory.py +++ b/ams/inventory.py @@ -7,6 +7,10 @@ from pathlib import Path import re +# Scene modules loaded by `SCENE:DLLS=`; each adds script types (QMAN: TEST). +PLUGINS = {"WORLD.DLL", "KOLOROWANKA.DLL", "MATRIX.DLL", "QMAN.DLL", "INERTIA.DLL", "BOARD.DLL"} + + def inventory(roots: list[Path], snapshot_dir: Path | None = None) -> dict: extracted = set() if snapshot_dir: @@ -29,7 +33,7 @@ def inventory(roots: list[Path], snapshot_dir: Path | None = None) -> dict: family, scope = name.removesuffix(".DLL"), "engine" elif name.startswith("BLOOMOO"): family, scope = "BLOOMOO", "engine" - elif name in ("WORLD.DLL", "KOLOROWANKA.DLL"): + elif name in PLUGINS: family, scope = name.removesuffix(".DLL"), "plugin" elif name == "SEKAI.DLL": family, scope = "SEKAI", "native_dependency" diff --git a/ams/tracker.py b/ams/tracker.py index bbdd335..9ab6f1f 100644 --- a/ams/tracker.py +++ b/ams/tracker.py @@ -345,11 +345,13 @@ def script_fields(snap: Snapshot, scopes: dict | None = None) -> dict[str, list[ def script_owners(snap: Snapshot, owners) -> dict[str, tuple[str, str]]: """Script scope -> (method owner, provenance) for one binary. - `runner_map` (CMC_RunnerFactory, Piklib 7.1+/BlooMoo) is the engine's own dispatch table and - wins: several types can share a runner (INTEGER/STRING/... -> Variable). Without it (Piklib - 6.1, plugins, older extractions) the C++ class or its name is the best available guess.""" + `runner_map` (CMC_RunnerFactory, Piklib 7.1+/BlooMoo; a scene module's types and the runner + their objects install) is the engine's own dispatch table and wins: several types can share + a runner (INTEGER/STRING/... -> Variable). Without it (Piklib 6.1, older extractions) the C++ + class or its name is the best available guess.""" runner_map = snap.raw.get("runner_map") or {} by_type_id = {e["type_id"]: e["runner"].removesuffix("_Runner") for e in runner_map.get("by_type_id", [])} + dispatched = "module" if runner_map.get("module") else "runner_factory" # a scene module's own resolve class_runner = {e["class"]: e["runner"].removesuffix("_Runner") for e in runner_map.get("by_class", [])} bases = {e["class"]: e["base_class"] for e in snap.field_inheritance} known_classes = set(class_runner) | set(bases) | set(bases.values()) @@ -359,7 +361,7 @@ def script_owners(snap: Snapshot, owners) -> dict[str, tuple[str, str]]: for t in snap.types: name = canonical(t["script_name"]) if t.get("type_id") in by_type_id: - mappings[name] = (by_type_id[t["type_id"]], "runner_factory") + mappings[name] = (by_type_id[t["type_id"]], dispatched) continue owner, mapping = t.get("cpp_class"), "explicit" # Known extractor failure: unrelated factory entries all mapped to Behaviour. diff --git a/ghidra_scripts/extract_engine_surface.py b/ghidra_scripts/extract_engine_surface.py index c0e04a2..1b7a927 100644 --- a/ghidra_scripts/extract_engine_surface.py +++ b/ghidra_scripts/extract_engine_surface.py @@ -606,12 +606,21 @@ def _parse_switch(program, func): byte_table = space.getAddress(int(mbt.group(2), 16)) break + base, count = _switch_bounds(instrs, jmp_addr, set([bounds_reg])) + if count is None: # unoptimised build: the index lives in a stack slot + base, count = _switch_bounds(instrs, jmp_addr, _index_aliases(instrs, jmp_addr, bounds_reg)) + return {"table": table, "base": base, "count": count, "default": default, + "byte_table": byte_table} + + +def _switch_bounds(instrs, jmp_addr, operands): + """(base, count) from the arithmetic and the bounds CMP applied to the index `operands`.""" base = 0 count = None for instr in instrs: if instr.getAddress().equals(jmp_addr): break - if instr.getNumOperands() == 0 or instr.getDefaultOperandRepresentation(0) != bounds_reg: + if instr.getNumOperands() == 0 or instr.getDefaultOperandRepresentation(0) not in operands: continue mn = instr.getMnemonicString() s = instr.getScalar(1) @@ -627,8 +636,27 @@ def _parse_switch(program, func): base = -_s32(s.getValue()) elif mn == "DEC": base = 1 - return {"table": table, "base": base, "count": count, "default": default, - "byte_table": byte_table} + return base, count + + +def _index_aliases(instrs, jmp_addr, reg): + """Operands that hold the switch index in an unoptimised build, which round-trips it through + a stack slot: `SUB ECX,1 ; MOV [EBP-0xc],ECX ; CMP [EBP-0xc],0xc3 ; MOV EDX,[EBP-0xc]`.""" + aliases = set([reg]) + for _ in range(3): + for instr in instrs: + if instr.getAddress().equals(jmp_addr): + break + if instr.getMnemonicString() != "MOV" or instr.getNumOperands() != 2: + continue + dst, src = instr.getDefaultOperandRepresentation(0), instr.getDefaultOperandRepresentation(1) + if dst in aliases and src.startswith("dword ptr ["): + aliases.add(src) + elif dst in aliases and re.match(r"E[A-Z]{2}$", src): + aliases.add(src) + elif src in aliases and dst.startswith("dword ptr ["): + aliases.add(dst) + return aliases def _name_dispatch_from_run(program, run_func, owner, runner, ids): @@ -2033,6 +2061,209 @@ def sha256_of(program): return None +# --------------------------------------------------------------------------- scene modules + +# A scene module (`SCENE:DLLS=`: World.dll, Matrix.dll, ...) adds script types through +# CMC_ExternObject::resolve, which compares TYPE with CXString::equals or operator==. +_MODULE_COMPARES = (OP_EQ, "equals", "equalsIgnoreCase") +# Engine SDK classes a module links in statically; their runners are not the module's. +_SDK_PREFIXES = ("CMC_", "CX") + + +def _is_sdk(func): + ns = func.getParentNamespace() + return ns is not None and ns.getName().startswith(_SDK_PREFIXES) + + +def _calls(program, func): + """Names of the functions `func` calls directly.""" + names = set() + it = program.getListing().getInstructions(func.getBody(), True) + while it.hasNext(): + instr = it.next() + if instr.getMnemonicString() == "CALL": + cf = call_target(program, instr) + if cf is not None: + names.add(cf.getName()) + return names + + +def host_library(program): + """The engine DLL a scene module imports (PIKLIB8.DLL, BlooMooDLL.dll, ...), or None.""" + for lib in program.getExternalManager().getExternalLibraryNames(): + if lib.upper().startswith(("PIKLIB", "BLOOMOO")): + return lib + return None + + +def extract_module_types(program, factory): + """The types one CMC_ExternObject::resolve builds: per branch the TYPE literal, a compare, + operator new(SIZE) and the constructor as the next call - the engine's ladder, except that + the literal is copied into a CXString inline, more than LOOKBACK instructions before the + compare, so the latest literal other than the "TYPE" key names the branch.""" + types = [] + last = pending = size = None + armed = False + recent = [] + it = program.getListing().getInstructions(factory.getBody(), True) + while it.hasNext(): + instr = it.next() + s = _imm_string(program, instr) + if s is not None and s != "TYPE" and _is_method_name(s): + last = s + if instr.getMnemonicString() == "CALL": + f = call_target(program, instr) + name = f.getName() if f is not None else None + if name in _MODULE_COMPARES: + pending = last + elif name == OP_NEW: + size = lookback_scalar(recent) + armed = pending is not None + elif armed and f is not None: + ctor = resolve_thunk(f) + types.append({"script_name": pending, + "cpp_class": f.getName() if f.getName().startswith(CMC_PREFIX) else None, + "ctor_addr": _hex(ctor.getEntryPoint()), "object_size": size, + "dispatch_addr": _hex(instr.getAddress()), "via_module_iface": True}) + pending = size = None + armed = False + recent.append(instr) + if len(recent) > LOOKBACK: + recent.pop(0) + return types + + +def find_module_factories(program): + """[(resolve, types)] of a scene module: functions outside the linked-in SDK that compare, + allocate and construct a CMC_Object-based type (its constructor hands a type id to the base).""" + out = [] + it = program.getFunctionManager().getFunctions(True) + while it.hasNext(): + f = it.next() + if f.isThunk() or _is_sdk(f): + continue + names = _calls(program, f) + if OP_NEW not in names or not names.intersection(_MODULE_COMPARES): + continue + types = [t for t in extract_module_types(program, f) if _ctor_type_id(program, t["ctor_addr"]) is not None] + if types: + out.append((f, types)) + return out + + +def _vtable_functions(program, table): + funcs = [] + for i in range(_MAX_VTABLE_SLOTS if table is not None else 0): + target = _read_ptr(program, table.add(i * 4)) + if target is None or not _executable(program, target): + break + f = _ensure_function(program, target) + if f is not None: + funcs.append(f) + return funcs + + +def _installed_tables(program, funcs, depth=3): + """Offsets of every vftable `funcs` and their callees (`depth` levels) install.""" + tables = set() + seen = set() + frontier = list(funcs) + for _ in range(depth + 1): + nxt = [] + for f in frontier: + body = resolve_thunk(f) + if body.isExternal() or body.getEntryPoint().getOffset() in seen: + continue + seen.add(body.getEntryPoint().getOffset()) + it = program.getListing().getInstructions(body.getBody(), True) + while it.hasNext(): + instr = it.next() + table = _vftable_store(program, instr) + if table is not None: + tables.add(table.getOffset()) + elif instr.getMnemonicString() == "CALL": + cf = call_target(program, instr) + if cf is not None: + nxt.append(cf) + frontier = nxt + return tables + + +def _module_runner(program, slots, tables): + """(vftable, prepare) of the runner among the vftables a type installs: the table whose + prepare slot chains to the engine's CMC_Runner::prepareMthHashSet, preferring a module + function over a linked-in SDK runner. Analysis often leaves that prepare undefined + (referenced from the vtable only), so it is defined here.""" + found = [] + for off in sorted(tables): + table = _addr(program, off) + prepare = _ensure_function(program, _read_ptr(program, table.add(slots["prepareMthHashSet"]))) + if prepare is not None and not prepare.isThunk() and "prepareMthHashSet" in _calls(program, prepare): + found.append((table, prepare)) + if len(found) > 1: # a linked-in SDK runner as well; Inertia names its own CMC_Inertia_Runner + found = [f for f in found if not _is_sdk(f[1])] or found + return found[0] if len(found) == 1 else None + + +def extract_module_runner_map(program, types): + """Runner of each module type, in `extract_runner_map`'s shape (marked `module`). The engine + asks the object for its runner, so a type's runner is the runner vftable its constructor or + one of its virtuals installs (Matrix and Board create it lazily). Annotates `types` with + `type_id`. Runner classes have no symbol; they are named after the type (CMC_Matrix_Runner).""" + slots = _runner_slots(program) + if "prepareMthHashSet" not in slots or "run" not in slots: + return None + runners = {} + by_type_id = [] + dec = _Decompiler(program) + try: + for t in types: + t["type_id"] = _ctor_type_id(program, t["ctor_addr"]) + ctor = _function_at(program, t["ctor_addr"]) + reach = [ctor] + _vtable_functions(program, _object_vftable(program, dec, ctor)) + found = _module_runner(program, slots, _installed_tables(program, reach)) + if found is None: + continue + table, prepare = found + if table.getOffset() not in runners: + cls = _vftable_class(program, table) + run_ = _ensure_function(program, _read_ptr(program, table.add(slots["run"]))) + runners[table.getOffset()] = { + "runner": cls or "CMC_%s_Runner" % t["script_name"].capitalize(), "global": None, + "class": cls, "vftable": _hex(table), "prepare_addr": _hex(prepare.getEntryPoint()), + "run_addr": _hex(run_.getEntryPoint()) if run_ else None} + by_type_id.append({"type_id": t["type_id"], "runner": runners[table.getOffset()]["runner"], + "global": None}) + finally: + dec.dispose() + return {"module": True, "slots": dict((k, "0x%x" % v) for k, v in slots.items()), + "runners": sorted(runners.values(), key=lambda r: r["runner"]), "by_type_id": by_type_id} + + +def module_surface(program, factories): + """Snapshot sections of a scene module. Only the module's own runners and types count: the + engine classes it links in statically are the engine's surface, extracted from the engine.""" + types = [t for _, found in factories for t in found] + runner_map = extract_module_runner_map(program, types) + methods, inheritance, dispatch, run_addrs = [], [], [], {} + for r in (runner_map or {}).get("runners", []): + own, runner, base = _extract_methods_from(program, _function_at(program, r["prepare_addr"]), r["runner"]) + methods.extend(own) + if base is not None: + inheritance.append({"runner": runner, "base_runner": base}) + run_func = _function_at(program, r["run_addr"]) + if run_func is not None: + run_addrs[r["runner"]] = r["run_addr"] + dispatch.extend(_dispatch_from_run(program, run_func, _owner_from_runner(r["runner"]), r["runner"])) + if runner_map: + annotate_method_facts(program, dispatch, run_addrs, runner_map, types) + annotate_type_vftables(program, types, dispatch) + return {"types": types, "methods": methods, "method_inheritance": inheritance, "events": [], + "fields": [], "field_inheritance": [], "struct_layout": [], "method_dispatch": dispatch, + "runner_map": runner_map, "type_events": extract_type_events(program, types), + "type_fields": extract_type_fields(program, types)} + + # --------------------------------------------------------------------------- main def find_factory(program): @@ -2049,16 +2280,8 @@ def find_factory(program): return None -def run(): - program = currentProgram # GhidraScript/pyghidra inject this global, not `program` - factory = find_factory(program) - if factory is None: - print("[!] factory not found (CMC_ObjectsContainer::resolve / CMC_Scene::resolve)" - " - is this a Piklib/BlooMoo DLL with text-script support? (added in Piklib 6.1)") - return - - engine, compiler = detect_engine(factory) # namespace lives on the symbol/stub - factory = resolve_thunk(factory) # MSVC8 exposes the symbol as a JMP stub +def engine_surface(program, factory): + """Snapshot sections of an engine DLL, from its type-dispatch factory.""" types = extract_types(program, factory) runner_map = extract_runner_map(program, types) or extract_class_runners(program) methods, method_inheritance = extract_methods(program, runner_map) @@ -2069,30 +2292,36 @@ def run(): if runner_map: annotate_method_facts(program, method_dispatch, run_addrs, runner_map, types) annotate_type_vftables(program, types, method_dispatch) - type_events = extract_type_events(program, types) - type_fields = extract_type_fields(program, types) + return {"types": types, "methods": methods, "method_inheritance": method_inheritance, + "events": events, "fields": fields, "field_inheritance": field_inheritance, + "struct_layout": struct_layout, "method_dispatch": method_dispatch, "runner_map": runner_map, + "type_events": extract_type_events(program, types), + "type_fields": extract_type_fields(program, types)} - snapshot = { - "schema_version": 7, - "binary": { - "name": program.getName(), - "sha256": sha256_of(program), - "engine": engine, - "compiler": compiler, - "factory_addr": "0x%x" % factory.getEntryPoint().getOffset(), - }, - "types": types, - "methods": methods, - "method_inheritance": method_inheritance, - "events": events, - "fields": fields, - "field_inheritance": field_inheritance, - "struct_layout": struct_layout, - "method_dispatch": method_dispatch, - "runner_map": runner_map, - "type_events": type_events, - "type_fields": type_fields, - } + +def run(): + program = currentProgram # GhidraScript/pyghidra inject this global, not `program` + factory = find_factory(program) + if factory is not None: + engine, compiler = detect_engine(factory) # namespace lives on the symbol/stub + factory = resolve_thunk(factory) # MSVC8 exposes the symbol as a JMP stub + surface = engine_surface(program, factory) + binary = {"factory_addr": _hex(factory.getEntryPoint())} + else: + modules = find_module_factories(program) + host = host_library(program) + if not modules or host is None: + print("[!] factory not found (CMC_ObjectsContainer::resolve / CMC_Scene::resolve, nor a" + " scene module's CMC_ExternObject::resolve) - is this a Piklib/BlooMoo DLL with" + " text-script support? (added in Piklib 6.1)") + return + engine, compiler = ("BlooMoo", "MSVC8") if host.upper().startswith("BLOOMOO") else ("Piklib", "MSVC6") + surface = module_surface(program, modules) + binary = {"factory_addr": _hex(modules[0][0].getEntryPoint()), "module_of": host} + + binary.update({"name": program.getName(), "sha256": sha256_of(program), "engine": engine, "compiler": compiler}) + snapshot = {"schema_version": 7, "binary": binary} + snapshot.update(surface) args = getScriptArgs() out_path = args[0] if len(args) > 0 else default_out_path(program) @@ -2102,10 +2331,11 @@ def run(): finally: fh.close() - print("[+] %s [%s/%s]: %d types, %d methods, %d events (%d per type), %d fields (%d per type, %d layout, %d dispatch, %d runners) -> %s" % ( - program.getName(), engine, compiler, len(types), len(methods), - len(events), len(type_events), len(fields), len(type_fields), len(struct_layout), len(method_dispatch), - len((runner_map or {}).get("runners", [])), out_path)) + print("[+] %s [%s/%s%s]: %d types, %d methods, %d events (%d per type), %d fields (%d per type, %d layout, %d dispatch, %d runners) -> %s" % ( + program.getName(), engine, compiler, " module of " + binary["module_of"] if "module_of" in binary else "", + len(snapshot["types"]), len(snapshot["methods"]), len(snapshot["events"]), len(snapshot["type_events"]), + len(snapshot["fields"]), len(snapshot["type_fields"]), len(snapshot["struct_layout"]), + len(snapshot["method_dispatch"]), len((snapshot["runner_map"] or {}).get("runners", [])), out_path)) run() diff --git a/tests/test_batch.py b/tests/test_batch.py index a2abf22..8810370 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -77,7 +77,7 @@ def test_lock_and_scope_limit(setup): with pytest.raises(RuntimeError, match='Another'): batch.run_queue(manifest, out, worker=success) data = json.loads(manifest.read_text()) - data['builds'][0]['scope'] = 'plugin' + data['builds'][0]['scope'] = 'native_dependency' manifest.write_text(json.dumps(data)) assert batch.run_queue(manifest, out, worker=success)['skipped_scope'] == 1 diff --git a/tests/test_tracker.py b/tests/test_tracker.py index b1e1b5f..38d4ad9 100644 --- a/tests/test_tracker.py +++ b/tests/test_tracker.py @@ -122,6 +122,18 @@ def test_runner_factory_maps_shared_and_unnamed_runners(tmp_path): assert "inherits" not in typ(result, "@") # built-ins do not chain to CMC_Runner +def test_scene_module_types_map_through_the_runner_their_objects_install(tmp_path): + module = Snapshot({"binary": {"name": "Matrix.dll", "sha256": "b" * 64, "module_of": "PIKLIB8.DLL"}, + "types": [{"script_name": "MATRIX", "cpp_class": None, "type_id": 10002}], + "methods": [{"owner": "CMC_Matrix", "runner": "CMC_Matrix_Runner", "name": "TICK", "id": 1}], + "method_inheritance": [{"runner": "CMC_Matrix_Runner", "base_runner": "CMC_Runner"}], + "runner_map": {"module": True, "by_type_id": [{"type_id": 10002, "runner": "CMC_Matrix_Runner"}]}}) + result = build_tracker([factory_snapshot(), module], emulator(tmp_path)) + tick = row(result, "MATRIX", "TICK")["original"][0] + assert (tick["owner"], tick["mapping"]) == ("CMC_Matrix", "module") + assert typ(result, "MATRIX")["inherits"] == ["*"] + + def test_builtin_statements_are_read_from_the_parser(tmp_path): parser = tmp_path / "core/src/pl/genschu/bloomooemulator/interpreter/parser" parser.mkdir(parents=True)