diff --git a/ams/acquire/batch.py b/ams/acquire/batch.py index afcb1d3..ee64aec 100644 --- a/ams/acquire/batch.py +++ b/ams/acquire/batch.py @@ -1,4 +1,4 @@ -"""Resumable, SHA-addressed engine extraction queue (POSIX).""" +"""Resumable, SHA-addressed extraction queue for engines and scene modules (POSIX).""" from __future__ import annotations import argparse @@ -115,6 +115,10 @@ def execute_worker(source, output, log, timeout, env): raise RuntimeError('Extractor exited with code ' + str(code)) +# Engines and the scene modules (SCENE:DLLS=) that add script types; the extractor handles both. +EXTRACTED_SCOPES = ('engine', 'plugin') + + def run_queue(manifest, out_dir, *, ghidra_home=None, timeout=1800, limit=None, retry_failed=False, worker=execute_worker): if timeout <= 0 or (limit is not None and limit < 0): @@ -146,7 +150,7 @@ def _run(builds, directory, ghidra_home, timeout, limit, retry_failed, worker): report = {'schema': 'extraction-batch/1', 'attempted': 0, 'succeeded': 0, 'pending': 0, 'skipped_success': 0, 'skipped_failed': 0, 'skipped_scope': 0, 'failures': []} for build in builds: - if build.get('scope') != 'engine': + if build.get('scope') not in EXTRACTED_SCOPES: report['skipped_scope'] += 1 continue sha = build['sha256'] diff --git a/ams/api/static/tracker.js b/ams/api/static/tracker.js index 95c1add..b930814 100644 --- a/ams/api/static/tracker.js +++ b/ams/api/static/tracker.js @@ -1,7 +1,7 @@ 'use strict'; const EVENT_STATUS={emitted:'emitowany w klasie typu',emitted_base:'emitowany w Variable',engine_literal:'literał w kodzie silnika',not_found:'brak literału w emulatorze'}; const FIELD_STATUS={read:'czytane w obsłudze typu',read_base:'czytane w Variable',engine_literal:'literał w innym kodzie',not_found:'brak literału w emulatorze'}; -const MAPPING={explicit:'Jawne (klasa C++ z fabryki)',runner_factory:'CMC_RunnerFactory (typ → runner)',class_constructor:'Runner tworzony w konstruktorze klasy',inferred:'Wywnioskowane z nazwy'}; +const MAPPING={explicit:'Jawne (klasa C++ z fabryki)',runner_factory:'CMC_RunnerFactory (typ → runner)',module:'Moduł sceny DLLS (typ → runner obiektu)',class_constructor:'Runner tworzony w konstruktorze klasy',inferred:'Wywnioskowane z nazwy'}; const $ = id => document.getElementById(id); const labels = {missing:'Brak rejestracji',registered:'Zarejestrowana',stub_candidate:'Podejrzenie stubu',partial:'Częściowa',implemented:'Zaimplementowana',unverified:'Niezweryfikowana',asm_reviewed:'Przegląd ASM',runtime_tested:'Test zachowania',stale:'Dowód wymaga aktualizacji'}; let data, selected = null; diff --git a/ams/inventory.py b/ams/inventory.py index b26d46b..13e9df4 100644 --- a/ams/inventory.py +++ b/ams/inventory.py @@ -7,6 +7,10 @@ from pathlib import Path import re +# Scene modules loaded by `SCENE:DLLS=`; each adds script types (QMAN: TEST). +PLUGINS = {"WORLD.DLL", "KOLOROWANKA.DLL", "MATRIX.DLL", "QMAN.DLL", "INERTIA.DLL", "BOARD.DLL"} + + def inventory(roots: list[Path], snapshot_dir: Path | None = None) -> dict: extracted = set() if snapshot_dir: @@ -29,7 +33,7 @@ def inventory(roots: list[Path], snapshot_dir: Path | None = None) -> dict: family, scope = name.removesuffix(".DLL"), "engine" elif name.startswith("BLOOMOO"): family, scope = "BLOOMOO", "engine" - elif name in ("WORLD.DLL", "KOLOROWANKA.DLL"): + elif name in PLUGINS: family, scope = name.removesuffix(".DLL"), "plugin" elif name == "SEKAI.DLL": family, scope = "SEKAI", "native_dependency" diff --git a/ams/tracker.py b/ams/tracker.py index bbdd335..9ab6f1f 100644 --- a/ams/tracker.py +++ b/ams/tracker.py @@ -345,11 +345,13 @@ def script_fields(snap: Snapshot, scopes: dict | None = None) -> dict[str, list[ def script_owners(snap: Snapshot, owners) -> dict[str, tuple[str, str]]: """Script scope -> (method owner, provenance) for one binary. - `runner_map` (CMC_RunnerFactory, Piklib 7.1+/BlooMoo) is the engine's own dispatch table and - wins: several types can share a runner (INTEGER/STRING/... -> Variable). Without it (Piklib - 6.1, plugins, older extractions) the C++ class or its name is the best available guess.""" + `runner_map` (CMC_RunnerFactory, Piklib 7.1+/BlooMoo; a scene module's types and the runner + their objects install) is the engine's own dispatch table and wins: several types can share + a runner (INTEGER/STRING/... -> Variable). Without it (Piklib 6.1, older extractions) the C++ + class or its name is the best available guess.""" runner_map = snap.raw.get("runner_map") or {} by_type_id = {e["type_id"]: e["runner"].removesuffix("_Runner") for e in runner_map.get("by_type_id", [])} + dispatched = "module" if runner_map.get("module") else "runner_factory" # a scene module's own resolve class_runner = {e["class"]: e["runner"].removesuffix("_Runner") for e in runner_map.get("by_class", [])} bases = {e["class"]: e["base_class"] for e in snap.field_inheritance} known_classes = set(class_runner) | set(bases) | set(bases.values()) @@ -359,7 +361,7 @@ def script_owners(snap: Snapshot, owners) -> dict[str, tuple[str, str]]: for t in snap.types: name = canonical(t["script_name"]) if t.get("type_id") in by_type_id: - mappings[name] = (by_type_id[t["type_id"]], "runner_factory") + mappings[name] = (by_type_id[t["type_id"]], dispatched) continue owner, mapping = t.get("cpp_class"), "explicit" # Known extractor failure: unrelated factory entries all mapped to Behaviour. diff --git a/ghidra_scripts/extract_engine_surface.py b/ghidra_scripts/extract_engine_surface.py index c0e04a2..1b7a927 100644 --- a/ghidra_scripts/extract_engine_surface.py +++ b/ghidra_scripts/extract_engine_surface.py @@ -606,12 +606,21 @@ def _parse_switch(program, func): byte_table = space.getAddress(int(mbt.group(2), 16)) break + base, count = _switch_bounds(instrs, jmp_addr, set([bounds_reg])) + if count is None: # unoptimised build: the index lives in a stack slot + base, count = _switch_bounds(instrs, jmp_addr, _index_aliases(instrs, jmp_addr, bounds_reg)) + return {"table": table, "base": base, "count": count, "default": default, + "byte_table": byte_table} + + +def _switch_bounds(instrs, jmp_addr, operands): + """(base, count) from the arithmetic and the bounds CMP applied to the index `operands`.""" base = 0 count = None for instr in instrs: if instr.getAddress().equals(jmp_addr): break - if instr.getNumOperands() == 0 or instr.getDefaultOperandRepresentation(0) != bounds_reg: + if instr.getNumOperands() == 0 or instr.getDefaultOperandRepresentation(0) not in operands: continue mn = instr.getMnemonicString() s = instr.getScalar(1) @@ -627,8 +636,27 @@ def _parse_switch(program, func): base = -_s32(s.getValue()) elif mn == "DEC": base = 1 - return {"table": table, "base": base, "count": count, "default": default, - "byte_table": byte_table} + return base, count + + +def _index_aliases(instrs, jmp_addr, reg): + """Operands that hold the switch index in an unoptimised build, which round-trips it through + a stack slot: `SUB ECX,1 ; MOV [EBP-0xc],ECX ; CMP [EBP-0xc],0xc3 ; MOV EDX,[EBP-0xc]`.""" + aliases = set([reg]) + for _ in range(3): + for instr in instrs: + if instr.getAddress().equals(jmp_addr): + break + if instr.getMnemonicString() != "MOV" or instr.getNumOperands() != 2: + continue + dst, src = instr.getDefaultOperandRepresentation(0), instr.getDefaultOperandRepresentation(1) + if dst in aliases and src.startswith("dword ptr ["): + aliases.add(src) + elif dst in aliases and re.match(r"E[A-Z]{2}$", src): + aliases.add(src) + elif src in aliases and dst.startswith("dword ptr ["): + aliases.add(dst) + return aliases def _name_dispatch_from_run(program, run_func, owner, runner, ids): @@ -2033,6 +2061,209 @@ def sha256_of(program): return None +# --------------------------------------------------------------------------- scene modules + +# A scene module (`SCENE:DLLS=`: World.dll, Matrix.dll, ...) adds script types through +# CMC_ExternObject::resolve, which compares TYPE with CXString::equals or operator==. +_MODULE_COMPARES = (OP_EQ, "equals", "equalsIgnoreCase") +# Engine SDK classes a module links in statically; their runners are not the module's. +_SDK_PREFIXES = ("CMC_", "CX") + + +def _is_sdk(func): + ns = func.getParentNamespace() + return ns is not None and ns.getName().startswith(_SDK_PREFIXES) + + +def _calls(program, func): + """Names of the functions `func` calls directly.""" + names = set() + it = program.getListing().getInstructions(func.getBody(), True) + while it.hasNext(): + instr = it.next() + if instr.getMnemonicString() == "CALL": + cf = call_target(program, instr) + if cf is not None: + names.add(cf.getName()) + return names + + +def host_library(program): + """The engine DLL a scene module imports (PIKLIB8.DLL, BlooMooDLL.dll, ...), or None.""" + for lib in program.getExternalManager().getExternalLibraryNames(): + if lib.upper().startswith(("PIKLIB", "BLOOMOO")): + return lib + return None + + +def extract_module_types(program, factory): + """The types one CMC_ExternObject::resolve builds: per branch the TYPE literal, a compare, + operator new(SIZE) and the constructor as the next call - the engine's ladder, except that + the literal is copied into a CXString inline, more than LOOKBACK instructions before the + compare, so the latest literal other than the "TYPE" key names the branch.""" + types = [] + last = pending = size = None + armed = False + recent = [] + it = program.getListing().getInstructions(factory.getBody(), True) + while it.hasNext(): + instr = it.next() + s = _imm_string(program, instr) + if s is not None and s != "TYPE" and _is_method_name(s): + last = s + if instr.getMnemonicString() == "CALL": + f = call_target(program, instr) + name = f.getName() if f is not None else None + if name in _MODULE_COMPARES: + pending = last + elif name == OP_NEW: + size = lookback_scalar(recent) + armed = pending is not None + elif armed and f is not None: + ctor = resolve_thunk(f) + types.append({"script_name": pending, + "cpp_class": f.getName() if f.getName().startswith(CMC_PREFIX) else None, + "ctor_addr": _hex(ctor.getEntryPoint()), "object_size": size, + "dispatch_addr": _hex(instr.getAddress()), "via_module_iface": True}) + pending = size = None + armed = False + recent.append(instr) + if len(recent) > LOOKBACK: + recent.pop(0) + return types + + +def find_module_factories(program): + """[(resolve, types)] of a scene module: functions outside the linked-in SDK that compare, + allocate and construct a CMC_Object-based type (its constructor hands a type id to the base).""" + out = [] + it = program.getFunctionManager().getFunctions(True) + while it.hasNext(): + f = it.next() + if f.isThunk() or _is_sdk(f): + continue + names = _calls(program, f) + if OP_NEW not in names or not names.intersection(_MODULE_COMPARES): + continue + types = [t for t in extract_module_types(program, f) if _ctor_type_id(program, t["ctor_addr"]) is not None] + if types: + out.append((f, types)) + return out + + +def _vtable_functions(program, table): + funcs = [] + for i in range(_MAX_VTABLE_SLOTS if table is not None else 0): + target = _read_ptr(program, table.add(i * 4)) + if target is None or not _executable(program, target): + break + f = _ensure_function(program, target) + if f is not None: + funcs.append(f) + return funcs + + +def _installed_tables(program, funcs, depth=3): + """Offsets of every vftable `funcs` and their callees (`depth` levels) install.""" + tables = set() + seen = set() + frontier = list(funcs) + for _ in range(depth + 1): + nxt = [] + for f in frontier: + body = resolve_thunk(f) + if body.isExternal() or body.getEntryPoint().getOffset() in seen: + continue + seen.add(body.getEntryPoint().getOffset()) + it = program.getListing().getInstructions(body.getBody(), True) + while it.hasNext(): + instr = it.next() + table = _vftable_store(program, instr) + if table is not None: + tables.add(table.getOffset()) + elif instr.getMnemonicString() == "CALL": + cf = call_target(program, instr) + if cf is not None: + nxt.append(cf) + frontier = nxt + return tables + + +def _module_runner(program, slots, tables): + """(vftable, prepare) of the runner among the vftables a type installs: the table whose + prepare slot chains to the engine's CMC_Runner::prepareMthHashSet, preferring a module + function over a linked-in SDK runner. Analysis often leaves that prepare undefined + (referenced from the vtable only), so it is defined here.""" + found = [] + for off in sorted(tables): + table = _addr(program, off) + prepare = _ensure_function(program, _read_ptr(program, table.add(slots["prepareMthHashSet"]))) + if prepare is not None and not prepare.isThunk() and "prepareMthHashSet" in _calls(program, prepare): + found.append((table, prepare)) + if len(found) > 1: # a linked-in SDK runner as well; Inertia names its own CMC_Inertia_Runner + found = [f for f in found if not _is_sdk(f[1])] or found + return found[0] if len(found) == 1 else None + + +def extract_module_runner_map(program, types): + """Runner of each module type, in `extract_runner_map`'s shape (marked `module`). The engine + asks the object for its runner, so a type's runner is the runner vftable its constructor or + one of its virtuals installs (Matrix and Board create it lazily). Annotates `types` with + `type_id`. Runner classes have no symbol; they are named after the type (CMC_Matrix_Runner).""" + slots = _runner_slots(program) + if "prepareMthHashSet" not in slots or "run" not in slots: + return None + runners = {} + by_type_id = [] + dec = _Decompiler(program) + try: + for t in types: + t["type_id"] = _ctor_type_id(program, t["ctor_addr"]) + ctor = _function_at(program, t["ctor_addr"]) + reach = [ctor] + _vtable_functions(program, _object_vftable(program, dec, ctor)) + found = _module_runner(program, slots, _installed_tables(program, reach)) + if found is None: + continue + table, prepare = found + if table.getOffset() not in runners: + cls = _vftable_class(program, table) + run_ = _ensure_function(program, _read_ptr(program, table.add(slots["run"]))) + runners[table.getOffset()] = { + "runner": cls or "CMC_%s_Runner" % t["script_name"].capitalize(), "global": None, + "class": cls, "vftable": _hex(table), "prepare_addr": _hex(prepare.getEntryPoint()), + "run_addr": _hex(run_.getEntryPoint()) if run_ else None} + by_type_id.append({"type_id": t["type_id"], "runner": runners[table.getOffset()]["runner"], + "global": None}) + finally: + dec.dispose() + return {"module": True, "slots": dict((k, "0x%x" % v) for k, v in slots.items()), + "runners": sorted(runners.values(), key=lambda r: r["runner"]), "by_type_id": by_type_id} + + +def module_surface(program, factories): + """Snapshot sections of a scene module. Only the module's own runners and types count: the + engine classes it links in statically are the engine's surface, extracted from the engine.""" + types = [t for _, found in factories for t in found] + runner_map = extract_module_runner_map(program, types) + methods, inheritance, dispatch, run_addrs = [], [], [], {} + for r in (runner_map or {}).get("runners", []): + own, runner, base = _extract_methods_from(program, _function_at(program, r["prepare_addr"]), r["runner"]) + methods.extend(own) + if base is not None: + inheritance.append({"runner": runner, "base_runner": base}) + run_func = _function_at(program, r["run_addr"]) + if run_func is not None: + run_addrs[r["runner"]] = r["run_addr"] + dispatch.extend(_dispatch_from_run(program, run_func, _owner_from_runner(r["runner"]), r["runner"])) + if runner_map: + annotate_method_facts(program, dispatch, run_addrs, runner_map, types) + annotate_type_vftables(program, types, dispatch) + return {"types": types, "methods": methods, "method_inheritance": inheritance, "events": [], + "fields": [], "field_inheritance": [], "struct_layout": [], "method_dispatch": dispatch, + "runner_map": runner_map, "type_events": extract_type_events(program, types), + "type_fields": extract_type_fields(program, types)} + + # --------------------------------------------------------------------------- main def find_factory(program): @@ -2049,16 +2280,8 @@ def find_factory(program): return None -def run(): - program = currentProgram # GhidraScript/pyghidra inject this global, not `program` - factory = find_factory(program) - if factory is None: - print("[!] factory not found (CMC_ObjectsContainer::resolve / CMC_Scene::resolve)" - " - is this a Piklib/BlooMoo DLL with text-script support? (added in Piklib 6.1)") - return - - engine, compiler = detect_engine(factory) # namespace lives on the symbol/stub - factory = resolve_thunk(factory) # MSVC8 exposes the symbol as a JMP stub +def engine_surface(program, factory): + """Snapshot sections of an engine DLL, from its type-dispatch factory.""" types = extract_types(program, factory) runner_map = extract_runner_map(program, types) or extract_class_runners(program) methods, method_inheritance = extract_methods(program, runner_map) @@ -2069,30 +2292,36 @@ def run(): if runner_map: annotate_method_facts(program, method_dispatch, run_addrs, runner_map, types) annotate_type_vftables(program, types, method_dispatch) - type_events = extract_type_events(program, types) - type_fields = extract_type_fields(program, types) + return {"types": types, "methods": methods, "method_inheritance": method_inheritance, + "events": events, "fields": fields, "field_inheritance": field_inheritance, + "struct_layout": struct_layout, "method_dispatch": method_dispatch, "runner_map": runner_map, + "type_events": extract_type_events(program, types), + "type_fields": extract_type_fields(program, types)} - snapshot = { - "schema_version": 7, - "binary": { - "name": program.getName(), - "sha256": sha256_of(program), - "engine": engine, - "compiler": compiler, - "factory_addr": "0x%x" % factory.getEntryPoint().getOffset(), - }, - "types": types, - "methods": methods, - "method_inheritance": method_inheritance, - "events": events, - "fields": fields, - "field_inheritance": field_inheritance, - "struct_layout": struct_layout, - "method_dispatch": method_dispatch, - "runner_map": runner_map, - "type_events": type_events, - "type_fields": type_fields, - } + +def run(): + program = currentProgram # GhidraScript/pyghidra inject this global, not `program` + factory = find_factory(program) + if factory is not None: + engine, compiler = detect_engine(factory) # namespace lives on the symbol/stub + factory = resolve_thunk(factory) # MSVC8 exposes the symbol as a JMP stub + surface = engine_surface(program, factory) + binary = {"factory_addr": _hex(factory.getEntryPoint())} + else: + modules = find_module_factories(program) + host = host_library(program) + if not modules or host is None: + print("[!] factory not found (CMC_ObjectsContainer::resolve / CMC_Scene::resolve, nor a" + " scene module's CMC_ExternObject::resolve) - is this a Piklib/BlooMoo DLL with" + " text-script support? (added in Piklib 6.1)") + return + engine, compiler = ("BlooMoo", "MSVC8") if host.upper().startswith("BLOOMOO") else ("Piklib", "MSVC6") + surface = module_surface(program, modules) + binary = {"factory_addr": _hex(modules[0][0].getEntryPoint()), "module_of": host} + + binary.update({"name": program.getName(), "sha256": sha256_of(program), "engine": engine, "compiler": compiler}) + snapshot = {"schema_version": 7, "binary": binary} + snapshot.update(surface) args = getScriptArgs() out_path = args[0] if len(args) > 0 else default_out_path(program) @@ -2102,10 +2331,11 @@ def run(): finally: fh.close() - print("[+] %s [%s/%s]: %d types, %d methods, %d events (%d per type), %d fields (%d per type, %d layout, %d dispatch, %d runners) -> %s" % ( - program.getName(), engine, compiler, len(types), len(methods), - len(events), len(type_events), len(fields), len(type_fields), len(struct_layout), len(method_dispatch), - len((runner_map or {}).get("runners", [])), out_path)) + print("[+] %s [%s/%s%s]: %d types, %d methods, %d events (%d per type), %d fields (%d per type, %d layout, %d dispatch, %d runners) -> %s" % ( + program.getName(), engine, compiler, " module of " + binary["module_of"] if "module_of" in binary else "", + len(snapshot["types"]), len(snapshot["methods"]), len(snapshot["events"]), len(snapshot["type_events"]), + len(snapshot["fields"]), len(snapshot["type_fields"]), len(snapshot["struct_layout"]), + len(snapshot["method_dispatch"]), len((snapshot["runner_map"] or {}).get("runners", [])), out_path)) run() diff --git a/tests/test_batch.py b/tests/test_batch.py index a2abf22..8810370 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -77,7 +77,7 @@ def test_lock_and_scope_limit(setup): with pytest.raises(RuntimeError, match='Another'): batch.run_queue(manifest, out, worker=success) data = json.loads(manifest.read_text()) - data['builds'][0]['scope'] = 'plugin' + data['builds'][0]['scope'] = 'native_dependency' manifest.write_text(json.dumps(data)) assert batch.run_queue(manifest, out, worker=success)['skipped_scope'] == 1 diff --git a/tests/test_tracker.py b/tests/test_tracker.py index b1e1b5f..38d4ad9 100644 --- a/tests/test_tracker.py +++ b/tests/test_tracker.py @@ -122,6 +122,18 @@ def test_runner_factory_maps_shared_and_unnamed_runners(tmp_path): assert "inherits" not in typ(result, "@") # built-ins do not chain to CMC_Runner +def test_scene_module_types_map_through_the_runner_their_objects_install(tmp_path): + module = Snapshot({"binary": {"name": "Matrix.dll", "sha256": "b" * 64, "module_of": "PIKLIB8.DLL"}, + "types": [{"script_name": "MATRIX", "cpp_class": None, "type_id": 10002}], + "methods": [{"owner": "CMC_Matrix", "runner": "CMC_Matrix_Runner", "name": "TICK", "id": 1}], + "method_inheritance": [{"runner": "CMC_Matrix_Runner", "base_runner": "CMC_Runner"}], + "runner_map": {"module": True, "by_type_id": [{"type_id": 10002, "runner": "CMC_Matrix_Runner"}]}}) + result = build_tracker([factory_snapshot(), module], emulator(tmp_path)) + tick = row(result, "MATRIX", "TICK")["original"][0] + assert (tick["owner"], tick["mapping"]) == ("CMC_Matrix", "module") + assert typ(result, "MATRIX")["inherits"] == ["*"] + + def test_builtin_statements_are_read_from_the_parser(tmp_path): parser = tmp_path / "core/src/pl/genschu/bloomooemulator/interpreter/parser" parser.mkdir(parents=True)