feat(extractor): extract scene modules (SCENE:DLLS=) like engines

A module DLL has no engine factory: its CMC_ExternObject::resolve compares
TYPE (CXString::equals or operator==), allocates and constructs. The
runner of each type is the vftable its constructor or a virtual installs
whose prepare slot chains to CMC_Runner::prepareMthHashSet (defined when
analysis left it out); registrations, dispatch, signatures, per-type
events and fields then come from the engine passes, limited to the
module's own runners. Snapshots record module_of (the host engine DLL).

An unoptimised run() keeps the switch index in a stack slot; its bounds
are followed through that slot only when the register match finds none.
The batch queue extracts plugins too, the inventory knows MATRIX, QMAN,
INERTIA and BOARD, and the tracker marks module-mapped types.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Patryk Gensch
2026-10-28 20:58:40 +01:00
co-authored by Claude Opus 5.5
parent f71cd9a90f
commit 4e65268a1e
7 changed files with 301 additions and 49 deletions
+6 -2
View File
@@ -1,4 +1,4 @@
"""Resumable, SHA-addressed engine extraction queue (POSIX)."""
"""Resumable, SHA-addressed extraction queue for engines and scene modules (POSIX)."""
from __future__ import annotations
import argparse
@@ -115,6 +115,10 @@ def execute_worker(source, output, log, timeout, env):
raise RuntimeError('Extractor exited with code ' + str(code))
# Engines and the scene modules (SCENE:DLLS=) that add script types; the extractor handles both.
EXTRACTED_SCOPES = ('engine', 'plugin')
def run_queue(manifest, out_dir, *, ghidra_home=None, timeout=1800, limit=None,
retry_failed=False, worker=execute_worker):
if timeout <= 0 or (limit is not None and limit < 0):
@@ -146,7 +150,7 @@ def _run(builds, directory, ghidra_home, timeout, limit, retry_failed, worker):
report = {'schema': 'extraction-batch/1', 'attempted': 0, 'succeeded': 0,
'pending': 0, 'skipped_success': 0, 'skipped_failed': 0, 'skipped_scope': 0, 'failures': []}
for build in builds:
if build.get('scope') != 'engine':
if build.get('scope') not in EXTRACTED_SCOPES:
report['skipped_scope'] += 1
continue
sha = build['sha256']
+1 -1
View File
@@ -1,7 +1,7 @@
'use strict';
const EVENT_STATUS={emitted:'emitowany w klasie typu',emitted_base:'emitowany w Variable',engine_literal:'literał w kodzie silnika',not_found:'brak literału w emulatorze'};
const FIELD_STATUS={read:'czytane w obsłudze typu',read_base:'czytane w Variable',engine_literal:'literał w innym kodzie',not_found:'brak literału w emulatorze'};
const MAPPING={explicit:'Jawne (klasa C++ z fabryki)',runner_factory:'CMC_RunnerFactory (typ → runner)',class_constructor:'Runner tworzony w konstruktorze klasy',inferred:'Wywnioskowane z nazwy'};
const MAPPING={explicit:'Jawne (klasa C++ z fabryki)',runner_factory:'CMC_RunnerFactory (typ → runner)',module:'Moduł sceny DLLS (typ → runner obiektu)',class_constructor:'Runner tworzony w konstruktorze klasy',inferred:'Wywnioskowane z nazwy'};
const $ = id => document.getElementById(id);
const labels = {missing:'Brak rejestracji',registered:'Zarejestrowana',stub_candidate:'Podejrzenie stubu',partial:'Częściowa',implemented:'Zaimplementowana',unverified:'Niezweryfikowana',asm_reviewed:'Przegląd ASM',runtime_tested:'Test zachowania',stale:'Dowód wymaga aktualizacji'};
let data, selected = null;
+5 -1
View File
@@ -7,6 +7,10 @@ from pathlib import Path
import re
# Scene modules loaded by `SCENE:DLLS=`; each adds script types (QMAN: TEST).
PLUGINS = {"WORLD.DLL", "KOLOROWANKA.DLL", "MATRIX.DLL", "QMAN.DLL", "INERTIA.DLL", "BOARD.DLL"}
def inventory(roots: list[Path], snapshot_dir: Path | None = None) -> dict:
extracted = set()
if snapshot_dir:
@@ -29,7 +33,7 @@ def inventory(roots: list[Path], snapshot_dir: Path | None = None) -> dict:
family, scope = name.removesuffix(".DLL"), "engine"
elif name.startswith("BLOOMOO"):
family, scope = "BLOOMOO", "engine"
elif name in ("WORLD.DLL", "KOLOROWANKA.DLL"):
elif name in PLUGINS:
family, scope = name.removesuffix(".DLL"), "plugin"
elif name == "SEKAI.DLL":
family, scope = "SEKAI", "native_dependency"
+6 -4
View File
@@ -345,11 +345,13 @@ def script_fields(snap: Snapshot, scopes: dict | None = None) -> dict[str, list[
def script_owners(snap: Snapshot, owners) -> dict[str, tuple[str, str]]:
"""Script scope -> (method owner, provenance) for one binary.
`runner_map` (CMC_RunnerFactory, Piklib 7.1+/BlooMoo) is the engine's own dispatch table and
wins: several types can share a runner (INTEGER/STRING/... -> Variable). Without it (Piklib
6.1, plugins, older extractions) the C++ class or its name is the best available guess."""
`runner_map` (CMC_RunnerFactory, Piklib 7.1+/BlooMoo; a scene module's types and the runner
their objects install) is the engine's own dispatch table and wins: several types can share
a runner (INTEGER/STRING/... -> Variable). Without it (Piklib 6.1, older extractions) the C++
class or its name is the best available guess."""
runner_map = snap.raw.get("runner_map") or {}
by_type_id = {e["type_id"]: e["runner"].removesuffix("_Runner") for e in runner_map.get("by_type_id", [])}
dispatched = "module" if runner_map.get("module") else "runner_factory" # a scene module's own resolve
class_runner = {e["class"]: e["runner"].removesuffix("_Runner") for e in runner_map.get("by_class", [])}
bases = {e["class"]: e["base_class"] for e in snap.field_inheritance}
known_classes = set(class_runner) | set(bases) | set(bases.values())
@@ -359,7 +361,7 @@ def script_owners(snap: Snapshot, owners) -> dict[str, tuple[str, str]]:
for t in snap.types:
name = canonical(t["script_name"])
if t.get("type_id") in by_type_id:
mappings[name] = (by_type_id[t["type_id"]], "runner_factory")
mappings[name] = (by_type_id[t["type_id"]], dispatched)
continue
owner, mapping = t.get("cpp_class"), "explicit"
# Known extractor failure: unrelated factory entries all mapped to Behaviour.
+270 -40
View File
@@ -606,12 +606,21 @@ def _parse_switch(program, func):
byte_table = space.getAddress(int(mbt.group(2), 16))
break
base, count = _switch_bounds(instrs, jmp_addr, set([bounds_reg]))
if count is None: # unoptimised build: the index lives in a stack slot
base, count = _switch_bounds(instrs, jmp_addr, _index_aliases(instrs, jmp_addr, bounds_reg))
return {"table": table, "base": base, "count": count, "default": default,
"byte_table": byte_table}
def _switch_bounds(instrs, jmp_addr, operands):
"""(base, count) from the arithmetic and the bounds CMP applied to the index `operands`."""
base = 0
count = None
for instr in instrs:
if instr.getAddress().equals(jmp_addr):
break
if instr.getNumOperands() == 0 or instr.getDefaultOperandRepresentation(0) != bounds_reg:
if instr.getNumOperands() == 0 or instr.getDefaultOperandRepresentation(0) not in operands:
continue
mn = instr.getMnemonicString()
s = instr.getScalar(1)
@@ -627,8 +636,27 @@ def _parse_switch(program, func):
base = -_s32(s.getValue())
elif mn == "DEC":
base = 1
return {"table": table, "base": base, "count": count, "default": default,
"byte_table": byte_table}
return base, count
def _index_aliases(instrs, jmp_addr, reg):
"""Operands that hold the switch index in an unoptimised build, which round-trips it through
a stack slot: `SUB ECX,1 ; MOV [EBP-0xc],ECX ; CMP [EBP-0xc],0xc3 ; MOV EDX,[EBP-0xc]`."""
aliases = set([reg])
for _ in range(3):
for instr in instrs:
if instr.getAddress().equals(jmp_addr):
break
if instr.getMnemonicString() != "MOV" or instr.getNumOperands() != 2:
continue
dst, src = instr.getDefaultOperandRepresentation(0), instr.getDefaultOperandRepresentation(1)
if dst in aliases and src.startswith("dword ptr ["):
aliases.add(src)
elif dst in aliases and re.match(r"E[A-Z]{2}$", src):
aliases.add(src)
elif src in aliases and dst.startswith("dword ptr ["):
aliases.add(dst)
return aliases
def _name_dispatch_from_run(program, run_func, owner, runner, ids):
@@ -2033,6 +2061,209 @@ def sha256_of(program):
return None
# --------------------------------------------------------------------------- scene modules
# A scene module (`SCENE:DLLS=`: World.dll, Matrix.dll, ...) adds script types through
# CMC_ExternObject::resolve, which compares TYPE with CXString::equals or operator==.
_MODULE_COMPARES = (OP_EQ, "equals", "equalsIgnoreCase")
# Engine SDK classes a module links in statically; their runners are not the module's.
_SDK_PREFIXES = ("CMC_", "CX")
def _is_sdk(func):
ns = func.getParentNamespace()
return ns is not None and ns.getName().startswith(_SDK_PREFIXES)
def _calls(program, func):
"""Names of the functions `func` calls directly."""
names = set()
it = program.getListing().getInstructions(func.getBody(), True)
while it.hasNext():
instr = it.next()
if instr.getMnemonicString() == "CALL":
cf = call_target(program, instr)
if cf is not None:
names.add(cf.getName())
return names
def host_library(program):
"""The engine DLL a scene module imports (PIKLIB8.DLL, BlooMooDLL.dll, ...), or None."""
for lib in program.getExternalManager().getExternalLibraryNames():
if lib.upper().startswith(("PIKLIB", "BLOOMOO")):
return lib
return None
def extract_module_types(program, factory):
"""The types one CMC_ExternObject::resolve builds: per branch the TYPE literal, a compare,
operator new(SIZE) and the constructor as the next call - the engine's ladder, except that
the literal is copied into a CXString inline, more than LOOKBACK instructions before the
compare, so the latest literal other than the "TYPE" key names the branch."""
types = []
last = pending = size = None
armed = False
recent = []
it = program.getListing().getInstructions(factory.getBody(), True)
while it.hasNext():
instr = it.next()
s = _imm_string(program, instr)
if s is not None and s != "TYPE" and _is_method_name(s):
last = s
if instr.getMnemonicString() == "CALL":
f = call_target(program, instr)
name = f.getName() if f is not None else None
if name in _MODULE_COMPARES:
pending = last
elif name == OP_NEW:
size = lookback_scalar(recent)
armed = pending is not None
elif armed and f is not None:
ctor = resolve_thunk(f)
types.append({"script_name": pending,
"cpp_class": f.getName() if f.getName().startswith(CMC_PREFIX) else None,
"ctor_addr": _hex(ctor.getEntryPoint()), "object_size": size,
"dispatch_addr": _hex(instr.getAddress()), "via_module_iface": True})
pending = size = None
armed = False
recent.append(instr)
if len(recent) > LOOKBACK:
recent.pop(0)
return types
def find_module_factories(program):
"""[(resolve, types)] of a scene module: functions outside the linked-in SDK that compare,
allocate and construct a CMC_Object-based type (its constructor hands a type id to the base)."""
out = []
it = program.getFunctionManager().getFunctions(True)
while it.hasNext():
f = it.next()
if f.isThunk() or _is_sdk(f):
continue
names = _calls(program, f)
if OP_NEW not in names or not names.intersection(_MODULE_COMPARES):
continue
types = [t for t in extract_module_types(program, f) if _ctor_type_id(program, t["ctor_addr"]) is not None]
if types:
out.append((f, types))
return out
def _vtable_functions(program, table):
funcs = []
for i in range(_MAX_VTABLE_SLOTS if table is not None else 0):
target = _read_ptr(program, table.add(i * 4))
if target is None or not _executable(program, target):
break
f = _ensure_function(program, target)
if f is not None:
funcs.append(f)
return funcs
def _installed_tables(program, funcs, depth=3):
"""Offsets of every vftable `funcs` and their callees (`depth` levels) install."""
tables = set()
seen = set()
frontier = list(funcs)
for _ in range(depth + 1):
nxt = []
for f in frontier:
body = resolve_thunk(f)
if body.isExternal() or body.getEntryPoint().getOffset() in seen:
continue
seen.add(body.getEntryPoint().getOffset())
it = program.getListing().getInstructions(body.getBody(), True)
while it.hasNext():
instr = it.next()
table = _vftable_store(program, instr)
if table is not None:
tables.add(table.getOffset())
elif instr.getMnemonicString() == "CALL":
cf = call_target(program, instr)
if cf is not None:
nxt.append(cf)
frontier = nxt
return tables
def _module_runner(program, slots, tables):
"""(vftable, prepare) of the runner among the vftables a type installs: the table whose
prepare slot chains to the engine's CMC_Runner::prepareMthHashSet, preferring a module
function over a linked-in SDK runner. Analysis often leaves that prepare undefined
(referenced from the vtable only), so it is defined here."""
found = []
for off in sorted(tables):
table = _addr(program, off)
prepare = _ensure_function(program, _read_ptr(program, table.add(slots["prepareMthHashSet"])))
if prepare is not None and not prepare.isThunk() and "prepareMthHashSet" in _calls(program, prepare):
found.append((table, prepare))
if len(found) > 1: # a linked-in SDK runner as well; Inertia names its own CMC_Inertia_Runner
found = [f for f in found if not _is_sdk(f[1])] or found
return found[0] if len(found) == 1 else None
def extract_module_runner_map(program, types):
"""Runner of each module type, in `extract_runner_map`'s shape (marked `module`). The engine
asks the object for its runner, so a type's runner is the runner vftable its constructor or
one of its virtuals installs (Matrix and Board create it lazily). Annotates `types` with
`type_id`. Runner classes have no symbol; they are named after the type (CMC_Matrix_Runner)."""
slots = _runner_slots(program)
if "prepareMthHashSet" not in slots or "run" not in slots:
return None
runners = {}
by_type_id = []
dec = _Decompiler(program)
try:
for t in types:
t["type_id"] = _ctor_type_id(program, t["ctor_addr"])
ctor = _function_at(program, t["ctor_addr"])
reach = [ctor] + _vtable_functions(program, _object_vftable(program, dec, ctor))
found = _module_runner(program, slots, _installed_tables(program, reach))
if found is None:
continue
table, prepare = found
if table.getOffset() not in runners:
cls = _vftable_class(program, table)
run_ = _ensure_function(program, _read_ptr(program, table.add(slots["run"])))
runners[table.getOffset()] = {
"runner": cls or "CMC_%s_Runner" % t["script_name"].capitalize(), "global": None,
"class": cls, "vftable": _hex(table), "prepare_addr": _hex(prepare.getEntryPoint()),
"run_addr": _hex(run_.getEntryPoint()) if run_ else None}
by_type_id.append({"type_id": t["type_id"], "runner": runners[table.getOffset()]["runner"],
"global": None})
finally:
dec.dispose()
return {"module": True, "slots": dict((k, "0x%x" % v) for k, v in slots.items()),
"runners": sorted(runners.values(), key=lambda r: r["runner"]), "by_type_id": by_type_id}
def module_surface(program, factories):
"""Snapshot sections of a scene module. Only the module's own runners and types count: the
engine classes it links in statically are the engine's surface, extracted from the engine."""
types = [t for _, found in factories for t in found]
runner_map = extract_module_runner_map(program, types)
methods, inheritance, dispatch, run_addrs = [], [], [], {}
for r in (runner_map or {}).get("runners", []):
own, runner, base = _extract_methods_from(program, _function_at(program, r["prepare_addr"]), r["runner"])
methods.extend(own)
if base is not None:
inheritance.append({"runner": runner, "base_runner": base})
run_func = _function_at(program, r["run_addr"])
if run_func is not None:
run_addrs[r["runner"]] = r["run_addr"]
dispatch.extend(_dispatch_from_run(program, run_func, _owner_from_runner(r["runner"]), r["runner"]))
if runner_map:
annotate_method_facts(program, dispatch, run_addrs, runner_map, types)
annotate_type_vftables(program, types, dispatch)
return {"types": types, "methods": methods, "method_inheritance": inheritance, "events": [],
"fields": [], "field_inheritance": [], "struct_layout": [], "method_dispatch": dispatch,
"runner_map": runner_map, "type_events": extract_type_events(program, types),
"type_fields": extract_type_fields(program, types)}
# --------------------------------------------------------------------------- main
def find_factory(program):
@@ -2049,16 +2280,8 @@ def find_factory(program):
return None
def run():
program = currentProgram # GhidraScript/pyghidra inject this global, not `program`
factory = find_factory(program)
if factory is None:
print("[!] factory not found (CMC_ObjectsContainer::resolve / CMC_Scene::resolve)"
" - is this a Piklib/BlooMoo DLL with text-script support? (added in Piklib 6.1)")
return
engine, compiler = detect_engine(factory) # namespace lives on the symbol/stub
factory = resolve_thunk(factory) # MSVC8 exposes the symbol as a JMP stub
def engine_surface(program, factory):
"""Snapshot sections of an engine DLL, from its type-dispatch factory."""
types = extract_types(program, factory)
runner_map = extract_runner_map(program, types) or extract_class_runners(program)
methods, method_inheritance = extract_methods(program, runner_map)
@@ -2069,30 +2292,36 @@ def run():
if runner_map:
annotate_method_facts(program, method_dispatch, run_addrs, runner_map, types)
annotate_type_vftables(program, types, method_dispatch)
type_events = extract_type_events(program, types)
type_fields = extract_type_fields(program, types)
return {"types": types, "methods": methods, "method_inheritance": method_inheritance,
"events": events, "fields": fields, "field_inheritance": field_inheritance,
"struct_layout": struct_layout, "method_dispatch": method_dispatch, "runner_map": runner_map,
"type_events": extract_type_events(program, types),
"type_fields": extract_type_fields(program, types)}
snapshot = {
"schema_version": 7,
"binary": {
"name": program.getName(),
"sha256": sha256_of(program),
"engine": engine,
"compiler": compiler,
"factory_addr": "0x%x" % factory.getEntryPoint().getOffset(),
},
"types": types,
"methods": methods,
"method_inheritance": method_inheritance,
"events": events,
"fields": fields,
"field_inheritance": field_inheritance,
"struct_layout": struct_layout,
"method_dispatch": method_dispatch,
"runner_map": runner_map,
"type_events": type_events,
"type_fields": type_fields,
}
def run():
program = currentProgram # GhidraScript/pyghidra inject this global, not `program`
factory = find_factory(program)
if factory is not None:
engine, compiler = detect_engine(factory) # namespace lives on the symbol/stub
factory = resolve_thunk(factory) # MSVC8 exposes the symbol as a JMP stub
surface = engine_surface(program, factory)
binary = {"factory_addr": _hex(factory.getEntryPoint())}
else:
modules = find_module_factories(program)
host = host_library(program)
if not modules or host is None:
print("[!] factory not found (CMC_ObjectsContainer::resolve / CMC_Scene::resolve, nor a"
" scene module's CMC_ExternObject::resolve) - is this a Piklib/BlooMoo DLL with"
" text-script support? (added in Piklib 6.1)")
return
engine, compiler = ("BlooMoo", "MSVC8") if host.upper().startswith("BLOOMOO") else ("Piklib", "MSVC6")
surface = module_surface(program, modules)
binary = {"factory_addr": _hex(modules[0][0].getEntryPoint()), "module_of": host}
binary.update({"name": program.getName(), "sha256": sha256_of(program), "engine": engine, "compiler": compiler})
snapshot = {"schema_version": 7, "binary": binary}
snapshot.update(surface)
args = getScriptArgs()
out_path = args[0] if len(args) > 0 else default_out_path(program)
@@ -2102,10 +2331,11 @@ def run():
finally:
fh.close()
print("[+] %s [%s/%s]: %d types, %d methods, %d events (%d per type), %d fields (%d per type, %d layout, %d dispatch, %d runners) -> %s" % (
program.getName(), engine, compiler, len(types), len(methods),
len(events), len(type_events), len(fields), len(type_fields), len(struct_layout), len(method_dispatch),
len((runner_map or {}).get("runners", [])), out_path))
print("[+] %s [%s/%s%s]: %d types, %d methods, %d events (%d per type), %d fields (%d per type, %d layout, %d dispatch, %d runners) -> %s" % (
program.getName(), engine, compiler, " module of " + binary["module_of"] if "module_of" in binary else "",
len(snapshot["types"]), len(snapshot["methods"]), len(snapshot["events"]), len(snapshot["type_events"]),
len(snapshot["fields"]), len(snapshot["type_fields"]), len(snapshot["struct_layout"]),
len(snapshot["method_dispatch"]), len((snapshot["runner_map"] or {}).get("runners", [])), out_path))
run()
+1 -1
View File
@@ -77,7 +77,7 @@ def test_lock_and_scope_limit(setup):
with pytest.raises(RuntimeError, match='Another'):
batch.run_queue(manifest, out, worker=success)
data = json.loads(manifest.read_text())
data['builds'][0]['scope'] = 'plugin'
data['builds'][0]['scope'] = 'native_dependency'
manifest.write_text(json.dumps(data))
assert batch.run_queue(manifest, out, worker=success)['skipped_scope'] == 1
+12
View File
@@ -122,6 +122,18 @@ def test_runner_factory_maps_shared_and_unnamed_runners(tmp_path):
assert "inherits" not in typ(result, "@") # built-ins do not chain to CMC_Runner
def test_scene_module_types_map_through_the_runner_their_objects_install(tmp_path):
module = Snapshot({"binary": {"name": "Matrix.dll", "sha256": "b" * 64, "module_of": "PIKLIB8.DLL"},
"types": [{"script_name": "MATRIX", "cpp_class": None, "type_id": 10002}],
"methods": [{"owner": "CMC_Matrix", "runner": "CMC_Matrix_Runner", "name": "TICK", "id": 1}],
"method_inheritance": [{"runner": "CMC_Matrix_Runner", "base_runner": "CMC_Runner"}],
"runner_map": {"module": True, "by_type_id": [{"type_id": 10002, "runner": "CMC_Matrix_Runner"}]}})
result = build_tracker([factory_snapshot(), module], emulator(tmp_path))
tick = row(result, "MATRIX", "TICK")["original"][0]
assert (tick["owner"], tick["mapping"]) == ("CMC_Matrix", "module")
assert typ(result, "MATRIX")["inherits"] == ["*"]
def test_builtin_statements_are_read_from_the_parser(tmp_path):
parser = tmp_path / "core/src/pl/genschu/bloomooemulator/interpreter/parser"
parser.mkdir(parents=True)