A module DLL has no engine factory: its CMC_ExternObject::resolve compares TYPE (CXString::equals or operator==), allocates and constructs. The runner of each type is the vftable its constructor or a virtual installs whose prepare slot chains to CMC_Runner::prepareMthHashSet (defined when analysis left it out); registrations, dispatch, signatures, per-type events and fields then come from the engine passes, limited to the module's own runners. Snapshots record module_of (the host engine DLL). An unoptimised run() keeps the switch index in a stack slot; its bounds are followed through that slot only when the register match finds none. The batch queue extracts plugins too, the inventory knows MATRIX, QMAN, INERTIA and BOARD, and the tracker marks module-mapped types. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
650 lines
35 KiB
Python
650 lines
35 KiB
Python
"""Evidence-aware script API tracker. Never treats registration as correctness.
|
|
|
|
The Java adapter deliberately recognises the repository's MethodSpec registrations,
|
|
not arbitrary Java. Unknown dispatch and incomplete native extraction stay explicit.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter, defaultdict
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
|
|
from .snapshot import Snapshot
|
|
|
|
ALIASES = {"BOOL": "BOOLEAN", "INT": "INTEGER"}
|
|
# Established factory mappings in the PIKLIB8/BlooMoo reference snapshots. When
|
|
# used to fill an unresolved mapping in another build, provenance stays inferred.
|
|
CLASS_ALIASES = {"CMC_CanvasObserver": "CANVAS_OBSERVER", "CMC_Groupped": "GROUP"}
|
|
ENTRY = re.compile(r'Map\s*\.\s*entry\s*\(\s*"([A-Za-z0-9_]+)"\s*,\s*MethodSpec\s*\.\s*of\s*\(')
|
|
JAVA_TOKENS = re.compile(r'"""[\s\S]*?"""|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'|//[^\n]*|/\*[\s\S]*?\*/')
|
|
SHA = re.compile(r"[0-9a-f]{64}")
|
|
|
|
|
|
def canonical(name: str) -> str:
|
|
name = name.upper()
|
|
return ALIASES.get(name, name)
|
|
|
|
|
|
def digest(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def binary_family(binary: dict) -> str:
|
|
library = binary.get("library", binary.get("name", "?").split("__")[-1]).upper()
|
|
if library.startswith("BLOOMOO"):
|
|
return "BlooMoo"
|
|
return {"PIKLIB61.DLL": "Piklib 6.1", "PIKLIB71.DLL": "Piklib 7.1",
|
|
"PIKLIB72.DLL": "Piklib 7.2", "PIKLIB8.DLL": "Piklib 8"}.get(library, library.removesuffix(".DLL"))
|
|
|
|
|
|
def code_mask(text: str) -> str:
|
|
"""Mask literals/comments without shifting source positions or line numbers."""
|
|
return JAVA_TOKENS.sub(lambda m: re.sub(r"[^\n]", " ", m.group()), text)
|
|
|
|
|
|
def closing(code: str, start: int, pair: str = "()") -> int:
|
|
depth = 0
|
|
for i in range(start, len(code)):
|
|
if code[i] == pair[0]:
|
|
depth += 1
|
|
elif code[i] == pair[1]:
|
|
depth -= 1
|
|
if depth == 0:
|
|
return i + 1
|
|
raise ValueError("Unbalanced " + pair)
|
|
|
|
|
|
BUILTINS_SOURCE = "pl/genschu/bloomooemulator/interpreter/parser/CodeParser.java"
|
|
BUILTINS_DISPATCH = re.compile(r"\bdispatchSpecialCall\s*\([^)]*\)\s*\{")
|
|
CASE_ARM = re.compile(r"\bcase\b([^;{}]*?)->|\bdefault\b\s*->")
|
|
|
|
|
|
SIGNAL_LITERAL = re.compile(r'"(ON[A-Z][A-Z_]*)"')
|
|
NOT_SIGNALS = {"ONEBREAK"} # the @ONEBREAK statement, not an event
|
|
|
|
|
|
def scan_signals(root: Path, source_root: Path, variable_root: Path) -> dict:
|
|
"""Where the emulator names each event: {scope: {event: [source]}}. A `<Type>Variable.java`
|
|
file is that type's scope, Variable.java is `*`, any other file is `engine`. Only string
|
|
literals count (comments ignored); names built at runtime are invisible, so absence is
|
|
"no literal found", never proof that an event is missing."""
|
|
scopes = {}
|
|
for path, text in sorted((p, p.read_text(encoding="utf-8")) for p in source_root.rglob("*.java")):
|
|
scope = "engine"
|
|
if path.parent == variable_root and path.name.endswith("Variable.java"):
|
|
if path.name == "Variable.java":
|
|
scope = "*"
|
|
else:
|
|
match = re.search(r"return\s+VariableType\.([A-Z_]+)\s*;", code_mask(text))
|
|
scope = canonical(match[1]) if match else "engine"
|
|
for token in JAVA_TOKENS.finditer(text):
|
|
literal = SIGNAL_LITERAL.fullmatch(token.group())
|
|
if literal and literal[1] not in NOT_SIGNALS:
|
|
scopes.setdefault(scope, {}).setdefault(literal[1], []).append(
|
|
{"path": str(path.relative_to(root)), "line": text.count("\n", 0, token.start()) + 1})
|
|
return scopes
|
|
|
|
|
|
def switch_arms(text: str, code: str, head: re.Pattern) -> list[tuple[list[str], int, int]]:
|
|
"""[(case labels, start, end)] of the first `switch` inside the method `head` matches.
|
|
Positions index `text`; `code` is its literal/comment mask. Default arms are skipped."""
|
|
found = head.search(code)
|
|
if found is None:
|
|
return []
|
|
switch = code.index("switch", found.end())
|
|
body_start = code.index("{", switch)
|
|
body_end = closing(code, body_start, "{}")
|
|
arms = list(CASE_ARM.finditer(code, body_start, body_end))
|
|
out = []
|
|
for arm, following in zip(arms, arms[1:] + [None]):
|
|
if arm.group(1) is not None:
|
|
labels = re.findall(r'"([A-Za-z_]+)"', text[arm.start(1):arm.end(1)])
|
|
out.append((labels, arm.start(), following.start() if following else body_end))
|
|
return out
|
|
|
|
|
|
CNV_SOURCE = "pl/genschu/bloomooemulator/loader/CNVParser.java"
|
|
CNV_CREATE = re.compile(r"\bcreateVariable\s*\([^)]*\)\s*\{")
|
|
PROPERTY_KEY = re.compile(r'":([A-Z][A-Z0-9_]*)"')
|
|
ATTRIBUTE_NAME = re.compile(r'"([A-Z][A-Z0-9_]*)"')
|
|
# context.getAttribute(name, "X"), getAttributeInHierarchy(name(), "X"), attributes().get(name, "X")
|
|
ATTRIBUTE_CALL = re.compile(r"(?:\bgetAttribute\w*|\battributes\(\)\s*\.\s*get\w*)\s*\([^;]*$")
|
|
|
|
|
|
def scan_properties(root: Path, source_root: Path, variable_root: Path) -> dict:
|
|
"""Where the emulator reads each CNV property: {scope: {property: [source]}}. A `":NAME"`
|
|
key inside a `case "TYPE"` arm of CNVParser.createVariable belongs to those types, and
|
|
an attribute read (`getAttribute`, `getAttributeInHierarchy`, `attributes().get`) in
|
|
`<Type>Variable.java` to that type, in Variable.java to `*`; anything else is `engine`.
|
|
Like signals, absence means only that no literal was found."""
|
|
scopes = {}
|
|
for path in sorted(source_root.rglob("*.java")):
|
|
text = path.read_text(encoding="utf-8")
|
|
code = code_mask(text)
|
|
scope = "engine"
|
|
if path.parent == variable_root and path.name == "Variable.java":
|
|
scope = "*"
|
|
elif path.parent == variable_root and path.name.endswith("Variable.java"):
|
|
match = re.search(r"return\s+VariableType\.([A-Z_]+)\s*;", code)
|
|
scope = canonical(match[1]) if match else "engine"
|
|
arms = switch_arms(text, code, CNV_CREATE) if path == source_root / CNV_SOURCE else []
|
|
for token in JAVA_TOKENS.finditer(text):
|
|
literal = PROPERTY_KEY.fullmatch(token.group())
|
|
if literal is None:
|
|
literal = ATTRIBUTE_NAME.fullmatch(token.group())
|
|
if literal is None or not ATTRIBUTE_CALL.search(code[max(0, token.start() - 160):token.start()]):
|
|
continue
|
|
owners = [canonical(label) for labels, start, end in arms if start <= token.start() < end
|
|
for label in labels] or [scope]
|
|
source = {"path": str(path.relative_to(root)), "line": text.count("\n", 0, token.start()) + 1}
|
|
for owner in owners:
|
|
scopes.setdefault(owner, {}).setdefault(literal[1], []).append(source)
|
|
return scopes
|
|
|
|
|
|
def scan_builtins(root: Path, source_root: Path) -> dict | None:
|
|
"""`@IF`, `@LOOP`, ... are parsed by CodeParser.dispatchSpecialCall, not registered as
|
|
MethodSpec: each `case "NAME", ... ->` arm of its switch is one built-in statement."""
|
|
path = source_root / BUILTINS_SOURCE
|
|
if not path.is_file():
|
|
return None
|
|
text = path.read_text(encoding="utf-8")
|
|
code = code_mask(text)
|
|
if BUILTINS_DISPATCH.search(code) is None:
|
|
return None
|
|
methods = {}
|
|
for labels, start, end in switch_arms(text, code, BUILTINS_DISPATCH):
|
|
body = text[start:end]
|
|
status = "registered"
|
|
notes = []
|
|
if re.search(r"not (?:yet )?implemented", body, re.I) or "UnsupportedOperationException" in body:
|
|
status = "stub_candidate"
|
|
notes.append("Heurystyka: wyjątek lub komunikat o braku implementacji; wymaga przeglądu.")
|
|
for name in labels:
|
|
methods[name.upper()] = {"status": status, "notes": list(notes), "source": {
|
|
"path": str(path.relative_to(root)), "line": text.count("\n", 0, start) + 1,
|
|
"owner": "@", "body_sha256": digest(body.encode()), "file_sha256": digest(text.encode())}}
|
|
return {"methods": methods, "mapping_notes": [
|
|
"Instrukcje @ są rozpoznawane w CodeParser.dispatchSpecialCall, nie przez MethodSpec."]}
|
|
|
|
|
|
def _git(root: Path, *args: str) -> str | None:
|
|
result = subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True)
|
|
return result.stdout.strip() if result.returncode == 0 else None
|
|
|
|
|
|
def scan_emulator(root: Path) -> dict:
|
|
root = root.resolve()
|
|
source_root = root / "core/src"
|
|
variable_root = source_root / "pl/genschu/bloomooemulator/interpreter/variable"
|
|
if not (variable_root / "Variable.java").is_file():
|
|
raise ValueError("AMS_EMULATOR_ROOT must point to the Rex-EMoolator repository")
|
|
tree = hashlib.sha256()
|
|
for path in sorted(source_root.rglob("*.java")):
|
|
tree.update(str(path.relative_to(root)).encode() + b"\0" + path.read_bytes() + b"\0")
|
|
types = {}
|
|
warnings = []
|
|
for path in sorted(variable_root.glob("*Variable.java")):
|
|
text = path.read_text(encoding="utf-8")
|
|
code = code_mask(text)
|
|
match = re.search(r"return\s+VariableType\.([A-Z_]+)\s*;", code)
|
|
if path.name == "Variable.java":
|
|
name = "*"
|
|
elif match:
|
|
name = canonical(match[1])
|
|
else:
|
|
warnings.append("Nie rozpoznano typu Java: " + path.name)
|
|
continue
|
|
notes = []
|
|
if name != "*" and re.search(r"\bMethodResult\s+callMethod\s*\(", code):
|
|
notes.append("Własny dispatcher: lista rejestracji nie opisuje wszystkich wywołań dynamicznych.")
|
|
methods = {}
|
|
for entry in ENTRY.finditer(text):
|
|
if not code[entry.start():entry.start() + 3].strip():
|
|
continue
|
|
end = closing(code, code.index("(", entry.start()))
|
|
body = text[entry.start():end]
|
|
body_code = code[entry.start():end]
|
|
status = "registered"
|
|
method_notes = []
|
|
if "UnsupportedOperationException" in body_code or re.search(r"not (?:yet )?implemented", body, re.I):
|
|
status = "stub_candidate"
|
|
method_notes.append("Heurystyka: wyjątek lub komunikat o braku implementacji; wymaga przeglądu.")
|
|
if re.search(r"->\s*MethodResult\.noReturn\(\)", body_code):
|
|
method_notes.append("Pusta operacja; może odpowiadać oryginałowi. Nie oznaczono automatycznie jako stub.")
|
|
method = entry[1].upper()
|
|
if method in methods:
|
|
raise ValueError(f"Duplicate registration: {name}.{method}")
|
|
methods[method] = {"status": status, "notes": method_notes, "source": {
|
|
"path": str(path.relative_to(root)), "line": text.count("\n", 0, entry.start()) + 1,
|
|
"owner": name, "body_sha256": digest(body.encode()), "file_sha256": digest(text.encode()),
|
|
}}
|
|
types[name] = {"methods": methods, "mapping_notes": notes}
|
|
builtins = scan_builtins(root, source_root)
|
|
if builtins is not None:
|
|
types["@"] = builtins
|
|
return {"root": str(root), "signals": scan_signals(root, source_root, variable_root),
|
|
"properties": scan_properties(root, source_root, variable_root), "commit": _git(root, "rev-parse", "HEAD"),
|
|
"dirty": bool(_git(root, "status", "--porcelain")),
|
|
"source_tree_sha256": tree.hexdigest(), "types": types, "warnings": warnings}
|
|
|
|
|
|
def owner_name(owner: str) -> str:
|
|
if owner in ("CMC", "CMC_Object"):
|
|
return "*"
|
|
if owner in CLASS_ALIASES:
|
|
return CLASS_ALIASES[owner]
|
|
return canonical(owner.removeprefix("CMC_"))
|
|
|
|
|
|
def signature(dispatch_row: dict) -> str | None:
|
|
"""`(int, string/int, …) → double` from the argument getters and result allocator the
|
|
extractor saw in the method body; None when the body was not analysed."""
|
|
if "params" not in dispatch_row:
|
|
return None
|
|
args = ["/".join(p["types"]) for p in dispatch_row["params"]]
|
|
if dispatch_row.get("variadic"):
|
|
args.append("…")
|
|
returns = "/".join(dispatch_row["returns"]) or "—"
|
|
return f"({', '.join(args)}) → {returns}"
|
|
|
|
|
|
BASE_SCOPE = "*"
|
|
# getRunner(0) is the runner behind `@IF`, `@LOOP`, ... statements, not an object type.
|
|
BUILTIN_SCOPE, BUILTIN_TYPE_ID = "@", 0
|
|
|
|
|
|
def _constructor_runner(name, cpp_class, class_runner, bases, known_classes):
|
|
"""Runner owner for engines whose class constructors create their runner (Piklib 6.1):
|
|
the nearest class up this snapshot's constructor chain that creates one."""
|
|
if not class_runner:
|
|
return None
|
|
if not cpp_class:
|
|
candidates = [c for c in known_classes if owner_name(c) == name]
|
|
cpp_class = candidates[0] if len(candidates) == 1 else None
|
|
seen = set()
|
|
while cpp_class and cpp_class not in seen:
|
|
if cpp_class in class_runner:
|
|
return class_runner[cpp_class]
|
|
seen.add(cpp_class)
|
|
cpp_class = bases.get(cpp_class)
|
|
return None
|
|
|
|
|
|
def script_events(snap: Snapshot, scopes: dict | None = None) -> dict[str, list[str]]:
|
|
"""Script type -> ordered event names for one binary. `type_events` (each type's own
|
|
vftable, schema 6) wins; otherwise the class list of the nearest class up this snapshot's
|
|
constructor chain that has one (INTEGER -> CMC_Variable). `scopes` (script_owners) adds
|
|
singletons such as SCENE, whose class is their runner's owner."""
|
|
out = {}
|
|
for e in sorted(snap.type_events, key=lambda e: e["order"]):
|
|
out.setdefault(canonical(e["type"]), []).append(e["name"].upper())
|
|
by_class = {}
|
|
for e in sorted(snap.events, key=lambda e: e.get("order") or 0):
|
|
by_class.setdefault(e["owner"], []).append(e["name"].upper())
|
|
bases = {e["class"]: e["base_class"] for e in snap.field_inheritance}
|
|
known = set(by_class) | set(bases) | set(bases.values())
|
|
for t in snap.types:
|
|
name = canonical(t["script_name"])
|
|
if name in out:
|
|
continue
|
|
cls = t.get("cpp_class") if t.get("cpp_class") != "CMC_Behaviour" or name == "BEHAVIOUR" else None
|
|
if not cls:
|
|
candidates = [c for c in known if owner_name(c) == name]
|
|
cls = candidates[0] if len(candidates) == 1 else None
|
|
seen = set()
|
|
while cls and cls not in seen and cls not in by_class:
|
|
seen.add(cls)
|
|
cls = bases.get(cls)
|
|
if cls in by_class:
|
|
out[name] = by_class[cls]
|
|
for name, (owner, _) in (scopes or {}).items():
|
|
if name not in out and owner in by_class:
|
|
out[name] = by_class[owner]
|
|
return out
|
|
|
|
|
|
def script_fields(snap: Snapshot, scopes: dict | None = None) -> dict[str, list[tuple[str, list[str]]]]:
|
|
"""Script type -> [(property, value types)] for one binary. `type_fields` (constructor
|
|
chain, schema 7) wins; otherwise the class fields of the type's class and every base up
|
|
this snapshot's constructor chain, base first - properties accumulate, unlike events.
|
|
`scopes` (script_owners) adds singletons such as SCENE."""
|
|
out = {}
|
|
for f in sorted(snap.type_fields, key=lambda f: f["order"]):
|
|
out.setdefault(canonical(f["type"]), []).append((f["name"].upper(), f.get("value_types", [])))
|
|
by_class = {}
|
|
for f in sorted(snap.fields, key=lambda f: f.get("order") or 0):
|
|
by_class.setdefault(f["owner"], []).append((f["name"].upper(), [f.get("type")]))
|
|
bases = {e["class"]: e["base_class"] for e in snap.field_inheritance}
|
|
known = set(by_class) | set(bases) | set(bases.values())
|
|
for t in snap.types:
|
|
name = canonical(t["script_name"])
|
|
if name in out:
|
|
continue
|
|
cls = t.get("cpp_class") if t.get("cpp_class") != "CMC_Behaviour" or name == "BEHAVIOUR" else None
|
|
if not cls:
|
|
candidates = [c for c in known if owner_name(c) == name]
|
|
cls = candidates[0] if len(candidates) == 1 else None
|
|
chain = []
|
|
while cls and cls not in chain:
|
|
chain.append(cls)
|
|
cls = bases.get(cls)
|
|
fields = [f for c in reversed(chain) for f in by_class.get(c, [])]
|
|
if fields:
|
|
out[name] = fields
|
|
for name, (owner, _) in (scopes or {}).items():
|
|
if name not in out and owner in by_class:
|
|
out[name] = by_class[owner]
|
|
return out
|
|
|
|
|
|
def script_owners(snap: Snapshot, owners) -> dict[str, tuple[str, str]]:
|
|
"""Script scope -> (method owner, provenance) for one binary.
|
|
|
|
`runner_map` (CMC_RunnerFactory, Piklib 7.1+/BlooMoo; a scene module's types and the runner
|
|
their objects install) is the engine's own dispatch table and wins: several types can share
|
|
a runner (INTEGER/STRING/... -> Variable). Without it (Piklib 6.1, older extractions) the C++
|
|
class or its name is the best available guess."""
|
|
runner_map = snap.raw.get("runner_map") or {}
|
|
by_type_id = {e["type_id"]: e["runner"].removesuffix("_Runner") for e in runner_map.get("by_type_id", [])}
|
|
dispatched = "module" if runner_map.get("module") else "runner_factory" # a scene module's own resolve
|
|
class_runner = {e["class"]: e["runner"].removesuffix("_Runner") for e in runner_map.get("by_class", [])}
|
|
bases = {e["class"]: e["base_class"] for e in snap.field_inheritance}
|
|
known_classes = set(class_runner) | set(bases) | set(bases.values())
|
|
mappings = {}
|
|
if BUILTIN_TYPE_ID in by_type_id:
|
|
mappings[BUILTIN_SCOPE] = (by_type_id[BUILTIN_TYPE_ID], "runner_factory")
|
|
for t in snap.types:
|
|
name = canonical(t["script_name"])
|
|
if t.get("type_id") in by_type_id:
|
|
mappings[name] = (by_type_id[t["type_id"]], dispatched)
|
|
continue
|
|
owner, mapping = t.get("cpp_class"), "explicit"
|
|
# Known extractor failure: unrelated factory entries all mapped to Behaviour.
|
|
if owner == "CMC_Behaviour" and name != "BEHAVIOUR":
|
|
owner = None
|
|
runner_owner = _constructor_runner(name, owner, class_runner, bases, known_classes)
|
|
if runner_owner:
|
|
mappings[name] = (runner_owner, "class_constructor")
|
|
continue
|
|
if not owner:
|
|
candidate = [o for o in owners if owner_name(o) == name]
|
|
owner, mapping = (candidate[0] if len(candidate) == 1 else None), "inferred"
|
|
if owner:
|
|
mappings[name] = (owner, mapping)
|
|
used = {owner for owner, _ in mappings.values()}
|
|
for owner in owners:
|
|
# Singletons (SYSTEM, SCENE, ...) and the base scope run without a factory type.
|
|
if owner not in used and owner_name(owner) not in mappings:
|
|
mappings[owner_name(owner)] = (owner, "inferred")
|
|
return mappings
|
|
|
|
|
|
def build_tracker(snapshots: list[Snapshot], emulator: dict, reviews: list[dict] | None = None) -> dict:
|
|
"""Join script APIs, keeping every binary and inherited declaration as evidence."""
|
|
warnings = list(emulator.get("warnings", []))
|
|
rows = {}
|
|
binaries = []
|
|
|
|
def type_row(name):
|
|
return rows.setdefault(name, {"name": name, "mapping_notes": [], "methods": {}})
|
|
|
|
def method_row(name, method):
|
|
return type_row(name)["methods"].setdefault(method, {
|
|
"name": method, "status": "missing", "verification": "unverified",
|
|
"source": None, "original": [], "reviews": [], "notes": [],
|
|
})
|
|
|
|
excluded = defaultdict(set) # (type, method) -> binaries whose code skips it for that type
|
|
seen = set()
|
|
for snap in snapshots:
|
|
sha = snap.binary.get("sha256")
|
|
if not isinstance(sha, str) or not SHA.fullmatch(sha.lower()):
|
|
sha = None
|
|
else:
|
|
sha = sha.lower()
|
|
# A content hash of the snapshot is NOT the identity of its DLL.
|
|
bid = sha or "snapshot:" + digest(json.dumps(snap.raw, sort_keys=True).encode())
|
|
if bid in seen:
|
|
warnings.append("Powtórzony snapshot pominięty: " + snap.binary.get("name", bid))
|
|
continue
|
|
seen.add(bid)
|
|
binaries.append({"id": bid, "sha256": sha, "name": snap.binary.get("name", "?"),
|
|
"engine": snap.binary.get("engine"), "identity": "binary" if sha else "snapshot_only",
|
|
"library": snap.binary.get("library", snap.binary.get("name", "?").split("__")[-1]),
|
|
"family": binary_family(snap.binary),
|
|
"sources": snap.binary.get("sources", []), "games": snap.binary.get("games", []),
|
|
"coverage": snap.raw.get("coverage", snap.raw.get("extraction", {}).get("scope",
|
|
snap.binary.get("extraction", {}).get("coverage"))),
|
|
"limitations": snap.raw.get("limitations", []) + snap.raw.get("extraction", {}).get("limitations", [])})
|
|
if not sha:
|
|
warnings.append(snap.binary.get("name", "?") + ": brak SHA-256 DLL; dowodów nie można przypisać do binarki.")
|
|
owners = defaultdict(list)
|
|
for m in snap.methods:
|
|
owners[m["owner"]].append(m)
|
|
id_names = defaultdict(set)
|
|
name_ids = defaultdict(set)
|
|
for m in snap.methods:
|
|
if m.get("id") is not None:
|
|
id_names[(m["owner"], m["id"])].add(m["name"].upper())
|
|
name_ids[(m["owner"], m["name"].upper())].add(m["id"])
|
|
dispatch = {(d["owner"], d.get("id")): d for d in snap.method_dispatch if d.get("id") is not None}
|
|
# Earliest Piklib 6.1 dispatches by name and has no method ids.
|
|
dispatch_by_name = {(d["owner"], d["name"]): d for d in snap.method_dispatch
|
|
if d.get("id") is None and d.get("name")}
|
|
parents = defaultdict(list)
|
|
for edge in snap.method_inheritance:
|
|
parents[edge["runner"].removesuffix("_Runner")].append(edge["base_runner"].removesuffix("_Runner"))
|
|
mappings = script_owners(snap, owners)
|
|
for name, names in script_events(snap, mappings).items():
|
|
events = type_row(name).setdefault("events", {})
|
|
for order, event in enumerate(names):
|
|
events.setdefault(event, {"name": event, "order": order, "original": [], "emulator": [],
|
|
"status": "not_found"})["original"].append(bid)
|
|
for name, props in script_fields(snap, mappings).items():
|
|
fields = type_row(name).setdefault("fields", {})
|
|
for order, (prop, value_types) in enumerate(props):
|
|
row = fields.setdefault(prop, {"name": prop, "order": order, "value_types": [],
|
|
"original": [], "emulator": [], "status": "not_found"})
|
|
if bid not in row["original"]:
|
|
row["original"].append(bid)
|
|
row["value_types"] = sorted(set(row["value_types"]) | {v for v in value_types if v})
|
|
for t in snap.types:
|
|
row = type_row(canonical(t["script_name"]))
|
|
if t.get("cpp_class_provenance"):
|
|
row["mapping_notes"].append(t["cpp_class_provenance"])
|
|
if canonical(t["script_name"]) not in mappings:
|
|
row["mapping_notes"].append(f"{snap.binary.get('name', '?')}: typ wykryty, klasa/metody nieustalone.")
|
|
sharing = defaultdict(set)
|
|
for name, (owner, _) in mappings.items():
|
|
sharing[owner].add(name)
|
|
for name, (owner, mapping) in mappings.items():
|
|
if len(sharing[owner]) > 1:
|
|
type_row(name)["mapping_notes"].append(
|
|
f"Wspólny runner {owner}_Runner z: {', '.join(sorted(sharing[owner] - {name}))}. "
|
|
"Rejestracja nie mówi, które metody działają dla tego typu.")
|
|
visited = set()
|
|
def add(current, inherited=False):
|
|
if current in visited:
|
|
return
|
|
visited.add(current)
|
|
if inherited and owner_name(current) == BASE_SCOPE and name != BASE_SCOPE:
|
|
# Base verbs (SEND, CLONE, ...) belong to every runner; they are listed once.
|
|
type_row(name).setdefault("inherits", []).append(BASE_SCOPE)
|
|
return
|
|
for m in owners[current]:
|
|
d = (dispatch.get((current, m.get("id")), {}) if m.get("id") is not None
|
|
else dispatch_by_name.get((current, m["name"].upper()), {}))
|
|
verdict = {canonical(k): v for k, v in (d.get("per_type") or {}).items()}.get(name)
|
|
if verdict == "excluded_by_type_check":
|
|
excluded[(name, m["name"].upper())].add(bid)
|
|
continue
|
|
row = method_row(name, m["name"].upper())
|
|
if len(name_ids[(current, m["name"].upper())]) > 1:
|
|
note = "Oryginał rejestruje tę nazwę z różnymi ID; skuteczny wybór wymaga sprawdzenia hashtable/dispatchera."
|
|
if note not in row["notes"]:
|
|
row["notes"].append(note)
|
|
if len(id_names[(current, m.get("id"))]) > 1:
|
|
note = "ID współdzielone przez nazwy: " + ", ".join(sorted(id_names[(current, m.get("id"))]))
|
|
if note not in row["notes"]:
|
|
row["notes"].append(note)
|
|
# Child declarations override names from ancestors for this binary.
|
|
if any(o["binary_id"] == bid and o["owner"] != current for o in row["original"]):
|
|
continue
|
|
if verdict == "trivial_virtual":
|
|
note = "W oryginale bez efektu dla tego typu: jego vftable zostawia wywoływane metody wirtualne puste."
|
|
if note not in row["notes"]:
|
|
row["notes"].append(note)
|
|
row["original"].append({"binary_id": bid, "owner": current, "method_id": m.get("id"),
|
|
"impl_addr": d.get("impl_addr"), "case_addr": d.get("case_addr"),
|
|
"signature": signature(d), "per_type": verdict,
|
|
"mapping": mapping, "inherited": inherited,
|
|
"registration_addr": m.get("registration_addr"), "string_addr": m.get("string_addr"),
|
|
"registration_call_addr": m.get("registration_call_addr"),
|
|
"id_push_addr": m.get("id_push_addr"),
|
|
"evidence_ids": m.get("evidence_ids", [])})
|
|
for parent in parents[current]:
|
|
add(parent, True)
|
|
add(owner)
|
|
for name, t in emulator["types"].items():
|
|
type_row(name)["mapping_notes"].extend(t["mapping_notes"])
|
|
for method, binding in t["methods"].items():
|
|
row = method_row(name, method)
|
|
notes = row["notes"] + binding.get("notes", [])
|
|
row.update(binding)
|
|
row["notes"] = notes
|
|
# Global fallback exists on concrete Variable implementations, not native-only types.
|
|
globals_ = emulator["types"].get("*", {}).get("methods", {})
|
|
for name, t in rows.items():
|
|
if name not in emulator["types"] or name == "*":
|
|
continue
|
|
for method, row in t["methods"].items():
|
|
if row["source"] is None and method in globals_:
|
|
row.update(globals_[method])
|
|
row["notes"] = row["notes"] + ["Obsługa przez globalny fallback Variable."]
|
|
for (name, method), binaries_ in excluded.items():
|
|
row = rows.get(name, {}).get("methods", {}).get(method)
|
|
if row is not None:
|
|
row["notes"].append(f"Oryginał ({len(binaries_)} binarek) pomija tę metodę dla tego typu: "
|
|
"warunek obj.getType() we wspólnym runnerze.")
|
|
signals = emulator.get("signals", {})
|
|
for name, t in rows.items():
|
|
for event, row in t.get("events", {}).items():
|
|
own, base, engine = (signals.get(scope, {}).get(event, []) for scope in (name, "*", "engine"))
|
|
row["emulator"] = own + base + engine
|
|
row["status"] = ("emitted" if own else "emitted_base" if base
|
|
else "engine_literal" if engine else "not_found")
|
|
properties = emulator.get("properties", {})
|
|
for name, t in rows.items():
|
|
for prop, row in t.get("fields", {}).items():
|
|
own, base, engine = (properties.get(scope, {}).get(prop, []) for scope in (name, "*", "engine"))
|
|
row["emulator"] = own + base + engine
|
|
row["status"] = ("read" if own else "read_base" if base
|
|
else "engine_literal" if engine else "not_found")
|
|
base_verbs = {m for m, row in rows.get(BASE_SCOPE, {}).get("methods", {}).items() if row["original"]}
|
|
for name, t in rows.items():
|
|
for method, row in t["methods"].items():
|
|
if name != BASE_SCOPE and method in base_verbs and row["source"] and not row["original"]:
|
|
row["notes"].append("W oryginale metoda bazowa (*) każdego runnera; emulator rejestruje ją lokalnie.")
|
|
# Reviews are declarative, version-controlled assertions; strict anchors prevent
|
|
# accidentally making an entire family green based on one stale binary/address.
|
|
for review in reviews or []:
|
|
validate_review(review)
|
|
row = rows.get(canonical(review["type"]), {}).get("methods", {}).get(review["method"].upper())
|
|
if row is None:
|
|
warnings.append("Przegląd nie pasuje do API: " + review["type"] + "." + review["method"])
|
|
continue
|
|
source = row["source"] or {}
|
|
current = (source.get("body_sha256") == review["body_sha256"]
|
|
and emulator["source_tree_sha256"] == review["source_tree_sha256"]
|
|
and emulator.get("commit") == review["emulator_commit"])
|
|
matches = any(o["binary_id"] == review["binary_sha256"] for o in row["original"])
|
|
row["reviews"].append({**review, "current": current and matches})
|
|
for t in rows.values():
|
|
for row in t["methods"].values():
|
|
current = [r for r in row["reviews"] if r["current"]]
|
|
targets = {o["binary_id"] for o in row["original"]}
|
|
verified = {r["binary_sha256"] for r in current if r["verdict"] == "matches"}
|
|
row["verified_variants"] = len(verified & targets)
|
|
if targets and targets <= verified:
|
|
row["verification"] = "runtime_tested" if all(any(r["binary_sha256"] == b and r["kind"] == "runtime" and r["verdict"] == "matches" for r in current) for b in targets) else "asm_reviewed"
|
|
row["status"] = "implemented"
|
|
elif any(r["verdict"] == "partial" for r in current):
|
|
row["status"] = "partial"
|
|
elif row["reviews"] and not current:
|
|
row["verification"] = "stale"
|
|
for row in t["methods"].values():
|
|
row["signatures"] = sorted({o["signature"] for o in row["original"] if o.get("signature")})
|
|
t["methods"] = sorted(t["methods"].values(), key=lambda m: m["name"])
|
|
t["mapping_notes"] = sorted(set(t["mapping_notes"]))
|
|
if "inherits" in t:
|
|
t["inherits"] = sorted(set(t["inherits"]))
|
|
t["events"] = sorted(t.get("events", {}).values(), key=lambda e: (e["order"], e["name"]))
|
|
t["fields"] = sorted(t.get("fields", {}).values(), key=lambda f: (f["order"], f["name"]))
|
|
all_rows = [m for t in rows.values() for m in t["methods"]]
|
|
all_events = [e for t in rows.values() for e in t["events"]]
|
|
all_fields = [f for t in rows.values() for f in t["fields"]]
|
|
return {"schema": "tracker/1", "emulator": {k: v for k, v in emulator.items() if k not in ("types", "warnings", "signals", "properties")},
|
|
"binaries": sorted(binaries, key=lambda b: (b["name"].lower(), b["id"])),
|
|
"types": sorted(rows.values(), key=lambda t: t["name"]),
|
|
"summary": {"types": len(rows), "methods": len(all_rows), "status": dict(Counter(m["status"] for m in all_rows)),
|
|
"verification": dict(Counter(m["verification"] for m in all_rows)),
|
|
"events": len(all_events), "event_status": dict(Counter(e["status"] for e in all_events)),
|
|
"fields": len(all_fields), "field_status": dict(Counter(f["status"] for f in all_fields))},
|
|
"warnings": warnings + ["Brak obserwacji w wariancie oznacza nieustalone, nie brak metody.",
|
|
"Rejestracja Java nie dowodzi poprawności. Liczniki obejmują pary typ/metoda, także odziedziczone.",
|
|
"Typ → runner pochodzi z CMC_RunnerFactory, gdy snapshot ma runner_map; inaczej z nazwy klasy (wnioskowane).",
|
|
"Zakres * to metody bazowe CMC_Runner, wspólne dla każdego typu (pole inherits); nie są powielane przy typach.",
|
|
"Zakres @ to wbudowane instrukcje @IF/@LOOP/... (runner typu 0).",
|
|
"Eventy: status emulatora to literał nazwy w kodzie (klasa typu / Variable / silnik); nazwy budowane dynamicznie nie są widoczne.",
|
|
"Pola: status emulatora to literał klucza (`:NAME` w gałęzi typu w CNVParser, odczyt atrybutu w klasie typu lub w Variable.java, albo w innym kodzie)."]}
|
|
|
|
|
|
def validate_review(review: dict) -> None:
|
|
required = ("type", "method", "binary_sha256", "rva", "emulator_commit", "body_sha256",
|
|
"source_tree_sha256", "reviewer", "date", "notes", "evidence", "kind", "verdict")
|
|
if any(not isinstance(review.get(k), str) or not review[k].strip() for k in required):
|
|
raise ValueError("Review requires nonempty fields: " + ", ".join(required))
|
|
for k in ("binary_sha256", "body_sha256", "source_tree_sha256"):
|
|
if not SHA.fullmatch(review[k]):
|
|
raise ValueError("Invalid review digest: " + k)
|
|
if not re.fullmatch(r"0x[0-9a-fA-F]+", review["rva"]):
|
|
raise ValueError("Review RVA must be hexadecimal (0x...)")
|
|
if review["kind"] not in ("asm", "runtime") or review["verdict"] not in ("matches", "partial", "differs"):
|
|
raise ValueError("Invalid review kind/verdict")
|
|
|
|
|
|
def load_reviews(path: Path | None) -> list[dict]:
|
|
if path is None:
|
|
return []
|
|
data = json.loads(path.read_text())
|
|
if data.get("schema") != "tracker-reviews/1" or not isinstance(data.get("reviews"), list):
|
|
raise ValueError("Expected tracker-reviews/1 with a reviews list")
|
|
for review in data["reviews"]:
|
|
validate_review(review)
|
|
# A single current assertion per binary/method/source; conflicting verdicts fail closed.
|
|
keys = [(r["type"], r["method"], r["binary_sha256"], r["source_tree_sha256"], r["emulator_commit"]) for r in data["reviews"]]
|
|
if len(keys) != len(set(keys)):
|
|
raise ValueError("Duplicate review for the same method, binary and source revision")
|
|
return data["reviews"]
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("snapshots", nargs="+")
|
|
parser.add_argument("--emulator-root", required=True, type=Path)
|
|
parser.add_argument("--reviews", type=Path)
|
|
parser.add_argument("--out", type=Path, required=True)
|
|
args = parser.parse_args(argv)
|
|
result = build_tracker([Snapshot.load(p) for p in args.snapshots], scan_emulator(args.emulator_root), load_reviews(args.reviews))
|
|
args.out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps(result["summary"]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|