The tracker lists each scope's original properties (constructor chain, or class fields up the chain for Piklib 6.1 and singletons such as SCENE) and where the emulator reads them: a CNVParser.createVariable case arm, an attribute read in the type's class or Variable.java, or other code. The switch-arm parser is shared with the @ statements. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
644 lines
34 KiB
Python
644 lines
34 KiB
Python
"""Evidence-aware script API tracker. Never treats registration as correctness.
|
|
|
|
The Java adapter deliberately recognises the repository's MethodSpec registrations,
|
|
not arbitrary Java. Unknown dispatch and incomplete native extraction stay explicit.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import Counter, defaultdict
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
|
|
from .snapshot import Snapshot
|
|
|
|
ALIASES = {"BOOL": "BOOLEAN", "INT": "INTEGER"}
|
|
# Established factory mappings in the PIKLIB8/BlooMoo reference snapshots. When
|
|
# used to fill an unresolved mapping in another build, provenance stays inferred.
|
|
CLASS_ALIASES = {"CMC_CanvasObserver": "CANVAS_OBSERVER", "CMC_Groupped": "GROUP"}
|
|
ENTRY = re.compile(r'Map\s*\.\s*entry\s*\(\s*"([A-Za-z0-9_]+)"\s*,\s*MethodSpec\s*\.\s*of\s*\(')
|
|
JAVA_TOKENS = re.compile(r'"""[\s\S]*?"""|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'|//[^\n]*|/\*[\s\S]*?\*/')
|
|
SHA = re.compile(r"[0-9a-f]{64}")
|
|
|
|
|
|
def canonical(name: str) -> str:
|
|
name = name.upper()
|
|
return ALIASES.get(name, name)
|
|
|
|
|
|
def digest(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def binary_family(binary: dict) -> str:
|
|
library = binary.get("library", binary.get("name", "?").split("__")[-1]).upper()
|
|
if library.startswith("BLOOMOO"):
|
|
return "BlooMoo"
|
|
return {"PIKLIB61.DLL": "Piklib 6.1", "PIKLIB71.DLL": "Piklib 7.1",
|
|
"PIKLIB72.DLL": "Piklib 7.2", "PIKLIB8.DLL": "Piklib 8"}.get(library, library.removesuffix(".DLL"))
|
|
|
|
|
|
def code_mask(text: str) -> str:
|
|
"""Mask literals/comments without shifting source positions or line numbers."""
|
|
return JAVA_TOKENS.sub(lambda m: re.sub(r"[^\n]", " ", m.group()), text)
|
|
|
|
|
|
def closing(code: str, start: int, pair: str = "()") -> int:
|
|
depth = 0
|
|
for i in range(start, len(code)):
|
|
if code[i] == pair[0]:
|
|
depth += 1
|
|
elif code[i] == pair[1]:
|
|
depth -= 1
|
|
if depth == 0:
|
|
return i + 1
|
|
raise ValueError("Unbalanced " + pair)
|
|
|
|
|
|
BUILTINS_SOURCE = "pl/genschu/bloomooemulator/interpreter/parser/CodeParser.java"
|
|
BUILTINS_DISPATCH = re.compile(r"\bdispatchSpecialCall\s*\([^)]*\)\s*\{")
|
|
CASE_ARM = re.compile(r"\bcase\b([^;{}]*?)->|\bdefault\b\s*->")
|
|
|
|
|
|
SIGNAL_LITERAL = re.compile(r'"(ON[A-Z][A-Z_]*)"')
|
|
NOT_SIGNALS = {"ONEBREAK"} # the @ONEBREAK statement, not an event
|
|
|
|
|
|
def scan_signals(root: Path, source_root: Path, variable_root: Path) -> dict:
|
|
"""Where the emulator names each event: {scope: {event: [source]}}. A `<Type>Variable.java`
|
|
file is that type's scope, Variable.java is `*`, any other file is `engine`. Only string
|
|
literals count (comments ignored); names built at runtime are invisible, so absence is
|
|
"no literal found", never proof that an event is missing."""
|
|
scopes = {}
|
|
for path, text in sorted((p, p.read_text(encoding="utf-8")) for p in source_root.rglob("*.java")):
|
|
scope = "engine"
|
|
if path.parent == variable_root and path.name.endswith("Variable.java"):
|
|
if path.name == "Variable.java":
|
|
scope = "*"
|
|
else:
|
|
match = re.search(r"return\s+VariableType\.([A-Z_]+)\s*;", code_mask(text))
|
|
scope = canonical(match[1]) if match else "engine"
|
|
for token in JAVA_TOKENS.finditer(text):
|
|
literal = SIGNAL_LITERAL.fullmatch(token.group())
|
|
if literal and literal[1] not in NOT_SIGNALS:
|
|
scopes.setdefault(scope, {}).setdefault(literal[1], []).append(
|
|
{"path": str(path.relative_to(root)), "line": text.count("\n", 0, token.start()) + 1})
|
|
return scopes
|
|
|
|
|
|
def switch_arms(text: str, code: str, head: re.Pattern) -> list[tuple[list[str], int, int]]:
|
|
"""[(case labels, start, end)] of the first `switch` inside the method `head` matches.
|
|
Positions index `text`; `code` is its literal/comment mask. Default arms are skipped."""
|
|
found = head.search(code)
|
|
if found is None:
|
|
return []
|
|
switch = code.index("switch", found.end())
|
|
body_start = code.index("{", switch)
|
|
body_end = closing(code, body_start, "{}")
|
|
arms = list(CASE_ARM.finditer(code, body_start, body_end))
|
|
out = []
|
|
for arm, following in zip(arms, arms[1:] + [None]):
|
|
if arm.group(1) is not None:
|
|
labels = re.findall(r'"([A-Za-z_]+)"', text[arm.start(1):arm.end(1)])
|
|
out.append((labels, arm.start(), following.start() if following else body_end))
|
|
return out
|
|
|
|
|
|
CNV_SOURCE = "pl/genschu/bloomooemulator/loader/CNVParser.java"
|
|
CNV_CREATE = re.compile(r"\bcreateVariable\s*\([^)]*\)\s*\{")
|
|
PROPERTY_KEY = re.compile(r'":([A-Z][A-Z0-9_]*)"')
|
|
ATTRIBUTE_NAME = re.compile(r'"([A-Z][A-Z0-9_]*)"')
|
|
# context.getAttribute(name, "X"), getAttributeInHierarchy(name(), "X"), attributes().get(name, "X")
|
|
ATTRIBUTE_CALL = re.compile(r"(?:\bgetAttribute\w*|\battributes\(\)\s*\.\s*get\w*)\s*\([^;]*$")
|
|
|
|
|
|
def scan_properties(root: Path, source_root: Path, variable_root: Path) -> dict:
|
|
"""Where the emulator reads each CNV property: {scope: {property: [source]}}. A `":NAME"`
|
|
key inside a `case "TYPE"` arm of CNVParser.createVariable belongs to those types, and
|
|
an attribute read (`getAttribute`, `getAttributeInHierarchy`, `attributes().get`) in
|
|
`<Type>Variable.java` to that type, in Variable.java to `*`; anything else is `engine`.
|
|
Like signals, absence means only that no literal was found."""
|
|
scopes = {}
|
|
for path in sorted(source_root.rglob("*.java")):
|
|
text = path.read_text(encoding="utf-8")
|
|
code = code_mask(text)
|
|
scope = "engine"
|
|
if path.parent == variable_root and path.name == "Variable.java":
|
|
scope = "*"
|
|
elif path.parent == variable_root and path.name.endswith("Variable.java"):
|
|
match = re.search(r"return\s+VariableType\.([A-Z_]+)\s*;", code)
|
|
scope = canonical(match[1]) if match else "engine"
|
|
arms = switch_arms(text, code, CNV_CREATE) if path == source_root / CNV_SOURCE else []
|
|
for token in JAVA_TOKENS.finditer(text):
|
|
literal = PROPERTY_KEY.fullmatch(token.group())
|
|
if literal is None:
|
|
literal = ATTRIBUTE_NAME.fullmatch(token.group())
|
|
if literal is None or not ATTRIBUTE_CALL.search(code[max(0, token.start() - 160):token.start()]):
|
|
continue
|
|
owners = [canonical(label) for labels, start, end in arms if start <= token.start() < end
|
|
for label in labels] or [scope]
|
|
source = {"path": str(path.relative_to(root)), "line": text.count("\n", 0, token.start()) + 1}
|
|
for owner in owners:
|
|
scopes.setdefault(owner, {}).setdefault(literal[1], []).append(source)
|
|
return scopes
|
|
|
|
|
|
def scan_builtins(root: Path, source_root: Path) -> dict | None:
|
|
"""`@IF`, `@LOOP`, ... are parsed by CodeParser.dispatchSpecialCall, not registered as
|
|
MethodSpec: each `case "NAME", ... ->` arm of its switch is one built-in statement."""
|
|
path = source_root / BUILTINS_SOURCE
|
|
if not path.is_file():
|
|
return None
|
|
text = path.read_text(encoding="utf-8")
|
|
code = code_mask(text)
|
|
if BUILTINS_DISPATCH.search(code) is None:
|
|
return None
|
|
methods = {}
|
|
for labels, start, end in switch_arms(text, code, BUILTINS_DISPATCH):
|
|
body = text[start:end]
|
|
status = "registered"
|
|
notes = []
|
|
if re.search(r"not (?:yet )?implemented", body, re.I) or "UnsupportedOperationException" in body:
|
|
status = "stub_candidate"
|
|
notes.append("Heurystyka: wyjątek lub komunikat o braku implementacji; wymaga przeglądu.")
|
|
for name in labels:
|
|
methods[name.upper()] = {"status": status, "notes": list(notes), "source": {
|
|
"path": str(path.relative_to(root)), "line": text.count("\n", 0, start) + 1,
|
|
"owner": "@", "body_sha256": digest(body.encode()), "file_sha256": digest(text.encode())}}
|
|
return {"methods": methods, "mapping_notes": [
|
|
"Instrukcje @ są rozpoznawane w CodeParser.dispatchSpecialCall, nie przez MethodSpec."]}
|
|
|
|
|
|
def _git(root: Path, *args: str) -> str | None:
|
|
result = subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True)
|
|
return result.stdout.strip() if result.returncode == 0 else None
|
|
|
|
|
|
def scan_emulator(root: Path) -> dict:
|
|
root = root.resolve()
|
|
source_root = root / "core/src"
|
|
variable_root = source_root / "pl/genschu/bloomooemulator/interpreter/variable"
|
|
if not (variable_root / "Variable.java").is_file():
|
|
raise ValueError("AMS_EMULATOR_ROOT must point to the Rex-EMoolator repository")
|
|
tree = hashlib.sha256()
|
|
for path in sorted(source_root.rglob("*.java")):
|
|
tree.update(str(path.relative_to(root)).encode() + b"\0" + path.read_bytes() + b"\0")
|
|
types = {}
|
|
warnings = []
|
|
for path in sorted(variable_root.glob("*Variable.java")):
|
|
text = path.read_text(encoding="utf-8")
|
|
code = code_mask(text)
|
|
match = re.search(r"return\s+VariableType\.([A-Z_]+)\s*;", code)
|
|
if path.name == "Variable.java":
|
|
name = "*"
|
|
elif match:
|
|
name = canonical(match[1])
|
|
else:
|
|
warnings.append("Nie rozpoznano typu Java: " + path.name)
|
|
continue
|
|
notes = []
|
|
if name != "*" and re.search(r"\bMethodResult\s+callMethod\s*\(", code):
|
|
notes.append("Własny dispatcher: lista rejestracji nie opisuje wszystkich wywołań dynamicznych.")
|
|
methods = {}
|
|
for entry in ENTRY.finditer(text):
|
|
if not code[entry.start():entry.start() + 3].strip():
|
|
continue
|
|
end = closing(code, code.index("(", entry.start()))
|
|
body = text[entry.start():end]
|
|
body_code = code[entry.start():end]
|
|
status = "registered"
|
|
method_notes = []
|
|
if "UnsupportedOperationException" in body_code or re.search(r"not (?:yet )?implemented", body, re.I):
|
|
status = "stub_candidate"
|
|
method_notes.append("Heurystyka: wyjątek lub komunikat o braku implementacji; wymaga przeglądu.")
|
|
if re.search(r"->\s*MethodResult\.noReturn\(\)", body_code):
|
|
method_notes.append("Pusta operacja; może odpowiadać oryginałowi. Nie oznaczono automatycznie jako stub.")
|
|
method = entry[1].upper()
|
|
if method in methods:
|
|
raise ValueError(f"Duplicate registration: {name}.{method}")
|
|
methods[method] = {"status": status, "notes": method_notes, "source": {
|
|
"path": str(path.relative_to(root)), "line": text.count("\n", 0, entry.start()) + 1,
|
|
"owner": name, "body_sha256": digest(body.encode()), "file_sha256": digest(text.encode()),
|
|
}}
|
|
types[name] = {"methods": methods, "mapping_notes": notes}
|
|
builtins = scan_builtins(root, source_root)
|
|
if builtins is not None:
|
|
types["@"] = builtins
|
|
return {"root": str(root), "signals": scan_signals(root, source_root, variable_root),
|
|
"properties": scan_properties(root, source_root, variable_root), "commit": _git(root, "rev-parse", "HEAD"),
|
|
"dirty": bool(_git(root, "status", "--porcelain")),
|
|
"source_tree_sha256": tree.hexdigest(), "types": types, "warnings": warnings}
|
|
|
|
|
|
def owner_name(owner: str) -> str:
|
|
if owner in ("CMC", "CMC_Object"):
|
|
return "*"
|
|
if owner in CLASS_ALIASES:
|
|
return CLASS_ALIASES[owner]
|
|
return canonical(owner.removeprefix("CMC_"))
|
|
|
|
|
|
def signature(dispatch_row: dict) -> str | None:
|
|
"""`(int, string/int, …) → double` from the argument getters and result allocator the
|
|
extractor saw in the method body; None when the body was not analysed."""
|
|
if "params" not in dispatch_row:
|
|
return None
|
|
args = ["/".join(p["types"]) for p in dispatch_row["params"]]
|
|
if dispatch_row.get("variadic"):
|
|
args.append("…")
|
|
returns = "/".join(dispatch_row["returns"]) or "—"
|
|
return f"({', '.join(args)}) → {returns}"
|
|
|
|
|
|
BASE_SCOPE = "*"
|
|
# getRunner(0) is the runner behind `@IF`, `@LOOP`, ... statements, not an object type.
|
|
BUILTIN_SCOPE, BUILTIN_TYPE_ID = "@", 0
|
|
|
|
|
|
def _constructor_runner(name, cpp_class, class_runner, bases, known_classes):
|
|
"""Runner owner for engines whose class constructors create their runner (Piklib 6.1):
|
|
the nearest class up this snapshot's constructor chain that creates one."""
|
|
if not class_runner:
|
|
return None
|
|
if not cpp_class:
|
|
candidates = [c for c in known_classes if owner_name(c) == name]
|
|
cpp_class = candidates[0] if len(candidates) == 1 else None
|
|
seen = set()
|
|
while cpp_class and cpp_class not in seen:
|
|
if cpp_class in class_runner:
|
|
return class_runner[cpp_class]
|
|
seen.add(cpp_class)
|
|
cpp_class = bases.get(cpp_class)
|
|
return None
|
|
|
|
|
|
def script_events(snap: Snapshot, scopes: dict | None = None) -> dict[str, list[str]]:
|
|
"""Script type -> ordered event names for one binary. `type_events` (each type's own
|
|
vftable, schema 6) wins; otherwise the class list of the nearest class up this snapshot's
|
|
constructor chain that has one (INTEGER -> CMC_Variable). `scopes` (script_owners) adds
|
|
singletons such as SCENE, whose class is their runner's owner."""
|
|
out = {}
|
|
for e in sorted(snap.type_events, key=lambda e: e["order"]):
|
|
out.setdefault(canonical(e["type"]), []).append(e["name"].upper())
|
|
by_class = {}
|
|
for e in sorted(snap.events, key=lambda e: e.get("order") or 0):
|
|
by_class.setdefault(e["owner"], []).append(e["name"].upper())
|
|
bases = {e["class"]: e["base_class"] for e in snap.field_inheritance}
|
|
known = set(by_class) | set(bases) | set(bases.values())
|
|
for t in snap.types:
|
|
name = canonical(t["script_name"])
|
|
if name in out:
|
|
continue
|
|
cls = t.get("cpp_class") if t.get("cpp_class") != "CMC_Behaviour" or name == "BEHAVIOUR" else None
|
|
if not cls:
|
|
candidates = [c for c in known if owner_name(c) == name]
|
|
cls = candidates[0] if len(candidates) == 1 else None
|
|
seen = set()
|
|
while cls and cls not in seen and cls not in by_class:
|
|
seen.add(cls)
|
|
cls = bases.get(cls)
|
|
if cls in by_class:
|
|
out[name] = by_class[cls]
|
|
for name, (owner, _) in (scopes or {}).items():
|
|
if name not in out and owner in by_class:
|
|
out[name] = by_class[owner]
|
|
return out
|
|
|
|
|
|
def script_fields(snap: Snapshot, scopes: dict | None = None) -> dict[str, list[tuple[str, list[str]]]]:
|
|
"""Script type -> [(property, value types)] for one binary. `type_fields` (constructor
|
|
chain, schema 7) wins; otherwise the class fields of the type's class and every base up
|
|
this snapshot's constructor chain, base first - properties accumulate, unlike events.
|
|
`scopes` (script_owners) adds singletons such as SCENE."""
|
|
out = {}
|
|
for f in sorted(snap.type_fields, key=lambda f: f["order"]):
|
|
out.setdefault(canonical(f["type"]), []).append((f["name"].upper(), f.get("value_types", [])))
|
|
by_class = {}
|
|
for f in sorted(snap.fields, key=lambda f: f.get("order") or 0):
|
|
by_class.setdefault(f["owner"], []).append((f["name"].upper(), [f.get("type")]))
|
|
bases = {e["class"]: e["base_class"] for e in snap.field_inheritance}
|
|
known = set(by_class) | set(bases) | set(bases.values())
|
|
for t in snap.types:
|
|
name = canonical(t["script_name"])
|
|
if name in out:
|
|
continue
|
|
cls = t.get("cpp_class") if t.get("cpp_class") != "CMC_Behaviour" or name == "BEHAVIOUR" else None
|
|
if not cls:
|
|
candidates = [c for c in known if owner_name(c) == name]
|
|
cls = candidates[0] if len(candidates) == 1 else None
|
|
chain = []
|
|
while cls and cls not in chain:
|
|
chain.append(cls)
|
|
cls = bases.get(cls)
|
|
fields = [f for c in reversed(chain) for f in by_class.get(c, [])]
|
|
if fields:
|
|
out[name] = fields
|
|
for name, (owner, _) in (scopes or {}).items():
|
|
if name not in out and owner in by_class:
|
|
out[name] = by_class[owner]
|
|
return out
|
|
|
|
|
|
def script_owners(snap: Snapshot, owners) -> dict[str, tuple[str, str]]:
|
|
"""Script scope -> (method owner, provenance) for one binary.
|
|
|
|
`runner_map` (CMC_RunnerFactory, Piklib 7.1+/BlooMoo) is the engine's own dispatch table and
|
|
wins: several types can share a runner (INTEGER/STRING/... -> Variable). Without it (Piklib
|
|
6.1, plugins, older extractions) the C++ class or its name is the best available guess."""
|
|
runner_map = snap.raw.get("runner_map") or {}
|
|
by_type_id = {e["type_id"]: e["runner"].removesuffix("_Runner") for e in runner_map.get("by_type_id", [])}
|
|
class_runner = {e["class"]: e["runner"].removesuffix("_Runner") for e in runner_map.get("by_class", [])}
|
|
bases = {e["class"]: e["base_class"] for e in snap.field_inheritance}
|
|
known_classes = set(class_runner) | set(bases) | set(bases.values())
|
|
mappings = {}
|
|
if BUILTIN_TYPE_ID in by_type_id:
|
|
mappings[BUILTIN_SCOPE] = (by_type_id[BUILTIN_TYPE_ID], "runner_factory")
|
|
for t in snap.types:
|
|
name = canonical(t["script_name"])
|
|
if t.get("type_id") in by_type_id:
|
|
mappings[name] = (by_type_id[t["type_id"]], "runner_factory")
|
|
continue
|
|
owner, mapping = t.get("cpp_class"), "explicit"
|
|
# Known extractor failure: unrelated factory entries all mapped to Behaviour.
|
|
if owner == "CMC_Behaviour" and name != "BEHAVIOUR":
|
|
owner = None
|
|
runner_owner = _constructor_runner(name, owner, class_runner, bases, known_classes)
|
|
if runner_owner:
|
|
mappings[name] = (runner_owner, "class_constructor")
|
|
continue
|
|
if not owner:
|
|
candidate = [o for o in owners if owner_name(o) == name]
|
|
owner, mapping = (candidate[0] if len(candidate) == 1 else None), "inferred"
|
|
if owner:
|
|
mappings[name] = (owner, mapping)
|
|
used = {owner for owner, _ in mappings.values()}
|
|
for owner in owners:
|
|
# Singletons (SYSTEM, SCENE, ...) and the base scope run without a factory type.
|
|
if owner not in used and owner_name(owner) not in mappings:
|
|
mappings[owner_name(owner)] = (owner, "inferred")
|
|
return mappings
|
|
|
|
|
|
def build_tracker(snapshots: list[Snapshot], emulator: dict, reviews: list[dict] | None = None) -> dict:
|
|
"""Join script APIs, keeping every binary and inherited declaration as evidence."""
|
|
warnings = list(emulator.get("warnings", []))
|
|
rows = {}
|
|
binaries = []
|
|
|
|
def type_row(name):
|
|
return rows.setdefault(name, {"name": name, "mapping_notes": [], "methods": {}})
|
|
|
|
def method_row(name, method):
|
|
return type_row(name)["methods"].setdefault(method, {
|
|
"name": method, "status": "missing", "verification": "unverified",
|
|
"source": None, "original": [], "reviews": [], "notes": [],
|
|
})
|
|
|
|
excluded = defaultdict(set) # (type, method) -> binaries whose code skips it for that type
|
|
seen = set()
|
|
for snap in snapshots:
|
|
sha = snap.binary.get("sha256")
|
|
if not isinstance(sha, str) or not SHA.fullmatch(sha.lower()):
|
|
sha = None
|
|
else:
|
|
sha = sha.lower()
|
|
# A content hash of the snapshot is NOT the identity of its DLL.
|
|
bid = sha or "snapshot:" + digest(json.dumps(snap.raw, sort_keys=True).encode())
|
|
if bid in seen:
|
|
warnings.append("Powtórzony snapshot pominięty: " + snap.binary.get("name", bid))
|
|
continue
|
|
seen.add(bid)
|
|
binaries.append({"id": bid, "sha256": sha, "name": snap.binary.get("name", "?"),
|
|
"engine": snap.binary.get("engine"), "identity": "binary" if sha else "snapshot_only",
|
|
"library": snap.binary.get("library", snap.binary.get("name", "?").split("__")[-1]),
|
|
"family": binary_family(snap.binary),
|
|
"sources": snap.binary.get("sources", []), "games": snap.binary.get("games", []),
|
|
"coverage": snap.raw.get("coverage", snap.raw.get("extraction", {}).get("scope",
|
|
snap.binary.get("extraction", {}).get("coverage"))),
|
|
"limitations": snap.raw.get("limitations", []) + snap.raw.get("extraction", {}).get("limitations", [])})
|
|
if not sha:
|
|
warnings.append(snap.binary.get("name", "?") + ": brak SHA-256 DLL; dowodów nie można przypisać do binarki.")
|
|
owners = defaultdict(list)
|
|
for m in snap.methods:
|
|
owners[m["owner"]].append(m)
|
|
id_names = defaultdict(set)
|
|
name_ids = defaultdict(set)
|
|
for m in snap.methods:
|
|
if m.get("id") is not None:
|
|
id_names[(m["owner"], m["id"])].add(m["name"].upper())
|
|
name_ids[(m["owner"], m["name"].upper())].add(m["id"])
|
|
dispatch = {(d["owner"], d.get("id")): d for d in snap.method_dispatch if d.get("id") is not None}
|
|
parents = defaultdict(list)
|
|
for edge in snap.method_inheritance:
|
|
parents[edge["runner"].removesuffix("_Runner")].append(edge["base_runner"].removesuffix("_Runner"))
|
|
mappings = script_owners(snap, owners)
|
|
for name, names in script_events(snap, mappings).items():
|
|
events = type_row(name).setdefault("events", {})
|
|
for order, event in enumerate(names):
|
|
events.setdefault(event, {"name": event, "order": order, "original": [], "emulator": [],
|
|
"status": "not_found"})["original"].append(bid)
|
|
for name, props in script_fields(snap, mappings).items():
|
|
fields = type_row(name).setdefault("fields", {})
|
|
for order, (prop, value_types) in enumerate(props):
|
|
row = fields.setdefault(prop, {"name": prop, "order": order, "value_types": [],
|
|
"original": [], "emulator": [], "status": "not_found"})
|
|
if bid not in row["original"]:
|
|
row["original"].append(bid)
|
|
row["value_types"] = sorted(set(row["value_types"]) | {v for v in value_types if v})
|
|
for t in snap.types:
|
|
row = type_row(canonical(t["script_name"]))
|
|
if t.get("cpp_class_provenance"):
|
|
row["mapping_notes"].append(t["cpp_class_provenance"])
|
|
if canonical(t["script_name"]) not in mappings:
|
|
row["mapping_notes"].append(f"{snap.binary.get('name', '?')}: typ wykryty, klasa/metody nieustalone.")
|
|
sharing = defaultdict(set)
|
|
for name, (owner, _) in mappings.items():
|
|
sharing[owner].add(name)
|
|
for name, (owner, mapping) in mappings.items():
|
|
if len(sharing[owner]) > 1:
|
|
type_row(name)["mapping_notes"].append(
|
|
f"Wspólny runner {owner}_Runner z: {', '.join(sorted(sharing[owner] - {name}))}. "
|
|
"Rejestracja nie mówi, które metody działają dla tego typu.")
|
|
visited = set()
|
|
def add(current, inherited=False):
|
|
if current in visited:
|
|
return
|
|
visited.add(current)
|
|
if inherited and owner_name(current) == BASE_SCOPE and name != BASE_SCOPE:
|
|
# Base verbs (SEND, CLONE, ...) belong to every runner; they are listed once.
|
|
type_row(name).setdefault("inherits", []).append(BASE_SCOPE)
|
|
return
|
|
for m in owners[current]:
|
|
d = dispatch.get((current, m.get("id")), {}) if m.get("id") is not None else {}
|
|
verdict = {canonical(k): v for k, v in (d.get("per_type") or {}).items()}.get(name)
|
|
if verdict == "excluded_by_type_check":
|
|
excluded[(name, m["name"].upper())].add(bid)
|
|
continue
|
|
row = method_row(name, m["name"].upper())
|
|
if len(name_ids[(current, m["name"].upper())]) > 1:
|
|
note = "Oryginał rejestruje tę nazwę z różnymi ID; skuteczny wybór wymaga sprawdzenia hashtable/dispatchera."
|
|
if note not in row["notes"]:
|
|
row["notes"].append(note)
|
|
if len(id_names[(current, m.get("id"))]) > 1:
|
|
note = "ID współdzielone przez nazwy: " + ", ".join(sorted(id_names[(current, m.get("id"))]))
|
|
if note not in row["notes"]:
|
|
row["notes"].append(note)
|
|
# Child declarations override names from ancestors for this binary.
|
|
if any(o["binary_id"] == bid and o["owner"] != current for o in row["original"]):
|
|
continue
|
|
if verdict == "trivial_virtual":
|
|
note = "W oryginale bez efektu dla tego typu: jego vftable zostawia wywoływane metody wirtualne puste."
|
|
if note not in row["notes"]:
|
|
row["notes"].append(note)
|
|
row["original"].append({"binary_id": bid, "owner": current, "method_id": m.get("id"),
|
|
"impl_addr": d.get("impl_addr"), "case_addr": d.get("case_addr"),
|
|
"signature": signature(d), "per_type": verdict,
|
|
"mapping": mapping, "inherited": inherited,
|
|
"registration_addr": m.get("registration_addr"), "string_addr": m.get("string_addr"),
|
|
"registration_call_addr": m.get("registration_call_addr"),
|
|
"id_push_addr": m.get("id_push_addr"),
|
|
"evidence_ids": m.get("evidence_ids", [])})
|
|
for parent in parents[current]:
|
|
add(parent, True)
|
|
add(owner)
|
|
for name, t in emulator["types"].items():
|
|
type_row(name)["mapping_notes"].extend(t["mapping_notes"])
|
|
for method, binding in t["methods"].items():
|
|
row = method_row(name, method)
|
|
notes = row["notes"] + binding.get("notes", [])
|
|
row.update(binding)
|
|
row["notes"] = notes
|
|
# Global fallback exists on concrete Variable implementations, not native-only types.
|
|
globals_ = emulator["types"].get("*", {}).get("methods", {})
|
|
for name, t in rows.items():
|
|
if name not in emulator["types"] or name == "*":
|
|
continue
|
|
for method, row in t["methods"].items():
|
|
if row["source"] is None and method in globals_:
|
|
row.update(globals_[method])
|
|
row["notes"] = row["notes"] + ["Obsługa przez globalny fallback Variable."]
|
|
for (name, method), binaries_ in excluded.items():
|
|
row = rows.get(name, {}).get("methods", {}).get(method)
|
|
if row is not None:
|
|
row["notes"].append(f"Oryginał ({len(binaries_)} binarek) pomija tę metodę dla tego typu: "
|
|
"warunek obj.getType() we wspólnym runnerze.")
|
|
signals = emulator.get("signals", {})
|
|
for name, t in rows.items():
|
|
for event, row in t.get("events", {}).items():
|
|
own, base, engine = (signals.get(scope, {}).get(event, []) for scope in (name, "*", "engine"))
|
|
row["emulator"] = own + base + engine
|
|
row["status"] = ("emitted" if own else "emitted_base" if base
|
|
else "engine_literal" if engine else "not_found")
|
|
properties = emulator.get("properties", {})
|
|
for name, t in rows.items():
|
|
for prop, row in t.get("fields", {}).items():
|
|
own, base, engine = (properties.get(scope, {}).get(prop, []) for scope in (name, "*", "engine"))
|
|
row["emulator"] = own + base + engine
|
|
row["status"] = ("read" if own else "read_base" if base
|
|
else "engine_literal" if engine else "not_found")
|
|
base_verbs = {m for m, row in rows.get(BASE_SCOPE, {}).get("methods", {}).items() if row["original"]}
|
|
for name, t in rows.items():
|
|
for method, row in t["methods"].items():
|
|
if name != BASE_SCOPE and method in base_verbs and row["source"] and not row["original"]:
|
|
row["notes"].append("W oryginale metoda bazowa (*) każdego runnera; emulator rejestruje ją lokalnie.")
|
|
# Reviews are declarative, version-controlled assertions; strict anchors prevent
|
|
# accidentally making an entire family green based on one stale binary/address.
|
|
for review in reviews or []:
|
|
validate_review(review)
|
|
row = rows.get(canonical(review["type"]), {}).get("methods", {}).get(review["method"].upper())
|
|
if row is None:
|
|
warnings.append("Przegląd nie pasuje do API: " + review["type"] + "." + review["method"])
|
|
continue
|
|
source = row["source"] or {}
|
|
current = (source.get("body_sha256") == review["body_sha256"]
|
|
and emulator["source_tree_sha256"] == review["source_tree_sha256"]
|
|
and emulator.get("commit") == review["emulator_commit"])
|
|
matches = any(o["binary_id"] == review["binary_sha256"] for o in row["original"])
|
|
row["reviews"].append({**review, "current": current and matches})
|
|
for t in rows.values():
|
|
for row in t["methods"].values():
|
|
current = [r for r in row["reviews"] if r["current"]]
|
|
targets = {o["binary_id"] for o in row["original"]}
|
|
verified = {r["binary_sha256"] for r in current if r["verdict"] == "matches"}
|
|
row["verified_variants"] = len(verified & targets)
|
|
if targets and targets <= verified:
|
|
row["verification"] = "runtime_tested" if all(any(r["binary_sha256"] == b and r["kind"] == "runtime" and r["verdict"] == "matches" for r in current) for b in targets) else "asm_reviewed"
|
|
row["status"] = "implemented"
|
|
elif any(r["verdict"] == "partial" for r in current):
|
|
row["status"] = "partial"
|
|
elif row["reviews"] and not current:
|
|
row["verification"] = "stale"
|
|
for row in t["methods"].values():
|
|
row["signatures"] = sorted({o["signature"] for o in row["original"] if o.get("signature")})
|
|
t["methods"] = sorted(t["methods"].values(), key=lambda m: m["name"])
|
|
t["mapping_notes"] = sorted(set(t["mapping_notes"]))
|
|
if "inherits" in t:
|
|
t["inherits"] = sorted(set(t["inherits"]))
|
|
t["events"] = sorted(t.get("events", {}).values(), key=lambda e: (e["order"], e["name"]))
|
|
t["fields"] = sorted(t.get("fields", {}).values(), key=lambda f: (f["order"], f["name"]))
|
|
all_rows = [m for t in rows.values() for m in t["methods"]]
|
|
all_events = [e for t in rows.values() for e in t["events"]]
|
|
all_fields = [f for t in rows.values() for f in t["fields"]]
|
|
return {"schema": "tracker/1", "emulator": {k: v for k, v in emulator.items() if k not in ("types", "warnings", "signals", "properties")},
|
|
"binaries": sorted(binaries, key=lambda b: (b["name"].lower(), b["id"])),
|
|
"types": sorted(rows.values(), key=lambda t: t["name"]),
|
|
"summary": {"types": len(rows), "methods": len(all_rows), "status": dict(Counter(m["status"] for m in all_rows)),
|
|
"verification": dict(Counter(m["verification"] for m in all_rows)),
|
|
"events": len(all_events), "event_status": dict(Counter(e["status"] for e in all_events)),
|
|
"fields": len(all_fields), "field_status": dict(Counter(f["status"] for f in all_fields))},
|
|
"warnings": warnings + ["Brak obserwacji w wariancie oznacza nieustalone, nie brak metody.",
|
|
"Rejestracja Java nie dowodzi poprawności. Liczniki obejmują pary typ/metoda, także odziedziczone.",
|
|
"Typ → runner pochodzi z CMC_RunnerFactory, gdy snapshot ma runner_map; inaczej z nazwy klasy (wnioskowane).",
|
|
"Zakres * to metody bazowe CMC_Runner, wspólne dla każdego typu (pole inherits); nie są powielane przy typach.",
|
|
"Zakres @ to wbudowane instrukcje @IF/@LOOP/... (runner typu 0).",
|
|
"Eventy: status emulatora to literał nazwy w kodzie (klasa typu / Variable / silnik); nazwy budowane dynamicznie nie są widoczne.",
|
|
"Pola: status emulatora to literał klucza (`:NAME` w gałęzi typu w CNVParser, odczyt atrybutu w klasie typu lub w Variable.java, albo w innym kodzie)."]}
|
|
|
|
|
|
def validate_review(review: dict) -> None:
|
|
required = ("type", "method", "binary_sha256", "rva", "emulator_commit", "body_sha256",
|
|
"source_tree_sha256", "reviewer", "date", "notes", "evidence", "kind", "verdict")
|
|
if any(not isinstance(review.get(k), str) or not review[k].strip() for k in required):
|
|
raise ValueError("Review requires nonempty fields: " + ", ".join(required))
|
|
for k in ("binary_sha256", "body_sha256", "source_tree_sha256"):
|
|
if not SHA.fullmatch(review[k]):
|
|
raise ValueError("Invalid review digest: " + k)
|
|
if not re.fullmatch(r"0x[0-9a-fA-F]+", review["rva"]):
|
|
raise ValueError("Review RVA must be hexadecimal (0x...)")
|
|
if review["kind"] not in ("asm", "runtime") or review["verdict"] not in ("matches", "partial", "differs"):
|
|
raise ValueError("Invalid review kind/verdict")
|
|
|
|
|
|
def load_reviews(path: Path | None) -> list[dict]:
|
|
if path is None:
|
|
return []
|
|
data = json.loads(path.read_text())
|
|
if data.get("schema") != "tracker-reviews/1" or not isinstance(data.get("reviews"), list):
|
|
raise ValueError("Expected tracker-reviews/1 with a reviews list")
|
|
for review in data["reviews"]:
|
|
validate_review(review)
|
|
# A single current assertion per binary/method/source; conflicting verdicts fail closed.
|
|
keys = [(r["type"], r["method"], r["binary_sha256"], r["source_tree_sha256"], r["emulator_commit"]) for r in data["reviews"]]
|
|
if len(keys) != len(set(keys)):
|
|
raise ValueError("Duplicate review for the same method, binary and source revision")
|
|
return data["reviews"]
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("snapshots", nargs="+")
|
|
parser.add_argument("--emulator-root", required=True, type=Path)
|
|
parser.add_argument("--reviews", type=Path)
|
|
parser.add_argument("--out", type=Path, required=True)
|
|
args = parser.parse_args(argv)
|
|
result = build_tracker([Snapshot.load(p) for p in args.snapshots], scan_emulator(args.emulator_root), load_reviews(args.reviews))
|
|
args.out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps(result["summary"]))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|