"""Evidence-aware script API tracker. Never treats registration as correctness. The Java adapter deliberately recognises the repository's MethodSpec registrations, not arbitrary Java. Unknown dispatch and incomplete native extraction stay explicit. """ from __future__ import annotations import argparse from collections import Counter, defaultdict import hashlib import json from pathlib import Path import re import subprocess from .snapshot import Snapshot ALIASES = {"BOOL": "BOOLEAN", "INT": "INTEGER"} # Established factory mappings in the PIKLIB8/BlooMoo reference snapshots. When # used to fill an unresolved mapping in another build, provenance stays inferred. CLASS_ALIASES = {"CMC_CanvasObserver": "CANVAS_OBSERVER", "CMC_Groupped": "GROUP"} ENTRY = re.compile(r'Map\s*\.\s*entry\s*\(\s*"([A-Za-z0-9_]+)"\s*,\s*MethodSpec\s*\.\s*of\s*\(') JAVA_TOKENS = re.compile(r'"""[\s\S]*?"""|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'|//[^\n]*|/\*[\s\S]*?\*/') SHA = re.compile(r"[0-9a-f]{64}") def canonical(name: str) -> str: name = name.upper() return ALIASES.get(name, name) def digest(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def binary_family(binary: dict) -> str: library = binary.get("library", binary.get("name", "?").split("__")[-1]).upper() if library.startswith("BLOOMOO"): return "BlooMoo" return {"PIKLIB61.DLL": "Piklib 6.1", "PIKLIB71.DLL": "Piklib 7.1", "PIKLIB72.DLL": "Piklib 7.2", "PIKLIB8.DLL": "Piklib 8"}.get(library, library.removesuffix(".DLL")) def code_mask(text: str) -> str: """Mask literals/comments without shifting source positions or line numbers.""" return JAVA_TOKENS.sub(lambda m: re.sub(r"[^\n]", " ", m.group()), text) def closing(code: str, start: int) -> int: depth = 0 for i in range(start, len(code)): if code[i] == "(": depth += 1 elif code[i] == ")": depth -= 1 if depth == 0: return i + 1 raise ValueError("Unbalanced MethodSpec registration") def _git(root: Path, *args: str) -> str | None: result = subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True) return result.stdout.strip() if result.returncode == 0 else None def scan_emulator(root: Path) -> dict: root = root.resolve() source_root = root / "core/src" variable_root = source_root / "pl/genschu/bloomooemulator/interpreter/variable" if not (variable_root / "Variable.java").is_file(): raise ValueError("AMS_EMULATOR_ROOT must point to the Rex-EMoolator repository") tree = hashlib.sha256() for path in sorted(source_root.rglob("*.java")): tree.update(str(path.relative_to(root)).encode() + b"\0" + path.read_bytes() + b"\0") types = {} warnings = [] for path in sorted(variable_root.glob("*Variable.java")): text = path.read_text(encoding="utf-8") code = code_mask(text) match = re.search(r"return\s+VariableType\.([A-Z_]+)\s*;", code) if path.name == "Variable.java": name = "*" elif match: name = canonical(match[1]) else: warnings.append("Nie rozpoznano typu Java: " + path.name) continue notes = [] if name != "*" and re.search(r"\bMethodResult\s+callMethod\s*\(", code): notes.append("Własny dispatcher: lista rejestracji nie opisuje wszystkich wywołań dynamicznych.") methods = {} for entry in ENTRY.finditer(text): if not code[entry.start():entry.start() + 3].strip(): continue end = closing(code, code.index("(", entry.start())) body = text[entry.start():end] body_code = code[entry.start():end] status = "registered" method_notes = [] if "UnsupportedOperationException" in body_code or re.search(r"not (?:yet )?implemented", body, re.I): status = "stub_candidate" method_notes.append("Heurystyka: wyjątek lub komunikat o braku implementacji; wymaga przeglądu.") if re.search(r"->\s*MethodResult\.noReturn\(\)", body_code): method_notes.append("Pusta operacja; może odpowiadać oryginałowi. Nie oznaczono automatycznie jako stub.") method = entry[1].upper() if method in methods: raise ValueError(f"Duplicate registration: {name}.{method}") methods[method] = {"status": status, "notes": method_notes, "source": { "path": str(path.relative_to(root)), "line": text.count("\n", 0, entry.start()) + 1, "owner": name, "body_sha256": digest(body.encode()), "file_sha256": digest(text.encode()), }} types[name] = {"methods": methods, "mapping_notes": notes} return {"root": str(root), "commit": _git(root, "rev-parse", "HEAD"), "dirty": bool(_git(root, "status", "--porcelain")), "source_tree_sha256": tree.hexdigest(), "types": types, "warnings": warnings} def owner_name(owner: str) -> str: if owner in ("CMC", "CMC_Object"): return "*" if owner in CLASS_ALIASES: return CLASS_ALIASES[owner] return canonical(owner.removeprefix("CMC_")) def build_tracker(snapshots: list[Snapshot], emulator: dict, reviews: list[dict] | None = None) -> dict: """Join script APIs, keeping every binary and inherited declaration as evidence.""" warnings = list(emulator.get("warnings", [])) rows = {} binaries = [] def type_row(name): return rows.setdefault(name, {"name": name, "mapping_notes": [], "methods": {}}) def method_row(name, method): return type_row(name)["methods"].setdefault(method, { "name": method, "status": "missing", "verification": "unverified", "source": None, "original": [], "reviews": [], "notes": [], }) seen = set() for snap in snapshots: sha = snap.binary.get("sha256") if not isinstance(sha, str) or not SHA.fullmatch(sha.lower()): sha = None else: sha = sha.lower() # A content hash of the snapshot is NOT the identity of its DLL. bid = sha or "snapshot:" + digest(json.dumps(snap.raw, sort_keys=True).encode()) if bid in seen: warnings.append("Powtórzony snapshot pominięty: " + snap.binary.get("name", bid)) continue seen.add(bid) binaries.append({"id": bid, "sha256": sha, "name": snap.binary.get("name", "?"), "engine": snap.binary.get("engine"), "identity": "binary" if sha else "snapshot_only", "library": snap.binary.get("library", snap.binary.get("name", "?").split("__")[-1]), "family": binary_family(snap.binary), "sources": snap.binary.get("sources", []), "games": snap.binary.get("games", []), "coverage": snap.raw.get("coverage", snap.raw.get("extraction", {}).get("scope", snap.binary.get("extraction", {}).get("coverage"))), "limitations": snap.raw.get("limitations", []) + snap.raw.get("extraction", {}).get("limitations", [])}) if not sha: warnings.append(snap.binary.get("name", "?") + ": brak SHA-256 DLL; dowodów nie można przypisać do binarki.") owners = defaultdict(list) for m in snap.methods: owners[m["owner"]].append(m) id_names = defaultdict(set) name_ids = defaultdict(set) for m in snap.methods: if m.get("id") is not None: id_names[(m["owner"], m["id"])].add(m["name"].upper()) name_ids[(m["owner"], m["name"].upper())].add(m["id"]) dispatch = {(d["owner"], d.get("id")): d for d in snap.method_dispatch if d.get("id") is not None} parents = defaultdict(list) for edge in snap.method_inheritance: parents[edge["runner"].removesuffix("_Runner")].append(edge["base_runner"].removesuffix("_Runner")) mappings = {} used_owners = set() for t in snap.types: name = canonical(t["script_name"]) row = type_row(name) owner = t.get("cpp_class") mapping = "explicit" if t.get("cpp_class_provenance"): row["mapping_notes"].append(t["cpp_class_provenance"]) # Known extractor failure: unrelated factory entries all mapped to Behaviour. if owner == "CMC_Behaviour" and name != "BEHAVIOUR": owner = None if not owner: candidate = [o for o in owners if owner_name(o) == name] owner = candidate[0] if len(candidate) == 1 else None mapping = "inferred" if owner: mappings[name] = (owner, mapping) used_owners.add(owner) else: row["mapping_notes"].append(f"{snap.binary.get('name', '?')}: typ wykryty, klasa/metody nieustalone.") for owner in owners: if owner not in used_owners: # Includes singleton classes and separate base scopes. Label inference explicitly. name = owner_name(owner) if name not in mappings: mappings[name] = (owner, "inferred") for name, (owner, mapping) in mappings.items(): visited = set() def add(current, inherited=False): if current in visited: return visited.add(current) for m in owners[current]: row = method_row(name, m["name"].upper()) if len(name_ids[(current, m["name"].upper())]) > 1: note = "Oryginał rejestruje tę nazwę z różnymi ID; skuteczny wybór wymaga sprawdzenia hashtable/dispatchera." if note not in row["notes"]: row["notes"].append(note) if len(id_names[(current, m.get("id"))]) > 1: note = "ID współdzielone przez nazwy: " + ", ".join(sorted(id_names[(current, m.get("id"))])) if note not in row["notes"]: row["notes"].append(note) # Child declarations override names from ancestors for this binary. if any(o["binary_id"] == bid and o["owner"] != current for o in row["original"]): continue d = dispatch.get((current, m.get("id")), {}) if m.get("id") is not None else {} row["original"].append({"binary_id": bid, "owner": current, "method_id": m.get("id"), "impl_addr": d.get("impl_addr"), "case_addr": d.get("case_addr"), "mapping": mapping, "inherited": inherited, "registration_addr": m.get("registration_addr"), "string_addr": m.get("string_addr"), "registration_call_addr": m.get("registration_call_addr"), "id_push_addr": m.get("id_push_addr"), "evidence_ids": m.get("evidence_ids", [])}) for parent in parents[current]: add(parent, True) add(owner) for name, t in emulator["types"].items(): type_row(name)["mapping_notes"].extend(t["mapping_notes"]) for method, binding in t["methods"].items(): row = method_row(name, method) notes = row["notes"] + binding.get("notes", []) row.update(binding) row["notes"] = notes # Global fallback exists on concrete Variable implementations, not native-only types. globals_ = emulator["types"].get("*", {}).get("methods", {}) for name, t in rows.items(): if name not in emulator["types"] or name == "*": continue for method, row in t["methods"].items(): if row["source"] is None and method in globals_: row.update(globals_[method]) row["notes"] = row["notes"] + ["Obsługa przez globalny fallback Variable."] # Reviews are declarative, version-controlled assertions; strict anchors prevent # accidentally making an entire family green based on one stale binary/address. for review in reviews or []: validate_review(review) row = rows.get(canonical(review["type"]), {}).get("methods", {}).get(review["method"].upper()) if row is None: warnings.append("Przegląd nie pasuje do API: " + review["type"] + "." + review["method"]) continue source = row["source"] or {} current = (source.get("body_sha256") == review["body_sha256"] and emulator["source_tree_sha256"] == review["source_tree_sha256"] and emulator.get("commit") == review["emulator_commit"]) matches = any(o["binary_id"] == review["binary_sha256"] for o in row["original"]) row["reviews"].append({**review, "current": current and matches}) for t in rows.values(): for row in t["methods"].values(): current = [r for r in row["reviews"] if r["current"]] targets = {o["binary_id"] for o in row["original"]} verified = {r["binary_sha256"] for r in current if r["verdict"] == "matches"} row["verified_variants"] = len(verified & targets) if targets and targets <= verified: row["verification"] = "runtime_tested" if all(any(r["binary_sha256"] == b and r["kind"] == "runtime" and r["verdict"] == "matches" for r in current) for b in targets) else "asm_reviewed" row["status"] = "implemented" elif any(r["verdict"] == "partial" for r in current): row["status"] = "partial" elif row["reviews"] and not current: row["verification"] = "stale" t["methods"] = sorted(t["methods"].values(), key=lambda m: m["name"]) t["mapping_notes"] = sorted(set(t["mapping_notes"])) all_rows = [m for t in rows.values() for m in t["methods"]] return {"schema": "tracker/1", "emulator": {k: v for k, v in emulator.items() if k not in ("types", "warnings")}, "binaries": sorted(binaries, key=lambda b: (b["name"].lower(), b["id"])), "types": sorted(rows.values(), key=lambda t: t["name"]), "summary": {"types": len(rows), "methods": len(all_rows), "status": dict(Counter(m["status"] for m in all_rows)), "verification": dict(Counter(m["verification"] for m in all_rows))}, "warnings": warnings + ["Brak obserwacji w wariancie oznacza nieustalone, nie brak metody.", "Rejestracja Java nie dowodzi poprawności. Liczniki obejmują pary typ/metoda, także odziedziczone.", "Dopasowania klas po nazwie są wnioskowane; zakres * to metody bazowe/globalne."]} def validate_review(review: dict) -> None: required = ("type", "method", "binary_sha256", "rva", "emulator_commit", "body_sha256", "source_tree_sha256", "reviewer", "date", "notes", "evidence", "kind", "verdict") if any(not isinstance(review.get(k), str) or not review[k].strip() for k in required): raise ValueError("Review requires nonempty fields: " + ", ".join(required)) for k in ("binary_sha256", "body_sha256", "source_tree_sha256"): if not SHA.fullmatch(review[k]): raise ValueError("Invalid review digest: " + k) if not re.fullmatch(r"0x[0-9a-fA-F]+", review["rva"]): raise ValueError("Review RVA must be hexadecimal (0x...)") if review["kind"] not in ("asm", "runtime") or review["verdict"] not in ("matches", "partial", "differs"): raise ValueError("Invalid review kind/verdict") def load_reviews(path: Path | None) -> list[dict]: if path is None: return [] data = json.loads(path.read_text()) if data.get("schema") != "tracker-reviews/1" or not isinstance(data.get("reviews"), list): raise ValueError("Expected tracker-reviews/1 with a reviews list") for review in data["reviews"]: validate_review(review) # A single current assertion per binary/method/source; conflicting verdicts fail closed. keys = [(r["type"], r["method"], r["binary_sha256"], r["source_tree_sha256"], r["emulator_commit"]) for r in data["reviews"]] if len(keys) != len(set(keys)): raise ValueError("Duplicate review for the same method, binary and source revision") return data["reviews"] def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("snapshots", nargs="+") parser.add_argument("--emulator-root", required=True, type=Path) parser.add_argument("--reviews", type=Path) parser.add_argument("--out", type=Path, required=True) args = parser.parse_args(argv) result = build_tracker([Snapshot.load(p) for p in args.snapshots], scan_emulator(args.emulator_root), load_reviews(args.reviews)) args.out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") print(json.dumps(result["summary"])) if __name__ == "__main__": main()