feat(usage): count how game scripts use each type and rank roadmap gaps by it

ams.usage reads game discs with 7-Zip (bsdtar misses the files on some
images), decrypts CNV/DEF/CLASS like the emulator's ScriptDecypher and
counts method calls, event handlers, properties and objects per declared
type. CLONE copies, database cursors, global objects, CLASS instances and
string-named targets are resolved; the rest stays per method name.

ams.roadmap --usage orders every gap by the games and script places that
use it, adds a priority table and lists direct usage the original's known
API does not explain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Patryk Gensch
2026-10-28 20:37:10 +01:00
co-authored by Claude Opus 5.5
parent 96c39d542a
commit ebbeeea144
3 changed files with 453 additions and 25 deletions
+129 -25
View File
@@ -1,56 +1,133 @@
"""Emulator roadmap from an Atlas export: what each script scope still lacks.
python -m ams.roadmap tracker.json -o docs/roadmap.md
python -m ams.roadmap tracker.json -o docs/roadmap.md [--usage data/script-usage.json]
Per scope: methods without a registration (with the signature read from the original),
emulator stubs, events and properties the emulator never names. Every item lists the engine
families whose DLLs have it - an item present in every family is core API, one present only
in BlooMoo is late. The report restates the tracker's limits: registration is not
correctness, and "never named" means no literal was found.
in BlooMoo is late. With `--usage` (python -m ams.usage) every item also says how many games
use it and in how many places of their scripts, gaps are ordered by that, and the report
lists the most used gaps and what the scripts use beyond the original's known API. The report
restates the tracker's limits: registration is not correctness, and "never named" means no
literal was found.
"""
from __future__ import annotations
import argparse
import json
from collections import Counter
from collections import Counter, defaultdict
from pathlib import Path
FAMILY_ORDER = ["Piklib 6.1", "Piklib 7.1", "Piklib 7.2", "Piklib 8", "BlooMoo"]
SHORT = {"Piklib 6.1": "6.1", "Piklib 7.1": "7.1", "Piklib 7.2": "7.2", "Piklib 8": "8", "BlooMoo": "BM"}
HIDDEN = {"INSTANCE"} # emulator helper, not an original type
HIDDEN = {"INSTANCE"} # emulator helper, not an original type; in the scripts a CLASS instance
KINDS = {"missing": "metoda", "stub": "stub", "event": "event", "field": "pole"}
USAGE_KIND = {"missing": "method", "stub": "method", "event": "event", "field": "field"}
USAGE_TABLES = (("methods", "method"), ("dynamic", "method"), ("events", "event"), ("fields", "field"))
def family_sort(found) -> list[str]:
return sorted(found, key=lambda f: (FAMILY_ORDER.index(f) if f in FAMILY_ORDER else len(FAMILY_ORDER), f))
def families(binary_ids, family_of) -> list[str]:
found = {family_of.get(b) for b in binary_ids} - {None}
return sorted(found, key=lambda f: (FAMILY_ORDER.index(f) if f in FAMILY_ORDER else len(FAMILY_ORDER), f))
return family_sort({family_of.get(b) for b in binary_ids} - {None})
def label(fams) -> str:
return ", ".join(SHORT.get(f, f) for f in fams) or "—"
def scope_gaps(scope: dict, family_of: dict) -> dict:
methods = scope["methods"]
def usage_index(usage: dict, types: dict) -> tuple[dict, dict, dict]:
"""Script usage of the tracked families: (scope, kind, name) -> {titles, places, families}.
kind is method / event / field / object. Within a title the build with the most places
counts, so several releases of one game are one title; places then add up across titles.
A method of the base runner resolves to `*`; a call whose target object is unknown counts
for the scope that has the method when exactly one does (or `*`). Returns the index, the
direct usage the original's known API does not explain (keyed by the script's type name),
and the number of builds and titles counted.
"""
lookup = {name.replace("_", ""): name for name in types}
known = {name: {"method": {m["name"] for m in t["methods"] if m["original"]},
"event": {e["name"] for e in t.get("events", [])},
"field": {f["name"] for f in t.get("fields", [])}} for name, t in types.items()}
base = known.get("*", {}).get("method", set())
owners = defaultdict(set)
for scope, names in known.items():
for method in names["method"]:
owners[method].add(scope)
best, outside_best, fams = defaultdict(Counter), defaultdict(Counter), defaultdict(set)
builds = [b for b in usage["builds"] if b.get("family") in FAMILY_ORDER]
for build in builds:
places, outside = Counter(), Counter()
for table, kind in USAGE_TABLES:
for type_name, names in build[table].items():
scope = lookup.get(type_name.replace("_", ""))
if scope in HIDDEN:
continue
for name, count in names.items():
if scope and name in known[scope][kind]:
places[(scope, kind, name)] += count
elif kind == "method" and name in base:
places[("*", kind, name)] += count
elif table != "dynamic": # a string-named target is a guess; only direct use is evidence
outside[(type_name, kind, name)] += count
for name, count in build["unresolved"]["calls"].items():
scopes = owners.get(name, set()) - HIDDEN
if "*" in scopes or len(scopes) == 1:
places[("*" if "*" in scopes else scopes.pop(), "method", name)] += count
for type_name, count in build["objects"].items():
scope = lookup.get(type_name.replace("_", ""))
if scope and scope not in HIDDEN:
places[(scope, "object", "")] += count
for table, counts in ((best, places), (outside_best, outside)):
for key, count in counts.items():
table[key][build["title"]] = max(table[key][build["title"]], count)
fams[key].add(build["family"])
def summary(table):
return {key: {"titles": len(titles), "places": sum(titles.values()), "families": family_sort(fams[key])}
for key, titles in table.items()}
return summary(best), summary(outside_best), {"builds": len(builds), "titles": len({b["title"] for b in builds})}
def scope_gaps(scope: dict, family_of: dict, used: dict | None = None) -> dict:
name, methods = scope["name"], scope["methods"]
original = [m for m in methods if m["original"]]
def item(name, binary_ids, detail=""):
return {"name": name, "families": families(binary_ids, family_of), "detail": detail}
missing = [item(m["name"], [o["binary_id"] for o in m["original"]], " / ".join(m.get("signatures") or []))
def item(kind, item_name, binary_ids, detail=""):
use = None if used is None else used.get((name, USAGE_KIND[kind], item_name), {"titles": 0, "places": 0})
return {"name": item_name, "kind": kind, "families": families(binary_ids, family_of), "detail": detail,
"use": use}
missing = [item("missing", m["name"], [o["binary_id"] for o in m["original"]], " / ".join(m.get("signatures") or []))
for m in original if m["status"] == "missing"]
stubs = [item(m["name"], [o["binary_id"] for o in m["original"]]) for m in methods if m["status"] == "stub_candidate"]
events = [item(e["name"], e["original"]) for e in scope.get("events", []) if e["status"] == "not_found"]
fields = [item(f["name"], f["original"], "/".join(f.get("value_types", [])))
stubs = [item("stub", m["name"], [o["binary_id"] for o in m["original"]])
for m in methods if m["status"] == "stub_candidate"]
events = [item("event", e["name"], e["original"]) for e in scope.get("events", []) if e["status"] == "not_found"]
fields = [item("field", f["name"], f["original"], "/".join(f.get("value_types", [])))
for f in scope.get("fields", []) if f["status"] == "not_found"]
for items in (missing, events, fields): # widest presence first: core API before late additions
items.sort(key=lambda i: (-len(i["families"]), i["name"]))
return {"name": scope["name"], "methods_total": len(original),
for items in (missing, stubs, events, fields): # most used, then widest presence: core API before late additions
items.sort(key=lambda i: (-(i["use"] or {}).get("titles", 0), -(i["use"] or {}).get("places", 0),
-len(i["families"]), i["name"]))
return {"name": name, "methods_total": len(original),
"methods_registered": sum(m["status"] not in ("missing", "stub_candidate") for m in original),
"events_total": len(scope.get("events", [])), "fields_total": len(scope.get("fields", [])),
"objects": None if used is None else used.get((name, "object", "")),
"missing": missing, "stubs": stubs, "events": events, "fields": fields}
def render(tracker: dict) -> str:
def used_text(use) -> str:
if use is None:
return ""
return " — gry: %d, miejsca: %d" % (use["titles"], use["places"]) if use["titles"] else " — nieużywane"
def render(tracker: dict, usage: dict | None = None) -> str:
family_of = {b["id"]: b.get("family") for b in tracker["binaries"]}
scopes = [scope_gaps(t, family_of) for t in tracker["types"] if t["name"] not in HIDDEN]
types = {t["name"]: t for t in tracker["types"]}
used, outside, counted = usage_index(usage, types) if usage else (None, None, None)
scopes = [scope_gaps(t, family_of, used) for t in tracker["types"] if t["name"] not in HIDDEN]
scopes = [s for s in scopes if s["missing"] or s["stubs"] or s["events"] or s["fields"]]
scopes.sort(key=lambda s: (-(len(s["missing"]) + len(s["stubs"]) + len(s["events"]) + len(s["fields"])), s["name"]))
emulator = tracker.get("emulator", {})
@@ -62,20 +139,45 @@ def render(tracker: dict) -> str:
"Binarki: " + ", ".join(f"{SHORT.get(f, f)} × {n}" for f, n in sorted(counts.items(),
key=lambda kv: FAMILY_ORDER.index(kv[0]) if kv[0] in FAMILY_ORDER else 99)) + ".",
"Rejestracja nie oznacza poprawności; „brak literału” znaczy tylko, że nazwy nie ma w kodzie emulatora.",
"W nawiasie: rodziny silnika, w których DLL pozycja występuje.", "",
"| Zakres | Metody (zarejestrowane / w oryginale) | Bez rejestracji | Stuby | Eventy bez literału | Pola bez literału |",
"|---|---:|---:|---:|---:|---:|"]
"W nawiasie: rodziny silnika, w których DLL pozycja występuje."]
if usage:
out += [f"Użycie w skryptach (`python -m ams.usage`): {counted['builds']} wydań, {counted['titles']} gier "
"z rodzin 6.1+. „Gry” to tytuły (z kilku wydań jednej gry liczy się to z największą liczbą), "
"„miejsca” to wystąpienia w skryptach, nie wykonania; wywołanie przez nazwę z łańcucha liczy się "
"dla typów obiektów, które skrypty do tej nazwy przypisują."]
out += ["", "| Zakres | Metody (zarejestrowane / w oryginale) | Bez rejestracji | Stuby | Eventy bez literału | Pola bez literału |",
"|---|---:|---:|---:|---:|---:|"]
for s in scopes:
out.append(f"| {s['name']} | {s['methods_registered']} / {s['methods_total']} | {len(s['missing'])} | "
f"{len(s['stubs'])} | {len(s['events'])} / {s['events_total']} | {len(s['fields'])} / {s['fields_total']} |")
if usage:
gaps = [dict(i, scope=s["name"]) for s in scopes for group in ("missing", "stubs", "events", "fields")
for i in s[group] if i["use"]["titles"]]
gaps.sort(key=lambda i: (-i["use"]["titles"], -i["use"]["places"], i["scope"], i["name"]))
out += ["", "## Priorytety: luki używane przez gry", "",
"| Zakres | Pozycja | Rodzaj | Gry | Miejsca | Rodziny DLL |", "|---|---|---|---:|---:|---|"]
out += [f"| {i['scope']} | `{i['name']}` | {KINDS[i['kind']]} | {i['use']['titles']} | {i['use']['places']} | "
f"{label(i['families'])} |" for i in gaps[:50]]
beyond = sorted(outside.items(), key=lambda kv: (-kv[1]["titles"], -kv[1]["places"], kv[0]))
out += ["", "## Użycia spoza znanego API oryginału", "",
"Skrypty używają tych nazw na obiektach danego typu, a analiza DLL nie przypisuje ich temu typowi: "
"luka analizatora, źle rozpoznany typ obiektu albo nazwa, którą silnik pomija. Metody grupy "
"(GROUP) emulator przekazuje jej elementom (`GroupVariable.callMethod`); skoro gry ich używają, "
"oryginał obsługuje je podobnie, a runner grupy ich nie wymienia."]
for title, kinds in (("Metody", {"method"}), ("Pola i eventy", {"field", "event"})):
out += ["", f"{title}:", "", "| Typ | Nazwa | Rodzaj | Gry | Miejsca | Rodziny gier |", "|---|---|---|---:|---:|---|"]
out += [f"| {t} | `{n}` | {dict(method='metoda', event='event', field='pole')[k]} | {u['titles']} | "
f"{u['places']} | {label(u['families'])} |" for (t, k, n), u in beyond if k in kinds][:30]
for s in scopes:
out += ["", f"## {s['name']}"]
if s["objects"]:
out.append(f"Obiekty w skryptach: {s['objects']['places']} (gry: {s['objects']['titles']}).")
for title, items in (("Metody bez rejestracji", s["missing"]), ("Stuby w emulatorze", s["stubs"]),
("Eventy bez literału", s["events"]), ("Pola bez literału", s["fields"])):
if items:
out.append(f"- **{title}:** " + "; ".join(
f"`{i['name']}`" + (f" {i['detail']}" if i["detail"] else "") + f" ({label(i['families'])})"
for i in items))
+ used_text(i["use"]) for i in items))
return "\n".join(out) + "\n"
@@ -83,8 +185,10 @@ def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("tracker", type=Path, help="Atlas export (python -m ams.tracker ... --out)")
parser.add_argument("-o", "--out", type=Path, required=True)
parser.add_argument("--usage", type=Path, help="script usage (python -m ams.usage ... -o)")
args = parser.parse_args(argv)
args.out.write_text(render(json.loads(args.tracker.read_text(encoding="utf-8"))), encoding="utf-8")
usage = json.loads(args.usage.read_text(encoding="utf-8")) if args.usage else None
args.out.write_text(render(json.loads(args.tracker.read_text(encoding="utf-8")), usage), encoding="utf-8")
if __name__ == "__main__":