feat(usage): count how game scripts use each type and rank roadmap gaps by it
ams.usage reads game discs with 7-Zip (bsdtar misses the files on some images), decrypts CNV/DEF/CLASS like the emulator's ScriptDecypher and counts method calls, event handlers, properties and objects per declared type. CLONE copies, database cursors, global objects, CLASS instances and string-named targets are resolved; the rest stays per method name. ams.roadmap --usage orders every gap by the games and script places that use it, adds a priority table and lists direct usage the original's known API does not explain. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
96c39d542a
commit
ebbeeea144
+129
-25
@@ -1,56 +1,133 @@
|
||||
"""Emulator roadmap from an Atlas export: what each script scope still lacks.
|
||||
|
||||
python -m ams.roadmap tracker.json -o docs/roadmap.md
|
||||
python -m ams.roadmap tracker.json -o docs/roadmap.md [--usage data/script-usage.json]
|
||||
|
||||
Per scope: methods without a registration (with the signature read from the original),
|
||||
emulator stubs, events and properties the emulator never names. Every item lists the engine
|
||||
families whose DLLs have it - an item present in every family is core API, one present only
|
||||
in BlooMoo is late. The report restates the tracker's limits: registration is not
|
||||
correctness, and "never named" means no literal was found.
|
||||
in BlooMoo is late. With `--usage` (python -m ams.usage) every item also says how many games
|
||||
use it and in how many places of their scripts, gaps are ordered by that, and the report
|
||||
lists the most used gaps and what the scripts use beyond the original's known API. The report
|
||||
restates the tracker's limits: registration is not correctness, and "never named" means no
|
||||
literal was found.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
FAMILY_ORDER = ["Piklib 6.1", "Piklib 7.1", "Piklib 7.2", "Piklib 8", "BlooMoo"]
|
||||
SHORT = {"Piklib 6.1": "6.1", "Piklib 7.1": "7.1", "Piklib 7.2": "7.2", "Piklib 8": "8", "BlooMoo": "BM"}
|
||||
HIDDEN = {"INSTANCE"} # emulator helper, not an original type
|
||||
HIDDEN = {"INSTANCE"} # emulator helper, not an original type; in the scripts a CLASS instance
|
||||
KINDS = {"missing": "metoda", "stub": "stub", "event": "event", "field": "pole"}
|
||||
USAGE_KIND = {"missing": "method", "stub": "method", "event": "event", "field": "field"}
|
||||
USAGE_TABLES = (("methods", "method"), ("dynamic", "method"), ("events", "event"), ("fields", "field"))
|
||||
|
||||
|
||||
def family_sort(found) -> list[str]:
|
||||
return sorted(found, key=lambda f: (FAMILY_ORDER.index(f) if f in FAMILY_ORDER else len(FAMILY_ORDER), f))
|
||||
|
||||
|
||||
def families(binary_ids, family_of) -> list[str]:
|
||||
found = {family_of.get(b) for b in binary_ids} - {None}
|
||||
return sorted(found, key=lambda f: (FAMILY_ORDER.index(f) if f in FAMILY_ORDER else len(FAMILY_ORDER), f))
|
||||
return family_sort({family_of.get(b) for b in binary_ids} - {None})
|
||||
|
||||
|
||||
def label(fams) -> str:
|
||||
return ", ".join(SHORT.get(f, f) for f in fams) or "—"
|
||||
|
||||
|
||||
def scope_gaps(scope: dict, family_of: dict) -> dict:
|
||||
methods = scope["methods"]
|
||||
def usage_index(usage: dict, types: dict) -> tuple[dict, dict, dict]:
|
||||
"""Script usage of the tracked families: (scope, kind, name) -> {titles, places, families}.
|
||||
|
||||
kind is method / event / field / object. Within a title the build with the most places
|
||||
counts, so several releases of one game are one title; places then add up across titles.
|
||||
A method of the base runner resolves to `*`; a call whose target object is unknown counts
|
||||
for the scope that has the method when exactly one does (or `*`). Returns the index, the
|
||||
direct usage the original's known API does not explain (keyed by the script's type name),
|
||||
and the number of builds and titles counted.
|
||||
"""
|
||||
lookup = {name.replace("_", ""): name for name in types}
|
||||
known = {name: {"method": {m["name"] for m in t["methods"] if m["original"]},
|
||||
"event": {e["name"] for e in t.get("events", [])},
|
||||
"field": {f["name"] for f in t.get("fields", [])}} for name, t in types.items()}
|
||||
base = known.get("*", {}).get("method", set())
|
||||
owners = defaultdict(set)
|
||||
for scope, names in known.items():
|
||||
for method in names["method"]:
|
||||
owners[method].add(scope)
|
||||
best, outside_best, fams = defaultdict(Counter), defaultdict(Counter), defaultdict(set)
|
||||
builds = [b for b in usage["builds"] if b.get("family") in FAMILY_ORDER]
|
||||
for build in builds:
|
||||
places, outside = Counter(), Counter()
|
||||
for table, kind in USAGE_TABLES:
|
||||
for type_name, names in build[table].items():
|
||||
scope = lookup.get(type_name.replace("_", ""))
|
||||
if scope in HIDDEN:
|
||||
continue
|
||||
for name, count in names.items():
|
||||
if scope and name in known[scope][kind]:
|
||||
places[(scope, kind, name)] += count
|
||||
elif kind == "method" and name in base:
|
||||
places[("*", kind, name)] += count
|
||||
elif table != "dynamic": # a string-named target is a guess; only direct use is evidence
|
||||
outside[(type_name, kind, name)] += count
|
||||
for name, count in build["unresolved"]["calls"].items():
|
||||
scopes = owners.get(name, set()) - HIDDEN
|
||||
if "*" in scopes or len(scopes) == 1:
|
||||
places[("*" if "*" in scopes else scopes.pop(), "method", name)] += count
|
||||
for type_name, count in build["objects"].items():
|
||||
scope = lookup.get(type_name.replace("_", ""))
|
||||
if scope and scope not in HIDDEN:
|
||||
places[(scope, "object", "")] += count
|
||||
for table, counts in ((best, places), (outside_best, outside)):
|
||||
for key, count in counts.items():
|
||||
table[key][build["title"]] = max(table[key][build["title"]], count)
|
||||
fams[key].add(build["family"])
|
||||
|
||||
def summary(table):
|
||||
return {key: {"titles": len(titles), "places": sum(titles.values()), "families": family_sort(fams[key])}
|
||||
for key, titles in table.items()}
|
||||
return summary(best), summary(outside_best), {"builds": len(builds), "titles": len({b["title"] for b in builds})}
|
||||
|
||||
|
||||
def scope_gaps(scope: dict, family_of: dict, used: dict | None = None) -> dict:
|
||||
name, methods = scope["name"], scope["methods"]
|
||||
original = [m for m in methods if m["original"]]
|
||||
def item(name, binary_ids, detail=""):
|
||||
return {"name": name, "families": families(binary_ids, family_of), "detail": detail}
|
||||
missing = [item(m["name"], [o["binary_id"] for o in m["original"]], " / ".join(m.get("signatures") or []))
|
||||
|
||||
def item(kind, item_name, binary_ids, detail=""):
|
||||
use = None if used is None else used.get((name, USAGE_KIND[kind], item_name), {"titles": 0, "places": 0})
|
||||
return {"name": item_name, "kind": kind, "families": families(binary_ids, family_of), "detail": detail,
|
||||
"use": use}
|
||||
missing = [item("missing", m["name"], [o["binary_id"] for o in m["original"]], " / ".join(m.get("signatures") or []))
|
||||
for m in original if m["status"] == "missing"]
|
||||
stubs = [item(m["name"], [o["binary_id"] for o in m["original"]]) for m in methods if m["status"] == "stub_candidate"]
|
||||
events = [item(e["name"], e["original"]) for e in scope.get("events", []) if e["status"] == "not_found"]
|
||||
fields = [item(f["name"], f["original"], "/".join(f.get("value_types", [])))
|
||||
stubs = [item("stub", m["name"], [o["binary_id"] for o in m["original"]])
|
||||
for m in methods if m["status"] == "stub_candidate"]
|
||||
events = [item("event", e["name"], e["original"]) for e in scope.get("events", []) if e["status"] == "not_found"]
|
||||
fields = [item("field", f["name"], f["original"], "/".join(f.get("value_types", [])))
|
||||
for f in scope.get("fields", []) if f["status"] == "not_found"]
|
||||
for items in (missing, events, fields): # widest presence first: core API before late additions
|
||||
items.sort(key=lambda i: (-len(i["families"]), i["name"]))
|
||||
return {"name": scope["name"], "methods_total": len(original),
|
||||
for items in (missing, stubs, events, fields): # most used, then widest presence: core API before late additions
|
||||
items.sort(key=lambda i: (-(i["use"] or {}).get("titles", 0), -(i["use"] or {}).get("places", 0),
|
||||
-len(i["families"]), i["name"]))
|
||||
return {"name": name, "methods_total": len(original),
|
||||
"methods_registered": sum(m["status"] not in ("missing", "stub_candidate") for m in original),
|
||||
"events_total": len(scope.get("events", [])), "fields_total": len(scope.get("fields", [])),
|
||||
"objects": None if used is None else used.get((name, "object", "")),
|
||||
"missing": missing, "stubs": stubs, "events": events, "fields": fields}
|
||||
|
||||
|
||||
def render(tracker: dict) -> str:
|
||||
def used_text(use) -> str:
|
||||
if use is None:
|
||||
return ""
|
||||
return " — gry: %d, miejsca: %d" % (use["titles"], use["places"]) if use["titles"] else " — nieużywane"
|
||||
|
||||
|
||||
def render(tracker: dict, usage: dict | None = None) -> str:
|
||||
family_of = {b["id"]: b.get("family") for b in tracker["binaries"]}
|
||||
scopes = [scope_gaps(t, family_of) for t in tracker["types"] if t["name"] not in HIDDEN]
|
||||
types = {t["name"]: t for t in tracker["types"]}
|
||||
used, outside, counted = usage_index(usage, types) if usage else (None, None, None)
|
||||
scopes = [scope_gaps(t, family_of, used) for t in tracker["types"] if t["name"] not in HIDDEN]
|
||||
scopes = [s for s in scopes if s["missing"] or s["stubs"] or s["events"] or s["fields"]]
|
||||
scopes.sort(key=lambda s: (-(len(s["missing"]) + len(s["stubs"]) + len(s["events"]) + len(s["fields"])), s["name"]))
|
||||
emulator = tracker.get("emulator", {})
|
||||
@@ -62,20 +139,45 @@ def render(tracker: dict) -> str:
|
||||
"Binarki: " + ", ".join(f"{SHORT.get(f, f)} × {n}" for f, n in sorted(counts.items(),
|
||||
key=lambda kv: FAMILY_ORDER.index(kv[0]) if kv[0] in FAMILY_ORDER else 99)) + ".",
|
||||
"Rejestracja nie oznacza poprawności; „brak literału” znaczy tylko, że nazwy nie ma w kodzie emulatora.",
|
||||
"W nawiasie: rodziny silnika, w których DLL pozycja występuje.", "",
|
||||
"| Zakres | Metody (zarejestrowane / w oryginale) | Bez rejestracji | Stuby | Eventy bez literału | Pola bez literału |",
|
||||
"|---|---:|---:|---:|---:|---:|"]
|
||||
"W nawiasie: rodziny silnika, w których DLL pozycja występuje."]
|
||||
if usage:
|
||||
out += [f"Użycie w skryptach (`python -m ams.usage`): {counted['builds']} wydań, {counted['titles']} gier "
|
||||
"z rodzin 6.1+. „Gry” to tytuły (z kilku wydań jednej gry liczy się to z największą liczbą), "
|
||||
"„miejsca” to wystąpienia w skryptach, nie wykonania; wywołanie przez nazwę z łańcucha liczy się "
|
||||
"dla typów obiektów, które skrypty do tej nazwy przypisują."]
|
||||
out += ["", "| Zakres | Metody (zarejestrowane / w oryginale) | Bez rejestracji | Stuby | Eventy bez literału | Pola bez literału |",
|
||||
"|---|---:|---:|---:|---:|---:|"]
|
||||
for s in scopes:
|
||||
out.append(f"| {s['name']} | {s['methods_registered']} / {s['methods_total']} | {len(s['missing'])} | "
|
||||
f"{len(s['stubs'])} | {len(s['events'])} / {s['events_total']} | {len(s['fields'])} / {s['fields_total']} |")
|
||||
if usage:
|
||||
gaps = [dict(i, scope=s["name"]) for s in scopes for group in ("missing", "stubs", "events", "fields")
|
||||
for i in s[group] if i["use"]["titles"]]
|
||||
gaps.sort(key=lambda i: (-i["use"]["titles"], -i["use"]["places"], i["scope"], i["name"]))
|
||||
out += ["", "## Priorytety: luki używane przez gry", "",
|
||||
"| Zakres | Pozycja | Rodzaj | Gry | Miejsca | Rodziny DLL |", "|---|---|---|---:|---:|---|"]
|
||||
out += [f"| {i['scope']} | `{i['name']}` | {KINDS[i['kind']]} | {i['use']['titles']} | {i['use']['places']} | "
|
||||
f"{label(i['families'])} |" for i in gaps[:50]]
|
||||
beyond = sorted(outside.items(), key=lambda kv: (-kv[1]["titles"], -kv[1]["places"], kv[0]))
|
||||
out += ["", "## Użycia spoza znanego API oryginału", "",
|
||||
"Skrypty używają tych nazw na obiektach danego typu, a analiza DLL nie przypisuje ich temu typowi: "
|
||||
"luka analizatora, źle rozpoznany typ obiektu albo nazwa, którą silnik pomija. Metody grupy "
|
||||
"(GROUP) emulator przekazuje jej elementom (`GroupVariable.callMethod`); skoro gry ich używają, "
|
||||
"oryginał obsługuje je podobnie, a runner grupy ich nie wymienia."]
|
||||
for title, kinds in (("Metody", {"method"}), ("Pola i eventy", {"field", "event"})):
|
||||
out += ["", f"{title}:", "", "| Typ | Nazwa | Rodzaj | Gry | Miejsca | Rodziny gier |", "|---|---|---|---:|---:|---|"]
|
||||
out += [f"| {t} | `{n}` | {dict(method='metoda', event='event', field='pole')[k]} | {u['titles']} | "
|
||||
f"{u['places']} | {label(u['families'])} |" for (t, k, n), u in beyond if k in kinds][:30]
|
||||
for s in scopes:
|
||||
out += ["", f"## {s['name']}"]
|
||||
if s["objects"]:
|
||||
out.append(f"Obiekty w skryptach: {s['objects']['places']} (gry: {s['objects']['titles']}).")
|
||||
for title, items in (("Metody bez rejestracji", s["missing"]), ("Stuby w emulatorze", s["stubs"]),
|
||||
("Eventy bez literału", s["events"]), ("Pola bez literału", s["fields"])):
|
||||
if items:
|
||||
out.append(f"- **{title}:** " + "; ".join(
|
||||
f"`{i['name']}`" + (f" {i['detail']}" if i["detail"] else "") + f" ({label(i['families'])})"
|
||||
for i in items))
|
||||
+ used_text(i["use"]) for i in items))
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
@@ -83,8 +185,10 @@ def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("tracker", type=Path, help="Atlas export (python -m ams.tracker ... --out)")
|
||||
parser.add_argument("-o", "--out", type=Path, required=True)
|
||||
parser.add_argument("--usage", type=Path, help="script usage (python -m ams.usage ... -o)")
|
||||
args = parser.parse_args(argv)
|
||||
args.out.write_text(render(json.loads(args.tracker.read_text(encoding="utf-8"))), encoding="utf-8")
|
||||
usage = json.loads(args.usage.read_text(encoding="utf-8")) if args.usage else None
|
||||
args.out.write_text(render(json.loads(args.tracker.read_text(encoding="utf-8")), usage), encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user