feat(usage): count how game scripts use each type and rank roadmap gaps by it

ams.usage reads game discs with 7-Zip (bsdtar misses the files on some
images), decrypts CNV/DEF/CLASS like the emulator's ScriptDecypher and
counts method calls, event handlers, properties and objects per declared
type. CLONE copies, database cursors, global objects, CLASS instances and
string-named targets are resolved; the rest stays per method name.

ams.roadmap --usage orders every gap by the games and script places that
use it, adds a priority table and lists direct usage the original's known
API does not explain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Patryk Gensch
2026-10-28 20:37:10 +01:00
co-authored by Claude Opus 5.5
parent 96c39d542a
commit ebbeeea144
3 changed files with 453 additions and 25 deletions
+129 -25
View File
@@ -1,56 +1,133 @@
"""Emulator roadmap from an Atlas export: what each script scope still lacks.
python -m ams.roadmap tracker.json -o docs/roadmap.md
python -m ams.roadmap tracker.json -o docs/roadmap.md [--usage data/script-usage.json]
Per scope: methods without a registration (with the signature read from the original),
emulator stubs, events and properties the emulator never names. Every item lists the engine
families whose DLLs have it - an item present in every family is core API, one present only
in BlooMoo is late. The report restates the tracker's limits: registration is not
correctness, and "never named" means no literal was found.
in BlooMoo is late. With `--usage` (python -m ams.usage) every item also says how many games
use it and in how many places of their scripts, gaps are ordered by that, and the report
lists the most used gaps and what the scripts use beyond the original's known API. The report
restates the tracker's limits: registration is not correctness, and "never named" means no
literal was found.
"""
from __future__ import annotations
import argparse
import json
from collections import Counter
from collections import Counter, defaultdict
from pathlib import Path
FAMILY_ORDER = ["Piklib 6.1", "Piklib 7.1", "Piklib 7.2", "Piklib 8", "BlooMoo"]
SHORT = {"Piklib 6.1": "6.1", "Piklib 7.1": "7.1", "Piklib 7.2": "7.2", "Piklib 8": "8", "BlooMoo": "BM"}
HIDDEN = {"INSTANCE"} # emulator helper, not an original type
HIDDEN = {"INSTANCE"} # emulator helper, not an original type; in the scripts a CLASS instance
KINDS = {"missing": "metoda", "stub": "stub", "event": "event", "field": "pole"}
USAGE_KIND = {"missing": "method", "stub": "method", "event": "event", "field": "field"}
USAGE_TABLES = (("methods", "method"), ("dynamic", "method"), ("events", "event"), ("fields", "field"))
def family_sort(found) -> list[str]:
return sorted(found, key=lambda f: (FAMILY_ORDER.index(f) if f in FAMILY_ORDER else len(FAMILY_ORDER), f))
def families(binary_ids, family_of) -> list[str]:
found = {family_of.get(b) for b in binary_ids} - {None}
return sorted(found, key=lambda f: (FAMILY_ORDER.index(f) if f in FAMILY_ORDER else len(FAMILY_ORDER), f))
return family_sort({family_of.get(b) for b in binary_ids} - {None})
def label(fams) -> str:
return ", ".join(SHORT.get(f, f) for f in fams) or "—"
def scope_gaps(scope: dict, family_of: dict) -> dict:
methods = scope["methods"]
def usage_index(usage: dict, types: dict) -> tuple[dict, dict, dict]:
"""Script usage of the tracked families: (scope, kind, name) -> {titles, places, families}.
kind is method / event / field / object. Within a title the build with the most places
counts, so several releases of one game are one title; places then add up across titles.
A method of the base runner resolves to `*`; a call whose target object is unknown counts
for the scope that has the method when exactly one does (or `*`). Returns the index, the
direct usage the original's known API does not explain (keyed by the script's type name),
and the number of builds and titles counted.
"""
lookup = {name.replace("_", ""): name for name in types}
known = {name: {"method": {m["name"] for m in t["methods"] if m["original"]},
"event": {e["name"] for e in t.get("events", [])},
"field": {f["name"] for f in t.get("fields", [])}} for name, t in types.items()}
base = known.get("*", {}).get("method", set())
owners = defaultdict(set)
for scope, names in known.items():
for method in names["method"]:
owners[method].add(scope)
best, outside_best, fams = defaultdict(Counter), defaultdict(Counter), defaultdict(set)
builds = [b for b in usage["builds"] if b.get("family") in FAMILY_ORDER]
for build in builds:
places, outside = Counter(), Counter()
for table, kind in USAGE_TABLES:
for type_name, names in build[table].items():
scope = lookup.get(type_name.replace("_", ""))
if scope in HIDDEN:
continue
for name, count in names.items():
if scope and name in known[scope][kind]:
places[(scope, kind, name)] += count
elif kind == "method" and name in base:
places[("*", kind, name)] += count
elif table != "dynamic": # a string-named target is a guess; only direct use is evidence
outside[(type_name, kind, name)] += count
for name, count in build["unresolved"]["calls"].items():
scopes = owners.get(name, set()) - HIDDEN
if "*" in scopes or len(scopes) == 1:
places[("*" if "*" in scopes else scopes.pop(), "method", name)] += count
for type_name, count in build["objects"].items():
scope = lookup.get(type_name.replace("_", ""))
if scope and scope not in HIDDEN:
places[(scope, "object", "")] += count
for table, counts in ((best, places), (outside_best, outside)):
for key, count in counts.items():
table[key][build["title"]] = max(table[key][build["title"]], count)
fams[key].add(build["family"])
def summary(table):
return {key: {"titles": len(titles), "places": sum(titles.values()), "families": family_sort(fams[key])}
for key, titles in table.items()}
return summary(best), summary(outside_best), {"builds": len(builds), "titles": len({b["title"] for b in builds})}
def scope_gaps(scope: dict, family_of: dict, used: dict | None = None) -> dict:
name, methods = scope["name"], scope["methods"]
original = [m for m in methods if m["original"]]
def item(name, binary_ids, detail=""):
return {"name": name, "families": families(binary_ids, family_of), "detail": detail}
missing = [item(m["name"], [o["binary_id"] for o in m["original"]], " / ".join(m.get("signatures") or []))
def item(kind, item_name, binary_ids, detail=""):
use = None if used is None else used.get((name, USAGE_KIND[kind], item_name), {"titles": 0, "places": 0})
return {"name": item_name, "kind": kind, "families": families(binary_ids, family_of), "detail": detail,
"use": use}
missing = [item("missing", m["name"], [o["binary_id"] for o in m["original"]], " / ".join(m.get("signatures") or []))
for m in original if m["status"] == "missing"]
stubs = [item(m["name"], [o["binary_id"] for o in m["original"]]) for m in methods if m["status"] == "stub_candidate"]
events = [item(e["name"], e["original"]) for e in scope.get("events", []) if e["status"] == "not_found"]
fields = [item(f["name"], f["original"], "/".join(f.get("value_types", [])))
stubs = [item("stub", m["name"], [o["binary_id"] for o in m["original"]])
for m in methods if m["status"] == "stub_candidate"]
events = [item("event", e["name"], e["original"]) for e in scope.get("events", []) if e["status"] == "not_found"]
fields = [item("field", f["name"], f["original"], "/".join(f.get("value_types", [])))
for f in scope.get("fields", []) if f["status"] == "not_found"]
for items in (missing, events, fields): # widest presence first: core API before late additions
items.sort(key=lambda i: (-len(i["families"]), i["name"]))
return {"name": scope["name"], "methods_total": len(original),
for items in (missing, stubs, events, fields): # most used, then widest presence: core API before late additions
items.sort(key=lambda i: (-(i["use"] or {}).get("titles", 0), -(i["use"] or {}).get("places", 0),
-len(i["families"]), i["name"]))
return {"name": name, "methods_total": len(original),
"methods_registered": sum(m["status"] not in ("missing", "stub_candidate") for m in original),
"events_total": len(scope.get("events", [])), "fields_total": len(scope.get("fields", [])),
"objects": None if used is None else used.get((name, "object", "")),
"missing": missing, "stubs": stubs, "events": events, "fields": fields}
def render(tracker: dict) -> str:
def used_text(use) -> str:
if use is None:
return ""
return " — gry: %d, miejsca: %d" % (use["titles"], use["places"]) if use["titles"] else " — nieużywane"
def render(tracker: dict, usage: dict | None = None) -> str:
family_of = {b["id"]: b.get("family") for b in tracker["binaries"]}
scopes = [scope_gaps(t, family_of) for t in tracker["types"] if t["name"] not in HIDDEN]
types = {t["name"]: t for t in tracker["types"]}
used, outside, counted = usage_index(usage, types) if usage else (None, None, None)
scopes = [scope_gaps(t, family_of, used) for t in tracker["types"] if t["name"] not in HIDDEN]
scopes = [s for s in scopes if s["missing"] or s["stubs"] or s["events"] or s["fields"]]
scopes.sort(key=lambda s: (-(len(s["missing"]) + len(s["stubs"]) + len(s["events"]) + len(s["fields"])), s["name"]))
emulator = tracker.get("emulator", {})
@@ -62,20 +139,45 @@ def render(tracker: dict) -> str:
"Binarki: " + ", ".join(f"{SHORT.get(f, f)} × {n}" for f, n in sorted(counts.items(),
key=lambda kv: FAMILY_ORDER.index(kv[0]) if kv[0] in FAMILY_ORDER else 99)) + ".",
"Rejestracja nie oznacza poprawności; „brak literału” znaczy tylko, że nazwy nie ma w kodzie emulatora.",
"W nawiasie: rodziny silnika, w których DLL pozycja występuje.", "",
"| Zakres | Metody (zarejestrowane / w oryginale) | Bez rejestracji | Stuby | Eventy bez literału | Pola bez literału |",
"|---|---:|---:|---:|---:|---:|"]
"W nawiasie: rodziny silnika, w których DLL pozycja występuje."]
if usage:
out += [f"Użycie w skryptach (`python -m ams.usage`): {counted['builds']} wydań, {counted['titles']} gier "
"z rodzin 6.1+. „Gry” to tytuły (z kilku wydań jednej gry liczy się to z największą liczbą), "
"„miejsca” to wystąpienia w skryptach, nie wykonania; wywołanie przez nazwę z łańcucha liczy się "
"dla typów obiektów, które skrypty do tej nazwy przypisują."]
out += ["", "| Zakres | Metody (zarejestrowane / w oryginale) | Bez rejestracji | Stuby | Eventy bez literału | Pola bez literału |",
"|---|---:|---:|---:|---:|---:|"]
for s in scopes:
out.append(f"| {s['name']} | {s['methods_registered']} / {s['methods_total']} | {len(s['missing'])} | "
f"{len(s['stubs'])} | {len(s['events'])} / {s['events_total']} | {len(s['fields'])} / {s['fields_total']} |")
if usage:
gaps = [dict(i, scope=s["name"]) for s in scopes for group in ("missing", "stubs", "events", "fields")
for i in s[group] if i["use"]["titles"]]
gaps.sort(key=lambda i: (-i["use"]["titles"], -i["use"]["places"], i["scope"], i["name"]))
out += ["", "## Priorytety: luki używane przez gry", "",
"| Zakres | Pozycja | Rodzaj | Gry | Miejsca | Rodziny DLL |", "|---|---|---|---:|---:|---|"]
out += [f"| {i['scope']} | `{i['name']}` | {KINDS[i['kind']]} | {i['use']['titles']} | {i['use']['places']} | "
f"{label(i['families'])} |" for i in gaps[:50]]
beyond = sorted(outside.items(), key=lambda kv: (-kv[1]["titles"], -kv[1]["places"], kv[0]))
out += ["", "## Użycia spoza znanego API oryginału", "",
"Skrypty używają tych nazw na obiektach danego typu, a analiza DLL nie przypisuje ich temu typowi: "
"luka analizatora, źle rozpoznany typ obiektu albo nazwa, którą silnik pomija. Metody grupy "
"(GROUP) emulator przekazuje jej elementom (`GroupVariable.callMethod`); skoro gry ich używają, "
"oryginał obsługuje je podobnie, a runner grupy ich nie wymienia."]
for title, kinds in (("Metody", {"method"}), ("Pola i eventy", {"field", "event"})):
out += ["", f"{title}:", "", "| Typ | Nazwa | Rodzaj | Gry | Miejsca | Rodziny gier |", "|---|---|---|---:|---:|---|"]
out += [f"| {t} | `{n}` | {dict(method='metoda', event='event', field='pole')[k]} | {u['titles']} | "
f"{u['places']} | {label(u['families'])} |" for (t, k, n), u in beyond if k in kinds][:30]
for s in scopes:
out += ["", f"## {s['name']}"]
if s["objects"]:
out.append(f"Obiekty w skryptach: {s['objects']['places']} (gry: {s['objects']['titles']}).")
for title, items in (("Metody bez rejestracji", s["missing"]), ("Stuby w emulatorze", s["stubs"]),
("Eventy bez literału", s["events"]), ("Pola bez literału", s["fields"])):
if items:
out.append(f"- **{title}:** " + "; ".join(
f"`{i['name']}`" + (f" {i['detail']}" if i["detail"] else "") + f" ({label(i['families'])})"
for i in items))
+ used_text(i["use"]) for i in items))
return "\n".join(out) + "\n"
@@ -83,8 +185,10 @@ def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("tracker", type=Path, help="Atlas export (python -m ams.tracker ... --out)")
parser.add_argument("-o", "--out", type=Path, required=True)
parser.add_argument("--usage", type=Path, help="script usage (python -m ams.usage ... -o)")
args = parser.parse_args(argv)
args.out.write_text(render(json.loads(args.tracker.read_text(encoding="utf-8"))), encoding="utf-8")
usage = json.loads(args.usage.read_text(encoding="utf-8")) if args.usage else None
args.out.write_text(render(json.loads(args.tracker.read_text(encoding="utf-8")), usage), encoding="utf-8")
if __name__ == "__main__":
+234
View File
@@ -0,0 +1,234 @@
"""How the games' scripts use each type: method calls, event handlers and properties.
python -m ams.usage "/Users/genszu/Gry Aidem Media" -o data/script-usage.json
python -m ams.usage GAME.iso -o usage.json --dump decoded/
A source is a disc image or a directory. A directory holding disc images yields one build per
image, titled by the image's folder, so several releases of one game count as one title; a
directory without images is one build of plain scripts. Images are read with 7-Zip. CNV, DEF
and CLASS scripts are decrypted like the emulator's ScriptDecypher ({<C:N>} / {<D:N>} header);
SEQ files describe sequences, not script objects, and are skipped. The engine family comes
from the engine DLL found next to the scripts.
`NAME^METHOD(` counts for the type NAME is declared with (`NAME:TYPE=`): in the same file
first, else anywhere in the build when every declaration agrees; NAME_N is a CLONE copy of
NAME, NAME_CURSOR the cursor of DATABASE NAME; else one of the engine's global objects
(RANDOM, MOUSE, ... as in the emulator's BuiltinVariableProvider). An object made by a
CLASS (`CLS^NEW("NAME")`) is an INSTANCE: its methods are the class's script behaviours, not
engine API. `*VAR^METHOD(` targets the
object a string names; it counts as `dynamic` for the types of the names the build assigns to
VAR (`VAR:VALUE=`, `VAR^SET("...")`). Other calls (parameters `$N`, THIS, undeclared or
ambiguous names, unknown strings) are kept per method name under `unresolved`. Counts are
places in the scripts, not executions.
"""
from __future__ import annotations
import argparse
import json
import re
import subprocess
import sys
import tempfile
from collections import Counter, defaultdict
from pathlib import Path
from ams.tracker import ALIASES, binary_family
IMAGES = {".iso"}
SCRIPTS = (".cnv", ".def", ".class")
ENGINE_DLL = re.compile(r"(?i)^(?:piklib|bloomoo)\w*\.dll$")
CHARSET = "windows-1250"
HEADER = re.compile(rb"\{<([CDcd])\s*:\s*(\d+)\s*>\}")
OBJECT = re.compile(r"OBJECT\s*=", re.I)
ENTRY = re.compile(r"([^\s:=#][^:=]*?)\s*:\s*([^=]+?)\s*=(.*)$")
CALL = re.compile(r"(\*?)([\w$]+)\^(\w+)\s*\(")
BUILTIN = re.compile(r"@(\w+)\s*\(")
SET_LITERAL = re.compile(r'([\w$]+)\^SET\s*\(\s*"([^"]*)"\s*\)', re.I)
NEW_LITERAL = re.compile(r'([\w$]+)\^NEW\s*\(\s*"([^"]+)"', re.I)
GLOBALS = {"RANDOM": "RAND", "RAND": "RAND", "MOUSE": "MOUSE", "KEYBOARD": "KEYBOARD", "SYSTEM": "SYSTEM"}
INSTANCE = "INSTANCE"
def decode(data: bytes) -> str:
"""ScriptDecypher.decypherIfNeeded: byte-wise variable shift, `<E>` is a newline, CR/LF are layout."""
header = HEADER.match(data)
if not header:
return data.decode(CHARSET, "replace")
movement, direction = int(header[2]), -1 if header[1] in b"Dd" else 1
if movement == 0:
return data[header.end():].decode(CHARSET, "replace")
out, step, i = bytearray(), 0, header.end()
while i < len(data):
if data[i:i + 3] == b"<E>":
out.append(10)
i += 3
continue
if data[i] not in (10, 13):
step = step % movement + 1
magnitude = (step + 1) // 2
out.append((data[i] + (-magnitude if step % 2 else magnitude) * direction) & 0xFF)
i += 1
return out.decode(CHARSET, "replace")
def entries(text: str) -> list[list[str]]:
"""[object, key, value] per `NAME:KEY=VALUE`; a line that starts no entry continues the previous value."""
rows = []
for line in text.splitlines():
line = line.strip()
if not line or line.startswith("#") or OBJECT.match(line):
continue
entry = ENTRY.match(line)
if entry:
rows.append([entry[1].upper(), entry[2].strip().upper(), entry[3]])
elif rows:
rows[-1][2] += "\n" + line
return rows
def type_name(value: str) -> str:
name = value.strip().upper()
return ALIASES.get(name, name)
def scan(files: dict[str, str]) -> dict:
"""Usage counts of one build from its decoded scripts (relative path -> text)."""
parsed = {path: entries(text) for path, text in files.items()}
local, declared, literals, objects = defaultdict(dict), defaultdict(set), defaultdict(set), Counter()
made = []
for path, rows in parsed.items():
for name, key, value in rows:
if key == "TYPE":
local[path][name] = type_name(value)
declared[name].add(type_name(value))
objects[type_name(value)] += 1
elif key == "VALUE":
literals[name].add(value.strip().strip('"').upper())
for target, literal in SET_LITERAL.findall(value):
literals[target.upper()].add(literal.upper())
made += NEW_LITERAL.findall(value)
for factory, name in made:
if "CLASS" in declared.get(factory.upper(), ()):
declared[name.upper()].add(INSTANCE)
def type_of(name, path):
if name in local[path]:
return local[path][name]
kinds = declared.get(name, ())
if kinds:
return next(iter(kinds)) if len(kinds) == 1 else None
base, _, suffix = name.rpartition("_")
if suffix.isdigit() and base in declared: # CLONE names the copies NAME_1, NAME_2, ...
return type_of(base, path)
if suffix == "CURSOR" and type_of(base, path) == "DATABASE":
return "DATABASE_CURSOR"
return GLOBALS.get(name)
methods, dynamic, events, fields = (defaultdict(Counter) for _ in range(4))
unresolved, reasons, targets = Counter(), Counter(), Counter()
for path, rows in parsed.items():
for name, key, value in rows:
owner = type_of(name, path)
head = key.split("^")[0]
if owner and head.startswith("ON"):
events[owner][head] += 1
elif owner and key != "TYPE":
fields[owner][key] += 1
for builtin in BUILTIN.findall(value):
methods["@"][builtin.upper()] += 1
for star, target, method in CALL.findall(value):
target, method = target.upper(), method.upper()
if star:
kinds = {type_of(literal, path) for literal in literals.get(target, ())} - {None}
for kind in kinds:
dynamic[kind][method] += 1
if kinds:
continue
reason = "dynamic"
elif "$" in target: # $N, or a name spliced from one
reason = "parameter"
elif target == "THIS":
reason = "this"
elif type_of(target, path):
methods[type_of(target, path)][method] += 1
continue
else:
reason = "ambiguous" if target in declared else "undeclared"
targets[target] += 1
unresolved[method] += 1
reasons[reason] += 1
def plain(table):
return {kind: dict(sorted(names.items())) for kind, names in sorted(table.items())}
return {"files": len(files), "objects": dict(sorted(objects.items())),
"methods": plain(methods), "dynamic": plain(dynamic), "events": plain(events), "fields": plain(fields),
"unresolved": {"calls": dict(sorted(unresolved.items())), "reasons": dict(sorted(reasons.items())),
"targets": dict(targets.most_common(25))}}
def family(dlls) -> str | None:
families = {binary_family({"library": dll}) for dll in dlls}
return families.pop() if len(families) == 1 else None
def seven_zip(*args: str) -> str:
done = subprocess.run(["7z", *args], capture_output=True, text=True)
if done.returncode > 1: # 1 = warnings
raise RuntimeError("7z %s: %s" % (" ".join(args[:2]), done.stderr.strip() or done.stdout[-500:]))
return done.stdout
def read_image(image: Path) -> tuple[dict[str, str], list[str]]:
listing = seven_zip("l", "-ba", "-slt", str(image))
dlls = sorted({Path(line[7:]).name.upper() for line in listing.splitlines()
if line.startswith("Path = ") and ENGINE_DLL.match(Path(line[7:]).name)})
with tempfile.TemporaryDirectory() as tmp:
seven_zip("x", "-y", "-ssc-", "-o" + tmp, str(image), *("-ir!*" + suffix for suffix in SCRIPTS))
files = {str(p.relative_to(tmp)): decode(p.read_bytes()) for p in sorted(Path(tmp).rglob("*")) if p.is_file()}
return files, dlls
def read_directory(directory: Path) -> tuple[dict[str, str], list[str]]:
paths = sorted(p for p in directory.rglob("*") if p.is_file())
files = {str(p.relative_to(directory)): decode(p.read_bytes()) for p in paths if p.suffix.lower() in SCRIPTS}
return files, sorted({p.name.upper() for p in paths if ENGINE_DLL.match(p.name)})
def builds(sources: list[Path]):
"""(source, title, reader) per build."""
for source in sources:
images = [source] if source.is_file() else sorted(p for p in source.rglob("*") if p.suffix.lower() in IMAGES)
if not images:
yield source, source.name, lambda directory=source: read_directory(directory)
for image in images:
relative = image.relative_to(source.parent if source.is_file() else source)
yield relative, image.parent.name, lambda image=image: read_image(image)
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("sources", nargs="+", type=Path, help="disc images or directories")
parser.add_argument("-o", "--out", type=Path, required=True)
parser.add_argument("--dump", type=Path, help="also write the decoded scripts here, as UTF-8")
args = parser.parse_args(argv)
result = {"schema": 1, "builds": []}
for source, title, read in builds(args.sources):
files, dlls = read()
if not files:
print("[usage] %s: no scripts" % source, file=sys.stderr)
continue
build = {"source": str(source), "title": title, "family": family(dlls), "engine_dlls": dlls, **scan(files)}
result["builds"].append(build)
print("[usage] %s: %d scripts, %s" % (source, len(files), build["family"] or "/".join(dlls) or "no engine DLL"),
file=sys.stderr)
if args.dump:
for path, text in files.items():
target = args.dump / title / Path(source).stem / path
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(text, encoding="utf-8")
args.out.write_text(json.dumps(result, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
if __name__ == "__main__":
main()