feat(usage): count how game scripts use each type and rank roadmap gaps by it
ams.usage reads game discs with 7-Zip (bsdtar misses the files on some images), decrypts CNV/DEF/CLASS like the emulator's ScriptDecypher and counts method calls, event handlers, properties and objects per declared type. CLONE copies, database cursors, global objects, CLASS instances and string-named targets are resolved; the rest stays per method name. ams.roadmap --usage orders every gap by the games and script places that use it, adds a priority table and lists direct usage the original's known API does not explain. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
96c39d542a
commit
ebbeeea144
+129
-25
@@ -1,56 +1,133 @@
|
|||||||
"""Emulator roadmap from an Atlas export: what each script scope still lacks.
|
"""Emulator roadmap from an Atlas export: what each script scope still lacks.
|
||||||
|
|
||||||
python -m ams.roadmap tracker.json -o docs/roadmap.md
|
python -m ams.roadmap tracker.json -o docs/roadmap.md [--usage data/script-usage.json]
|
||||||
|
|
||||||
Per scope: methods without a registration (with the signature read from the original),
|
Per scope: methods without a registration (with the signature read from the original),
|
||||||
emulator stubs, events and properties the emulator never names. Every item lists the engine
|
emulator stubs, events and properties the emulator never names. Every item lists the engine
|
||||||
families whose DLLs have it - an item present in every family is core API, one present only
|
families whose DLLs have it - an item present in every family is core API, one present only
|
||||||
in BlooMoo is late. The report restates the tracker's limits: registration is not
|
in BlooMoo is late. With `--usage` (python -m ams.usage) every item also says how many games
|
||||||
correctness, and "never named" means no literal was found.
|
use it and in how many places of their scripts, gaps are ordered by that, and the report
|
||||||
|
lists the most used gaps and what the scripts use beyond the original's known API. The report
|
||||||
|
restates the tracker's limits: registration is not correctness, and "never named" means no
|
||||||
|
literal was found.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
from collections import Counter
|
from collections import Counter, defaultdict
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
FAMILY_ORDER = ["Piklib 6.1", "Piklib 7.1", "Piklib 7.2", "Piklib 8", "BlooMoo"]
|
FAMILY_ORDER = ["Piklib 6.1", "Piklib 7.1", "Piklib 7.2", "Piklib 8", "BlooMoo"]
|
||||||
SHORT = {"Piklib 6.1": "6.1", "Piklib 7.1": "7.1", "Piklib 7.2": "7.2", "Piklib 8": "8", "BlooMoo": "BM"}
|
SHORT = {"Piklib 6.1": "6.1", "Piklib 7.1": "7.1", "Piklib 7.2": "7.2", "Piklib 8": "8", "BlooMoo": "BM"}
|
||||||
HIDDEN = {"INSTANCE"} # emulator helper, not an original type
|
HIDDEN = {"INSTANCE"} # emulator helper, not an original type; in the scripts a CLASS instance
|
||||||
|
KINDS = {"missing": "metoda", "stub": "stub", "event": "event", "field": "pole"}
|
||||||
|
USAGE_KIND = {"missing": "method", "stub": "method", "event": "event", "field": "field"}
|
||||||
|
USAGE_TABLES = (("methods", "method"), ("dynamic", "method"), ("events", "event"), ("fields", "field"))
|
||||||
|
|
||||||
|
|
||||||
|
def family_sort(found) -> list[str]:
|
||||||
|
return sorted(found, key=lambda f: (FAMILY_ORDER.index(f) if f in FAMILY_ORDER else len(FAMILY_ORDER), f))
|
||||||
|
|
||||||
|
|
||||||
def families(binary_ids, family_of) -> list[str]:
|
def families(binary_ids, family_of) -> list[str]:
|
||||||
found = {family_of.get(b) for b in binary_ids} - {None}
|
return family_sort({family_of.get(b) for b in binary_ids} - {None})
|
||||||
return sorted(found, key=lambda f: (FAMILY_ORDER.index(f) if f in FAMILY_ORDER else len(FAMILY_ORDER), f))
|
|
||||||
|
|
||||||
|
|
||||||
def label(fams) -> str:
|
def label(fams) -> str:
|
||||||
return ", ".join(SHORT.get(f, f) for f in fams) or "—"
|
return ", ".join(SHORT.get(f, f) for f in fams) or "—"
|
||||||
|
|
||||||
|
|
||||||
def scope_gaps(scope: dict, family_of: dict) -> dict:
|
def usage_index(usage: dict, types: dict) -> tuple[dict, dict, dict]:
|
||||||
methods = scope["methods"]
|
"""Script usage of the tracked families: (scope, kind, name) -> {titles, places, families}.
|
||||||
|
|
||||||
|
kind is method / event / field / object. Within a title the build with the most places
|
||||||
|
counts, so several releases of one game are one title; places then add up across titles.
|
||||||
|
A method of the base runner resolves to `*`; a call whose target object is unknown counts
|
||||||
|
for the scope that has the method when exactly one does (or `*`). Returns the index, the
|
||||||
|
direct usage the original's known API does not explain (keyed by the script's type name),
|
||||||
|
and the number of builds and titles counted.
|
||||||
|
"""
|
||||||
|
lookup = {name.replace("_", ""): name for name in types}
|
||||||
|
known = {name: {"method": {m["name"] for m in t["methods"] if m["original"]},
|
||||||
|
"event": {e["name"] for e in t.get("events", [])},
|
||||||
|
"field": {f["name"] for f in t.get("fields", [])}} for name, t in types.items()}
|
||||||
|
base = known.get("*", {}).get("method", set())
|
||||||
|
owners = defaultdict(set)
|
||||||
|
for scope, names in known.items():
|
||||||
|
for method in names["method"]:
|
||||||
|
owners[method].add(scope)
|
||||||
|
best, outside_best, fams = defaultdict(Counter), defaultdict(Counter), defaultdict(set)
|
||||||
|
builds = [b for b in usage["builds"] if b.get("family") in FAMILY_ORDER]
|
||||||
|
for build in builds:
|
||||||
|
places, outside = Counter(), Counter()
|
||||||
|
for table, kind in USAGE_TABLES:
|
||||||
|
for type_name, names in build[table].items():
|
||||||
|
scope = lookup.get(type_name.replace("_", ""))
|
||||||
|
if scope in HIDDEN:
|
||||||
|
continue
|
||||||
|
for name, count in names.items():
|
||||||
|
if scope and name in known[scope][kind]:
|
||||||
|
places[(scope, kind, name)] += count
|
||||||
|
elif kind == "method" and name in base:
|
||||||
|
places[("*", kind, name)] += count
|
||||||
|
elif table != "dynamic": # a string-named target is a guess; only direct use is evidence
|
||||||
|
outside[(type_name, kind, name)] += count
|
||||||
|
for name, count in build["unresolved"]["calls"].items():
|
||||||
|
scopes = owners.get(name, set()) - HIDDEN
|
||||||
|
if "*" in scopes or len(scopes) == 1:
|
||||||
|
places[("*" if "*" in scopes else scopes.pop(), "method", name)] += count
|
||||||
|
for type_name, count in build["objects"].items():
|
||||||
|
scope = lookup.get(type_name.replace("_", ""))
|
||||||
|
if scope and scope not in HIDDEN:
|
||||||
|
places[(scope, "object", "")] += count
|
||||||
|
for table, counts in ((best, places), (outside_best, outside)):
|
||||||
|
for key, count in counts.items():
|
||||||
|
table[key][build["title"]] = max(table[key][build["title"]], count)
|
||||||
|
fams[key].add(build["family"])
|
||||||
|
|
||||||
|
def summary(table):
|
||||||
|
return {key: {"titles": len(titles), "places": sum(titles.values()), "families": family_sort(fams[key])}
|
||||||
|
for key, titles in table.items()}
|
||||||
|
return summary(best), summary(outside_best), {"builds": len(builds), "titles": len({b["title"] for b in builds})}
|
||||||
|
|
||||||
|
|
||||||
|
def scope_gaps(scope: dict, family_of: dict, used: dict | None = None) -> dict:
|
||||||
|
name, methods = scope["name"], scope["methods"]
|
||||||
original = [m for m in methods if m["original"]]
|
original = [m for m in methods if m["original"]]
|
||||||
def item(name, binary_ids, detail=""):
|
|
||||||
return {"name": name, "families": families(binary_ids, family_of), "detail": detail}
|
def item(kind, item_name, binary_ids, detail=""):
|
||||||
missing = [item(m["name"], [o["binary_id"] for o in m["original"]], " / ".join(m.get("signatures") or []))
|
use = None if used is None else used.get((name, USAGE_KIND[kind], item_name), {"titles": 0, "places": 0})
|
||||||
|
return {"name": item_name, "kind": kind, "families": families(binary_ids, family_of), "detail": detail,
|
||||||
|
"use": use}
|
||||||
|
missing = [item("missing", m["name"], [o["binary_id"] for o in m["original"]], " / ".join(m.get("signatures") or []))
|
||||||
for m in original if m["status"] == "missing"]
|
for m in original if m["status"] == "missing"]
|
||||||
stubs = [item(m["name"], [o["binary_id"] for o in m["original"]]) for m in methods if m["status"] == "stub_candidate"]
|
stubs = [item("stub", m["name"], [o["binary_id"] for o in m["original"]])
|
||||||
events = [item(e["name"], e["original"]) for e in scope.get("events", []) if e["status"] == "not_found"]
|
for m in methods if m["status"] == "stub_candidate"]
|
||||||
fields = [item(f["name"], f["original"], "/".join(f.get("value_types", [])))
|
events = [item("event", e["name"], e["original"]) for e in scope.get("events", []) if e["status"] == "not_found"]
|
||||||
|
fields = [item("field", f["name"], f["original"], "/".join(f.get("value_types", [])))
|
||||||
for f in scope.get("fields", []) if f["status"] == "not_found"]
|
for f in scope.get("fields", []) if f["status"] == "not_found"]
|
||||||
for items in (missing, events, fields): # widest presence first: core API before late additions
|
for items in (missing, stubs, events, fields): # most used, then widest presence: core API before late additions
|
||||||
items.sort(key=lambda i: (-len(i["families"]), i["name"]))
|
items.sort(key=lambda i: (-(i["use"] or {}).get("titles", 0), -(i["use"] or {}).get("places", 0),
|
||||||
return {"name": scope["name"], "methods_total": len(original),
|
-len(i["families"]), i["name"]))
|
||||||
|
return {"name": name, "methods_total": len(original),
|
||||||
"methods_registered": sum(m["status"] not in ("missing", "stub_candidate") for m in original),
|
"methods_registered": sum(m["status"] not in ("missing", "stub_candidate") for m in original),
|
||||||
"events_total": len(scope.get("events", [])), "fields_total": len(scope.get("fields", [])),
|
"events_total": len(scope.get("events", [])), "fields_total": len(scope.get("fields", [])),
|
||||||
|
"objects": None if used is None else used.get((name, "object", "")),
|
||||||
"missing": missing, "stubs": stubs, "events": events, "fields": fields}
|
"missing": missing, "stubs": stubs, "events": events, "fields": fields}
|
||||||
|
|
||||||
|
|
||||||
def render(tracker: dict) -> str:
|
def used_text(use) -> str:
|
||||||
|
if use is None:
|
||||||
|
return ""
|
||||||
|
return " — gry: %d, miejsca: %d" % (use["titles"], use["places"]) if use["titles"] else " — nieużywane"
|
||||||
|
|
||||||
|
|
||||||
|
def render(tracker: dict, usage: dict | None = None) -> str:
|
||||||
family_of = {b["id"]: b.get("family") for b in tracker["binaries"]}
|
family_of = {b["id"]: b.get("family") for b in tracker["binaries"]}
|
||||||
scopes = [scope_gaps(t, family_of) for t in tracker["types"] if t["name"] not in HIDDEN]
|
types = {t["name"]: t for t in tracker["types"]}
|
||||||
|
used, outside, counted = usage_index(usage, types) if usage else (None, None, None)
|
||||||
|
scopes = [scope_gaps(t, family_of, used) for t in tracker["types"] if t["name"] not in HIDDEN]
|
||||||
scopes = [s for s in scopes if s["missing"] or s["stubs"] or s["events"] or s["fields"]]
|
scopes = [s for s in scopes if s["missing"] or s["stubs"] or s["events"] or s["fields"]]
|
||||||
scopes.sort(key=lambda s: (-(len(s["missing"]) + len(s["stubs"]) + len(s["events"]) + len(s["fields"])), s["name"]))
|
scopes.sort(key=lambda s: (-(len(s["missing"]) + len(s["stubs"]) + len(s["events"]) + len(s["fields"])), s["name"]))
|
||||||
emulator = tracker.get("emulator", {})
|
emulator = tracker.get("emulator", {})
|
||||||
@@ -62,20 +139,45 @@ def render(tracker: dict) -> str:
|
|||||||
"Binarki: " + ", ".join(f"{SHORT.get(f, f)} × {n}" for f, n in sorted(counts.items(),
|
"Binarki: " + ", ".join(f"{SHORT.get(f, f)} × {n}" for f, n in sorted(counts.items(),
|
||||||
key=lambda kv: FAMILY_ORDER.index(kv[0]) if kv[0] in FAMILY_ORDER else 99)) + ".",
|
key=lambda kv: FAMILY_ORDER.index(kv[0]) if kv[0] in FAMILY_ORDER else 99)) + ".",
|
||||||
"Rejestracja nie oznacza poprawności; „brak literału” znaczy tylko, że nazwy nie ma w kodzie emulatora.",
|
"Rejestracja nie oznacza poprawności; „brak literału” znaczy tylko, że nazwy nie ma w kodzie emulatora.",
|
||||||
"W nawiasie: rodziny silnika, w których DLL pozycja występuje.", "",
|
"W nawiasie: rodziny silnika, w których DLL pozycja występuje."]
|
||||||
"| Zakres | Metody (zarejestrowane / w oryginale) | Bez rejestracji | Stuby | Eventy bez literału | Pola bez literału |",
|
if usage:
|
||||||
"|---|---:|---:|---:|---:|---:|"]
|
out += [f"Użycie w skryptach (`python -m ams.usage`): {counted['builds']} wydań, {counted['titles']} gier "
|
||||||
|
"z rodzin 6.1+. „Gry” to tytuły (z kilku wydań jednej gry liczy się to z największą liczbą), "
|
||||||
|
"„miejsca” to wystąpienia w skryptach, nie wykonania; wywołanie przez nazwę z łańcucha liczy się "
|
||||||
|
"dla typów obiektów, które skrypty do tej nazwy przypisują."]
|
||||||
|
out += ["", "| Zakres | Metody (zarejestrowane / w oryginale) | Bez rejestracji | Stuby | Eventy bez literału | Pola bez literału |",
|
||||||
|
"|---|---:|---:|---:|---:|---:|"]
|
||||||
for s in scopes:
|
for s in scopes:
|
||||||
out.append(f"| {s['name']} | {s['methods_registered']} / {s['methods_total']} | {len(s['missing'])} | "
|
out.append(f"| {s['name']} | {s['methods_registered']} / {s['methods_total']} | {len(s['missing'])} | "
|
||||||
f"{len(s['stubs'])} | {len(s['events'])} / {s['events_total']} | {len(s['fields'])} / {s['fields_total']} |")
|
f"{len(s['stubs'])} | {len(s['events'])} / {s['events_total']} | {len(s['fields'])} / {s['fields_total']} |")
|
||||||
|
if usage:
|
||||||
|
gaps = [dict(i, scope=s["name"]) for s in scopes for group in ("missing", "stubs", "events", "fields")
|
||||||
|
for i in s[group] if i["use"]["titles"]]
|
||||||
|
gaps.sort(key=lambda i: (-i["use"]["titles"], -i["use"]["places"], i["scope"], i["name"]))
|
||||||
|
out += ["", "## Priorytety: luki używane przez gry", "",
|
||||||
|
"| Zakres | Pozycja | Rodzaj | Gry | Miejsca | Rodziny DLL |", "|---|---|---|---:|---:|---|"]
|
||||||
|
out += [f"| {i['scope']} | `{i['name']}` | {KINDS[i['kind']]} | {i['use']['titles']} | {i['use']['places']} | "
|
||||||
|
f"{label(i['families'])} |" for i in gaps[:50]]
|
||||||
|
beyond = sorted(outside.items(), key=lambda kv: (-kv[1]["titles"], -kv[1]["places"], kv[0]))
|
||||||
|
out += ["", "## Użycia spoza znanego API oryginału", "",
|
||||||
|
"Skrypty używają tych nazw na obiektach danego typu, a analiza DLL nie przypisuje ich temu typowi: "
|
||||||
|
"luka analizatora, źle rozpoznany typ obiektu albo nazwa, którą silnik pomija. Metody grupy "
|
||||||
|
"(GROUP) emulator przekazuje jej elementom (`GroupVariable.callMethod`); skoro gry ich używają, "
|
||||||
|
"oryginał obsługuje je podobnie, a runner grupy ich nie wymienia."]
|
||||||
|
for title, kinds in (("Metody", {"method"}), ("Pola i eventy", {"field", "event"})):
|
||||||
|
out += ["", f"{title}:", "", "| Typ | Nazwa | Rodzaj | Gry | Miejsca | Rodziny gier |", "|---|---|---|---:|---:|---|"]
|
||||||
|
out += [f"| {t} | `{n}` | {dict(method='metoda', event='event', field='pole')[k]} | {u['titles']} | "
|
||||||
|
f"{u['places']} | {label(u['families'])} |" for (t, k, n), u in beyond if k in kinds][:30]
|
||||||
for s in scopes:
|
for s in scopes:
|
||||||
out += ["", f"## {s['name']}"]
|
out += ["", f"## {s['name']}"]
|
||||||
|
if s["objects"]:
|
||||||
|
out.append(f"Obiekty w skryptach: {s['objects']['places']} (gry: {s['objects']['titles']}).")
|
||||||
for title, items in (("Metody bez rejestracji", s["missing"]), ("Stuby w emulatorze", s["stubs"]),
|
for title, items in (("Metody bez rejestracji", s["missing"]), ("Stuby w emulatorze", s["stubs"]),
|
||||||
("Eventy bez literału", s["events"]), ("Pola bez literału", s["fields"])):
|
("Eventy bez literału", s["events"]), ("Pola bez literału", s["fields"])):
|
||||||
if items:
|
if items:
|
||||||
out.append(f"- **{title}:** " + "; ".join(
|
out.append(f"- **{title}:** " + "; ".join(
|
||||||
f"`{i['name']}`" + (f" {i['detail']}" if i["detail"] else "") + f" ({label(i['families'])})"
|
f"`{i['name']}`" + (f" {i['detail']}" if i["detail"] else "") + f" ({label(i['families'])})"
|
||||||
for i in items))
|
+ used_text(i["use"]) for i in items))
|
||||||
return "\n".join(out) + "\n"
|
return "\n".join(out) + "\n"
|
||||||
|
|
||||||
|
|
||||||
@@ -83,8 +185,10 @@ def main(argv=None):
|
|||||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
parser.add_argument("tracker", type=Path, help="Atlas export (python -m ams.tracker ... --out)")
|
parser.add_argument("tracker", type=Path, help="Atlas export (python -m ams.tracker ... --out)")
|
||||||
parser.add_argument("-o", "--out", type=Path, required=True)
|
parser.add_argument("-o", "--out", type=Path, required=True)
|
||||||
|
parser.add_argument("--usage", type=Path, help="script usage (python -m ams.usage ... -o)")
|
||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
args.out.write_text(render(json.loads(args.tracker.read_text(encoding="utf-8"))), encoding="utf-8")
|
usage = json.loads(args.usage.read_text(encoding="utf-8")) if args.usage else None
|
||||||
|
args.out.write_text(render(json.loads(args.tracker.read_text(encoding="utf-8")), usage), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
+234
@@ -0,0 +1,234 @@
|
|||||||
|
"""How the games' scripts use each type: method calls, event handlers and properties.
|
||||||
|
|
||||||
|
python -m ams.usage "/Users/genszu/Gry Aidem Media" -o data/script-usage.json
|
||||||
|
python -m ams.usage GAME.iso -o usage.json --dump decoded/
|
||||||
|
|
||||||
|
A source is a disc image or a directory. A directory holding disc images yields one build per
|
||||||
|
image, titled by the image's folder, so several releases of one game count as one title; a
|
||||||
|
directory without images is one build of plain scripts. Images are read with 7-Zip. CNV, DEF
|
||||||
|
and CLASS scripts are decrypted like the emulator's ScriptDecypher ({<C:N>} / {<D:N>} header);
|
||||||
|
SEQ files describe sequences, not script objects, and are skipped. The engine family comes
|
||||||
|
from the engine DLL found next to the scripts.
|
||||||
|
|
||||||
|
`NAME^METHOD(` counts for the type NAME is declared with (`NAME:TYPE=`): in the same file
|
||||||
|
first, else anywhere in the build when every declaration agrees; NAME_N is a CLONE copy of
|
||||||
|
NAME, NAME_CURSOR the cursor of DATABASE NAME; else one of the engine's global objects
|
||||||
|
(RANDOM, MOUSE, ... as in the emulator's BuiltinVariableProvider). An object made by a
|
||||||
|
CLASS (`CLS^NEW("NAME")`) is an INSTANCE: its methods are the class's script behaviours, not
|
||||||
|
engine API. `*VAR^METHOD(` targets the
|
||||||
|
object a string names; it counts as `dynamic` for the types of the names the build assigns to
|
||||||
|
VAR (`VAR:VALUE=`, `VAR^SET("...")`). Other calls (parameters `$N`, THIS, undeclared or
|
||||||
|
ambiguous names, unknown strings) are kept per method name under `unresolved`. Counts are
|
||||||
|
places in the scripts, not executions.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ams.tracker import ALIASES, binary_family
|
||||||
|
|
||||||
|
IMAGES = {".iso"}
|
||||||
|
SCRIPTS = (".cnv", ".def", ".class")
|
||||||
|
ENGINE_DLL = re.compile(r"(?i)^(?:piklib|bloomoo)\w*\.dll$")
|
||||||
|
CHARSET = "windows-1250"
|
||||||
|
HEADER = re.compile(rb"\{<([CDcd])\s*:\s*(\d+)\s*>\}")
|
||||||
|
|
||||||
|
OBJECT = re.compile(r"OBJECT\s*=", re.I)
|
||||||
|
ENTRY = re.compile(r"([^\s:=#][^:=]*?)\s*:\s*([^=]+?)\s*=(.*)$")
|
||||||
|
CALL = re.compile(r"(\*?)([\w$]+)\^(\w+)\s*\(")
|
||||||
|
BUILTIN = re.compile(r"@(\w+)\s*\(")
|
||||||
|
SET_LITERAL = re.compile(r'([\w$]+)\^SET\s*\(\s*"([^"]*)"\s*\)', re.I)
|
||||||
|
NEW_LITERAL = re.compile(r'([\w$]+)\^NEW\s*\(\s*"([^"]+)"', re.I)
|
||||||
|
GLOBALS = {"RANDOM": "RAND", "RAND": "RAND", "MOUSE": "MOUSE", "KEYBOARD": "KEYBOARD", "SYSTEM": "SYSTEM"}
|
||||||
|
INSTANCE = "INSTANCE"
|
||||||
|
|
||||||
|
|
||||||
|
def decode(data: bytes) -> str:
|
||||||
|
"""ScriptDecypher.decypherIfNeeded: byte-wise variable shift, `<E>` is a newline, CR/LF are layout."""
|
||||||
|
header = HEADER.match(data)
|
||||||
|
if not header:
|
||||||
|
return data.decode(CHARSET, "replace")
|
||||||
|
movement, direction = int(header[2]), -1 if header[1] in b"Dd" else 1
|
||||||
|
if movement == 0:
|
||||||
|
return data[header.end():].decode(CHARSET, "replace")
|
||||||
|
out, step, i = bytearray(), 0, header.end()
|
||||||
|
while i < len(data):
|
||||||
|
if data[i:i + 3] == b"<E>":
|
||||||
|
out.append(10)
|
||||||
|
i += 3
|
||||||
|
continue
|
||||||
|
if data[i] not in (10, 13):
|
||||||
|
step = step % movement + 1
|
||||||
|
magnitude = (step + 1) // 2
|
||||||
|
out.append((data[i] + (-magnitude if step % 2 else magnitude) * direction) & 0xFF)
|
||||||
|
i += 1
|
||||||
|
return out.decode(CHARSET, "replace")
|
||||||
|
|
||||||
|
|
||||||
|
def entries(text: str) -> list[list[str]]:
|
||||||
|
"""[object, key, value] per `NAME:KEY=VALUE`; a line that starts no entry continues the previous value."""
|
||||||
|
rows = []
|
||||||
|
for line in text.splitlines():
|
||||||
|
line = line.strip()
|
||||||
|
if not line or line.startswith("#") or OBJECT.match(line):
|
||||||
|
continue
|
||||||
|
entry = ENTRY.match(line)
|
||||||
|
if entry:
|
||||||
|
rows.append([entry[1].upper(), entry[2].strip().upper(), entry[3]])
|
||||||
|
elif rows:
|
||||||
|
rows[-1][2] += "\n" + line
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def type_name(value: str) -> str:
|
||||||
|
name = value.strip().upper()
|
||||||
|
return ALIASES.get(name, name)
|
||||||
|
|
||||||
|
|
||||||
|
def scan(files: dict[str, str]) -> dict:
|
||||||
|
"""Usage counts of one build from its decoded scripts (relative path -> text)."""
|
||||||
|
parsed = {path: entries(text) for path, text in files.items()}
|
||||||
|
local, declared, literals, objects = defaultdict(dict), defaultdict(set), defaultdict(set), Counter()
|
||||||
|
made = []
|
||||||
|
for path, rows in parsed.items():
|
||||||
|
for name, key, value in rows:
|
||||||
|
if key == "TYPE":
|
||||||
|
local[path][name] = type_name(value)
|
||||||
|
declared[name].add(type_name(value))
|
||||||
|
objects[type_name(value)] += 1
|
||||||
|
elif key == "VALUE":
|
||||||
|
literals[name].add(value.strip().strip('"').upper())
|
||||||
|
for target, literal in SET_LITERAL.findall(value):
|
||||||
|
literals[target.upper()].add(literal.upper())
|
||||||
|
made += NEW_LITERAL.findall(value)
|
||||||
|
for factory, name in made:
|
||||||
|
if "CLASS" in declared.get(factory.upper(), ()):
|
||||||
|
declared[name.upper()].add(INSTANCE)
|
||||||
|
|
||||||
|
def type_of(name, path):
|
||||||
|
if name in local[path]:
|
||||||
|
return local[path][name]
|
||||||
|
kinds = declared.get(name, ())
|
||||||
|
if kinds:
|
||||||
|
return next(iter(kinds)) if len(kinds) == 1 else None
|
||||||
|
base, _, suffix = name.rpartition("_")
|
||||||
|
if suffix.isdigit() and base in declared: # CLONE names the copies NAME_1, NAME_2, ...
|
||||||
|
return type_of(base, path)
|
||||||
|
if suffix == "CURSOR" and type_of(base, path) == "DATABASE":
|
||||||
|
return "DATABASE_CURSOR"
|
||||||
|
return GLOBALS.get(name)
|
||||||
|
|
||||||
|
methods, dynamic, events, fields = (defaultdict(Counter) for _ in range(4))
|
||||||
|
unresolved, reasons, targets = Counter(), Counter(), Counter()
|
||||||
|
for path, rows in parsed.items():
|
||||||
|
for name, key, value in rows:
|
||||||
|
owner = type_of(name, path)
|
||||||
|
head = key.split("^")[0]
|
||||||
|
if owner and head.startswith("ON"):
|
||||||
|
events[owner][head] += 1
|
||||||
|
elif owner and key != "TYPE":
|
||||||
|
fields[owner][key] += 1
|
||||||
|
for builtin in BUILTIN.findall(value):
|
||||||
|
methods["@"][builtin.upper()] += 1
|
||||||
|
for star, target, method in CALL.findall(value):
|
||||||
|
target, method = target.upper(), method.upper()
|
||||||
|
if star:
|
||||||
|
kinds = {type_of(literal, path) for literal in literals.get(target, ())} - {None}
|
||||||
|
for kind in kinds:
|
||||||
|
dynamic[kind][method] += 1
|
||||||
|
if kinds:
|
||||||
|
continue
|
||||||
|
reason = "dynamic"
|
||||||
|
elif "$" in target: # $N, or a name spliced from one
|
||||||
|
reason = "parameter"
|
||||||
|
elif target == "THIS":
|
||||||
|
reason = "this"
|
||||||
|
elif type_of(target, path):
|
||||||
|
methods[type_of(target, path)][method] += 1
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
reason = "ambiguous" if target in declared else "undeclared"
|
||||||
|
targets[target] += 1
|
||||||
|
unresolved[method] += 1
|
||||||
|
reasons[reason] += 1
|
||||||
|
|
||||||
|
def plain(table):
|
||||||
|
return {kind: dict(sorted(names.items())) for kind, names in sorted(table.items())}
|
||||||
|
return {"files": len(files), "objects": dict(sorted(objects.items())),
|
||||||
|
"methods": plain(methods), "dynamic": plain(dynamic), "events": plain(events), "fields": plain(fields),
|
||||||
|
"unresolved": {"calls": dict(sorted(unresolved.items())), "reasons": dict(sorted(reasons.items())),
|
||||||
|
"targets": dict(targets.most_common(25))}}
|
||||||
|
|
||||||
|
|
||||||
|
def family(dlls) -> str | None:
|
||||||
|
families = {binary_family({"library": dll}) for dll in dlls}
|
||||||
|
return families.pop() if len(families) == 1 else None
|
||||||
|
|
||||||
|
|
||||||
|
def seven_zip(*args: str) -> str:
|
||||||
|
done = subprocess.run(["7z", *args], capture_output=True, text=True)
|
||||||
|
if done.returncode > 1: # 1 = warnings
|
||||||
|
raise RuntimeError("7z %s: %s" % (" ".join(args[:2]), done.stderr.strip() or done.stdout[-500:]))
|
||||||
|
return done.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def read_image(image: Path) -> tuple[dict[str, str], list[str]]:
|
||||||
|
listing = seven_zip("l", "-ba", "-slt", str(image))
|
||||||
|
dlls = sorted({Path(line[7:]).name.upper() for line in listing.splitlines()
|
||||||
|
if line.startswith("Path = ") and ENGINE_DLL.match(Path(line[7:]).name)})
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
seven_zip("x", "-y", "-ssc-", "-o" + tmp, str(image), *("-ir!*" + suffix for suffix in SCRIPTS))
|
||||||
|
files = {str(p.relative_to(tmp)): decode(p.read_bytes()) for p in sorted(Path(tmp).rglob("*")) if p.is_file()}
|
||||||
|
return files, dlls
|
||||||
|
|
||||||
|
|
||||||
|
def read_directory(directory: Path) -> tuple[dict[str, str], list[str]]:
|
||||||
|
paths = sorted(p for p in directory.rglob("*") if p.is_file())
|
||||||
|
files = {str(p.relative_to(directory)): decode(p.read_bytes()) for p in paths if p.suffix.lower() in SCRIPTS}
|
||||||
|
return files, sorted({p.name.upper() for p in paths if ENGINE_DLL.match(p.name)})
|
||||||
|
|
||||||
|
|
||||||
|
def builds(sources: list[Path]):
|
||||||
|
"""(source, title, reader) per build."""
|
||||||
|
for source in sources:
|
||||||
|
images = [source] if source.is_file() else sorted(p for p in source.rglob("*") if p.suffix.lower() in IMAGES)
|
||||||
|
if not images:
|
||||||
|
yield source, source.name, lambda directory=source: read_directory(directory)
|
||||||
|
for image in images:
|
||||||
|
relative = image.relative_to(source.parent if source.is_file() else source)
|
||||||
|
yield relative, image.parent.name, lambda image=image: read_image(image)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
|
parser.add_argument("sources", nargs="+", type=Path, help="disc images or directories")
|
||||||
|
parser.add_argument("-o", "--out", type=Path, required=True)
|
||||||
|
parser.add_argument("--dump", type=Path, help="also write the decoded scripts here, as UTF-8")
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
result = {"schema": 1, "builds": []}
|
||||||
|
for source, title, read in builds(args.sources):
|
||||||
|
files, dlls = read()
|
||||||
|
if not files:
|
||||||
|
print("[usage] %s: no scripts" % source, file=sys.stderr)
|
||||||
|
continue
|
||||||
|
build = {"source": str(source), "title": title, "family": family(dlls), "engine_dlls": dlls, **scan(files)}
|
||||||
|
result["builds"].append(build)
|
||||||
|
print("[usage] %s: %d scripts, %s" % (source, len(files), build["family"] or "/".join(dlls) or "no engine DLL"),
|
||||||
|
file=sys.stderr)
|
||||||
|
if args.dump:
|
||||||
|
for path, text in files.items():
|
||||||
|
target = args.dump / title / Path(source).stem / path
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_text(text, encoding="utf-8")
|
||||||
|
args.out.write_text(json.dumps(result, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
from ams.roadmap import render
|
||||||
|
from ams.usage import decode, scan
|
||||||
|
|
||||||
|
|
||||||
|
def encrypt(text: str, movement: int, direction: str = "C") -> bytes:
|
||||||
|
"""Inverse of ScriptDecypher, for tests: shift back, a newline becomes <E>, layout CR/LF added."""
|
||||||
|
sign, out, step = -1 if direction == "D" else 1, bytearray(), 0
|
||||||
|
for byte in text.encode("windows-1250"):
|
||||||
|
if byte == 10:
|
||||||
|
out += b"<E>\r\n"
|
||||||
|
continue
|
||||||
|
step = step % movement + 1
|
||||||
|
magnitude = (step + 1) // 2
|
||||||
|
out.append((byte - (-magnitude if step % 2 else magnitude) * sign) & 0xFF)
|
||||||
|
return b"{<%s:%d>}" % (direction.encode(), movement) + bytes(out)
|
||||||
|
|
||||||
|
|
||||||
|
def test_decode_inverts_both_cipher_directions_and_reads_plain_scripts():
|
||||||
|
text = "OBJECT=ŻABA\nŻABA:TYPE=ANIMO\n"
|
||||||
|
assert decode(encrypt(text, 6)) == text
|
||||||
|
assert decode(encrypt(text, 3, "D")) == text
|
||||||
|
assert decode(text.encode("windows-1250")) == text
|
||||||
|
|
||||||
|
|
||||||
|
SCENE = """# comment
|
||||||
|
OBJECT=ANN
|
||||||
|
ANN:TYPE=ANIMO
|
||||||
|
ANN:FILENAME=X.ANN
|
||||||
|
ANN:ONFINISHED^PLAY={ANN^HIDE();}
|
||||||
|
OBJECT=SNAME
|
||||||
|
SNAME:TYPE=STRING
|
||||||
|
SNAME:VALUE=ANN
|
||||||
|
OBJECT=CLS
|
||||||
|
CLS:TYPE=CLASS
|
||||||
|
OBJECT=BEH
|
||||||
|
BEH:TYPE=BEHAVIOUR
|
||||||
|
BEH:CODE={ANN^PLAY("X");*SNAME^SHOW();@IF(B,"_","TRUE","BEH","");RANDOM^GET(5);$1^STOP();
|
||||||
|
CLS^NEW("INST");INST^FOO();ANN_2^SETFRAME(1);BOOM^PLAY();}
|
||||||
|
"""
|
||||||
|
OTHER = """OBJECT=ANN
|
||||||
|
ANN:TYPE=IMAGE
|
||||||
|
OBJECT=X
|
||||||
|
X:TYPE=BEHAVIOUR
|
||||||
|
X:CODE={ANN^SHOW();*SNAME^SHOW();}
|
||||||
|
"""
|
||||||
|
THIRD = "OBJECT=Y\nY:TYPE=BEHAVIOUR\nY:CODE={ANN^GETWIDTH();}\n"
|
||||||
|
|
||||||
|
|
||||||
|
def test_scan_attributes_calls_events_and_fields_to_declared_types():
|
||||||
|
usage = scan({"a.cnv": SCENE, "b.cnv": OTHER, "c.cnv": THIRD})
|
||||||
|
assert usage["methods"]["ANIMO"] == {"HIDE": 1, "PLAY": 1, "SETFRAME": 1} # ANN_2: a CLONE copy
|
||||||
|
assert usage["methods"]["IMAGE"] == {"SHOW": 1} # the same file's declaration wins
|
||||||
|
assert usage["dynamic"] == {"ANIMO": {"SHOW": 1}, "IMAGE": {"SHOW": 1}} # SNAME holds "ANN"
|
||||||
|
assert usage["methods"]["@"] == {"IF": 1} and usage["methods"]["RAND"] == {"GET": 1}
|
||||||
|
assert usage["methods"]["INSTANCE"] == {"FOO": 1} and usage["methods"]["CLASS"] == {"NEW": 1}
|
||||||
|
assert usage["events"] == {"ANIMO": {"ONFINISHED": 1}}
|
||||||
|
assert usage["fields"]["ANIMO"] == {"FILENAME": 1} and usage["fields"]["STRING"] == {"VALUE": 1}
|
||||||
|
assert usage["unresolved"]["calls"] == {"GETWIDTH": 1, "PLAY": 1, "STOP": 1}
|
||||||
|
assert usage["unresolved"]["reasons"] == {"ambiguous": 1, "parameter": 1, "undeclared": 1}
|
||||||
|
assert usage["objects"]["BEHAVIOUR"] == 3 and usage["files"] == 3
|
||||||
|
|
||||||
|
|
||||||
|
def build(title, family, **tables):
|
||||||
|
return {"title": title, "family": family, "objects": tables.pop("objects", {}),
|
||||||
|
"methods": {}, "dynamic": {}, "events": {}, "fields": {},
|
||||||
|
"unresolved": {"calls": {}, "reasons": {}, "targets": {}}, **tables}
|
||||||
|
|
||||||
|
|
||||||
|
def test_roadmap_orders_gaps_by_games_that_use_them():
|
||||||
|
tracker = {"emulator": {}, "binaries": [{"id": "a", "family": "Piklib 8"}],
|
||||||
|
"types": [{"name": "*", "methods": [{"name": "GETNAME", "status": "missing", "original": [{"binary_id": "a"}]}]},
|
||||||
|
{"name": "CANVAS_OBSERVER", "methods": [
|
||||||
|
{"name": "REDRAW", "status": "missing", "original": [{"binary_id": "a"}]},
|
||||||
|
{"name": "SAVE", "status": "missing", "original": [{"binary_id": "a"}]},
|
||||||
|
{"name": "ADD", "status": "registered", "original": [{"binary_id": "a"}]}],
|
||||||
|
"events": [{"name": "ONUPDATE", "status": "not_found", "original": ["a"]}], "fields": []}]}
|
||||||
|
usage = {"builds": [
|
||||||
|
build("Gra A", "Piklib 8", objects={"CANVASOBSERVER": 1},
|
||||||
|
methods={"CANVASOBSERVER": {"SAVE": 2, "ADD": 9, "PASTE": 1}}, dynamic={"CANVAS_OBSERVER": {"BLUR": 4}}),
|
||||||
|
build("Gra A", "Piklib 8", methods={"CANVAS_OBSERVER": {"SAVE": 3}}), # another release: counts once
|
||||||
|
build("Gra B", "BlooMoo", methods={"CANVAS_OBSERVER": {"SAVE": 1}}, events={"CANVAS_OBSERVER": {"ONUPDATE": 2}},
|
||||||
|
unresolved={"calls": {"GETNAME": 5}, "reasons": {}, "targets": {}}),
|
||||||
|
build("Gra C", "Piklib 5", methods={"CANVAS_OBSERVER": {"REDRAW": 7}})]} # not a tracked family
|
||||||
|
text = render(tracker, usage)
|
||||||
|
assert "3 wydań, 2 gier" in text
|
||||||
|
assert "`SAVE` (8) — gry: 2, miejsca: 4; `REDRAW` (8) — nieużywane" in text
|
||||||
|
assert "| CANVAS_OBSERVER | `SAVE` | metoda | 2 | 4 | 8 |" in text
|
||||||
|
assert "| * | `GETNAME` | metoda | 1 | 5 | 8 |" in text # unknown target, base method
|
||||||
|
assert "| CANVASOBSERVER | `PASTE` | metoda | 1 | 1 | 8 |" in text and "BLUR" not in text
|
||||||
|
assert "Obiekty w skryptach: 1 (gry: 1)." in text
|
||||||
Reference in New Issue
Block a user