feat(usage): count how game scripts use each type and rank roadmap gaps by it

ams.usage reads game discs with 7-Zip (bsdtar misses the files on some
images), decrypts CNV/DEF/CLASS like the emulator's ScriptDecypher and
counts method calls, event handlers, properties and objects per declared
type. CLONE copies, database cursors, global objects, CLASS instances and
string-named targets are resolved; the rest stays per method name.

ams.roadmap --usage orders every gap by the games and script places that
use it, adds a priority table and lists direct usage the original's known
API does not explain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Patryk Gensch
2026-10-28 20:37:10 +01:00
co-authored by Claude Opus 5.5
parent 96c39d542a
commit ebbeeea144
3 changed files with 453 additions and 25 deletions
+234
View File
@@ -0,0 +1,234 @@
"""How the games' scripts use each type: method calls, event handlers and properties.
python -m ams.usage "/Users/genszu/Gry Aidem Media" -o data/script-usage.json
python -m ams.usage GAME.iso -o usage.json --dump decoded/
A source is a disc image or a directory. A directory holding disc images yields one build per
image, titled by the image's folder, so several releases of one game count as one title; a
directory without images is one build of plain scripts. Images are read with 7-Zip. CNV, DEF
and CLASS scripts are decrypted like the emulator's ScriptDecypher ({<C:N>} / {<D:N>} header);
SEQ files describe sequences, not script objects, and are skipped. The engine family comes
from the engine DLL found next to the scripts.
`NAME^METHOD(` counts for the type NAME is declared with (`NAME:TYPE=`): in the same file
first, else anywhere in the build when every declaration agrees; NAME_N is a CLONE copy of
NAME, NAME_CURSOR the cursor of DATABASE NAME; else one of the engine's global objects
(RANDOM, MOUSE, ... as in the emulator's BuiltinVariableProvider). An object made by a
CLASS (`CLS^NEW("NAME")`) is an INSTANCE: its methods are the class's script behaviours, not
engine API. `*VAR^METHOD(` targets the
object a string names; it counts as `dynamic` for the types of the names the build assigns to
VAR (`VAR:VALUE=`, `VAR^SET("...")`). Other calls (parameters `$N`, THIS, undeclared or
ambiguous names, unknown strings) are kept per method name under `unresolved`. Counts are
places in the scripts, not executions.
"""
from __future__ import annotations
import argparse
import json
import re
import subprocess
import sys
import tempfile
from collections import Counter, defaultdict
from pathlib import Path
from ams.tracker import ALIASES, binary_family
IMAGES = {".iso"}
SCRIPTS = (".cnv", ".def", ".class")
ENGINE_DLL = re.compile(r"(?i)^(?:piklib|bloomoo)\w*\.dll$")
CHARSET = "windows-1250"
HEADER = re.compile(rb"\{<([CDcd])\s*:\s*(\d+)\s*>\}")
OBJECT = re.compile(r"OBJECT\s*=", re.I)
ENTRY = re.compile(r"([^\s:=#][^:=]*?)\s*:\s*([^=]+?)\s*=(.*)$")
CALL = re.compile(r"(\*?)([\w$]+)\^(\w+)\s*\(")
BUILTIN = re.compile(r"@(\w+)\s*\(")
SET_LITERAL = re.compile(r'([\w$]+)\^SET\s*\(\s*"([^"]*)"\s*\)', re.I)
NEW_LITERAL = re.compile(r'([\w$]+)\^NEW\s*\(\s*"([^"]+)"', re.I)
GLOBALS = {"RANDOM": "RAND", "RAND": "RAND", "MOUSE": "MOUSE", "KEYBOARD": "KEYBOARD", "SYSTEM": "SYSTEM"}
INSTANCE = "INSTANCE"
def decode(data: bytes) -> str:
"""ScriptDecypher.decypherIfNeeded: byte-wise variable shift, `<E>` is a newline, CR/LF are layout."""
header = HEADER.match(data)
if not header:
return data.decode(CHARSET, "replace")
movement, direction = int(header[2]), -1 if header[1] in b"Dd" else 1
if movement == 0:
return data[header.end():].decode(CHARSET, "replace")
out, step, i = bytearray(), 0, header.end()
while i < len(data):
if data[i:i + 3] == b"<E>":
out.append(10)
i += 3
continue
if data[i] not in (10, 13):
step = step % movement + 1
magnitude = (step + 1) // 2
out.append((data[i] + (-magnitude if step % 2 else magnitude) * direction) & 0xFF)
i += 1
return out.decode(CHARSET, "replace")
def entries(text: str) -> list[list[str]]:
"""[object, key, value] per `NAME:KEY=VALUE`; a line that starts no entry continues the previous value."""
rows = []
for line in text.splitlines():
line = line.strip()
if not line or line.startswith("#") or OBJECT.match(line):
continue
entry = ENTRY.match(line)
if entry:
rows.append([entry[1].upper(), entry[2].strip().upper(), entry[3]])
elif rows:
rows[-1][2] += "\n" + line
return rows
def type_name(value: str) -> str:
name = value.strip().upper()
return ALIASES.get(name, name)
def scan(files: dict[str, str]) -> dict:
"""Usage counts of one build from its decoded scripts (relative path -> text)."""
parsed = {path: entries(text) for path, text in files.items()}
local, declared, literals, objects = defaultdict(dict), defaultdict(set), defaultdict(set), Counter()
made = []
for path, rows in parsed.items():
for name, key, value in rows:
if key == "TYPE":
local[path][name] = type_name(value)
declared[name].add(type_name(value))
objects[type_name(value)] += 1
elif key == "VALUE":
literals[name].add(value.strip().strip('"').upper())
for target, literal in SET_LITERAL.findall(value):
literals[target.upper()].add(literal.upper())
made += NEW_LITERAL.findall(value)
for factory, name in made:
if "CLASS" in declared.get(factory.upper(), ()):
declared[name.upper()].add(INSTANCE)
def type_of(name, path):
if name in local[path]:
return local[path][name]
kinds = declared.get(name, ())
if kinds:
return next(iter(kinds)) if len(kinds) == 1 else None
base, _, suffix = name.rpartition("_")
if suffix.isdigit() and base in declared: # CLONE names the copies NAME_1, NAME_2, ...
return type_of(base, path)
if suffix == "CURSOR" and type_of(base, path) == "DATABASE":
return "DATABASE_CURSOR"
return GLOBALS.get(name)
methods, dynamic, events, fields = (defaultdict(Counter) for _ in range(4))
unresolved, reasons, targets = Counter(), Counter(), Counter()
for path, rows in parsed.items():
for name, key, value in rows:
owner = type_of(name, path)
head = key.split("^")[0]
if owner and head.startswith("ON"):
events[owner][head] += 1
elif owner and key != "TYPE":
fields[owner][key] += 1
for builtin in BUILTIN.findall(value):
methods["@"][builtin.upper()] += 1
for star, target, method in CALL.findall(value):
target, method = target.upper(), method.upper()
if star:
kinds = {type_of(literal, path) for literal in literals.get(target, ())} - {None}
for kind in kinds:
dynamic[kind][method] += 1
if kinds:
continue
reason = "dynamic"
elif "$" in target: # $N, or a name spliced from one
reason = "parameter"
elif target == "THIS":
reason = "this"
elif type_of(target, path):
methods[type_of(target, path)][method] += 1
continue
else:
reason = "ambiguous" if target in declared else "undeclared"
targets[target] += 1
unresolved[method] += 1
reasons[reason] += 1
def plain(table):
return {kind: dict(sorted(names.items())) for kind, names in sorted(table.items())}
return {"files": len(files), "objects": dict(sorted(objects.items())),
"methods": plain(methods), "dynamic": plain(dynamic), "events": plain(events), "fields": plain(fields),
"unresolved": {"calls": dict(sorted(unresolved.items())), "reasons": dict(sorted(reasons.items())),
"targets": dict(targets.most_common(25))}}
def family(dlls) -> str | None:
families = {binary_family({"library": dll}) for dll in dlls}
return families.pop() if len(families) == 1 else None
def seven_zip(*args: str) -> str:
done = subprocess.run(["7z", *args], capture_output=True, text=True)
if done.returncode > 1: # 1 = warnings
raise RuntimeError("7z %s: %s" % (" ".join(args[:2]), done.stderr.strip() or done.stdout[-500:]))
return done.stdout
def read_image(image: Path) -> tuple[dict[str, str], list[str]]:
listing = seven_zip("l", "-ba", "-slt", str(image))
dlls = sorted({Path(line[7:]).name.upper() for line in listing.splitlines()
if line.startswith("Path = ") and ENGINE_DLL.match(Path(line[7:]).name)})
with tempfile.TemporaryDirectory() as tmp:
seven_zip("x", "-y", "-ssc-", "-o" + tmp, str(image), *("-ir!*" + suffix for suffix in SCRIPTS))
files = {str(p.relative_to(tmp)): decode(p.read_bytes()) for p in sorted(Path(tmp).rglob("*")) if p.is_file()}
return files, dlls
def read_directory(directory: Path) -> tuple[dict[str, str], list[str]]:
paths = sorted(p for p in directory.rglob("*") if p.is_file())
files = {str(p.relative_to(directory)): decode(p.read_bytes()) for p in paths if p.suffix.lower() in SCRIPTS}
return files, sorted({p.name.upper() for p in paths if ENGINE_DLL.match(p.name)})
def builds(sources: list[Path]):
"""(source, title, reader) per build."""
for source in sources:
images = [source] if source.is_file() else sorted(p for p in source.rglob("*") if p.suffix.lower() in IMAGES)
if not images:
yield source, source.name, lambda directory=source: read_directory(directory)
for image in images:
relative = image.relative_to(source.parent if source.is_file() else source)
yield relative, image.parent.name, lambda image=image: read_image(image)
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("sources", nargs="+", type=Path, help="disc images or directories")
parser.add_argument("-o", "--out", type=Path, required=True)
parser.add_argument("--dump", type=Path, help="also write the decoded scripts here, as UTF-8")
args = parser.parse_args(argv)
result = {"schema": 1, "builds": []}
for source, title, read in builds(args.sources):
files, dlls = read()
if not files:
print("[usage] %s: no scripts" % source, file=sys.stderr)
continue
build = {"source": str(source), "title": title, "family": family(dlls), "engine_dlls": dlls, **scan(files)}
result["builds"].append(build)
print("[usage] %s: %d scripts, %s" % (source, len(files), build["family"] or "/".join(dlls) or "no engine DLL"),
file=sys.stderr)
if args.dump:
for path, text in files.items():
target = args.dump / title / Path(source).stem / path
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(text, encoding="utf-8")
args.out.write_text(json.dumps(result, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
if __name__ == "__main__":
main()