feat(usage): count how game scripts use each type and rank roadmap gaps by it
ams.usage reads game discs with 7-Zip (bsdtar misses the files on some images), decrypts CNV/DEF/CLASS like the emulator's ScriptDecypher and counts method calls, event handlers, properties and objects per declared type. CLONE copies, database cursors, global objects, CLASS instances and string-named targets are resolved; the rest stays per method name. ams.roadmap --usage orders every gap by the games and script places that use it, adds a priority table and lists direct usage the original's known API does not explain. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
96c39d542a
commit
ebbeeea144
+234
@@ -0,0 +1,234 @@
|
||||
"""How the games' scripts use each type: method calls, event handlers and properties.
|
||||
|
||||
python -m ams.usage "/Users/genszu/Gry Aidem Media" -o data/script-usage.json
|
||||
python -m ams.usage GAME.iso -o usage.json --dump decoded/
|
||||
|
||||
A source is a disc image or a directory. A directory holding disc images yields one build per
|
||||
image, titled by the image's folder, so several releases of one game count as one title; a
|
||||
directory without images is one build of plain scripts. Images are read with 7-Zip. CNV, DEF
|
||||
and CLASS scripts are decrypted like the emulator's ScriptDecypher ({<C:N>} / {<D:N>} header);
|
||||
SEQ files describe sequences, not script objects, and are skipped. The engine family comes
|
||||
from the engine DLL found next to the scripts.
|
||||
|
||||
`NAME^METHOD(` counts for the type NAME is declared with (`NAME:TYPE=`): in the same file
|
||||
first, else anywhere in the build when every declaration agrees; NAME_N is a CLONE copy of
|
||||
NAME, NAME_CURSOR the cursor of DATABASE NAME; else one of the engine's global objects
|
||||
(RANDOM, MOUSE, ... as in the emulator's BuiltinVariableProvider). An object made by a
|
||||
CLASS (`CLS^NEW("NAME")`) is an INSTANCE: its methods are the class's script behaviours, not
|
||||
engine API. `*VAR^METHOD(` targets the
|
||||
object a string names; it counts as `dynamic` for the types of the names the build assigns to
|
||||
VAR (`VAR:VALUE=`, `VAR^SET("...")`). Other calls (parameters `$N`, THIS, undeclared or
|
||||
ambiguous names, unknown strings) are kept per method name under `unresolved`. Counts are
|
||||
places in the scripts, not executions.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from ams.tracker import ALIASES, binary_family
|
||||
|
||||
IMAGES = {".iso"}
|
||||
SCRIPTS = (".cnv", ".def", ".class")
|
||||
ENGINE_DLL = re.compile(r"(?i)^(?:piklib|bloomoo)\w*\.dll$")
|
||||
CHARSET = "windows-1250"
|
||||
HEADER = re.compile(rb"\{<([CDcd])\s*:\s*(\d+)\s*>\}")
|
||||
|
||||
OBJECT = re.compile(r"OBJECT\s*=", re.I)
|
||||
ENTRY = re.compile(r"([^\s:=#][^:=]*?)\s*:\s*([^=]+?)\s*=(.*)$")
|
||||
CALL = re.compile(r"(\*?)([\w$]+)\^(\w+)\s*\(")
|
||||
BUILTIN = re.compile(r"@(\w+)\s*\(")
|
||||
SET_LITERAL = re.compile(r'([\w$]+)\^SET\s*\(\s*"([^"]*)"\s*\)', re.I)
|
||||
NEW_LITERAL = re.compile(r'([\w$]+)\^NEW\s*\(\s*"([^"]+)"', re.I)
|
||||
GLOBALS = {"RANDOM": "RAND", "RAND": "RAND", "MOUSE": "MOUSE", "KEYBOARD": "KEYBOARD", "SYSTEM": "SYSTEM"}
|
||||
INSTANCE = "INSTANCE"
|
||||
|
||||
|
||||
def decode(data: bytes) -> str:
|
||||
"""ScriptDecypher.decypherIfNeeded: byte-wise variable shift, `<E>` is a newline, CR/LF are layout."""
|
||||
header = HEADER.match(data)
|
||||
if not header:
|
||||
return data.decode(CHARSET, "replace")
|
||||
movement, direction = int(header[2]), -1 if header[1] in b"Dd" else 1
|
||||
if movement == 0:
|
||||
return data[header.end():].decode(CHARSET, "replace")
|
||||
out, step, i = bytearray(), 0, header.end()
|
||||
while i < len(data):
|
||||
if data[i:i + 3] == b"<E>":
|
||||
out.append(10)
|
||||
i += 3
|
||||
continue
|
||||
if data[i] not in (10, 13):
|
||||
step = step % movement + 1
|
||||
magnitude = (step + 1) // 2
|
||||
out.append((data[i] + (-magnitude if step % 2 else magnitude) * direction) & 0xFF)
|
||||
i += 1
|
||||
return out.decode(CHARSET, "replace")
|
||||
|
||||
|
||||
def entries(text: str) -> list[list[str]]:
|
||||
"""[object, key, value] per `NAME:KEY=VALUE`; a line that starts no entry continues the previous value."""
|
||||
rows = []
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or OBJECT.match(line):
|
||||
continue
|
||||
entry = ENTRY.match(line)
|
||||
if entry:
|
||||
rows.append([entry[1].upper(), entry[2].strip().upper(), entry[3]])
|
||||
elif rows:
|
||||
rows[-1][2] += "\n" + line
|
||||
return rows
|
||||
|
||||
|
||||
def type_name(value: str) -> str:
|
||||
name = value.strip().upper()
|
||||
return ALIASES.get(name, name)
|
||||
|
||||
|
||||
def scan(files: dict[str, str]) -> dict:
|
||||
"""Usage counts of one build from its decoded scripts (relative path -> text)."""
|
||||
parsed = {path: entries(text) for path, text in files.items()}
|
||||
local, declared, literals, objects = defaultdict(dict), defaultdict(set), defaultdict(set), Counter()
|
||||
made = []
|
||||
for path, rows in parsed.items():
|
||||
for name, key, value in rows:
|
||||
if key == "TYPE":
|
||||
local[path][name] = type_name(value)
|
||||
declared[name].add(type_name(value))
|
||||
objects[type_name(value)] += 1
|
||||
elif key == "VALUE":
|
||||
literals[name].add(value.strip().strip('"').upper())
|
||||
for target, literal in SET_LITERAL.findall(value):
|
||||
literals[target.upper()].add(literal.upper())
|
||||
made += NEW_LITERAL.findall(value)
|
||||
for factory, name in made:
|
||||
if "CLASS" in declared.get(factory.upper(), ()):
|
||||
declared[name.upper()].add(INSTANCE)
|
||||
|
||||
def type_of(name, path):
|
||||
if name in local[path]:
|
||||
return local[path][name]
|
||||
kinds = declared.get(name, ())
|
||||
if kinds:
|
||||
return next(iter(kinds)) if len(kinds) == 1 else None
|
||||
base, _, suffix = name.rpartition("_")
|
||||
if suffix.isdigit() and base in declared: # CLONE names the copies NAME_1, NAME_2, ...
|
||||
return type_of(base, path)
|
||||
if suffix == "CURSOR" and type_of(base, path) == "DATABASE":
|
||||
return "DATABASE_CURSOR"
|
||||
return GLOBALS.get(name)
|
||||
|
||||
methods, dynamic, events, fields = (defaultdict(Counter) for _ in range(4))
|
||||
unresolved, reasons, targets = Counter(), Counter(), Counter()
|
||||
for path, rows in parsed.items():
|
||||
for name, key, value in rows:
|
||||
owner = type_of(name, path)
|
||||
head = key.split("^")[0]
|
||||
if owner and head.startswith("ON"):
|
||||
events[owner][head] += 1
|
||||
elif owner and key != "TYPE":
|
||||
fields[owner][key] += 1
|
||||
for builtin in BUILTIN.findall(value):
|
||||
methods["@"][builtin.upper()] += 1
|
||||
for star, target, method in CALL.findall(value):
|
||||
target, method = target.upper(), method.upper()
|
||||
if star:
|
||||
kinds = {type_of(literal, path) for literal in literals.get(target, ())} - {None}
|
||||
for kind in kinds:
|
||||
dynamic[kind][method] += 1
|
||||
if kinds:
|
||||
continue
|
||||
reason = "dynamic"
|
||||
elif "$" in target: # $N, or a name spliced from one
|
||||
reason = "parameter"
|
||||
elif target == "THIS":
|
||||
reason = "this"
|
||||
elif type_of(target, path):
|
||||
methods[type_of(target, path)][method] += 1
|
||||
continue
|
||||
else:
|
||||
reason = "ambiguous" if target in declared else "undeclared"
|
||||
targets[target] += 1
|
||||
unresolved[method] += 1
|
||||
reasons[reason] += 1
|
||||
|
||||
def plain(table):
|
||||
return {kind: dict(sorted(names.items())) for kind, names in sorted(table.items())}
|
||||
return {"files": len(files), "objects": dict(sorted(objects.items())),
|
||||
"methods": plain(methods), "dynamic": plain(dynamic), "events": plain(events), "fields": plain(fields),
|
||||
"unresolved": {"calls": dict(sorted(unresolved.items())), "reasons": dict(sorted(reasons.items())),
|
||||
"targets": dict(targets.most_common(25))}}
|
||||
|
||||
|
||||
def family(dlls) -> str | None:
|
||||
families = {binary_family({"library": dll}) for dll in dlls}
|
||||
return families.pop() if len(families) == 1 else None
|
||||
|
||||
|
||||
def seven_zip(*args: str) -> str:
|
||||
done = subprocess.run(["7z", *args], capture_output=True, text=True)
|
||||
if done.returncode > 1: # 1 = warnings
|
||||
raise RuntimeError("7z %s: %s" % (" ".join(args[:2]), done.stderr.strip() or done.stdout[-500:]))
|
||||
return done.stdout
|
||||
|
||||
|
||||
def read_image(image: Path) -> tuple[dict[str, str], list[str]]:
|
||||
listing = seven_zip("l", "-ba", "-slt", str(image))
|
||||
dlls = sorted({Path(line[7:]).name.upper() for line in listing.splitlines()
|
||||
if line.startswith("Path = ") and ENGINE_DLL.match(Path(line[7:]).name)})
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
seven_zip("x", "-y", "-ssc-", "-o" + tmp, str(image), *("-ir!*" + suffix for suffix in SCRIPTS))
|
||||
files = {str(p.relative_to(tmp)): decode(p.read_bytes()) for p in sorted(Path(tmp).rglob("*")) if p.is_file()}
|
||||
return files, dlls
|
||||
|
||||
|
||||
def read_directory(directory: Path) -> tuple[dict[str, str], list[str]]:
|
||||
paths = sorted(p for p in directory.rglob("*") if p.is_file())
|
||||
files = {str(p.relative_to(directory)): decode(p.read_bytes()) for p in paths if p.suffix.lower() in SCRIPTS}
|
||||
return files, sorted({p.name.upper() for p in paths if ENGINE_DLL.match(p.name)})
|
||||
|
||||
|
||||
def builds(sources: list[Path]):
|
||||
"""(source, title, reader) per build."""
|
||||
for source in sources:
|
||||
images = [source] if source.is_file() else sorted(p for p in source.rglob("*") if p.suffix.lower() in IMAGES)
|
||||
if not images:
|
||||
yield source, source.name, lambda directory=source: read_directory(directory)
|
||||
for image in images:
|
||||
relative = image.relative_to(source.parent if source.is_file() else source)
|
||||
yield relative, image.parent.name, lambda image=image: read_image(image)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("sources", nargs="+", type=Path, help="disc images or directories")
|
||||
parser.add_argument("-o", "--out", type=Path, required=True)
|
||||
parser.add_argument("--dump", type=Path, help="also write the decoded scripts here, as UTF-8")
|
||||
args = parser.parse_args(argv)
|
||||
result = {"schema": 1, "builds": []}
|
||||
for source, title, read in builds(args.sources):
|
||||
files, dlls = read()
|
||||
if not files:
|
||||
print("[usage] %s: no scripts" % source, file=sys.stderr)
|
||||
continue
|
||||
build = {"source": str(source), "title": title, "family": family(dlls), "engine_dlls": dlls, **scan(files)}
|
||||
result["builds"].append(build)
|
||||
print("[usage] %s: %d scripts, %s" % (source, len(files), build["family"] or "/".join(dlls) or "no engine DLL"),
|
||||
file=sys.stderr)
|
||||
if args.dump:
|
||||
for path, text in files.items():
|
||||
target = args.dump / title / Path(source).stem / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(text, encoding="utf-8")
|
||||
args.out.write_text(json.dumps(result, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user