"""How the games' scripts use each type: method calls, event handlers and properties. python -m ams.usage "/Users/genszu/Gry Aidem Media" -o data/script-usage.json python -m ams.usage GAME.iso -o usage.json --dump decoded/ A source is a disc image or a directory. A directory holding disc images yields one build per image, titled by the image's folder, so several releases of one game count as one title; a directory without images is one build of plain scripts. Images are read with 7-Zip. CNV, DEF and CLASS scripts are decrypted like the emulator's ScriptDecypher ({} / {} header); SEQ files describe sequences, not script objects, and are skipped. The engine family comes from the engine DLL found next to the scripts. `NAME^METHOD(` counts for the type NAME is declared with (`NAME:TYPE=`): in the same file first, else anywhere in the build when every declaration agrees; NAME_N is a CLONE copy of NAME, NAME_CURSOR the cursor of DATABASE NAME; else one of the engine's global objects (RANDOM, MOUSE, ... as in the emulator's BuiltinVariableProvider). An object made by a CLASS (`CLS^NEW("NAME")`) is an INSTANCE: its methods are the class's script behaviours, not engine API. `*VAR^METHOD(` targets the object a string names; it counts as `dynamic` for the types of the names the build assigns to VAR (`VAR:VALUE=`, `VAR^SET("...")`). Other calls (parameters `$N`, THIS, undeclared or ambiguous names, unknown strings) are kept per method name under `unresolved`. Counts are places in the scripts, not executions. """ from __future__ import annotations import argparse import json import re import subprocess import sys import tempfile from collections import Counter, defaultdict from pathlib import Path from ams.tracker import ALIASES, binary_family IMAGES = {".iso"} SCRIPTS = (".cnv", ".def", ".class") ENGINE_DLL = re.compile(r"(?i)^(?:piklib|bloomoo)\w*\.dll$") CHARSET = "windows-1250" HEADER = re.compile(rb"\{<([CDcd])\s*:\s*(\d+)\s*>\}") OBJECT = re.compile(r"OBJECT\s*=", re.I) ENTRY = re.compile(r"([^\s:=#][^:=]*?)\s*:\s*([^=]+?)\s*=(.*)$") CALL = re.compile(r"(\*?)([\w$]+)\^(\w+)\s*\(") BUILTIN = re.compile(r"@(\w+)\s*\(") SET_LITERAL = re.compile(r'([\w$]+)\^SET\s*\(\s*"([^"]*)"\s*\)', re.I) NEW_LITERAL = re.compile(r'([\w$]+)\^NEW\s*\(\s*"([^"]+)"', re.I) GLOBALS = {"RANDOM": "RAND", "RAND": "RAND", "MOUSE": "MOUSE", "KEYBOARD": "KEYBOARD", "SYSTEM": "SYSTEM"} INSTANCE = "INSTANCE" def decode(data: bytes) -> str: """ScriptDecypher.decypherIfNeeded: byte-wise variable shift, `` is a newline, CR/LF are layout.""" header = HEADER.match(data) if not header: return data.decode(CHARSET, "replace") movement, direction = int(header[2]), -1 if header[1] in b"Dd" else 1 if movement == 0: return data[header.end():].decode(CHARSET, "replace") out, step, i = bytearray(), 0, header.end() while i < len(data): if data[i:i + 3] == b"": out.append(10) i += 3 continue if data[i] not in (10, 13): step = step % movement + 1 magnitude = (step + 1) // 2 out.append((data[i] + (-magnitude if step % 2 else magnitude) * direction) & 0xFF) i += 1 return out.decode(CHARSET, "replace") def entries(text: str) -> list[list[str]]: """[object, key, value] per `NAME:KEY=VALUE`; a line that starts no entry continues the previous value.""" rows = [] for line in text.splitlines(): line = line.strip() if not line or line.startswith("#") or OBJECT.match(line): continue entry = ENTRY.match(line) if entry: rows.append([entry[1].upper(), entry[2].strip().upper(), entry[3]]) elif rows: rows[-1][2] += "\n" + line return rows def type_name(value: str) -> str: name = value.strip().upper() return ALIASES.get(name, name) def scan(files: dict[str, str]) -> dict: """Usage counts of one build from its decoded scripts (relative path -> text).""" parsed = {path: entries(text) for path, text in files.items()} local, declared, literals, objects = defaultdict(dict), defaultdict(set), defaultdict(set), Counter() made = [] for path, rows in parsed.items(): for name, key, value in rows: if key == "TYPE": local[path][name] = type_name(value) declared[name].add(type_name(value)) objects[type_name(value)] += 1 elif key == "VALUE": literals[name].add(value.strip().strip('"').upper()) for target, literal in SET_LITERAL.findall(value): literals[target.upper()].add(literal.upper()) made += NEW_LITERAL.findall(value) for factory, name in made: if "CLASS" in declared.get(factory.upper(), ()): declared[name.upper()].add(INSTANCE) def type_of(name, path): if name in local[path]: return local[path][name] kinds = declared.get(name, ()) if kinds: return next(iter(kinds)) if len(kinds) == 1 else None base, _, suffix = name.rpartition("_") if suffix.isdigit() and base in declared: # CLONE names the copies NAME_1, NAME_2, ... return type_of(base, path) if suffix == "CURSOR" and type_of(base, path) == "DATABASE": return "DATABASE_CURSOR" return GLOBALS.get(name) methods, dynamic, events, fields = (defaultdict(Counter) for _ in range(4)) unresolved, reasons, targets = Counter(), Counter(), Counter() for path, rows in parsed.items(): for name, key, value in rows: owner = type_of(name, path) head = key.split("^")[0] if owner and head.startswith("ON"): events[owner][head] += 1 elif owner and key != "TYPE": fields[owner][key] += 1 for builtin in BUILTIN.findall(value): methods["@"][builtin.upper()] += 1 for star, target, method in CALL.findall(value): target, method = target.upper(), method.upper() if star: kinds = {type_of(literal, path) for literal in literals.get(target, ())} - {None} for kind in kinds: dynamic[kind][method] += 1 if kinds: continue reason = "dynamic" elif "$" in target: # $N, or a name spliced from one reason = "parameter" elif target == "THIS": reason = "this" elif type_of(target, path): methods[type_of(target, path)][method] += 1 continue else: reason = "ambiguous" if target in declared else "undeclared" targets[target] += 1 unresolved[method] += 1 reasons[reason] += 1 def plain(table): return {kind: dict(sorted(names.items())) for kind, names in sorted(table.items())} return {"files": len(files), "objects": dict(sorted(objects.items())), "methods": plain(methods), "dynamic": plain(dynamic), "events": plain(events), "fields": plain(fields), "unresolved": {"calls": dict(sorted(unresolved.items())), "reasons": dict(sorted(reasons.items())), "targets": dict(targets.most_common(25))}} def family(dlls) -> str | None: families = {binary_family({"library": dll}) for dll in dlls} return families.pop() if len(families) == 1 else None def seven_zip(*args: str) -> str: done = subprocess.run(["7z", *args], capture_output=True, text=True) if done.returncode > 1: # 1 = warnings raise RuntimeError("7z %s: %s" % (" ".join(args[:2]), done.stderr.strip() or done.stdout[-500:])) return done.stdout def read_image(image: Path) -> tuple[dict[str, str], list[str]]: listing = seven_zip("l", "-ba", "-slt", str(image)) dlls = sorted({Path(line[7:]).name.upper() for line in listing.splitlines() if line.startswith("Path = ") and ENGINE_DLL.match(Path(line[7:]).name)}) with tempfile.TemporaryDirectory() as tmp: seven_zip("x", "-y", "-ssc-", "-o" + tmp, str(image), *("-ir!*" + suffix for suffix in SCRIPTS)) files = {str(p.relative_to(tmp)): decode(p.read_bytes()) for p in sorted(Path(tmp).rglob("*")) if p.is_file()} return files, dlls def read_directory(directory: Path) -> tuple[dict[str, str], list[str]]: paths = sorted(p for p in directory.rglob("*") if p.is_file()) files = {str(p.relative_to(directory)): decode(p.read_bytes()) for p in paths if p.suffix.lower() in SCRIPTS} return files, sorted({p.name.upper() for p in paths if ENGINE_DLL.match(p.name)}) def builds(sources: list[Path]): """(source, title, reader) per build.""" for source in sources: images = [source] if source.is_file() else sorted(p for p in source.rglob("*") if p.suffix.lower() in IMAGES) if not images: yield source, source.name, lambda directory=source: read_directory(directory) for image in images: relative = image.relative_to(source.parent if source.is_file() else source) yield relative, image.parent.name, lambda image=image: read_image(image) def main(argv=None): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("sources", nargs="+", type=Path, help="disc images or directories") parser.add_argument("-o", "--out", type=Path, required=True) parser.add_argument("--dump", type=Path, help="also write the decoded scripts here, as UTF-8") args = parser.parse_args(argv) result = {"schema": 1, "builds": []} for source, title, read in builds(args.sources): files, dlls = read() if not files: print("[usage] %s: no scripts" % source, file=sys.stderr) continue build = {"source": str(source), "title": title, "family": family(dlls), "engine_dlls": dlls, **scan(files)} result["builds"].append(build) print("[usage] %s: %d scripts, %s" % (source, len(files), build["family"] or "/".join(dlls) or "no engine DLL"), file=sys.stderr) if args.dump: for path, text in files.items(): target = args.dump / title / Path(source).stem / path target.parent.mkdir(parents=True, exist_ok=True) target.write_text(text, encoding="utf-8") args.out.write_text(json.dumps(result, ensure_ascii=False, indent=1) + "\n", encoding="utf-8") if __name__ == "__main__": main()