feat: add compatibility table endpoint and rendering for game libraries

This commit is contained in:
Patryk Gensch
2026-06-18 23:32:31 +02:00
parent 8875540186
commit 2c785dc87c
8 changed files with 575 additions and 4 deletions

1
.gitignore vendored
View File

@@ -12,6 +12,7 @@
__pycache__/
*.pyc
.venv/
*.egg-info/
# macOS
.DS_Store

View File

@@ -200,7 +200,8 @@ python -m ams.api.importer --game "Reksio i UFO" snapshots/PIKLIB8.dll.snapshot.
uvicorn ams.api.app:create_app --factory --reload # serwer
```
Endpointy: `POST/GET /games`, `POST/GET /snapshots` (import deduplikowany po sha256),
`GET /diff?old=&new=[&owner=]`, `GET /snapshots/{id}/similar`, `POST/GET /jobs`, `GET /health`.
`GET /diff?old=&new=[&owner=]`, `GET /snapshots/{id}/similar`, `GET /compat?format=md|json`,
`POST/GET /jobs`, `GET /health`.
Testy: `pytest` (28, w tym integracyjne na golden pair).
### Podobne wersje
@@ -211,6 +212,35 @@ Miara jest *cross-compiler*: golden pair PIKLIB8 (MSVC6) ↔ bloomoodll (MSVC8)
(types 95% / methods 87% / events 77% / fields 90%), tam gdzie fuzzy-hash binarki daje 0.
Fuzzy (ssdeep) leci jako sygnał poboczny „prawie ten sam plik", gdy snapshot ma `binary.fuzzy`.
## Tablica kompatybilności (eksport pod dokumentację)
Krosowy widok całego katalogu **grupowany po nazwie biblioteki** (`PIKLIB8.dll`, `bloomoodll.dll`…):
która gra wiezie którą bibliotekę. Haczyk: ta sama nazwa może wystąpić w kilku grach, a *mimo to*
nie być tym samym silnikiem. Dlatego w obrębie jednej nazwy klastrujemy **warianty po powierzchni
silnika** (te same zbiory tożsamości co `diff`/`similarity` — kompilator-agnostyczne), a każdy
wariant inny niż referencyjny (`A`) dostaje **gwiazdkę** `*` z deltą *czym* się różni. Byte-różnice
z rekompilacji nie generują gwiazdki — liczy się API skryptowe.
```bash
# z plików snapshot (bez DB/serwera); Label= ustawia kolumnę-grę
python -m ams.compat \
"Reksio i UFO=snapshots/PIKLIB8.dll.snapshot.json" \
"Reksio i Kapitan Nemo=snapshots/bloomoodll.dll.snapshot.json" \
--format md -o compat.md # albo --format json
```
Z katalogu (DB) przez API — pobiera gotowy plik (`Content-Disposition: attachment`):
```bash
curl -OJ "http://127.0.0.1:8000/compat?format=md" # → compat.md
curl -OJ "http://127.0.0.1:8000/compat?format=json" # → compat.json (grupy + warianty + diffy)
```
W UI: przycisk **⬇ kompat** w panelu *Gry / wersje*. Markdown to macierz `biblioteka × gra`
(`✓` = obecna; `A`/`B*`/… gdy nazwa ma kilka wariantów powierzchni) plus sekcja *Warianty* z
przypisem: dla każdego rozbieżnego wariantu overlap % i `types/methods/events/fields +N/-N/~N`
względem referencyjnego.
## Front — Command Center
Po starcie serwera otwórz **http://127.0.0.1:8000/** (`/``/ui/`). Statyczny UI bez build-stepu

View File

@@ -15,7 +15,7 @@ from fastapi.staticfiles import StaticFiles
from .. import __version__
from .db import configure, init_db
from .routes import diff, games, jobs, snapshots
from .routes import compat, diff, games, jobs, snapshots
_STATIC = Path(__file__).parent / "static"
@@ -29,6 +29,7 @@ def create_app(database_url: str | None = None) -> FastAPI:
app.include_router(snapshots.router)
app.include_router(diff.router)
app.include_router(jobs.router)
app.include_router(compat.router)
@app.get("/health", tags=["meta"])
def health() -> dict[str, str]:

49
ams/api/routes/compat.py Normal file
View File

@@ -0,0 +1,49 @@
"""Catalog-wide compatibility table, grouped by library name, as a downloadable file.
GET /compat?format=md -> Markdown table (attachment compat.md)
GET /compat?format=json -> structured JSON (attachment compat.json)
Builds `ams.compat` entries from every catalogued snapshot, labelling each by its game (or
'(bez gry)' when unlinked). Same engine library name across games clusters into surface variants,
so divergent copies get an asterisk — exactly what the docs table needs.
"""
from __future__ import annotations
from fastapi import APIRouter, Depends, Query
from fastapi.responses import Response
from sqlalchemy import select
from sqlalchemy.orm import Session
from ...compat import Entry, compatibility_table, render_json, render_markdown
from ...snapshot import Snapshot as Surface
from .. import models
from ..db import get_db
router = APIRouter(tags=["compat"])
_NO_GAME = "(bez gry)"
@router.get("/compat")
def get_compat(
format: str = Query("md", pattern="^(md|json)$", description="md = Markdown, json = structured"),
db: Session = Depends(get_db),
) -> Response:
entries = [
Entry(game=snap.game.name if snap.game else _NO_GAME, snapshot=Surface(snap.data))
for snap in db.scalars(select(models.Snapshot).order_by(models.Snapshot.id))
]
table = compatibility_table(entries)
if format == "json":
return Response(
content=render_json(table),
media_type="application/json",
headers={"Content-Disposition": 'attachment; filename="compat.json"'},
)
return Response(
content=render_markdown(table),
media_type="text/markdown; charset=utf-8",
headers={"Content-Disposition": 'attachment; filename="compat.md"'},
)

View File

@@ -16,7 +16,11 @@
<aside id="sidebar" class="sidebar">
<div class="panel-title">
Gry / wersje
<span class="panel-actions">
<a id="dl-compat" class="mini-btn" href="/compat?format=md" download="compat.md"
title="Pobierz tablicę kompatybilności (Markdown) — grupowaną po nazwie biblioteki">⬇ kompat</a>
<button id="upload-toggle" class="mini-btn" title="Wgraj grę (ISO / ZIP / DLL)">+ wgraj</button>
</span>
</div>
<form id="upload-form" class="upload" hidden>

View File

@@ -19,9 +19,10 @@ body { background: var(--bg); color: var(--fg); font: 13px/1.45 var(--mono); }
letter-spacing: 1px; position: sticky; top: 0; background: var(--panel); border-bottom: 1px solid var(--border); }
.panel-title { display: flex; align-items: center; justify-content: space-between; }
.panel-actions { display: flex; gap: 6px; }
.mini-btn { background: #16202c; border: 1px solid var(--border); color: var(--accent);
border-radius: 5px; padding: 2px 8px; cursor: pointer; font-family: var(--mono); font-size: 11px;
letter-spacing: 0; text-transform: none; }
letter-spacing: 0; text-transform: none; text-decoration: none; display: inline-block; }
.mini-btn:hover { border-color: var(--accent); }
.upload { display: flex; flex-direction: column; gap: 7px; padding: 10px 12px;

336
ams/compat.py Normal file
View File

@@ -0,0 +1,336 @@
"""Cross-binary **compatibility table**, grouped by library (DLL) name.
A documentation aid: take a set of catalogued snapshots and lay out, per engine library
(``PIKLIB8.dll``, ``bloomoodll.dll`` …), which games ship it. The catch the user cares about:
the *same* library name can appear in several games yet not actually be the same engine —
``PIKLIB8.dll`` in game A and game B may diverge. So within each library name we **cluster
variants by engine surface** (the compiler-agnostic identity sets that `ams.similarity` /
`ams.diff` already key on) and flag every non-reference variant with an **asterisk**, carrying a
``vs_reference`` delta so the footnote can say *what* differs.
The builder is pure (lists of snapshots in, JSON-able dict out); `render_markdown` /
`render_json` turn that into the downloadable file. Both the CLI (`python -m ams.compat`) and the
`/compat` API endpoint sit on top of this.
"""
from __future__ import annotations
import hashlib
import json
from collections import Counter
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Any, Iterable
from .diff import compute_diff
from .similarity import _AXES as _SIM_AXES # axis -> (accessor, identity-key fn)
from .similarity import _keys, surface_similarity
from .snapshot import Snapshot
# The four script-facing axes we cluster on (same set similarity scores over).
_AXES = list(_SIM_AXES)
@dataclass
class Entry:
"""One catalogued binary: the column label (game) plus its surface snapshot."""
game: str
snapshot: Snapshot
# --- surface signature -------------------------------------------------------------------------
def surface_signature(snap: Snapshot) -> str:
"""A stable hash of the engine surface: the sorted identity sets of every axis.
Two snapshots with the same signature expose the same script-facing API — i.e. they are
*compatible* even across recompiles/compilers, so they collapse into one variant. Any
difference in the sets (a method added, a field retyped, a type gone) yields a new signature
and thus an asterisk."""
payload = {axis: sorted(list(k) for k in _keys(snap, axis)) for axis in _AXES}
blob = json.dumps(payload, sort_keys=True, ensure_ascii=False).encode("utf-8")
return hashlib.sha1(blob).hexdigest()
def _surface_counts(snap: Snapshot) -> dict[str, int]:
"""Identity-set size per axis (mirrors the signature's notion of 'how big is the surface')."""
return {axis: len(_keys(snap, axis)) for axis in _AXES}
def _axis_counts(block: dict[str, Any]) -> dict[str, int]:
return {
"added": len(block["added"]),
"removed": len(block["removed"]),
"changed": len(block["changed"]),
}
def _vs_reference(reference: Snapshot, variant: Snapshot) -> dict[str, Any]:
"""How a divergent variant differs from its library's reference variant: per-axis +/-/~ counts
plus the pooled surface-overlap score, reusing the existing diff/similarity engines."""
diff = compute_diff(reference, variant)
return {
"overall": surface_similarity(reference, variant)["overall"],
"axes": {axis: _axis_counts(diff[axis]) for axis in _AXES},
"moved_methods": len(diff["moved_methods"]),
}
# --- table builder -----------------------------------------------------------------------------
def _library_name(snap: Snapshot) -> str:
return snap.binary.get("name") or "?"
def _variant_id(index: int) -> str:
"""0 -> A, 1 -> B, …, 26 -> AA (spreadsheet-style), so cells read 'A', 'B*', 'C*'."""
out = ""
index += 1
while index:
index, rem = divmod(index - 1, 26)
out = chr(ord("A") + rem) + out
return out
def _build_library(name: str, entries: list[Entry]) -> dict[str, Any]:
"""Cluster one library name's entries into variants and assemble its table row."""
# group by surface signature -> the entries that share it
by_sig: dict[str, list[Entry]] = {}
for e in entries:
by_sig.setdefault(surface_signature(e.snapshot), []).append(e)
# deterministic order: biggest cluster first, ties broken by earliest game name. The first
# is the reference variant (no asterisk); the rest diverge.
clusters = sorted(
by_sig.values(),
key=lambda es: (-len(es), min(e.game for e in es)),
)
reference_snap = clusters[0][0].snapshot
variants: list[dict[str, Any]] = []
variant_of_game: dict[str, list[str]] = {}
for i, cluster in enumerate(clusters):
vid = _variant_id(i)
is_ref = i == 0
sample = cluster[0].snapshot
members = [
{
"game": e.game,
"engine": e.snapshot.binary.get("engine"),
"compiler": e.snapshot.binary.get("compiler"),
"sha256": e.snapshot.binary.get("sha256"),
}
for e in sorted(cluster, key=lambda e: e.game)
]
for e in cluster:
variant_of_game.setdefault(e.game, []).append(vid)
variants.append(
{
"id": vid,
"reference": is_ref,
"surface": _surface_counts(sample),
"members": members,
"vs_reference": None if is_ref else _vs_reference(reference_snap, sample),
}
)
cells: dict[str, dict[str, Any]] = {}
for game, vids in variant_of_game.items():
uniq = sorted(set(vids))
cells[game] = {
"present": True,
"variants": uniq,
"diverges": any(v != "A" for v in uniq),
}
return {
"name": name,
"variant_count": len(variants),
"diverges": len(variants) > 1,
"cells": cells,
"variants": variants,
}
def compatibility_table(entries: Iterable[Entry]) -> dict[str, Any]:
"""Build the full compatibility table from catalogued entries.
Groups by library name (case-insensitive, keeping the most common spelling for display),
clusters each group into surface variants, and returns a JSON-able dict: a `games` column
list and a `libraries` list of rows. Libraries with >1 variant are the ones carrying
asterisks."""
entries = list(entries)
by_lib: dict[str, list[Entry]] = {}
spellings: dict[str, Counter] = {}
for e in entries:
name = _library_name(e.snapshot)
key = name.casefold()
by_lib.setdefault(key, []).append(e)
spellings.setdefault(key, Counter())[name] += 1
libraries = [
_build_library(spellings[key].most_common(1)[0][0], es)
for key, es in by_lib.items()
]
libraries.sort(key=lambda lib: lib["name"].casefold())
games = sorted({e.game for e in entries})
return {
"schema": "compat/1",
"generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"games": games,
"libraries": libraries,
}
# --- renderers ---------------------------------------------------------------------------------
def render_json(table: dict[str, Any]) -> str:
return json.dumps(table, indent=2, sort_keys=True, ensure_ascii=False)
def _cell_marker(lib: dict[str, Any], game: str) -> str:
"""Matrix cell: empty if absent; ✓ for a single-variant library; else the variant letter(s)
with an asterisk on every non-reference (divergent) variant."""
cell = lib["cells"].get(game)
if not cell:
return ""
if lib["variant_count"] == 1:
return ""
return " / ".join(v + ("" if v == "A" else "\\*") for v in cell["variants"])
def _short(sha: str | None) -> str:
return (sha or "?")[:8]
def render_markdown(table: dict[str, Any]) -> str:
games = table["games"]
libs = table["libraries"]
out: list[str] = ["# Tablica kompatybilności silników", ""]
out.append(
"Grupowane po nazwie biblioteki. Komórka: pusta = brak; `✓` = obecna (jedyny wariant); "
"litera wariantu (`A`, `B\\*`, …) gdy ta sama nazwa biblioteki ma **różne powierzchnie "
"silnika** — `\\*` oznacza rozbieżność względem wariantu referencyjnego (`A`). "
"Szczegóły rozbieżności w sekcji *Warianty* niżej."
)
out.append("")
out.append("_Wygenerowano: {0} · gier: {1} · bibliotek: {2}_".format(
table.get("generated_at", "?"), len(games), len(libs)))
out.append("")
if not libs:
out.append("_(katalog pusty)_")
return "\n".join(out)
# --- matrix: library × game ---
header = "| Biblioteka | " + " | ".join(games) + " |"
sep = "|" + "---|" * (len(games) + 1)
out.append(header)
out.append(sep)
for lib in libs:
mark = "" if lib["variant_count"] == 1 else " \\*"
row = ["**{0}**{1}".format(lib["name"], mark)]
row += [_cell_marker(lib, g) for g in games]
out.append("| " + " | ".join(row) + " |")
out.append("")
# --- variant breakdown: only the interesting (multi-variant) libraries ---
divergent = [lib for lib in libs if lib["variant_count"] > 1]
if divergent:
out.append("## Warianty (\\* = rozbieżność powierzchni)")
out.append("")
for lib in divergent:
out.append("### {0}{1} warianty".format(lib["name"], lib["variant_count"]))
for v in lib["variants"]:
s = v["surface"]
tag = "referencyjny" if v["reference"] else "rozbieżny \\*"
games_str = ", ".join(
"{0} ({1}/{2}, {3})".format(
m["game"], m["engine"] or "?", m["compiler"] or "?", _short(m["sha256"]))
for m in v["members"]
)
out.append(
"- **{0}** ({1}) — typy {2} / metody {3} / eventy {4} / pola {5}".format(
v["id"], tag, s["types"], s["methods"], s["events"], s["fields"]))
out.append(" - gry: {0}".format(games_str))
vs = v["vs_reference"]
if vs:
ax = vs["axes"]
deltas = ", ".join(
"{0} +{1}/-{2}/~{3}".format(a, ax[a]["added"], ax[a]["removed"], ax[a]["changed"])
for a in _AXES
)
moved = " · przeniesionych metod {0}".format(vs["moved_methods"]) if vs["moved_methods"] else ""
out.append(
" - vs **A**: overlap {0}% · {1}{2}".format(vs["overall"], deltas, moved))
out.append("")
return "\n".join(out).rstrip() + "\n"
# --- CLI: build the table from snapshot.json files (no DB/server) -------------------------------
def _label_and_path(token: str) -> tuple[str | None, str]:
"""Parse a CLI token: 'Game Name=path.json' pins the column label, a bare 'path.json' lets the
label default (acquisition source, else the file's basename). The '=' must precede a real
file so paths that happen to contain '=' still work."""
import os
if "=" in token:
lhs, rhs = token.split("=", 1)
if os.path.exists(rhs):
return lhs.strip(), rhs
return None, token
def _default_label(path: str, snap: Snapshot) -> str:
import os
acq = snap.binary.get("acquisition") or {}
if acq.get("source"):
return str(acq["source"])
base = os.path.basename(path)
for suffix in (".snapshot.json", ".json"):
if base.endswith(suffix):
return base[: -len(suffix)]
return base
def main(argv: list[str] | None = None) -> int:
import argparse
p = argparse.ArgumentParser(
prog="ams.compat",
description="Build a cross-binary compatibility table grouped by library name.",
)
p.add_argument(
"snapshots",
nargs="+",
help="snapshot.json files; prefix with 'Label=' to set the game/column label",
)
p.add_argument("--format", choices=["md", "json"], default="md", help="output format")
p.add_argument("-o", "--out", help="write here instead of stdout")
args = p.parse_args(argv)
entries: list[Entry] = []
for token in args.snapshots:
label, path = _label_and_path(token)
snap = Snapshot.load(path)
entries.append(Entry(game=label or _default_label(path, snap), snapshot=snap))
table = compatibility_table(entries)
text = render_json(table) if args.format == "json" else render_markdown(table)
if args.out:
with open(args.out, "w", encoding="utf-8") as fh:
fh.write(text if text.endswith("\n") else text + "\n")
print("[+] {0} libraries, {1} games -> {2}".format(
len(table["libraries"]), len(table["games"]), args.out))
else:
print(text)
return 0
if __name__ == "__main__":
import sys
sys.exit(main())

149
tests/test_compat.py Normal file
View File

@@ -0,0 +1,149 @@
"""Compatibility table: variant clustering, the divergence asterisk, renderers, and /compat."""
from __future__ import annotations
import json
import pytest
from ams.compat import (
Entry,
compatibility_table,
render_json,
render_markdown,
surface_signature,
)
from ams.snapshot import Snapshot
def _snap(name: str, methods: list[str], *, sha: str | None = None, events: list[str] | None = None):
return Snapshot(
{
"binary": {"name": name, "engine": "Piklib8", "compiler": "MSVC6",
"sha256": sha or ("sha-" + name + "-" + "".join(methods))},
"types": [{"script_name": "ANIMO"}],
"events": [{"owner": "CMC_Animo", "name": e, "order": i} for i, e in enumerate(events or [])],
"fields": [],
"methods": [{"owner": "CMC_Animo", "name": m, "id": i} for i, m in enumerate(methods)],
}
)
# --- signature ---------------------------------------------------------------------------------
def test_signature_ignores_identity_order_and_addresses():
a = _snap("X.dll", ["SHOW", "HIDE"])
b = _snap("X.dll", ["HIDE", "SHOW"]) # same surface, different list order
assert surface_signature(a) == surface_signature(b)
def test_signature_differs_on_surface_change():
a = _snap("X.dll", ["SHOW", "HIDE"])
b = _snap("X.dll", ["SHOW", "HIDE", "PAUSE"])
assert surface_signature(a) != surface_signature(b)
# --- clustering / asterisk ---------------------------------------------------------------------
def test_same_surface_collapses_to_one_variant():
# same library name, byte-different files, identical surface -> compatible, no asterisk
t = compatibility_table([
Entry("Game A", _snap("PIKLIB8.dll", ["SHOW", "HIDE"], sha="aaa")),
Entry("Game B", _snap("PIKLIB8.dll", ["SHOW", "HIDE"], sha="bbb")),
])
lib = t["libraries"][0]
assert lib["name"] == "PIKLIB8.dll"
assert lib["variant_count"] == 1
assert lib["diverges"] is False
assert all(c["diverges"] is False for c in lib["cells"].values())
def test_divergent_surface_raises_asterisk():
t = compatibility_table([
Entry("Game A", _snap("PIKLIB8.dll", ["SHOW", "HIDE"])),
Entry("Game B", _snap("PIKLIB8.dll", ["SHOW", "HIDE"])), # identical -> reference (bigger)
Entry("Game C", _snap("PIKLIB8.dll", ["SHOW", "HIDE", "PAUSE"])),
])
lib = t["libraries"][0]
assert lib["variant_count"] == 2
assert lib["diverges"] is True
ref = next(v for v in lib["variants"] if v["reference"])
div = next(v for v in lib["variants"] if not v["reference"])
assert {m["game"] for m in ref["members"]} == {"Game A", "Game B"} # the majority cluster
assert [m["game"] for m in div["members"]] == ["Game C"]
assert div["vs_reference"]["axes"]["methods"] == {"added": 1, "removed": 0, "changed": 0}
assert 0 < div["vs_reference"]["overall"] < 100
assert lib["cells"]["Game A"]["diverges"] is False
assert lib["cells"]["Game C"]["diverges"] is True
def test_grouping_is_case_insensitive_keeps_common_spelling():
t = compatibility_table([
Entry("Game A", _snap("Piklib8.dll", ["SHOW"])),
Entry("Game B", _snap("PIKLIB8.dll", ["SHOW"])),
Entry("Game C", _snap("PIKLIB8.dll", ["SHOW"])),
])
assert len(t["libraries"]) == 1
assert t["libraries"][0]["name"] == "PIKLIB8.dll" # most common spelling
# --- renderers ---------------------------------------------------------------------------------
def test_markdown_has_matrix_and_asterisk():
t = compatibility_table([
Entry("Game A", _snap("PIKLIB8.dll", ["SHOW", "HIDE"])),
Entry("Game B", _snap("PIKLIB8.dll", ["SHOW", "HIDE"])),
Entry("Game C", _snap("PIKLIB8.dll", ["SHOW", "HIDE", "PAUSE"])),
])
md = render_markdown(t)
assert "| Biblioteka |" in md
assert "PIKLIB8.dll" in md
assert "\\*" in md # the divergence marker survives to the table
assert "vs **A**" in md # the footnote carries the delta
def test_json_round_trips():
t = compatibility_table([Entry("Game A", _snap("X.dll", ["SHOW"]))])
assert json.loads(render_json(t)) == t
def test_empty_catalog():
t = compatibility_table([])
assert t["libraries"] == []
assert t["games"] == []
assert "katalog pusty" in render_markdown(t)
# --- endpoint ----------------------------------------------------------------------------------
@pytest.fixture()
def client(tmp_path):
pytest.importorskip("fastapi")
from fastapi.testclient import TestClient
from ams.api.app import create_app
return TestClient(create_app(database_url="sqlite:///{0}/compat.db".format(tmp_path)))
def _raw(name: str, methods: list[str], sha: str) -> dict:
return _snap(name, methods, sha=sha).raw
def test_compat_endpoint_md_and_json(client):
client.post("/snapshots", params={"game": "Reksio i UFO"},
json=_raw("PIKLIB8.dll", ["SHOW", "HIDE"], "s1"))
client.post("/snapshots", params={"game": "Reksio i Nemo"},
json=_raw("PIKLIB8.dll", ["SHOW", "HIDE", "PAUSE"], "s2"))
md = client.get("/compat", params={"format": "md"})
assert md.status_code == 200
assert "attachment" in md.headers["content-disposition"]
assert "compat.md" in md.headers["content-disposition"]
assert "PIKLIB8.dll" in md.text and "\\*" in md.text
js = client.get("/compat", params={"format": "json"})
assert js.status_code == 200
assert "compat.json" in js.headers["content-disposition"]
body = js.json()
lib = body["libraries"][0]
assert lib["name"] == "PIKLIB8.dll"
assert lib["variant_count"] == 2
assert set(body["games"]) == {"Reksio i UFO", "Reksio i Nemo"}