From 10c81c439027007289fea475e5e9e6a180cb4167 Mon Sep 17 00:00:00 2001 From: Patryk Gensch <43010113+patryk025@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:24:15 +0200 Subject: [PATCH] feat(tracker): Atlas joining DLL snapshots with emulator registrations Matches original script APIs against MethodSpec registrations in Rex-EMoolator and keeps review evidence separate from registration. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 + ams/api/app.py | 3 +- ams/api/routes/tracker.py | 84 +++++++++ ams/api/static/index.html | 1 + ams/api/static/tracker.css | 5 + ams/api/static/tracker.html | 21 +++ ams/api/static/tracker.js | 41 +++++ ams/serve_tracker.py | 27 +++ ams/tracker.py | 332 ++++++++++++++++++++++++++++++++++ docs/tracker.md | 75 ++++++++ tests/test_tracker.py | 174 ++++++++++++++++++ tests/test_tracker_catalog.py | 72 ++++++++ tracker-reviews.json | 4 + 13 files changed, 846 insertions(+), 1 deletion(-) create mode 100644 ams/api/routes/tracker.py create mode 100644 ams/api/static/tracker.css create mode 100644 ams/api/static/tracker.html create mode 100644 ams/api/static/tracker.js create mode 100644 ams/serve_tracker.py create mode 100644 ams/tracker.py create mode 100644 docs/tracker.md create mode 100644 tests/test_tracker.py create mode 100644 tests/test_tracker_catalog.py create mode 100644 tracker-reviews.json diff --git a/README.md b/README.md index 545ad42..d0e0acf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # aidem_media_playground +**[Atlas API — tracker implementacji emulatora i dowodów](docs/tracker.md)**, +dostępny pod `/ui/tracker.html`. [Kolejka ekstrakcji silników](docs/batch.md), +statyczne dowody pluginów: [WORLD](docs/world-analysis.md), [KOLOROWANKA](docs/kolorowanka-pilot.md). + +```sh +.venv/bin/python -m ams.serve_tracker --emulator-root /Users/genszu/IdeaProjects/Rex-EMoolator +``` + Narzędzie do **analizy różnicowej** silników gier Aidem Media (Piklib / BlooMoo). Cel: katalog gier (z ISO/ZIP), wersje silnika z hashami, oraz ekstrakcja i porównywanie "powierzchni silnika" — typów, metod, eventów i pól klas `CMC_*` — między wersjami. diff --git a/ams/api/app.py b/ams/api/app.py index a70fec6..a2e668d 100644 --- a/ams/api/app.py +++ b/ams/api/app.py @@ -15,7 +15,7 @@ from fastapi.staticfiles import StaticFiles from .. import __version__ from .db import configure, init_db -from .routes import compat, diff, games, jobs, snapshots +from .routes import compat, diff, games, jobs, snapshots, tracker _STATIC = Path(__file__).parent / "static" @@ -30,6 +30,7 @@ def create_app(database_url: str | None = None) -> FastAPI: app.include_router(diff.router) app.include_router(jobs.router) app.include_router(compat.router) + app.include_router(tracker.router) @app.get("/health", tags=["meta"]) def health() -> dict[str, str]: diff --git a/ams/api/routes/tracker.py b/ams/api/routes/tracker.py new file mode 100644 index 0000000..cf9cb1e --- /dev/null +++ b/ams/api/routes/tracker.py @@ -0,0 +1,84 @@ +"""Read-only tracker. Filesystem paths come from server configuration, never requests.""" +import os +import json +from pathlib import Path + +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import JSONResponse +from sqlalchemy import select +from sqlalchemy.orm import Session + +from ...snapshot import Snapshot as Surface +from ...tracker import build_tracker, load_reviews, scan_emulator +from .. import models +from ..db import get_db + +router = APIRouter(tags=["tracker"]) + + +def order_snapshots(snapshots): + """Prefer reproducible batch results, retaining explicit conflict diagnostics.""" + ordered = sorted(snapshots, key=lambda s: bool(s.binary.get("extraction")), reverse=True) + seen, warnings, distinct = {}, [], [] + for snap in ordered: + sha = str(snap.binary.get("sha256", "")).lower() + if not sha: + distinct.append(snap) + continue + if sha in seen and seen[sha].raw == snap.raw: + continue + distinct.append(snap) + if sha in seen and seen[sha].raw != snap.raw: + warnings.append(f"Różne snapshoty tej samej DLL ({sha}): wybrano pierwszy, z pierwszeństwem metadanych ekstrakcji; wyników nie scalono.") + seen.setdefault(sha, snap) + return distinct, list(dict.fromkeys(warnings)) + + +def inventory_status(path, binaries): + manifest = json.loads(Path(path).read_text(encoding="utf-8")) + builds = manifest["builds"] + if not isinstance(builds, list): + raise ValueError("inventory builds must be a list") + loaded = {str(b.get("sha256", "")).lower() for b in binaries if b.get("sha256")} + scopes, rows, seen = {}, [], set() + for build in builds: + sha = str(build["sha256"]).lower() + if sha in seen: + raise ValueError("Duplicate inventory SHA-256: " + sha) + seen.add(sha) + scope = build.get("scope", "unknown") + present = sha in loaded + rows.append({**build, "snapshot_loaded": present}) + counts = scopes.setdefault(scope, {"target": 0, "loaded": 0, "pending": 0}) + counts["target"] += 1 + counts["loaded" if present else "pending"] += 1 + return {"builds": rows, "scopes": scopes, "loaded_snapshots": len(binaries), + "unlisted_snapshots": len(loaded - seen), + "note": "Wczytany snapshot nie oznacza kompletnej ekstrakcji ani zgodności funkcjonalnej."} + + +@router.get("/tracker") +def get_tracker(download: bool = False, db: Session = Depends(get_db)): + root = os.environ.get("AMS_EMULATOR_ROOT") + if not root: + raise HTTPException(503, "Ustaw AMS_EMULATOR_ROOT na katalog Rex-EMoolator i uruchom serwer ponownie.") + try: + snapshots = [Surface(s.data) for s in db.scalars(select(models.Snapshot).order_by(models.Snapshot.id))] + # Explicitly configured disk snapshots may supplement a DB; identities are deduplicated. + directory = os.environ.get("AMS_TRACKER_SNAPSHOTS") + if directory: + path = Path(directory) + if not path.is_dir(): + raise ValueError("AMS_TRACKER_SNAPSHOTS is not a directory") + snapshots.extend(Surface.load(str(p)) for p in sorted(path.rglob("*.snapshot.json"))) + snapshots, catalog_warnings = order_snapshots(snapshots) + reviews = os.environ.get("AMS_TRACKER_REVIEWS") + result = build_tracker(snapshots, scan_emulator(Path(root)), load_reviews(Path(reviews) if reviews else None)) + result["warnings"].extend(catalog_warnings) + inventory = os.environ.get("AMS_TRACKER_INVENTORY") + if inventory: + result["inventory"] = inventory_status(inventory, result["binaries"]) + except (OSError, ValueError, KeyError, TypeError) as exc: + raise HTTPException(503, "Konfiguracja lub dane trackera: " + str(exc)) from exc + headers = {"Content-Disposition": 'attachment; filename="tracker.json"'} if download else {} + return JSONResponse(result, headers=headers) diff --git a/ams/api/static/index.html b/ams/api/static/index.html index 8f20b57..f98b56a 100644 --- a/ams/api/static/index.html +++ b/ams/api/static/index.html @@ -9,6 +9,7 @@
⌖ Aidem Media Command Center
+ Atlas API ↗
Piklib / BlooMoo — analiza różnicowa powierzchni silnika
diff --git a/ams/api/static/tracker.css b/ams/api/static/tracker.css new file mode 100644 index 0000000..4b3e32f --- /dev/null +++ b/ams/api/static/tracker.css @@ -0,0 +1,5 @@ +:root{color-scheme:dark;--bg:#0c131e;--panel:#131e2c;--line:#2a394a;--text:#e4edf5;--muted:#9aafc3;--teal:#54d6ba;--amber:#efbd69}*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text);font:14px/1.55 system-ui,sans-serif}header,main{max-width:1550px;margin:auto;padding:30px 40px}header{display:flex;align-items:center;justify-content:space-between;border-bottom:1px solid var(--line);gap:20px}h1{font-size:34px;letter-spacing:-1px;margin:14px 0 3px}h1 span{display:block;color:var(--teal);font-size:12px;letter-spacing:2px;text-transform:uppercase;margin-top:6px}p{margin:8px 0 16px}header p,.muted,.footnote{color:var(--muted)}a{color:var(--teal)}.back{text-decoration:none;font-size:12px}button,.button,input,select{font:inherit;color:inherit;background:#192738;border:1px solid var(--line);border-radius:7px;padding:9px 12px}button,.button{cursor:pointer}.button{text-decoration:none;white-space:nowrap}button:hover,.button:hover{border-color:var(--teal)}:focus-visible{outline:2px solid var(--teal);outline-offset:3px}.filters{display:flex;flex-wrap:wrap;align-items:end;gap:16px;margin:24px 0 10px}.filters label{display:grid;gap:6px;font-size:12px;color:var(--muted)}.filters label:first-child{flex:1;min-width:230px}.filters select{max-width:330px}.filters .check{display:flex;align-items:center;padding:10px 0}.stats{display:grid;grid-template-columns:repeat(4,1fr);gap:14px;margin:25px 0}.stat{border:1px solid var(--line);border-radius:10px;background:var(--panel);padding:18px}.stat strong{display:block;font-size:30px;line-height:1.3}.stat span{color:var(--muted);font-size:12px}.layout{display:grid;grid-template-columns:290px minmax(0,1fr);gap:25px}.section-heading{display:flex;justify-content:space-between;align-items:center;gap:10px;margin-bottom:12px}h2{font-size:16px;margin:0}.quiet{padding:5px 9px;font-size:12px}.types{display:grid;gap:8px;max-height:720px;overflow:auto;padding:3px}.type{display:block;width:100%;text-align:left;background:var(--panel);padding:13px}.type[aria-pressed=true]{border-color:var(--teal);background:#18332f}.type strong{display:block}.type small{color:var(--muted)}.bar{height:4px;background:#314052;border-radius:3px;margin:7px 0 3px;overflow:hidden}.bar i{display:block;height:100%;background:var(--teal)}.bar.review i{background:var(--amber)}.table-scroll{overflow:auto;border:1px solid var(--line);border-radius:10px}table{width:100%;border-collapse:collapse;text-align:left;white-space:nowrap}th{font-size:11px;text-transform:uppercase;color:var(--muted);background:var(--panel);padding:13px}td{padding:13px;border-top:1px solid var(--line);font-size:12px}td:first-child small{display:block;color:var(--muted)}td:first-child strong{font-family:ui-monospace,monospace}.badge{display:inline-block;border-radius:4px;padding:3px 7px;background:#233145;color:#c0d0e0}.badge.implemented,.badge.asm_reviewed,.badge.runtime_tested{color:var(--teal);background:#183a33}.badge.missing,.badge.stub_candidate,.badge.stale{color:var(--amber);background:#3b3020}.footnote{font-size:12px;margin-top:28px;max-width:1000px}.warning{white-space:pre-wrap;border-left:3px solid var(--amber);background:#30291e;padding:12px;margin-top:15px}#message{color:var(--muted);padding:25px 0}dialog{background:var(--panel);color:var(--text);border:1px solid var(--line);border-radius:12px;width:min(790px,94vw);max-height:90vh;padding:24px}dialog::backdrop{background:#030911bb}dialog h3{font-size:14px;color:var(--teal);margin-top:25px}dialog pre{white-space:pre-wrap;overflow-wrap:anywhere;background:var(--bg);padding:13px;border-radius:7px;font-size:12px}dialog article{border-top:1px solid var(--line);padding:10px 0;overflow-wrap:anywhere}dialog dl{display:grid;grid-template-columns:130px 1fr;gap:7px}dialog dt{color:var(--muted)}dialog dd{margin:0;overflow-wrap:anywhere}.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0)}[hidden]{display:none!important}@media(max-width:950px){header,main{padding:24px}.layout{grid-template-columns:1fr}.types{grid-template-columns:repeat(auto-fill,minmax(210px,1fr));max-height:300px}.stats{grid-template-columns:repeat(2,1fr)}}@media(max-width:550px){header{align-items:flex-start;flex-direction:column}header,main{padding:20px}.filters label,.filters select{width:100%;max-width:none}.stats{gap:8px}.stat{padding:12px}.stat strong{font-size:25px}dialog dl{grid-template-columns:1fr}} + +.layout>section,.stat,.filters label{min-width:0}#revision,.warning{overflow-wrap:anywhere}.table-scroll{max-width:100%}@media(max-width:950px){.layout{grid-template-columns:minmax(0,1fr)}} + +.table-scroll{position:relative} diff --git a/ams/api/static/tracker.html b/ams/api/static/tracker.html new file mode 100644 index 0000000..8118f6b --- /dev/null +++ b/ams/api/static/tracker.html @@ -0,0 +1,21 @@ + + +Atlas API · Aidem Media + +
← Command Center

Atlas APIPiklib · BlooMoo · biblioteki

Mapa implementacji i dowodów zgodności z oryginałem.

Eksport JSON ↗
+
+
Wczytywanie katalogu i implementacji emulatora…
+ +
+

+ + diff --git a/ams/api/static/tracker.js b/ams/api/static/tracker.js new file mode 100644 index 0000000..dbf1aed --- /dev/null +++ b/ams/api/static/tracker.js @@ -0,0 +1,41 @@ +'use strict'; +const $ = id => document.getElementById(id); +const labels = {missing:'Brak rejestracji',registered:'Zarejestrowana',stub_candidate:'Podejrzenie stubu',partial:'Częściowa',implemented:'Zaimplementowana',unverified:'Niezweryfikowana',asm_reviewed:'Przegląd ASM',runtime_tested:'Test zachowania',stale:'Dowód wymaga aktualizacji'}; +let data, selected = null; +const el = (tag, text, cls) => {const n = document.createElement(tag); if(text !== undefined) n.textContent = text; if(cls) n.className = cls; return n;}; +const family = b => String(b.family || (typeof b.engine === 'object' ? b.engine?.family || b.engine?.name : b.engine) || b.library || 'Nieustalona'); +const binaryName = b => String(b.library || b.name || '?').split(/[\\/]/).pop().replace(/^.*__[0-9a-f]+__/i,''); +const inFamily = b => !$('family').value || family(b) === $('family').value; +function variants(){const previous=$('binary').value;$('binary').replaceChildren(new Option('Wszystkie warianty',''));for(const b of data.binaries.filter(inFamily)){const games=(b.games||[]).join(', ');const opt=el('option',`${binaryName(b)}${games ? ' · '+games : ''} · ${String(b.sha256||b.id).slice(0,10)}`);opt.value=b.id;$('binary').append(opt);}if([...$('binary').options].some(o=>o.value===previous))$('binary').value=previous;} +const original = m => (m.original || []).filter(o => (!$('binary').value || o.binary_id === $('binary').value) && data.binaries.some(b=>b.id===o.binary_id && inFamily(b))); +const registered = m => m.status !== 'missing'; +const reviewed = m => ['asm_reviewed','runtime_tested'].includes(m.verification); +const typeName = name => name === '*' ? 'Zakres globalny (*)' : name; +function matches(t,m){const q=$('search').value.trim().toUpperCase();return (!q || `${t.name} ${m.name}`.toUpperCase().includes(q)) && (!$('status').value || m.status === $('status').value) && (!$('gaps').checked || ['missing','stub_candidate'].includes(m.status));} +function stat(number,label){const n=el('div',undefined,'stat');n.append(el('strong',number),el('span',label));return n;} +function badge(value){return el('span',labels[value] || value,'badge '+value);} +function render(){ + const query=$('search').value.trim().toUpperCase(); + const groups=data.types.map(t=>({...t,unresolved:t.methods.length===0,methods:t.methods.filter(m=>matches(t,m))})).filter(t=>t.methods.length || (t.unresolved && (!query || t.name.toUpperCase().includes(query)) && !$('status').value && !$('gaps').checked)); + const active=groups.filter(t=>!selected || t.name===selected), methods=active.flatMap(t=>t.methods); + $('scope-note').textContent=($('binary').value || $('family').value) ? 'Wybrany wariant lub rodzina ogranicza dowody oryginału. Metody bez dopasowania pozostają widoczne jako „nieustalone”; liczniki obejmują widoczne pary typ / metoda. Weryfikacja jest zbiorcza i nie zmienia się po wyborze wariantu.' : 'Widok jest sumą API wariantów. Rejestracja metody nie potwierdza jej poprawności. Liczniki obejmują widoczne pary typ / metoda, w tym dziedziczone.'; + $('stats').replaceChildren(stat(methods.length,'par typ / metoda (także dziedziczone)'),stat(methods.filter(registered).length,'zarejestrowanych w emulatorze'),stat(methods.filter(m=>m.status==='implemented').length,'oznaczonych jako zaimplementowane'),stat(methods.filter(reviewed).length,'z aktualnym dowodem ASM / testu')); + $('types').replaceChildren(); + for(const t of groups){const b=el('button',undefined,'type');b.type='button';b.setAttribute('aria-pressed',String(selected===t.name));const n=t.methods.length,r=t.methods.filter(registered).length,v=t.methods.filter(reviewed).length;b.append(el('strong',typeName(t.name)),el('small',n ? `${n} metod · ${r} rejestracji · ${v} dowodów` : 'Metody nierozpoznane — typ wykryty'));for(const [amount,cls,title] of [[r,'','Rejestracja'],[v,'review','Weryfikacja']]){const bar=el('div',undefined,'bar '+cls);bar.title=`${title}: ${amount} / ${n}`;const fill=el('i');fill.style.width=`${n ? 100*amount/n : 0}%`;bar.append(fill);b.append(bar);}b.onclick=()=>{selected=selected===t.name?null:t.name;render();};$('types').append(b);} + const chosen=data.types.find(t=>t.name===selected);$('type-notes').hidden=!chosen?.mapping_notes?.length;$('type-notes').textContent=(chosen?.mapping_notes||[]).map(note=>typeof note==='string'?note:JSON.stringify(note)).join('\n');$('methods-heading').textContent=selected?typeName(selected):'Wszystkie metody';$('count').textContent=`${methods.length} pozycji`;$('rows').replaceChildren(); + for(const t of active)for(const m of t.methods){const tr=el('tr'),name=el('td');name.append(el('small',typeName(t.name)),el('strong',m.name));const s=el('td'),v=el('td');s.append(badge(m.status));v.append(badge(m.verification));const found=original(m);const availability=el('td',found.length?`Wykryta · ${new Set(found.map(o=>o.binary_id)).size} wariantów`:'Nieustalone');const action=el('td'),b=el('button','Dowody ↗','quiet');b.setAttribute('aria-label',`Dowody ${t.name}.${m.name}`);b.onclick=()=>detail(t,m);action.append(b);tr.append(name,s,v,availability,action);$('rows').append(tr);} + $('empty').hidden=methods.length>0;$('empty').textContent=chosen && !chosen.methods.length ? 'Typ został wykryty, ale metody nie zostały jeszcze rozpoznane. Brak wpisów nie oznacza braku API.' : 'Brak metod dla wybranych filtrów.'; +} +function detail(t,m){$('detail-title').textContent=`${typeName(t.name)} · ${m.name}`;const body=$('detail-body');body.replaceChildren();const statuses=el('p');statuses.append(badge(m.status),document.createTextNode(' '),badge(m.verification));body.append(statuses); + const section=(title,text)=>{body.append(el('h3',title));if(text !== undefined)body.append(el('pre',text));}; + section('Implementacja emulatora',m.source?`${m.source.path}:${m.source.line}\nWłaściciel: ${m.source.owner || '—'}\nHash treści: ${m.source.body_sha256 || '—'}`:'Nie znaleziono rejestracji w emulatorze.'); + section('Dowody z oryginału');const entries=original(m);if(!entries.length)body.append(el('p','Nieustalone — ekstrakcja nie dostarcza dopasowania w wybranym zakresie.')); + for(const o of entries){const b=data.binaries.find(b=>b.id===o.binary_id);const a=el('article');a.append(el('strong',b?binaryName(b):o.binary_id));const dl=el('dl');for(const [key,value] of [['SHA-256',b?.sha256],['Silnik',typeof b?.engine==='object'?JSON.stringify(b.engine):b?.engine],['Właściciel',o.owner],['Id metody',o.method_id],['Implementacja',o.impl_addr],['Gałąź dispatchera',o.case_addr],['Funkcja rejestrująca',o.registration_addr],['Wywołanie rejestracji',o.registration_call_addr],['Przekazanie ID',o.id_push_addr],['Napis',o.string_addr],['Id dowodów',(o.evidence_ids||[]).join(', ')||'—'],['Dopasowanie',o.mapping==='explicit'?'Jawne':'Wywnioskowane'],['Dziedziczenie',o.inherited?'Tak':'Nie']]){dl.append(el('dt',key),el('dd',value===null || value===undefined?'—':String(value)));}a.append(dl);if(o.registration_addr || o.evidence_ids?.length)a.append(el('p','Dowód rejestracji ASM potwierdza rozpoznanie API; nie stanowi weryfikacji zachowania implementacji.','muted'));if(b?.limitations?.length){a.append(el('strong','Ograniczenia ekstrakcji'));a.append(el('pre',b.limitations.map(item=>typeof item==='string'?item:JSON.stringify(item,null,2)).join('\n')));}if(b?.coverage !== undefined)a.append(el('pre','Zakres ekstrakcji: '+(typeof b.coverage==='string'?b.coverage:JSON.stringify(b.coverage,null,2))));body.append(a);} + section('Przeglądy i testy');body.append(el('p','Poniżej wszystkie zapisane przeglądy metody. Zbiorczy status weryfikacji nie oznacza potwierdzenia każdego wariantu; zakres dowodu określają jego dane.'));if(!(m.reviews||[]).length)body.append(el('p','Brak zapisanych dowodów weryfikacji zachowania.'));for(const review of m.reviews||[])body.append(el('pre',typeof review==='string'?review:JSON.stringify(review,null,2))); + const notes=[...(t.mapping_notes||[]),...(m.notes||[])];if(notes.length)section('Uwagi',notes.map(n=>typeof n==='string'?n:JSON.stringify(n,null,2)).join('\n')); + $('details').showModal(); +} +$('close').onclick=()=>$('details').close();$('all-types').onclick=()=>{selected=null;render();};for(const id of ['search','family','binary','status','gaps'])$(id).addEventListener(id==='search'?'input':'change',()=>{if(data){if(id==='family')variants();render();}}); +async function load(){try{const res=await fetch('/tracker');if(!res.ok){const text=await res.text();let reason=text;try{const parsed=JSON.parse(text);reason=typeof parsed.detail==='string'?parsed.detail:JSON.stringify(parsed.detail||parsed);}catch{}throw new Error(res.status===503?`Katalog nie jest jeszcze skonfigurowany. ${reason}`:`Błąd API (${res.status}): ${reason}`);}data=await res.json();if(!Array.isArray(data.types)||!Array.isArray(data.binaries))throw new Error('Odpowiedź nie zawiera katalogu typów i wariantów.');for(const name of [...new Set(data.binaries.map(family))].sort())$('family').append(new Option(name,name));variants();$('revision').textContent=`Emulator: ${data.emulator?.commit || 'rewizja nieznana'}${data.emulator?.dirty?' · lokalne zmiany':''} · ${data.binaries.length} wariantów · ${data.types.length} typów`; + if(data.warnings?.length){$('warnings').hidden=false;$('warnings-summary').textContent=`Uwagi do katalogu (${data.warnings.length})`;$('warnings-body').textContent=[...new Set(data.warnings.map(w=>typeof w==='string'?w:JSON.stringify(w)))].join('\n');}if(data.inventory){const names={engine:'Silniki',plugin:'Biblioteki skryptowe',native_dependency:'Biblioteki natywne',native:'Biblioteki natywne',excluded_early_engine:'Wcześniejsze silniki (poza zakresem)'};$('inventory').hidden=false;$('inventory').textContent=Object.entries(data.inventory.scopes).map(([scope,c])=>scope.startsWith('excluded') ? `${names[scope]||scope}: ${c.target}` : `${names[scope]||scope}: ${c.loaded} / ${c.target} snapshotów, ${c.pending} oczekuje`).join(' · ')+'. Snapshot nie oznacza kompletnej ekstrakcji ani zgodności zachowania.';}$('message').hidden=true;$('workspace').hidden=false;render();}catch(error){$('message').replaceChildren(el('p',error.message));const retry=el('button','Spróbuj ponownie');retry.onclick=()=>{data=null;$('binary').replaceChildren(new Option('Wszystkie warianty',''));$('family').replaceChildren(new Option('Wszystkie rodziny',''));$('message').textContent='Wczytywanie…';load();};$('message').append(retry);}} +load(); diff --git a/ams/serve_tracker.py b/ams/serve_tracker.py new file mode 100644 index 0000000..ef30c1d --- /dev/null +++ b/ams/serve_tracker.py @@ -0,0 +1,27 @@ +"""Start the local Atlas with this repository's snapshots and review registry.""" +import argparse +import os +from pathlib import Path + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--emulator-root', required=True, type=Path) + p.add_argument('--inventory', type=Path) + p.add_argument('--port', type=int, default=8766) + a = p.parse_args() + root = Path(__file__).resolve().parents[1] + if not (a.emulator_root / 'core/src').is_dir(): + p.error('--emulator-root must point to Rex-EMoolator') + os.environ['AMS_EMULATOR_ROOT'] = str(a.emulator_root.resolve()) + os.environ['AMS_TRACKER_SNAPSHOTS'] = str(root / 'snapshots') + os.environ['AMS_TRACKER_REVIEWS'] = str(root / 'tracker-reviews.json') + inventory = a.inventory or root / 'data/binary-inventory.json' + if inventory.is_file(): + os.environ['AMS_TRACKER_INVENTORY'] = str(inventory.resolve()) + import uvicorn + uvicorn.run('ams.api.app:create_app', factory=True, host='127.0.0.1', port=a.port) + + +if __name__ == '__main__': + main() diff --git a/ams/tracker.py b/ams/tracker.py new file mode 100644 index 0000000..9fec824 --- /dev/null +++ b/ams/tracker.py @@ -0,0 +1,332 @@ +"""Evidence-aware script API tracker. Never treats registration as correctness. + +The Java adapter deliberately recognises the repository's MethodSpec registrations, +not arbitrary Java. Unknown dispatch and incomplete native extraction stay explicit. +""" +from __future__ import annotations + +import argparse +from collections import Counter, defaultdict +import hashlib +import json +from pathlib import Path +import re +import subprocess + +from .snapshot import Snapshot + +ALIASES = {"BOOL": "BOOLEAN", "INT": "INTEGER"} +# Established factory mappings in the PIKLIB8/BlooMoo reference snapshots. When +# used to fill an unresolved mapping in another build, provenance stays inferred. +CLASS_ALIASES = {"CMC_CanvasObserver": "CANVAS_OBSERVER", "CMC_Groupped": "GROUP"} +ENTRY = re.compile(r'Map\s*\.\s*entry\s*\(\s*"([A-Za-z0-9_]+)"\s*,\s*MethodSpec\s*\.\s*of\s*\(') +JAVA_TOKENS = re.compile(r'"""[\s\S]*?"""|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'|//[^\n]*|/\*[\s\S]*?\*/') +SHA = re.compile(r"[0-9a-f]{64}") + + +def canonical(name: str) -> str: + name = name.upper() + return ALIASES.get(name, name) + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def binary_family(binary: dict) -> str: + library = binary.get("library", binary.get("name", "?").split("__")[-1]).upper() + if library.startswith("BLOOMOO"): + return "BlooMoo" + return {"PIKLIB61.DLL": "Piklib 6.1", "PIKLIB71.DLL": "Piklib 7.1", + "PIKLIB72.DLL": "Piklib 7.2", "PIKLIB8.DLL": "Piklib 8"}.get(library, library.removesuffix(".DLL")) + + +def code_mask(text: str) -> str: + """Mask literals/comments without shifting source positions or line numbers.""" + return JAVA_TOKENS.sub(lambda m: re.sub(r"[^\n]", " ", m.group()), text) + + +def closing(code: str, start: int) -> int: + depth = 0 + for i in range(start, len(code)): + if code[i] == "(": + depth += 1 + elif code[i] == ")": + depth -= 1 + if depth == 0: + return i + 1 + raise ValueError("Unbalanced MethodSpec registration") + + +def _git(root: Path, *args: str) -> str | None: + result = subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True) + return result.stdout.strip() if result.returncode == 0 else None + + +def scan_emulator(root: Path) -> dict: + root = root.resolve() + source_root = root / "core/src" + variable_root = source_root / "pl/genschu/bloomooemulator/interpreter/variable" + if not (variable_root / "Variable.java").is_file(): + raise ValueError("AMS_EMULATOR_ROOT must point to the Rex-EMoolator repository") + tree = hashlib.sha256() + for path in sorted(source_root.rglob("*.java")): + tree.update(str(path.relative_to(root)).encode() + b"\0" + path.read_bytes() + b"\0") + types = {} + warnings = [] + for path in sorted(variable_root.glob("*Variable.java")): + text = path.read_text(encoding="utf-8") + code = code_mask(text) + match = re.search(r"return\s+VariableType\.([A-Z_]+)\s*;", code) + if path.name == "Variable.java": + name = "*" + elif match: + name = canonical(match[1]) + else: + warnings.append("Nie rozpoznano typu Java: " + path.name) + continue + notes = [] + if name != "*" and re.search(r"\bMethodResult\s+callMethod\s*\(", code): + notes.append("Własny dispatcher: lista rejestracji nie opisuje wszystkich wywołań dynamicznych.") + methods = {} + for entry in ENTRY.finditer(text): + if not code[entry.start():entry.start() + 3].strip(): + continue + end = closing(code, code.index("(", entry.start())) + body = text[entry.start():end] + body_code = code[entry.start():end] + status = "registered" + method_notes = [] + if "UnsupportedOperationException" in body_code or re.search(r"not (?:yet )?implemented", body, re.I): + status = "stub_candidate" + method_notes.append("Heurystyka: wyjątek lub komunikat o braku implementacji; wymaga przeglądu.") + if re.search(r"->\s*MethodResult\.noReturn\(\)", body_code): + method_notes.append("Pusta operacja; może odpowiadać oryginałowi. Nie oznaczono automatycznie jako stub.") + method = entry[1].upper() + if method in methods: + raise ValueError(f"Duplicate registration: {name}.{method}") + methods[method] = {"status": status, "notes": method_notes, "source": { + "path": str(path.relative_to(root)), "line": text.count("\n", 0, entry.start()) + 1, + "owner": name, "body_sha256": digest(body.encode()), "file_sha256": digest(text.encode()), + }} + types[name] = {"methods": methods, "mapping_notes": notes} + return {"root": str(root), "commit": _git(root, "rev-parse", "HEAD"), + "dirty": bool(_git(root, "status", "--porcelain")), + "source_tree_sha256": tree.hexdigest(), "types": types, "warnings": warnings} + + +def owner_name(owner: str) -> str: + if owner in ("CMC", "CMC_Object"): + return "*" + if owner in CLASS_ALIASES: + return CLASS_ALIASES[owner] + return canonical(owner.removeprefix("CMC_")) + + +def build_tracker(snapshots: list[Snapshot], emulator: dict, reviews: list[dict] | None = None) -> dict: + """Join script APIs, keeping every binary and inherited declaration as evidence.""" + warnings = list(emulator.get("warnings", [])) + rows = {} + binaries = [] + + def type_row(name): + return rows.setdefault(name, {"name": name, "mapping_notes": [], "methods": {}}) + + def method_row(name, method): + return type_row(name)["methods"].setdefault(method, { + "name": method, "status": "missing", "verification": "unverified", + "source": None, "original": [], "reviews": [], "notes": [], + }) + + seen = set() + for snap in snapshots: + sha = snap.binary.get("sha256") + if not isinstance(sha, str) or not SHA.fullmatch(sha.lower()): + sha = None + else: + sha = sha.lower() + # A content hash of the snapshot is NOT the identity of its DLL. + bid = sha or "snapshot:" + digest(json.dumps(snap.raw, sort_keys=True).encode()) + if bid in seen: + warnings.append("Powtórzony snapshot pominięty: " + snap.binary.get("name", bid)) + continue + seen.add(bid) + binaries.append({"id": bid, "sha256": sha, "name": snap.binary.get("name", "?"), + "engine": snap.binary.get("engine"), "identity": "binary" if sha else "snapshot_only", + "library": snap.binary.get("library", snap.binary.get("name", "?").split("__")[-1]), + "family": binary_family(snap.binary), + "sources": snap.binary.get("sources", []), "games": snap.binary.get("games", []), + "coverage": snap.raw.get("coverage", snap.raw.get("extraction", {}).get("scope", + snap.binary.get("extraction", {}).get("coverage"))), + "limitations": snap.raw.get("limitations", []) + snap.raw.get("extraction", {}).get("limitations", [])}) + if not sha: + warnings.append(snap.binary.get("name", "?") + ": brak SHA-256 DLL; dowodów nie można przypisać do binarki.") + owners = defaultdict(list) + for m in snap.methods: + owners[m["owner"]].append(m) + id_names = defaultdict(set) + name_ids = defaultdict(set) + for m in snap.methods: + if m.get("id") is not None: + id_names[(m["owner"], m["id"])].add(m["name"].upper()) + name_ids[(m["owner"], m["name"].upper())].add(m["id"]) + dispatch = {(d["owner"], d.get("id")): d for d in snap.method_dispatch if d.get("id") is not None} + parents = defaultdict(list) + for edge in snap.method_inheritance: + parents[edge["runner"].removesuffix("_Runner")].append(edge["base_runner"].removesuffix("_Runner")) + mappings = {} + used_owners = set() + for t in snap.types: + name = canonical(t["script_name"]) + row = type_row(name) + owner = t.get("cpp_class") + mapping = "explicit" + if t.get("cpp_class_provenance"): + row["mapping_notes"].append(t["cpp_class_provenance"]) + # Known extractor failure: unrelated factory entries all mapped to Behaviour. + if owner == "CMC_Behaviour" and name != "BEHAVIOUR": + owner = None + if not owner: + candidate = [o for o in owners if owner_name(o) == name] + owner = candidate[0] if len(candidate) == 1 else None + mapping = "inferred" + if owner: + mappings[name] = (owner, mapping) + used_owners.add(owner) + else: + row["mapping_notes"].append(f"{snap.binary.get('name', '?')}: typ wykryty, klasa/metody nieustalone.") + for owner in owners: + if owner not in used_owners: + # Includes singleton classes and separate base scopes. Label inference explicitly. + name = owner_name(owner) + if name not in mappings: + mappings[name] = (owner, "inferred") + for name, (owner, mapping) in mappings.items(): + visited = set() + def add(current, inherited=False): + if current in visited: + return + visited.add(current) + for m in owners[current]: + row = method_row(name, m["name"].upper()) + if len(name_ids[(current, m["name"].upper())]) > 1: + note = "Oryginał rejestruje tę nazwę z różnymi ID; skuteczny wybór wymaga sprawdzenia hashtable/dispatchera." + if note not in row["notes"]: + row["notes"].append(note) + if len(id_names[(current, m.get("id"))]) > 1: + note = "ID współdzielone przez nazwy: " + ", ".join(sorted(id_names[(current, m.get("id"))])) + if note not in row["notes"]: + row["notes"].append(note) + # Child declarations override names from ancestors for this binary. + if any(o["binary_id"] == bid and o["owner"] != current for o in row["original"]): + continue + d = dispatch.get((current, m.get("id")), {}) if m.get("id") is not None else {} + row["original"].append({"binary_id": bid, "owner": current, "method_id": m.get("id"), + "impl_addr": d.get("impl_addr"), "case_addr": d.get("case_addr"), + "mapping": mapping, "inherited": inherited, + "registration_addr": m.get("registration_addr"), "string_addr": m.get("string_addr"), + "registration_call_addr": m.get("registration_call_addr"), + "id_push_addr": m.get("id_push_addr"), + "evidence_ids": m.get("evidence_ids", [])}) + for parent in parents[current]: + add(parent, True) + add(owner) + for name, t in emulator["types"].items(): + type_row(name)["mapping_notes"].extend(t["mapping_notes"]) + for method, binding in t["methods"].items(): + row = method_row(name, method) + notes = row["notes"] + binding.get("notes", []) + row.update(binding) + row["notes"] = notes + # Global fallback exists on concrete Variable implementations, not native-only types. + globals_ = emulator["types"].get("*", {}).get("methods", {}) + for name, t in rows.items(): + if name not in emulator["types"] or name == "*": + continue + for method, row in t["methods"].items(): + if row["source"] is None and method in globals_: + row.update(globals_[method]) + row["notes"] = row["notes"] + ["Obsługa przez globalny fallback Variable."] + # Reviews are declarative, version-controlled assertions; strict anchors prevent + # accidentally making an entire family green based on one stale binary/address. + for review in reviews or []: + validate_review(review) + row = rows.get(canonical(review["type"]), {}).get("methods", {}).get(review["method"].upper()) + if row is None: + warnings.append("Przegląd nie pasuje do API: " + review["type"] + "." + review["method"]) + continue + source = row["source"] or {} + current = (source.get("body_sha256") == review["body_sha256"] + and emulator["source_tree_sha256"] == review["source_tree_sha256"] + and emulator.get("commit") == review["emulator_commit"]) + matches = any(o["binary_id"] == review["binary_sha256"] for o in row["original"]) + row["reviews"].append({**review, "current": current and matches}) + for t in rows.values(): + for row in t["methods"].values(): + current = [r for r in row["reviews"] if r["current"]] + targets = {o["binary_id"] for o in row["original"]} + verified = {r["binary_sha256"] for r in current if r["verdict"] == "matches"} + row["verified_variants"] = len(verified & targets) + if targets and targets <= verified: + row["verification"] = "runtime_tested" if all(any(r["binary_sha256"] == b and r["kind"] == "runtime" and r["verdict"] == "matches" for r in current) for b in targets) else "asm_reviewed" + row["status"] = "implemented" + elif any(r["verdict"] == "partial" for r in current): + row["status"] = "partial" + elif row["reviews"] and not current: + row["verification"] = "stale" + t["methods"] = sorted(t["methods"].values(), key=lambda m: m["name"]) + t["mapping_notes"] = sorted(set(t["mapping_notes"])) + all_rows = [m for t in rows.values() for m in t["methods"]] + return {"schema": "tracker/1", "emulator": {k: v for k, v in emulator.items() if k not in ("types", "warnings")}, + "binaries": sorted(binaries, key=lambda b: (b["name"].lower(), b["id"])), + "types": sorted(rows.values(), key=lambda t: t["name"]), + "summary": {"types": len(rows), "methods": len(all_rows), "status": dict(Counter(m["status"] for m in all_rows)), + "verification": dict(Counter(m["verification"] for m in all_rows))}, + "warnings": warnings + ["Brak obserwacji w wariancie oznacza nieustalone, nie brak metody.", + "Rejestracja Java nie dowodzi poprawności. Liczniki obejmują pary typ/metoda, także odziedziczone.", + "Dopasowania klas po nazwie są wnioskowane; zakres * to metody bazowe/globalne."]} + + +def validate_review(review: dict) -> None: + required = ("type", "method", "binary_sha256", "rva", "emulator_commit", "body_sha256", + "source_tree_sha256", "reviewer", "date", "notes", "evidence", "kind", "verdict") + if any(not isinstance(review.get(k), str) or not review[k].strip() for k in required): + raise ValueError("Review requires nonempty fields: " + ", ".join(required)) + for k in ("binary_sha256", "body_sha256", "source_tree_sha256"): + if not SHA.fullmatch(review[k]): + raise ValueError("Invalid review digest: " + k) + if not re.fullmatch(r"0x[0-9a-fA-F]+", review["rva"]): + raise ValueError("Review RVA must be hexadecimal (0x...)") + if review["kind"] not in ("asm", "runtime") or review["verdict"] not in ("matches", "partial", "differs"): + raise ValueError("Invalid review kind/verdict") + + +def load_reviews(path: Path | None) -> list[dict]: + if path is None: + return [] + data = json.loads(path.read_text()) + if data.get("schema") != "tracker-reviews/1" or not isinstance(data.get("reviews"), list): + raise ValueError("Expected tracker-reviews/1 with a reviews list") + for review in data["reviews"]: + validate_review(review) + # A single current assertion per binary/method/source; conflicting verdicts fail closed. + keys = [(r["type"], r["method"], r["binary_sha256"], r["source_tree_sha256"], r["emulator_commit"]) for r in data["reviews"]] + if len(keys) != len(set(keys)): + raise ValueError("Duplicate review for the same method, binary and source revision") + return data["reviews"] + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("snapshots", nargs="+") + parser.add_argument("--emulator-root", required=True, type=Path) + parser.add_argument("--reviews", type=Path) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args(argv) + result = build_tracker([Snapshot.load(p) for p in args.snapshots], scan_emulator(args.emulator_root), load_reviews(args.reviews)) + args.out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + print(json.dumps(result["summary"])) + + +if __name__ == "__main__": + main() diff --git a/docs/tracker.md b/docs/tracker.md new file mode 100644 index 0000000..838ae48 --- /dev/null +++ b/docs/tracker.md @@ -0,0 +1,75 @@ +# Atlas API — tracker silników i bibliotek + +Atlas łączy snapshoty oryginalnych DLL z rejestracjami `MethodSpec` w Rex-EMoolator. +Nie uruchamia emulatora ani binarek. Nie utożsamia wykrycia metody z poprawną implementacją. + +## Uruchomienie + +Z katalogu tego repozytorium, z zainstalowanymi zależnościami `.[api]`: + +```sh +.venv/bin/python -m ams.serve_tracker \ + --emulator-root /ścieżka/do/Rex-EMoolator +``` + +Otwórz `http://127.0.0.1:8766/ui/tracker.html`. Command Center ma odnośnik do Atlasu. +`GET /tracker` oraz `GET /tracker?download=1` udostępniają te same dane jako JSON. +Snapshoty z bazy i jawnie wskazanego katalogu są łączone i deduplikowane po SHA DLL. +Bez SHA używany jest identyfikator snapshotu, jawnie niebędący tożsamością binarki. +Katalog snapshotów jest przeszukiwany rekurencyjnie. Wyniki batcha mają pierwszeństwo przed starszymi snapshotami tego samego SHA; konflikt jest widoczny w uwagach, a dane nie są automatycznie scalane. +Launcher domyślnie wczytuje `data/binary-inventory.json`, aby pokazać pokrycie zbioru DLL. Filtr rodziny i wariantu ogranicza dowody oryginału; nadal pokazuje sumę metod API z oznaczeniem „nieustalone” dla braku obserwacji w wybranym zakresie. + +Eksport bez serwera: + +```sh +.venv/bin/python -m ams.tracker snapshots/batch/*.snapshot.json snapshots/*.snapshot.json \ + --emulator-root /ścieżka/do/Rex-EMoolator \ + --reviews tracker-reviews.json --out /tmp/tracker.json +``` + +Atlas odczytuje snapshoty z dysku przy każdym pobraniu katalogu. Command Center, diff i eksport zgodności korzystają z bazy SQLite; nowe wyniki można do niej dodać poleceniem `.venv/bin/python -m ams.api.importer snapshots/batch/*.snapshot.json`. + +## Interpretacja + +- **Zarejestrowana**: znaleziono wpis `MethodSpec`, bez oceny poprawności. +- **Podejrzenie stubu**: heurystyka wykryła wyjątek lub komunikat o braku implementacji. Może dotyczyć tylko gałęzi; wymaga przeglądu. +- **Brak rejestracji**: adapter nie znalazł wpisu; dynamiczny dispatcher może obsługiwać metodę inaczej. +- **Zaimplementowana / przegląd ASM / test zachowania**: aktualne deklaracje przeglądu obejmują wszystkie obserwowane warianty danej pary typ/metoda. +- **Nieustalone w oryginale**: brak obserwacji, nigdy dowód nieobecności. + +Licznik to pary typ/metoda, z rozwinięciem rozpoznanego dziedziczenia. Nie jest liczbą unikalnych ciał Java ani procentem zgodności całego silnika. Zakres `*` pokazuje metody bazowe/globalne; nazwy klas bez dopasowania fabryki są wnioskowane (mogą obejmować klasy bazowe i singletony). Typy bez rozpoznanych metod pozostają widoczne. +Alias `BOOL` jest łączony z `BOOLEAN`, `INT` z `INTEGER`; nazwy metod są normalizowane do uppercase. ID metody pozostaje lokalne dla konkretnej klasy i binarki. Relacje dziedziczenia pochodzą wyłącznie ze snapshotu, nie są zgadywane z innej wersji. + +Adapter rozpoznaje bieżący wzorzec rejestracji Java; nie jest ogólnym parserem języka Java. Komentarze i literały nie tworzą fałszywych wpisów, nawiasy są dopasowywane leksykalnie. Własne dispatchery `GROUP` / `INSTANCE` są oznaczone do przeglądu. Celowe no-op nie są automatycznie stubami. + +## Dowody poprawności + +`tracker-reviews.json` jest wersjonowanym rejestrem przeglądów; początkowo pustym. Ekstrakcja rejestracji silników, WORLD i KOLOROWANKA nie wypełnia go automatycznie. +Każdy wpis wymaga pól tekstowych: + +| Pole | Znaczenie | +|---|---| +| `type`, `method` | Para API | +| `binary_sha256`, `rva` | Dokładna binarka i adres względny, np. `0xc950` | +| `emulator_commit`, `body_sha256`, `source_tree_sha256` | Rewizja i hashe z eksportu trackera | +| `kind` | `asm` lub `runtime` | +| `verdict` | `matches`, `partial`, `differs` | +| `reviewer`, `date` | Autor i data przeglądu | +| `notes`, `evidence` | Zakres porównania i odnośnik do trwałego dowodu/testu | + +`matches` oznacza świadomą ocenę całej metody dla wskazanego wariantu; dla sprawdzenia pojedynczego przypadku użyj `partial`. Sam adres lub dekompilat nie wystarcza do takiej oceny. Zmiana commita lub dowolnego źródła Java w `core/src` unieważnia przegląd konserwatywnie (także zmiana helpera). Można zachować historyczne wpisy dla starszych rewizji. Sprzeczne, zduplikowane przeglądy tej samej pary i rewizji są odrzucane. +Adresy `impl_addr`, `case_addr`, `registration_addr` w snapshotach są VA, nie RVA. Należy odjąć potwierdzoną bazę obrazu przed zapisaniem `rva`. + +## Kolejka dalszej analizy + +```sh +.venv/bin/python -m ams.inventory /ścieżka/do/_dll_repository \ + /ścieżka/do/World.dll /ścieżka/do/Kolorowanka.dll \ + --snapshots snapshots --out /tmp/binary-inventory.json +``` + +Manifest zachowuje wszystkie ścieżki identycznych DLL i proponuje pliki wynikowe po pełnym SHA. Nie uruchamia analiz. Klasyfikacja nazw wymaga przeglądu przy przemianowanych DLL. Starsze Pikliby są wyłączone z zakresu, Sekai oznaczone jako zależność natywna. + +Kolejka wszystkich unikalnych silników: [batch.md](batch.md). Statyczne dowody pluginów: [WORLD](world-analysis.md) i [KOLOROWANKA](kolorowanka-pilot.md). + +Pozostałe etapy: uogólnienie ekstrakcji pluginów na kolejne warianty; uzupełnienie niepełnych mapowań klas, dziedziczenia, pól i zdarzeń; rozstrzygnięcie podwójnej rejestracji WORLD.SETPOSITION; analiza natywnego Sekai; porównanie zachowania i rejestracja przeglądów. Historyczny snapshot 7.1 bez SHA pozostaje oznaczony jako nieustalony; nowe ekstrakcje mają pełną tożsamość DLL. Istniejący ogólny pipeline `acquire(directory)` nadal wybiera jeden silnik — do masowej analizy służy `ams.acquire.batch`. diff --git a/tests/test_tracker.py b/tests/test_tracker.py new file mode 100644 index 0000000..d7315f0 --- /dev/null +++ b/tests/test_tracker.py @@ -0,0 +1,174 @@ +from copy import deepcopy +import json +from pathlib import Path + +import pytest + +from ams.snapshot import Snapshot +from ams.tracker import build_tracker, load_reviews, scan_emulator + +SHA = "a" * 64 + + +def emulator(tmp_path): + directory = tmp_path / "core/src/pl/genschu/bloomooemulator/interpreter/variable" + directory.mkdir(parents=True) + (directory / "Variable.java").write_text('''interface Variable { + Map.entry("CLONE", MethodSpec.of((s,a,c) -> clone(s))); + }''') + (directory / "BoolVariable.java").write_text('''record BoolVariable() { + VariableType type() { return VariableType.BOOLEAN; } + // Map.entry("FAKE", MethodSpec.of((s,a,c) -> null)); + String misleading = "Map.entry(\\"FAKE2\\", MethodSpec.of("; + Map.entry("SET", MethodSpec.of((s,a,c) -> { + String x = ") /*"; // ) + return new Value(a.get(0)); + })); + Map.entry("TODO", MethodSpec.of((s,a,c) -> { + throw new UnsupportedOperationException("todo"); + })); + Map.entry("NOOP", MethodSpec.of((s,a,c) -> MethodResult.noReturn())); + }''') + result = scan_emulator(tmp_path) + result["commit"] = "f" * 40 + return result + + +def native(sha=SHA): + return Snapshot({"binary": {"name": "test.dll", "sha256": sha}, + "types": [{"script_name": "BOOL", "cpp_class": "CMC_Bool"}, + {"script_name": "UNKNOWN", "cpp_class": None}], + "methods": [{"owner": "CMC_Bool", "name": "set", "id": 1}, + {"owner": "CMC", "name": "CLONE", "id": 1}, + {"owner": "CMC", "name": "SET", "id": 2}], + "method_inheritance": [{"runner": "CMC_Bool_Runner", "base_runner": "CMC_Runner"}], + "method_dispatch": [{"owner": "CMC_Bool", "id": 1, "impl_addr": "0x10001000"}]}) + + +def row(result, typ="BOOLEAN", method="SET"): + return next(m for t in result["types"] if t["name"] == typ for m in t["methods"] if m["name"] == method) + + +def review(emu, **changes): + return {"type": "BOOLEAN", "method": "SET", "binary_sha256": SHA, "rva": "0x1000", + "emulator_commit": emu["commit"], "body_sha256": emu["types"]["BOOLEAN"]["methods"]["SET"]["source"]["body_sha256"], + "source_tree_sha256": emu["source_tree_sha256"], "reviewer": "test", "date": "2026-10-10", + "notes": "Checked integer input", "evidence": "evidence/example.json#method", "kind": "asm", + "verdict": "matches", **changes} + + +def test_java_adapter_ignores_comments_literals_and_balances(tmp_path): + emu = emulator(tmp_path) + methods = emu["types"]["BOOLEAN"]["methods"] + assert set(methods) == {"SET", "TODO", "NOOP"} + assert methods["TODO"]["status"] == "stub_candidate" + assert methods["NOOP"]["status"] == "registered" + assert methods["SET"]["source"]["line"] == 5 + + +def test_inheritance_aliases_and_globals_are_not_implementation_proof(tmp_path): + result = build_tracker([native()], emulator(tmp_path)) + assert row(result)["status"] == "registered" + assert row(result)["verification"] == "unverified" + assert row(result)["original"][0]["owner"] == "CMC_Bool" + assert len(row(result)["original"]) == 1 # child wins over SET on base + assert row(result, method="CLONE")["source"]["owner"] == "*" + assert row(result, method="CLONE")["original"][0]["inherited"] + assert next(t for t in result["types"] if t["name"] == "UNKNOWN")["methods"] == [] + + +def test_unknown_hash_not_fabricated_and_method_ids_are_binary_local(tmp_path): + a, b = native(), native(None) + b.raw["method_dispatch"][0]["impl_addr"] = "0x10009999" + result = build_tracker([a,b], emulator(tmp_path)) + assert {o["impl_addr"] for o in row(result)["original"]} == {"0x10001000", "0x10009999"} + assert any(b["sha256"] is None and b["id"].startswith("snapshot:") for b in result["binaries"]) + + +def test_review_stales_on_source_or_commit_change_and_covers_only_one_binary(tmp_path): + emu = emulator(tmp_path) + r = review(emu) + assert row(build_tracker([native()], emu, [r]))["verification"] == "asm_reviewed" + changed = deepcopy(emu) + changed["source_tree_sha256"] = "b" * 64 # includes helper changes, not just lambda + assert row(build_tracker([native()], changed, [r]))["verification"] == "stale" + changed = deepcopy(emu) + changed["commit"] = "d" * 40 + assert row(build_tracker([native()], changed, [r]))["verification"] == "stale" + multiple = build_tracker([native(),native("c"*64)], emu, [r]) + assert row(multiple)["verification"] == "unverified" + assert row(multiple)["verified_variants"] == 1 + + +def test_review_without_required_anchors_rejected(tmp_path): + path = tmp_path / "reviews.json" + path.write_text(json.dumps({"schema":"tracker-reviews/1", "reviews":[{"type":"BOOLEAN"}]})) + with pytest.raises(ValueError): load_reviews(path) + emu = emulator(tmp_path / "repo") + path.write_text(json.dumps({"schema":"tracker-reviews/1", "reviews":[review(emu),review(emu)]})) + with pytest.raises(ValueError, match="Duplicate"): load_reviews(path) + + +def test_kolorowanka_registration_is_not_behavior_verification(tmp_path): + snap = Snapshot.load(str(Path(__file__).parents[1] / "snapshots/kolorowanka.snapshot.json")) + result = build_tracker([snap], emulator(tmp_path)) + t = next(t for t in result["types"] if t["name"] == "KOLOROWANKA") + assert len(t["methods"]) == 23 + assert all(m["verification"] == "unverified" for m in t["methods"]) + assert row(result,"KOLOROWANKA","GETENDCOLOR")["original"][0]["evidence_ids"] + + +def test_duplicate_native_name_and_id_preserved(tmp_path): + snap = native() + snap.raw["methods"].extend([ + {"owner": "CMC_Bool", "name": "SET", "id": 3}, + {"owner": "CMC_Bool", "name": "OTHER", "id": 3}, + ]) + result = build_tracker([snap], emulator(tmp_path)) + assert {o["method_id"] for o in row(result)["original"]} == {1, 3} + assert len(row(result)["notes"]) == 2 + assert len(row(result, method="OTHER")["notes"]) == 1 + + +def test_world_snapshot_joins_and_keeps_registration_ambiguity(tmp_path): + snap = Snapshot.load(str(Path(__file__).parents[1] / "snapshots/world.snapshot.json")) + result = build_tracker([snap], emulator(tmp_path)) + world = next(t for t in result["types"] if t["name"] == "WORLD") + assert len(world["methods"]) == 66 + position = row(result, "WORLD", "SETPOSITION") + assert {o["method_id"] for o in position["original"]} == {24, 43} + assert len(position["notes"]) == 2 + assert result["binaries"][0]["limitations"] + + +def test_known_class_aliases_do_not_create_false_missing_types(tmp_path): + snap = Snapshot({"binary": {"sha256": SHA}, "types": [ + {"script_name": "CANVAS_OBSERVER", "cpp_class": None}, + {"script_name": "GROUP", "cpp_class": None}], "methods": [ + {"owner": "CMC_CanvasObserver", "name": "REFRESH", "id": None}, + {"owner": "CMC_Groupped", "name": "ADD", "id": None}]}) + result = build_tracker([snap], emulator(tmp_path)) + assert row(result, "CANVAS_OBSERVER", "REFRESH")["original"][0]["mapping"] == "inferred" + assert row(result, "GROUP", "ADD")["original"] + assert not {"CANVASOBSERVER", "GROUPPED"} & {t["name"] for t in result["types"]} + + +def test_api_configuration_and_download(tmp_path, monkeypatch): + from fastapi.testclient import TestClient + from ams.api.app import create_app + monkeypatch.delenv("AMS_EMULATOR_ROOT", raising=False) + monkeypatch.delenv("AMS_TRACKER_REVIEWS", raising=False) + monkeypatch.delenv("AMS_TRACKER_SNAPSHOTS", raising=False) + client = TestClient(create_app("sqlite:///" + str(tmp_path / "test.db"))) + assert client.get("/tracker").status_code == 503 + emulator(tmp_path / "repo") + monkeypatch.setenv("AMS_EMULATOR_ROOT", str(tmp_path / "repo")) + snaps = tmp_path / "snapshots" + snaps.mkdir() + (snaps / "test.snapshot.json").write_text(json.dumps(native().raw)) + monkeypatch.setenv("AMS_TRACKER_SNAPSHOTS", str(snaps)) + response = client.get("/tracker?download=1") + assert response.status_code == 200 + assert "attachment" in response.headers["content-disposition"] + assert response.json()["schema"] == "tracker/1" + assert client.get("/ui/tracker.html").status_code == 200 diff --git a/tests/test_tracker_catalog.py b/tests/test_tracker_catalog.py new file mode 100644 index 0000000..6b0ab8d --- /dev/null +++ b/tests/test_tracker_catalog.py @@ -0,0 +1,72 @@ +import json + +import pytest + +pytest.importorskip('fastapi') +from fastapi.testclient import TestClient +from ams.api.app import create_app +from ams.api.routes.tracker import inventory_status, order_snapshots +from ams.snapshot import Snapshot + + +def test_fresh_snapshot_preferred_and_conflict_reported(): + old = Snapshot({'binary': {'sha256': 'a', 'name': 'golden'}}) + fresh = Snapshot({'binary': {'sha256': 'a', 'name': 'batch', 'extraction': {'fingerprint': 'v2'}}}) + ordered, warnings = order_snapshots([old, fresh]) + assert ordered == [fresh, old] + assert len(warnings) == 1 + assert 'nie scalono' in warnings[0] + assert order_snapshots([old, old]) == ([old], []) + + +def test_batch_library_and_game_aliases_reach_compat(tmp_path): + from ams.compat import _library_name + payload = {'binary': {'sha256': 'a'*64, 'name': 'Game__hash__PIKLIB8.DLL', + 'library': 'PIKLIB8.DLL', 'games': ['Game A','Game B']}, 'types': []} + assert _library_name(Snapshot(payload)) == 'PIKLIB8.DLL' + with TestClient(create_app(database_url=f'sqlite:///{tmp_path}/compat.db')) as client: + assert client.post('/snapshots',json=payload).status_code == 201 + data = client.get('/compat?format=json').json() + assert data['games'] == ['Game A', 'Game B'] + assert len(data['libraries']) == 1 + assert data['libraries'][0]['name'] == 'PIKLIB8.DLL' + + +def test_inventory_counts_by_hash_and_scope(tmp_path): + path = tmp_path / 'inventory.json' + path.write_text(json.dumps({'builds': [ + {'sha256': 'AAA', 'scope': 'engine', 'analysis': 'pending'}, + {'sha256': 'bbb', 'scope': 'engine'}, + {'sha256': 'ccc', 'scope': 'plugin'}, + {'sha256': 'ddd', 'scope': 'native_dependency'}, + ]})) + result = inventory_status(path, [{'sha256': 'aaa'}, {'sha256': 'eee'}]) + assert result['scopes']['engine'] == {'target': 2, 'loaded': 1, 'pending': 1} + assert result['scopes']['plugin']['pending'] == 1 + assert result['scopes']['native_dependency']['pending'] == 1 + assert result['builds'][0]['snapshot_loaded'] is True + assert result['unlisted_snapshots'] == 1 + + +def test_recursive_endpoint_prefers_batch(tmp_path, monkeypatch): + from ams.api.routes import tracker + snapshots = tmp_path / 'snapshots' + nested = snapshots / 'batch' / 'by-hash' + nested.mkdir(parents=True) + sha = 'a' * 64 + (snapshots / 'golden.snapshot.json').write_text(json.dumps({'binary': {'sha256': sha, 'name': 'old'}})) + (nested / 'fresh.snapshot.json').write_text(json.dumps({'binary': {'sha256': sha, 'name': 'fresh', 'extraction': {'fingerprint': 'v2'}}})) + inventory = tmp_path / 'inventory.json' + inventory.write_text(json.dumps({'builds': [{'sha256': sha, 'scope': 'engine'}]})) + monkeypatch.setenv('AMS_EMULATOR_ROOT', str(tmp_path)) + monkeypatch.setenv('AMS_TRACKER_SNAPSHOTS', str(snapshots)) + monkeypatch.setenv('AMS_TRACKER_INVENTORY', str(inventory)) + monkeypatch.delenv('AMS_TRACKER_REVIEWS', raising=False) + monkeypatch.setattr(tracker, 'scan_emulator', lambda root: {'types': {}, 'warnings': []}) + with TestClient(create_app(database_url=f'sqlite:///{tmp_path}/test.db')) as client: + response = client.get('/tracker') + assert response.status_code == 200, response.text + result = response.json() + assert [b['name'] for b in result['binaries']] == ['fresh'] + assert result['inventory']['scopes']['engine']['loaded'] == 1 + assert any('nie scalono' in w for w in result['warnings']) diff --git a/tracker-reviews.json b/tracker-reviews.json new file mode 100644 index 0000000..29d8409 --- /dev/null +++ b/tracker-reviews.json @@ -0,0 +1,4 @@ +{ + "schema": "tracker-reviews/1", + "reviews": [] +}