feat(tracker): Atlas joining DLL snapshots with emulator registrations

Matches original script APIs against MethodSpec registrations in
Rex-EMoolator and keeps review evidence separate from registration.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Patryk Gensch
2026-10-24 18:24:15 +02:00
co-authored by Claude Opus 5.5
parent b0013c5114
commit 10c81c4390
13 changed files with 846 additions and 1 deletions
+2 -1
View File
@@ -15,7 +15,7 @@ from fastapi.staticfiles import StaticFiles
from .. import __version__
from .db import configure, init_db
from .routes import compat, diff, games, jobs, snapshots
from .routes import compat, diff, games, jobs, snapshots, tracker
_STATIC = Path(__file__).parent / "static"
@@ -30,6 +30,7 @@ def create_app(database_url: str | None = None) -> FastAPI:
app.include_router(diff.router)
app.include_router(jobs.router)
app.include_router(compat.router)
app.include_router(tracker.router)
@app.get("/health", tags=["meta"])
def health() -> dict[str, str]:
+84
View File
@@ -0,0 +1,84 @@
"""Read-only tracker. Filesystem paths come from server configuration, never requests."""
import os
import json
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import JSONResponse
from sqlalchemy import select
from sqlalchemy.orm import Session
from ...snapshot import Snapshot as Surface
from ...tracker import build_tracker, load_reviews, scan_emulator
from .. import models
from ..db import get_db
router = APIRouter(tags=["tracker"])
def order_snapshots(snapshots):
"""Prefer reproducible batch results, retaining explicit conflict diagnostics."""
ordered = sorted(snapshots, key=lambda s: bool(s.binary.get("extraction")), reverse=True)
seen, warnings, distinct = {}, [], []
for snap in ordered:
sha = str(snap.binary.get("sha256", "")).lower()
if not sha:
distinct.append(snap)
continue
if sha in seen and seen[sha].raw == snap.raw:
continue
distinct.append(snap)
if sha in seen and seen[sha].raw != snap.raw:
warnings.append(f"Różne snapshoty tej samej DLL ({sha}): wybrano pierwszy, z pierwszeństwem metadanych ekstrakcji; wyników nie scalono.")
seen.setdefault(sha, snap)
return distinct, list(dict.fromkeys(warnings))
def inventory_status(path, binaries):
manifest = json.loads(Path(path).read_text(encoding="utf-8"))
builds = manifest["builds"]
if not isinstance(builds, list):
raise ValueError("inventory builds must be a list")
loaded = {str(b.get("sha256", "")).lower() for b in binaries if b.get("sha256")}
scopes, rows, seen = {}, [], set()
for build in builds:
sha = str(build["sha256"]).lower()
if sha in seen:
raise ValueError("Duplicate inventory SHA-256: " + sha)
seen.add(sha)
scope = build.get("scope", "unknown")
present = sha in loaded
rows.append({**build, "snapshot_loaded": present})
counts = scopes.setdefault(scope, {"target": 0, "loaded": 0, "pending": 0})
counts["target"] += 1
counts["loaded" if present else "pending"] += 1
return {"builds": rows, "scopes": scopes, "loaded_snapshots": len(binaries),
"unlisted_snapshots": len(loaded - seen),
"note": "Wczytany snapshot nie oznacza kompletnej ekstrakcji ani zgodności funkcjonalnej."}
@router.get("/tracker")
def get_tracker(download: bool = False, db: Session = Depends(get_db)):
root = os.environ.get("AMS_EMULATOR_ROOT")
if not root:
raise HTTPException(503, "Ustaw AMS_EMULATOR_ROOT na katalog Rex-EMoolator i uruchom serwer ponownie.")
try:
snapshots = [Surface(s.data) for s in db.scalars(select(models.Snapshot).order_by(models.Snapshot.id))]
# Explicitly configured disk snapshots may supplement a DB; identities are deduplicated.
directory = os.environ.get("AMS_TRACKER_SNAPSHOTS")
if directory:
path = Path(directory)
if not path.is_dir():
raise ValueError("AMS_TRACKER_SNAPSHOTS is not a directory")
snapshots.extend(Surface.load(str(p)) for p in sorted(path.rglob("*.snapshot.json")))
snapshots, catalog_warnings = order_snapshots(snapshots)
reviews = os.environ.get("AMS_TRACKER_REVIEWS")
result = build_tracker(snapshots, scan_emulator(Path(root)), load_reviews(Path(reviews) if reviews else None))
result["warnings"].extend(catalog_warnings)
inventory = os.environ.get("AMS_TRACKER_INVENTORY")
if inventory:
result["inventory"] = inventory_status(inventory, result["binaries"])
except (OSError, ValueError, KeyError, TypeError) as exc:
raise HTTPException(503, "Konfiguracja lub dane trackera: " + str(exc)) from exc
headers = {"Content-Disposition": 'attachment; filename="tracker.json"'} if download else {}
return JSONResponse(result, headers=headers)
+1
View File
@@ -9,6 +9,7 @@
<body>
<header class="topbar">
<div class="brand">⌖ Aidem&nbsp;Media <span>Command Center</span></div>
<a class="mini-btn" href="tracker.html">Atlas API ↗</a>
<div class="sub">Piklib / BlooMoo — analiza różnicowa powierzchni silnika</div>
</header>
+5
View File
@@ -0,0 +1,5 @@
:root{color-scheme:dark;--bg:#0c131e;--panel:#131e2c;--line:#2a394a;--text:#e4edf5;--muted:#9aafc3;--teal:#54d6ba;--amber:#efbd69}*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text);font:14px/1.55 system-ui,sans-serif}header,main{max-width:1550px;margin:auto;padding:30px 40px}header{display:flex;align-items:center;justify-content:space-between;border-bottom:1px solid var(--line);gap:20px}h1{font-size:34px;letter-spacing:-1px;margin:14px 0 3px}h1 span{display:block;color:var(--teal);font-size:12px;letter-spacing:2px;text-transform:uppercase;margin-top:6px}p{margin:8px 0 16px}header p,.muted,.footnote{color:var(--muted)}a{color:var(--teal)}.back{text-decoration:none;font-size:12px}button,.button,input,select{font:inherit;color:inherit;background:#192738;border:1px solid var(--line);border-radius:7px;padding:9px 12px}button,.button{cursor:pointer}.button{text-decoration:none;white-space:nowrap}button:hover,.button:hover{border-color:var(--teal)}:focus-visible{outline:2px solid var(--teal);outline-offset:3px}.filters{display:flex;flex-wrap:wrap;align-items:end;gap:16px;margin:24px 0 10px}.filters label{display:grid;gap:6px;font-size:12px;color:var(--muted)}.filters label:first-child{flex:1;min-width:230px}.filters select{max-width:330px}.filters .check{display:flex;align-items:center;padding:10px 0}.stats{display:grid;grid-template-columns:repeat(4,1fr);gap:14px;margin:25px 0}.stat{border:1px solid var(--line);border-radius:10px;background:var(--panel);padding:18px}.stat strong{display:block;font-size:30px;line-height:1.3}.stat span{color:var(--muted);font-size:12px}.layout{display:grid;grid-template-columns:290px minmax(0,1fr);gap:25px}.section-heading{display:flex;justify-content:space-between;align-items:center;gap:10px;margin-bottom:12px}h2{font-size:16px;margin:0}.quiet{padding:5px 9px;font-size:12px}.types{display:grid;gap:8px;max-height:720px;overflow:auto;padding:3px}.type{display:block;width:100%;text-align:left;background:var(--panel);padding:13px}.type[aria-pressed=true]{border-color:var(--teal);background:#18332f}.type strong{display:block}.type small{color:var(--muted)}.bar{height:4px;background:#314052;border-radius:3px;margin:7px 0 3px;overflow:hidden}.bar i{display:block;height:100%;background:var(--teal)}.bar.review i{background:var(--amber)}.table-scroll{overflow:auto;border:1px solid var(--line);border-radius:10px}table{width:100%;border-collapse:collapse;text-align:left;white-space:nowrap}th{font-size:11px;text-transform:uppercase;color:var(--muted);background:var(--panel);padding:13px}td{padding:13px;border-top:1px solid var(--line);font-size:12px}td:first-child small{display:block;color:var(--muted)}td:first-child strong{font-family:ui-monospace,monospace}.badge{display:inline-block;border-radius:4px;padding:3px 7px;background:#233145;color:#c0d0e0}.badge.implemented,.badge.asm_reviewed,.badge.runtime_tested{color:var(--teal);background:#183a33}.badge.missing,.badge.stub_candidate,.badge.stale{color:var(--amber);background:#3b3020}.footnote{font-size:12px;margin-top:28px;max-width:1000px}.warning{white-space:pre-wrap;border-left:3px solid var(--amber);background:#30291e;padding:12px;margin-top:15px}#message{color:var(--muted);padding:25px 0}dialog{background:var(--panel);color:var(--text);border:1px solid var(--line);border-radius:12px;width:min(790px,94vw);max-height:90vh;padding:24px}dialog::backdrop{background:#030911bb}dialog h3{font-size:14px;color:var(--teal);margin-top:25px}dialog pre{white-space:pre-wrap;overflow-wrap:anywhere;background:var(--bg);padding:13px;border-radius:7px;font-size:12px}dialog article{border-top:1px solid var(--line);padding:10px 0;overflow-wrap:anywhere}dialog dl{display:grid;grid-template-columns:130px 1fr;gap:7px}dialog dt{color:var(--muted)}dialog dd{margin:0;overflow-wrap:anywhere}.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0)}[hidden]{display:none!important}@media(max-width:950px){header,main{padding:24px}.layout{grid-template-columns:1fr}.types{grid-template-columns:repeat(auto-fill,minmax(210px,1fr));max-height:300px}.stats{grid-template-columns:repeat(2,1fr)}}@media(max-width:550px){header{align-items:flex-start;flex-direction:column}header,main{padding:20px}.filters label,.filters select{width:100%;max-width:none}.stats{gap:8px}.stat{padding:12px}.stat strong{font-size:25px}dialog dl{grid-template-columns:1fr}}
.layout>section,.stat,.filters label{min-width:0}#revision,.warning{overflow-wrap:anywhere}.table-scroll{max-width:100%}@media(max-width:950px){.layout{grid-template-columns:minmax(0,1fr)}}
.table-scroll{position:relative}
+21
View File
@@ -0,0 +1,21 @@
<!doctype html>
<html lang="pl">
<head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Atlas API · Aidem Media</title><link rel="stylesheet" href="tracker.css"></head>
<body>
<header><div><a href="index.html" class="back">← Command Center</a><h1>Atlas API<span>Piklib · BlooMoo · biblioteki</span></h1><p>Mapa implementacji i dowodów zgodności z oryginałem.</p></div><a class="button" href="/tracker?download=1" download="tracker.json">Eksport JSON ↗</a></header>
<main>
<div id="message" role="status" aria-live="polite">Wczytywanie katalogu i implementacji emulatora…</div>
<section id="workspace" hidden>
<div id="revision" class="muted"></div>
<p id="inventory" class="muted" hidden></p>
<details id="warnings" class="warning" hidden><summary id="warnings-summary">Uwagi do katalogu</summary><div id="warnings-body"></div></details>
<div class="filters"><label>Szukaj typu lub metody<input id="search" type="search" placeholder="np. ANIMO, SETPOSITION" autocomplete="off"></label><label>Rodzina<select id="family"><option value="">Wszystkie rodziny</option></select></label><label>Wariant oryginału<select id="binary"><option value="">Wszystkie warianty</option></select></label><label>Implementacja<select id="status"><option value="">Wszystkie stany</option><option value="missing">Brak rejestracji</option><option value="registered">Zarejestrowana</option><option value="stub_candidate">Podejrzenie stubu</option><option value="partial">Częściowa</option><option value="implemented">Zaimplementowana</option></select></label><label class="check"><input id="gaps" type="checkbox"> Tylko braki i podejrzenia stubów</label></div>
<p class="muted" id="scope-note">Widok jest sumą API wariantów. Rejestracja metody nie potwierdza jej poprawności.</p>
<div id="stats" class="stats"></div>
<div class="layout"><section aria-labelledby="types-heading"><div class="section-heading"><h2 id="types-heading">Typy</h2><button id="all-types" class="quiet">Wszystkie</button></div><div id="types" class="types"></div></section><section class="methods" aria-labelledby="methods-heading"><div class="section-heading"><h2 id="methods-heading">Metody</h2><span id="count" class="muted"></span></div><div id="type-notes" class="warning" hidden></div><div class="table-scroll"><table><thead><tr><th>Typ / metoda</th><th>Implementacja</th><th>Weryfikacja</th><th>Oryginał</th><th><span class="sr-only">Szczegóły</span></th></tr></thead><tbody id="rows"></tbody></table></div><p id="empty" hidden>Brak metod dla wybranych filtrów.</p></section></div>
<p class="footnote">„Nieustalone” oznacza brak dowodu w wynikach ekstrakcji, a nie nieobecność metody w bibliotece. Stan „zarejestrowana” wynika z kodu; stub jest kandydatem do przeglądu. Potwierdzenia ASM i testy są osobnym wymiarem.</p>
</section>
</main>
<dialog id="details"><div class="section-heading"><h2 id="detail-title"></h2><button id="close" aria-label="Zamknij szczegóły">Zamknij ×</button></div><div id="detail-body"></div></dialog>
<script src="tracker.js"></script>
</body></html>
+41
View File
@@ -0,0 +1,41 @@
'use strict';
const $ = id => document.getElementById(id);
const labels = {missing:'Brak rejestracji',registered:'Zarejestrowana',stub_candidate:'Podejrzenie stubu',partial:'Częściowa',implemented:'Zaimplementowana',unverified:'Niezweryfikowana',asm_reviewed:'Przegląd ASM',runtime_tested:'Test zachowania',stale:'Dowód wymaga aktualizacji'};
let data, selected = null;
const el = (tag, text, cls) => {const n = document.createElement(tag); if(text !== undefined) n.textContent = text; if(cls) n.className = cls; return n;};
const family = b => String(b.family || (typeof b.engine === 'object' ? b.engine?.family || b.engine?.name : b.engine) || b.library || 'Nieustalona');
const binaryName = b => String(b.library || b.name || '?').split(/[\\/]/).pop().replace(/^.*__[0-9a-f]+__/i,'');
const inFamily = b => !$('family').value || family(b) === $('family').value;
function variants(){const previous=$('binary').value;$('binary').replaceChildren(new Option('Wszystkie warianty',''));for(const b of data.binaries.filter(inFamily)){const games=(b.games||[]).join(', ');const opt=el('option',`${binaryName(b)}${games ? ' · '+games : ''} · ${String(b.sha256||b.id).slice(0,10)}`);opt.value=b.id;$('binary').append(opt);}if([...$('binary').options].some(o=>o.value===previous))$('binary').value=previous;}
const original = m => (m.original || []).filter(o => (!$('binary').value || o.binary_id === $('binary').value) && data.binaries.some(b=>b.id===o.binary_id && inFamily(b)));
const registered = m => m.status !== 'missing';
const reviewed = m => ['asm_reviewed','runtime_tested'].includes(m.verification);
const typeName = name => name === '*' ? 'Zakres globalny (*)' : name;
function matches(t,m){const q=$('search').value.trim().toUpperCase();return (!q || `${t.name} ${m.name}`.toUpperCase().includes(q)) && (!$('status').value || m.status === $('status').value) && (!$('gaps').checked || ['missing','stub_candidate'].includes(m.status));}
function stat(number,label){const n=el('div',undefined,'stat');n.append(el('strong',number),el('span',label));return n;}
function badge(value){return el('span',labels[value] || value,'badge '+value);}
function render(){
const query=$('search').value.trim().toUpperCase();
const groups=data.types.map(t=>({...t,unresolved:t.methods.length===0,methods:t.methods.filter(m=>matches(t,m))})).filter(t=>t.methods.length || (t.unresolved && (!query || t.name.toUpperCase().includes(query)) && !$('status').value && !$('gaps').checked));
const active=groups.filter(t=>!selected || t.name===selected), methods=active.flatMap(t=>t.methods);
$('scope-note').textContent=($('binary').value || $('family').value) ? 'Wybrany wariant lub rodzina ogranicza dowody oryginału. Metody bez dopasowania pozostają widoczne jako „nieustalone”; liczniki obejmują widoczne pary typ / metoda. Weryfikacja jest zbiorcza i nie zmienia się po wyborze wariantu.' : 'Widok jest sumą API wariantów. Rejestracja metody nie potwierdza jej poprawności. Liczniki obejmują widoczne pary typ / metoda, w tym dziedziczone.';
$('stats').replaceChildren(stat(methods.length,'par typ / metoda (także dziedziczone)'),stat(methods.filter(registered).length,'zarejestrowanych w emulatorze'),stat(methods.filter(m=>m.status==='implemented').length,'oznaczonych jako zaimplementowane'),stat(methods.filter(reviewed).length,'z aktualnym dowodem ASM / testu'));
$('types').replaceChildren();
for(const t of groups){const b=el('button',undefined,'type');b.type='button';b.setAttribute('aria-pressed',String(selected===t.name));const n=t.methods.length,r=t.methods.filter(registered).length,v=t.methods.filter(reviewed).length;b.append(el('strong',typeName(t.name)),el('small',n ? `${n} metod · ${r} rejestracji · ${v} dowodów` : 'Metody nierozpoznane — typ wykryty'));for(const [amount,cls,title] of [[r,'','Rejestracja'],[v,'review','Weryfikacja']]){const bar=el('div',undefined,'bar '+cls);bar.title=`${title}: ${amount} / ${n}`;const fill=el('i');fill.style.width=`${n ? 100*amount/n : 0}%`;bar.append(fill);b.append(bar);}b.onclick=()=>{selected=selected===t.name?null:t.name;render();};$('types').append(b);}
const chosen=data.types.find(t=>t.name===selected);$('type-notes').hidden=!chosen?.mapping_notes?.length;$('type-notes').textContent=(chosen?.mapping_notes||[]).map(note=>typeof note==='string'?note:JSON.stringify(note)).join('\n');$('methods-heading').textContent=selected?typeName(selected):'Wszystkie metody';$('count').textContent=`${methods.length} pozycji`;$('rows').replaceChildren();
for(const t of active)for(const m of t.methods){const tr=el('tr'),name=el('td');name.append(el('small',typeName(t.name)),el('strong',m.name));const s=el('td'),v=el('td');s.append(badge(m.status));v.append(badge(m.verification));const found=original(m);const availability=el('td',found.length?`Wykryta · ${new Set(found.map(o=>o.binary_id)).size} wariantów`:'Nieustalone');const action=el('td'),b=el('button','Dowody ↗','quiet');b.setAttribute('aria-label',`Dowody ${t.name}.${m.name}`);b.onclick=()=>detail(t,m);action.append(b);tr.append(name,s,v,availability,action);$('rows').append(tr);}
$('empty').hidden=methods.length>0;$('empty').textContent=chosen && !chosen.methods.length ? 'Typ został wykryty, ale metody nie zostały jeszcze rozpoznane. Brak wpisów nie oznacza braku API.' : 'Brak metod dla wybranych filtrów.';
}
function detail(t,m){$('detail-title').textContent=`${typeName(t.name)} · ${m.name}`;const body=$('detail-body');body.replaceChildren();const statuses=el('p');statuses.append(badge(m.status),document.createTextNode(' '),badge(m.verification));body.append(statuses);
const section=(title,text)=>{body.append(el('h3',title));if(text !== undefined)body.append(el('pre',text));};
section('Implementacja emulatora',m.source?`${m.source.path}:${m.source.line}\nWłaściciel: ${m.source.owner || '—'}\nHash treści: ${m.source.body_sha256 || '—'}`:'Nie znaleziono rejestracji w emulatorze.');
section('Dowody z oryginału');const entries=original(m);if(!entries.length)body.append(el('p','Nieustalone — ekstrakcja nie dostarcza dopasowania w wybranym zakresie.'));
for(const o of entries){const b=data.binaries.find(b=>b.id===o.binary_id);const a=el('article');a.append(el('strong',b?binaryName(b):o.binary_id));const dl=el('dl');for(const [key,value] of [['SHA-256',b?.sha256],['Silnik',typeof b?.engine==='object'?JSON.stringify(b.engine):b?.engine],['Właściciel',o.owner],['Id metody',o.method_id],['Implementacja',o.impl_addr],['Gałąź dispatchera',o.case_addr],['Funkcja rejestrująca',o.registration_addr],['Wywołanie rejestracji',o.registration_call_addr],['Przekazanie ID',o.id_push_addr],['Napis',o.string_addr],['Id dowodów',(o.evidence_ids||[]).join(', ')||'—'],['Dopasowanie',o.mapping==='explicit'?'Jawne':'Wywnioskowane'],['Dziedziczenie',o.inherited?'Tak':'Nie']]){dl.append(el('dt',key),el('dd',value===null || value===undefined?'—':String(value)));}a.append(dl);if(o.registration_addr || o.evidence_ids?.length)a.append(el('p','Dowód rejestracji ASM potwierdza rozpoznanie API; nie stanowi weryfikacji zachowania implementacji.','muted'));if(b?.limitations?.length){a.append(el('strong','Ograniczenia ekstrakcji'));a.append(el('pre',b.limitations.map(item=>typeof item==='string'?item:JSON.stringify(item,null,2)).join('\n')));}if(b?.coverage !== undefined)a.append(el('pre','Zakres ekstrakcji: '+(typeof b.coverage==='string'?b.coverage:JSON.stringify(b.coverage,null,2))));body.append(a);}
section('Przeglądy i testy');body.append(el('p','Poniżej wszystkie zapisane przeglądy metody. Zbiorczy status weryfikacji nie oznacza potwierdzenia każdego wariantu; zakres dowodu określają jego dane.'));if(!(m.reviews||[]).length)body.append(el('p','Brak zapisanych dowodów weryfikacji zachowania.'));for(const review of m.reviews||[])body.append(el('pre',typeof review==='string'?review:JSON.stringify(review,null,2)));
const notes=[...(t.mapping_notes||[]),...(m.notes||[])];if(notes.length)section('Uwagi',notes.map(n=>typeof n==='string'?n:JSON.stringify(n,null,2)).join('\n'));
$('details').showModal();
}
$('close').onclick=()=>$('details').close();$('all-types').onclick=()=>{selected=null;render();};for(const id of ['search','family','binary','status','gaps'])$(id).addEventListener(id==='search'?'input':'change',()=>{if(data){if(id==='family')variants();render();}});
async function load(){try{const res=await fetch('/tracker');if(!res.ok){const text=await res.text();let reason=text;try{const parsed=JSON.parse(text);reason=typeof parsed.detail==='string'?parsed.detail:JSON.stringify(parsed.detail||parsed);}catch{}throw new Error(res.status===503?`Katalog nie jest jeszcze skonfigurowany. ${reason}`:`Błąd API (${res.status}): ${reason}`);}data=await res.json();if(!Array.isArray(data.types)||!Array.isArray(data.binaries))throw new Error('Odpowiedź nie zawiera katalogu typów i wariantów.');for(const name of [...new Set(data.binaries.map(family))].sort())$('family').append(new Option(name,name));variants();$('revision').textContent=`Emulator: ${data.emulator?.commit || 'rewizja nieznana'}${data.emulator?.dirty?' · lokalne zmiany':''} · ${data.binaries.length} wariantów · ${data.types.length} typów`;
if(data.warnings?.length){$('warnings').hidden=false;$('warnings-summary').textContent=`Uwagi do katalogu (${data.warnings.length})`;$('warnings-body').textContent=[...new Set(data.warnings.map(w=>typeof w==='string'?w:JSON.stringify(w)))].join('\n');}if(data.inventory){const names={engine:'Silniki',plugin:'Biblioteki skryptowe',native_dependency:'Biblioteki natywne',native:'Biblioteki natywne',excluded_early_engine:'Wcześniejsze silniki (poza zakresem)'};$('inventory').hidden=false;$('inventory').textContent=Object.entries(data.inventory.scopes).map(([scope,c])=>scope.startsWith('excluded') ? `${names[scope]||scope}: ${c.target}` : `${names[scope]||scope}: ${c.loaded} / ${c.target} snapshotów, ${c.pending} oczekuje`).join(' · ')+'. Snapshot nie oznacza kompletnej ekstrakcji ani zgodności zachowania.';}$('message').hidden=true;$('workspace').hidden=false;render();}catch(error){$('message').replaceChildren(el('p',error.message));const retry=el('button','Spróbuj ponownie');retry.onclick=()=>{data=null;$('binary').replaceChildren(new Option('Wszystkie warianty',''));$('family').replaceChildren(new Option('Wszystkie rodziny',''));$('message').textContent='Wczytywanie…';load();};$('message').append(retry);}}
load();
+27
View File
@@ -0,0 +1,27 @@
"""Start the local Atlas with this repository's snapshots and review registry."""
import argparse
import os
from pathlib import Path
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--emulator-root', required=True, type=Path)
p.add_argument('--inventory', type=Path)
p.add_argument('--port', type=int, default=8766)
a = p.parse_args()
root = Path(__file__).resolve().parents[1]
if not (a.emulator_root / 'core/src').is_dir():
p.error('--emulator-root must point to Rex-EMoolator')
os.environ['AMS_EMULATOR_ROOT'] = str(a.emulator_root.resolve())
os.environ['AMS_TRACKER_SNAPSHOTS'] = str(root / 'snapshots')
os.environ['AMS_TRACKER_REVIEWS'] = str(root / 'tracker-reviews.json')
inventory = a.inventory or root / 'data/binary-inventory.json'
if inventory.is_file():
os.environ['AMS_TRACKER_INVENTORY'] = str(inventory.resolve())
import uvicorn
uvicorn.run('ams.api.app:create_app', factory=True, host='127.0.0.1', port=a.port)
if __name__ == '__main__':
main()
+332
View File
@@ -0,0 +1,332 @@
"""Evidence-aware script API tracker. Never treats registration as correctness.
The Java adapter deliberately recognises the repository's MethodSpec registrations,
not arbitrary Java. Unknown dispatch and incomplete native extraction stay explicit.
"""
from __future__ import annotations
import argparse
from collections import Counter, defaultdict
import hashlib
import json
from pathlib import Path
import re
import subprocess
from .snapshot import Snapshot
ALIASES = {"BOOL": "BOOLEAN", "INT": "INTEGER"}
# Established factory mappings in the PIKLIB8/BlooMoo reference snapshots. When
# used to fill an unresolved mapping in another build, provenance stays inferred.
CLASS_ALIASES = {"CMC_CanvasObserver": "CANVAS_OBSERVER", "CMC_Groupped": "GROUP"}
ENTRY = re.compile(r'Map\s*\.\s*entry\s*\(\s*"([A-Za-z0-9_]+)"\s*,\s*MethodSpec\s*\.\s*of\s*\(')
JAVA_TOKENS = re.compile(r'"""[\s\S]*?"""|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'|//[^\n]*|/\*[\s\S]*?\*/')
SHA = re.compile(r"[0-9a-f]{64}")
def canonical(name: str) -> str:
name = name.upper()
return ALIASES.get(name, name)
def digest(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def binary_family(binary: dict) -> str:
library = binary.get("library", binary.get("name", "?").split("__")[-1]).upper()
if library.startswith("BLOOMOO"):
return "BlooMoo"
return {"PIKLIB61.DLL": "Piklib 6.1", "PIKLIB71.DLL": "Piklib 7.1",
"PIKLIB72.DLL": "Piklib 7.2", "PIKLIB8.DLL": "Piklib 8"}.get(library, library.removesuffix(".DLL"))
def code_mask(text: str) -> str:
"""Mask literals/comments without shifting source positions or line numbers."""
return JAVA_TOKENS.sub(lambda m: re.sub(r"[^\n]", " ", m.group()), text)
def closing(code: str, start: int) -> int:
depth = 0
for i in range(start, len(code)):
if code[i] == "(":
depth += 1
elif code[i] == ")":
depth -= 1
if depth == 0:
return i + 1
raise ValueError("Unbalanced MethodSpec registration")
def _git(root: Path, *args: str) -> str | None:
result = subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True)
return result.stdout.strip() if result.returncode == 0 else None
def scan_emulator(root: Path) -> dict:
root = root.resolve()
source_root = root / "core/src"
variable_root = source_root / "pl/genschu/bloomooemulator/interpreter/variable"
if not (variable_root / "Variable.java").is_file():
raise ValueError("AMS_EMULATOR_ROOT must point to the Rex-EMoolator repository")
tree = hashlib.sha256()
for path in sorted(source_root.rglob("*.java")):
tree.update(str(path.relative_to(root)).encode() + b"\0" + path.read_bytes() + b"\0")
types = {}
warnings = []
for path in sorted(variable_root.glob("*Variable.java")):
text = path.read_text(encoding="utf-8")
code = code_mask(text)
match = re.search(r"return\s+VariableType\.([A-Z_]+)\s*;", code)
if path.name == "Variable.java":
name = "*"
elif match:
name = canonical(match[1])
else:
warnings.append("Nie rozpoznano typu Java: " + path.name)
continue
notes = []
if name != "*" and re.search(r"\bMethodResult\s+callMethod\s*\(", code):
notes.append("Własny dispatcher: lista rejestracji nie opisuje wszystkich wywołań dynamicznych.")
methods = {}
for entry in ENTRY.finditer(text):
if not code[entry.start():entry.start() + 3].strip():
continue
end = closing(code, code.index("(", entry.start()))
body = text[entry.start():end]
body_code = code[entry.start():end]
status = "registered"
method_notes = []
if "UnsupportedOperationException" in body_code or re.search(r"not (?:yet )?implemented", body, re.I):
status = "stub_candidate"
method_notes.append("Heurystyka: wyjątek lub komunikat o braku implementacji; wymaga przeglądu.")
if re.search(r"->\s*MethodResult\.noReturn\(\)", body_code):
method_notes.append("Pusta operacja; może odpowiadać oryginałowi. Nie oznaczono automatycznie jako stub.")
method = entry[1].upper()
if method in methods:
raise ValueError(f"Duplicate registration: {name}.{method}")
methods[method] = {"status": status, "notes": method_notes, "source": {
"path": str(path.relative_to(root)), "line": text.count("\n", 0, entry.start()) + 1,
"owner": name, "body_sha256": digest(body.encode()), "file_sha256": digest(text.encode()),
}}
types[name] = {"methods": methods, "mapping_notes": notes}
return {"root": str(root), "commit": _git(root, "rev-parse", "HEAD"),
"dirty": bool(_git(root, "status", "--porcelain")),
"source_tree_sha256": tree.hexdigest(), "types": types, "warnings": warnings}
def owner_name(owner: str) -> str:
if owner in ("CMC", "CMC_Object"):
return "*"
if owner in CLASS_ALIASES:
return CLASS_ALIASES[owner]
return canonical(owner.removeprefix("CMC_"))
def build_tracker(snapshots: list[Snapshot], emulator: dict, reviews: list[dict] | None = None) -> dict:
"""Join script APIs, keeping every binary and inherited declaration as evidence."""
warnings = list(emulator.get("warnings", []))
rows = {}
binaries = []
def type_row(name):
return rows.setdefault(name, {"name": name, "mapping_notes": [], "methods": {}})
def method_row(name, method):
return type_row(name)["methods"].setdefault(method, {
"name": method, "status": "missing", "verification": "unverified",
"source": None, "original": [], "reviews": [], "notes": [],
})
seen = set()
for snap in snapshots:
sha = snap.binary.get("sha256")
if not isinstance(sha, str) or not SHA.fullmatch(sha.lower()):
sha = None
else:
sha = sha.lower()
# A content hash of the snapshot is NOT the identity of its DLL.
bid = sha or "snapshot:" + digest(json.dumps(snap.raw, sort_keys=True).encode())
if bid in seen:
warnings.append("Powtórzony snapshot pominięty: " + snap.binary.get("name", bid))
continue
seen.add(bid)
binaries.append({"id": bid, "sha256": sha, "name": snap.binary.get("name", "?"),
"engine": snap.binary.get("engine"), "identity": "binary" if sha else "snapshot_only",
"library": snap.binary.get("library", snap.binary.get("name", "?").split("__")[-1]),
"family": binary_family(snap.binary),
"sources": snap.binary.get("sources", []), "games": snap.binary.get("games", []),
"coverage": snap.raw.get("coverage", snap.raw.get("extraction", {}).get("scope",
snap.binary.get("extraction", {}).get("coverage"))),
"limitations": snap.raw.get("limitations", []) + snap.raw.get("extraction", {}).get("limitations", [])})
if not sha:
warnings.append(snap.binary.get("name", "?") + ": brak SHA-256 DLL; dowodów nie można przypisać do binarki.")
owners = defaultdict(list)
for m in snap.methods:
owners[m["owner"]].append(m)
id_names = defaultdict(set)
name_ids = defaultdict(set)
for m in snap.methods:
if m.get("id") is not None:
id_names[(m["owner"], m["id"])].add(m["name"].upper())
name_ids[(m["owner"], m["name"].upper())].add(m["id"])
dispatch = {(d["owner"], d.get("id")): d for d in snap.method_dispatch if d.get("id") is not None}
parents = defaultdict(list)
for edge in snap.method_inheritance:
parents[edge["runner"].removesuffix("_Runner")].append(edge["base_runner"].removesuffix("_Runner"))
mappings = {}
used_owners = set()
for t in snap.types:
name = canonical(t["script_name"])
row = type_row(name)
owner = t.get("cpp_class")
mapping = "explicit"
if t.get("cpp_class_provenance"):
row["mapping_notes"].append(t["cpp_class_provenance"])
# Known extractor failure: unrelated factory entries all mapped to Behaviour.
if owner == "CMC_Behaviour" and name != "BEHAVIOUR":
owner = None
if not owner:
candidate = [o for o in owners if owner_name(o) == name]
owner = candidate[0] if len(candidate) == 1 else None
mapping = "inferred"
if owner:
mappings[name] = (owner, mapping)
used_owners.add(owner)
else:
row["mapping_notes"].append(f"{snap.binary.get('name', '?')}: typ wykryty, klasa/metody nieustalone.")
for owner in owners:
if owner not in used_owners:
# Includes singleton classes and separate base scopes. Label inference explicitly.
name = owner_name(owner)
if name not in mappings:
mappings[name] = (owner, "inferred")
for name, (owner, mapping) in mappings.items():
visited = set()
def add(current, inherited=False):
if current in visited:
return
visited.add(current)
for m in owners[current]:
row = method_row(name, m["name"].upper())
if len(name_ids[(current, m["name"].upper())]) > 1:
note = "Oryginał rejestruje tę nazwę z różnymi ID; skuteczny wybór wymaga sprawdzenia hashtable/dispatchera."
if note not in row["notes"]:
row["notes"].append(note)
if len(id_names[(current, m.get("id"))]) > 1:
note = "ID współdzielone przez nazwy: " + ", ".join(sorted(id_names[(current, m.get("id"))]))
if note not in row["notes"]:
row["notes"].append(note)
# Child declarations override names from ancestors for this binary.
if any(o["binary_id"] == bid and o["owner"] != current for o in row["original"]):
continue
d = dispatch.get((current, m.get("id")), {}) if m.get("id") is not None else {}
row["original"].append({"binary_id": bid, "owner": current, "method_id": m.get("id"),
"impl_addr": d.get("impl_addr"), "case_addr": d.get("case_addr"),
"mapping": mapping, "inherited": inherited,
"registration_addr": m.get("registration_addr"), "string_addr": m.get("string_addr"),
"registration_call_addr": m.get("registration_call_addr"),
"id_push_addr": m.get("id_push_addr"),
"evidence_ids": m.get("evidence_ids", [])})
for parent in parents[current]:
add(parent, True)
add(owner)
for name, t in emulator["types"].items():
type_row(name)["mapping_notes"].extend(t["mapping_notes"])
for method, binding in t["methods"].items():
row = method_row(name, method)
notes = row["notes"] + binding.get("notes", [])
row.update(binding)
row["notes"] = notes
# Global fallback exists on concrete Variable implementations, not native-only types.
globals_ = emulator["types"].get("*", {}).get("methods", {})
for name, t in rows.items():
if name not in emulator["types"] or name == "*":
continue
for method, row in t["methods"].items():
if row["source"] is None and method in globals_:
row.update(globals_[method])
row["notes"] = row["notes"] + ["Obsługa przez globalny fallback Variable."]
# Reviews are declarative, version-controlled assertions; strict anchors prevent
# accidentally making an entire family green based on one stale binary/address.
for review in reviews or []:
validate_review(review)
row = rows.get(canonical(review["type"]), {}).get("methods", {}).get(review["method"].upper())
if row is None:
warnings.append("Przegląd nie pasuje do API: " + review["type"] + "." + review["method"])
continue
source = row["source"] or {}
current = (source.get("body_sha256") == review["body_sha256"]
and emulator["source_tree_sha256"] == review["source_tree_sha256"]
and emulator.get("commit") == review["emulator_commit"])
matches = any(o["binary_id"] == review["binary_sha256"] for o in row["original"])
row["reviews"].append({**review, "current": current and matches})
for t in rows.values():
for row in t["methods"].values():
current = [r for r in row["reviews"] if r["current"]]
targets = {o["binary_id"] for o in row["original"]}
verified = {r["binary_sha256"] for r in current if r["verdict"] == "matches"}
row["verified_variants"] = len(verified & targets)
if targets and targets <= verified:
row["verification"] = "runtime_tested" if all(any(r["binary_sha256"] == b and r["kind"] == "runtime" and r["verdict"] == "matches" for r in current) for b in targets) else "asm_reviewed"
row["status"] = "implemented"
elif any(r["verdict"] == "partial" for r in current):
row["status"] = "partial"
elif row["reviews"] and not current:
row["verification"] = "stale"
t["methods"] = sorted(t["methods"].values(), key=lambda m: m["name"])
t["mapping_notes"] = sorted(set(t["mapping_notes"]))
all_rows = [m for t in rows.values() for m in t["methods"]]
return {"schema": "tracker/1", "emulator": {k: v for k, v in emulator.items() if k not in ("types", "warnings")},
"binaries": sorted(binaries, key=lambda b: (b["name"].lower(), b["id"])),
"types": sorted(rows.values(), key=lambda t: t["name"]),
"summary": {"types": len(rows), "methods": len(all_rows), "status": dict(Counter(m["status"] for m in all_rows)),
"verification": dict(Counter(m["verification"] for m in all_rows))},
"warnings": warnings + ["Brak obserwacji w wariancie oznacza nieustalone, nie brak metody.",
"Rejestracja Java nie dowodzi poprawności. Liczniki obejmują pary typ/metoda, także odziedziczone.",
"Dopasowania klas po nazwie są wnioskowane; zakres * to metody bazowe/globalne."]}
def validate_review(review: dict) -> None:
required = ("type", "method", "binary_sha256", "rva", "emulator_commit", "body_sha256",
"source_tree_sha256", "reviewer", "date", "notes", "evidence", "kind", "verdict")
if any(not isinstance(review.get(k), str) or not review[k].strip() for k in required):
raise ValueError("Review requires nonempty fields: " + ", ".join(required))
for k in ("binary_sha256", "body_sha256", "source_tree_sha256"):
if not SHA.fullmatch(review[k]):
raise ValueError("Invalid review digest: " + k)
if not re.fullmatch(r"0x[0-9a-fA-F]+", review["rva"]):
raise ValueError("Review RVA must be hexadecimal (0x...)")
if review["kind"] not in ("asm", "runtime") or review["verdict"] not in ("matches", "partial", "differs"):
raise ValueError("Invalid review kind/verdict")
def load_reviews(path: Path | None) -> list[dict]:
if path is None:
return []
data = json.loads(path.read_text())
if data.get("schema") != "tracker-reviews/1" or not isinstance(data.get("reviews"), list):
raise ValueError("Expected tracker-reviews/1 with a reviews list")
for review in data["reviews"]:
validate_review(review)
# A single current assertion per binary/method/source; conflicting verdicts fail closed.
keys = [(r["type"], r["method"], r["binary_sha256"], r["source_tree_sha256"], r["emulator_commit"]) for r in data["reviews"]]
if len(keys) != len(set(keys)):
raise ValueError("Duplicate review for the same method, binary and source revision")
return data["reviews"]
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("snapshots", nargs="+")
parser.add_argument("--emulator-root", required=True, type=Path)
parser.add_argument("--reviews", type=Path)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args(argv)
result = build_tracker([Snapshot.load(p) for p in args.snapshots], scan_emulator(args.emulator_root), load_reviews(args.reviews))
args.out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(result["summary"]))
if __name__ == "__main__":
main()