feat(tracker): Atlas joining DLL snapshots with emulator registrations
Matches original script APIs against MethodSpec registrations in Rex-EMoolator and keeps review evidence separate from registration. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
b0013c5114
commit
10c81c4390
+2
-1
@@ -15,7 +15,7 @@ from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from .. import __version__
|
||||
from .db import configure, init_db
|
||||
from .routes import compat, diff, games, jobs, snapshots
|
||||
from .routes import compat, diff, games, jobs, snapshots, tracker
|
||||
|
||||
_STATIC = Path(__file__).parent / "static"
|
||||
|
||||
@@ -30,6 +30,7 @@ def create_app(database_url: str | None = None) -> FastAPI:
|
||||
app.include_router(diff.router)
|
||||
app.include_router(jobs.router)
|
||||
app.include_router(compat.router)
|
||||
app.include_router(tracker.router)
|
||||
|
||||
@app.get("/health", tags=["meta"])
|
||||
def health() -> dict[str, str]:
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Read-only tracker. Filesystem paths come from server configuration, never requests."""
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...snapshot import Snapshot as Surface
|
||||
from ...tracker import build_tracker, load_reviews, scan_emulator
|
||||
from .. import models
|
||||
from ..db import get_db
|
||||
|
||||
router = APIRouter(tags=["tracker"])
|
||||
|
||||
|
||||
def order_snapshots(snapshots):
|
||||
"""Prefer reproducible batch results, retaining explicit conflict diagnostics."""
|
||||
ordered = sorted(snapshots, key=lambda s: bool(s.binary.get("extraction")), reverse=True)
|
||||
seen, warnings, distinct = {}, [], []
|
||||
for snap in ordered:
|
||||
sha = str(snap.binary.get("sha256", "")).lower()
|
||||
if not sha:
|
||||
distinct.append(snap)
|
||||
continue
|
||||
if sha in seen and seen[sha].raw == snap.raw:
|
||||
continue
|
||||
distinct.append(snap)
|
||||
if sha in seen and seen[sha].raw != snap.raw:
|
||||
warnings.append(f"Różne snapshoty tej samej DLL ({sha}): wybrano pierwszy, z pierwszeństwem metadanych ekstrakcji; wyników nie scalono.")
|
||||
seen.setdefault(sha, snap)
|
||||
return distinct, list(dict.fromkeys(warnings))
|
||||
|
||||
|
||||
def inventory_status(path, binaries):
|
||||
manifest = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
builds = manifest["builds"]
|
||||
if not isinstance(builds, list):
|
||||
raise ValueError("inventory builds must be a list")
|
||||
loaded = {str(b.get("sha256", "")).lower() for b in binaries if b.get("sha256")}
|
||||
scopes, rows, seen = {}, [], set()
|
||||
for build in builds:
|
||||
sha = str(build["sha256"]).lower()
|
||||
if sha in seen:
|
||||
raise ValueError("Duplicate inventory SHA-256: " + sha)
|
||||
seen.add(sha)
|
||||
scope = build.get("scope", "unknown")
|
||||
present = sha in loaded
|
||||
rows.append({**build, "snapshot_loaded": present})
|
||||
counts = scopes.setdefault(scope, {"target": 0, "loaded": 0, "pending": 0})
|
||||
counts["target"] += 1
|
||||
counts["loaded" if present else "pending"] += 1
|
||||
return {"builds": rows, "scopes": scopes, "loaded_snapshots": len(binaries),
|
||||
"unlisted_snapshots": len(loaded - seen),
|
||||
"note": "Wczytany snapshot nie oznacza kompletnej ekstrakcji ani zgodności funkcjonalnej."}
|
||||
|
||||
|
||||
@router.get("/tracker")
|
||||
def get_tracker(download: bool = False, db: Session = Depends(get_db)):
|
||||
root = os.environ.get("AMS_EMULATOR_ROOT")
|
||||
if not root:
|
||||
raise HTTPException(503, "Ustaw AMS_EMULATOR_ROOT na katalog Rex-EMoolator i uruchom serwer ponownie.")
|
||||
try:
|
||||
snapshots = [Surface(s.data) for s in db.scalars(select(models.Snapshot).order_by(models.Snapshot.id))]
|
||||
# Explicitly configured disk snapshots may supplement a DB; identities are deduplicated.
|
||||
directory = os.environ.get("AMS_TRACKER_SNAPSHOTS")
|
||||
if directory:
|
||||
path = Path(directory)
|
||||
if not path.is_dir():
|
||||
raise ValueError("AMS_TRACKER_SNAPSHOTS is not a directory")
|
||||
snapshots.extend(Surface.load(str(p)) for p in sorted(path.rglob("*.snapshot.json")))
|
||||
snapshots, catalog_warnings = order_snapshots(snapshots)
|
||||
reviews = os.environ.get("AMS_TRACKER_REVIEWS")
|
||||
result = build_tracker(snapshots, scan_emulator(Path(root)), load_reviews(Path(reviews) if reviews else None))
|
||||
result["warnings"].extend(catalog_warnings)
|
||||
inventory = os.environ.get("AMS_TRACKER_INVENTORY")
|
||||
if inventory:
|
||||
result["inventory"] = inventory_status(inventory, result["binaries"])
|
||||
except (OSError, ValueError, KeyError, TypeError) as exc:
|
||||
raise HTTPException(503, "Konfiguracja lub dane trackera: " + str(exc)) from exc
|
||||
headers = {"Content-Disposition": 'attachment; filename="tracker.json"'} if download else {}
|
||||
return JSONResponse(result, headers=headers)
|
||||
@@ -9,6 +9,7 @@
|
||||
<body>
|
||||
<header class="topbar">
|
||||
<div class="brand">⌖ Aidem Media <span>Command Center</span></div>
|
||||
<a class="mini-btn" href="tracker.html">Atlas API ↗</a>
|
||||
<div class="sub">Piklib / BlooMoo — analiza różnicowa powierzchni silnika</div>
|
||||
</header>
|
||||
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
:root{color-scheme:dark;--bg:#0c131e;--panel:#131e2c;--line:#2a394a;--text:#e4edf5;--muted:#9aafc3;--teal:#54d6ba;--amber:#efbd69}*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text);font:14px/1.55 system-ui,sans-serif}header,main{max-width:1550px;margin:auto;padding:30px 40px}header{display:flex;align-items:center;justify-content:space-between;border-bottom:1px solid var(--line);gap:20px}h1{font-size:34px;letter-spacing:-1px;margin:14px 0 3px}h1 span{display:block;color:var(--teal);font-size:12px;letter-spacing:2px;text-transform:uppercase;margin-top:6px}p{margin:8px 0 16px}header p,.muted,.footnote{color:var(--muted)}a{color:var(--teal)}.back{text-decoration:none;font-size:12px}button,.button,input,select{font:inherit;color:inherit;background:#192738;border:1px solid var(--line);border-radius:7px;padding:9px 12px}button,.button{cursor:pointer}.button{text-decoration:none;white-space:nowrap}button:hover,.button:hover{border-color:var(--teal)}:focus-visible{outline:2px solid var(--teal);outline-offset:3px}.filters{display:flex;flex-wrap:wrap;align-items:end;gap:16px;margin:24px 0 10px}.filters label{display:grid;gap:6px;font-size:12px;color:var(--muted)}.filters label:first-child{flex:1;min-width:230px}.filters select{max-width:330px}.filters .check{display:flex;align-items:center;padding:10px 0}.stats{display:grid;grid-template-columns:repeat(4,1fr);gap:14px;margin:25px 0}.stat{border:1px solid var(--line);border-radius:10px;background:var(--panel);padding:18px}.stat strong{display:block;font-size:30px;line-height:1.3}.stat span{color:var(--muted);font-size:12px}.layout{display:grid;grid-template-columns:290px minmax(0,1fr);gap:25px}.section-heading{display:flex;justify-content:space-between;align-items:center;gap:10px;margin-bottom:12px}h2{font-size:16px;margin:0}.quiet{padding:5px 9px;font-size:12px}.types{display:grid;gap:8px;max-height:720px;overflow:auto;padding:3px}.type{display:block;width:100%;text-align:left;background:var(--panel);padding:13px}.type[aria-pressed=true]{border-color:var(--teal);background:#18332f}.type strong{display:block}.type small{color:var(--muted)}.bar{height:4px;background:#314052;border-radius:3px;margin:7px 0 3px;overflow:hidden}.bar i{display:block;height:100%;background:var(--teal)}.bar.review i{background:var(--amber)}.table-scroll{overflow:auto;border:1px solid var(--line);border-radius:10px}table{width:100%;border-collapse:collapse;text-align:left;white-space:nowrap}th{font-size:11px;text-transform:uppercase;color:var(--muted);background:var(--panel);padding:13px}td{padding:13px;border-top:1px solid var(--line);font-size:12px}td:first-child small{display:block;color:var(--muted)}td:first-child strong{font-family:ui-monospace,monospace}.badge{display:inline-block;border-radius:4px;padding:3px 7px;background:#233145;color:#c0d0e0}.badge.implemented,.badge.asm_reviewed,.badge.runtime_tested{color:var(--teal);background:#183a33}.badge.missing,.badge.stub_candidate,.badge.stale{color:var(--amber);background:#3b3020}.footnote{font-size:12px;margin-top:28px;max-width:1000px}.warning{white-space:pre-wrap;border-left:3px solid var(--amber);background:#30291e;padding:12px;margin-top:15px}#message{color:var(--muted);padding:25px 0}dialog{background:var(--panel);color:var(--text);border:1px solid var(--line);border-radius:12px;width:min(790px,94vw);max-height:90vh;padding:24px}dialog::backdrop{background:#030911bb}dialog h3{font-size:14px;color:var(--teal);margin-top:25px}dialog pre{white-space:pre-wrap;overflow-wrap:anywhere;background:var(--bg);padding:13px;border-radius:7px;font-size:12px}dialog article{border-top:1px solid var(--line);padding:10px 0;overflow-wrap:anywhere}dialog dl{display:grid;grid-template-columns:130px 1fr;gap:7px}dialog dt{color:var(--muted)}dialog dd{margin:0;overflow-wrap:anywhere}.sr-only{position:absolute;width:1px;height:1px;overflow:hidden;clip:rect(0,0,0,0)}[hidden]{display:none!important}@media(max-width:950px){header,main{padding:24px}.layout{grid-template-columns:1fr}.types{grid-template-columns:repeat(auto-fill,minmax(210px,1fr));max-height:300px}.stats{grid-template-columns:repeat(2,1fr)}}@media(max-width:550px){header{align-items:flex-start;flex-direction:column}header,main{padding:20px}.filters label,.filters select{width:100%;max-width:none}.stats{gap:8px}.stat{padding:12px}.stat strong{font-size:25px}dialog dl{grid-template-columns:1fr}}
|
||||
|
||||
.layout>section,.stat,.filters label{min-width:0}#revision,.warning{overflow-wrap:anywhere}.table-scroll{max-width:100%}@media(max-width:950px){.layout{grid-template-columns:minmax(0,1fr)}}
|
||||
|
||||
.table-scroll{position:relative}
|
||||
@@ -0,0 +1,21 @@
|
||||
<!doctype html>
|
||||
<html lang="pl">
|
||||
<head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Atlas API · Aidem Media</title><link rel="stylesheet" href="tracker.css"></head>
|
||||
<body>
|
||||
<header><div><a href="index.html" class="back">← Command Center</a><h1>Atlas API<span>Piklib · BlooMoo · biblioteki</span></h1><p>Mapa implementacji i dowodów zgodności z oryginałem.</p></div><a class="button" href="/tracker?download=1" download="tracker.json">Eksport JSON ↗</a></header>
|
||||
<main>
|
||||
<div id="message" role="status" aria-live="polite">Wczytywanie katalogu i implementacji emulatora…</div>
|
||||
<section id="workspace" hidden>
|
||||
<div id="revision" class="muted"></div>
|
||||
<p id="inventory" class="muted" hidden></p>
|
||||
<details id="warnings" class="warning" hidden><summary id="warnings-summary">Uwagi do katalogu</summary><div id="warnings-body"></div></details>
|
||||
<div class="filters"><label>Szukaj typu lub metody<input id="search" type="search" placeholder="np. ANIMO, SETPOSITION" autocomplete="off"></label><label>Rodzina<select id="family"><option value="">Wszystkie rodziny</option></select></label><label>Wariant oryginału<select id="binary"><option value="">Wszystkie warianty</option></select></label><label>Implementacja<select id="status"><option value="">Wszystkie stany</option><option value="missing">Brak rejestracji</option><option value="registered">Zarejestrowana</option><option value="stub_candidate">Podejrzenie stubu</option><option value="partial">Częściowa</option><option value="implemented">Zaimplementowana</option></select></label><label class="check"><input id="gaps" type="checkbox"> Tylko braki i podejrzenia stubów</label></div>
|
||||
<p class="muted" id="scope-note">Widok jest sumą API wariantów. Rejestracja metody nie potwierdza jej poprawności.</p>
|
||||
<div id="stats" class="stats"></div>
|
||||
<div class="layout"><section aria-labelledby="types-heading"><div class="section-heading"><h2 id="types-heading">Typy</h2><button id="all-types" class="quiet">Wszystkie</button></div><div id="types" class="types"></div></section><section class="methods" aria-labelledby="methods-heading"><div class="section-heading"><h2 id="methods-heading">Metody</h2><span id="count" class="muted"></span></div><div id="type-notes" class="warning" hidden></div><div class="table-scroll"><table><thead><tr><th>Typ / metoda</th><th>Implementacja</th><th>Weryfikacja</th><th>Oryginał</th><th><span class="sr-only">Szczegóły</span></th></tr></thead><tbody id="rows"></tbody></table></div><p id="empty" hidden>Brak metod dla wybranych filtrów.</p></section></div>
|
||||
<p class="footnote">„Nieustalone” oznacza brak dowodu w wynikach ekstrakcji, a nie nieobecność metody w bibliotece. Stan „zarejestrowana” wynika z kodu; stub jest kandydatem do przeglądu. Potwierdzenia ASM i testy są osobnym wymiarem.</p>
|
||||
</section>
|
||||
</main>
|
||||
<dialog id="details"><div class="section-heading"><h2 id="detail-title"></h2><button id="close" aria-label="Zamknij szczegóły">Zamknij ×</button></div><div id="detail-body"></div></dialog>
|
||||
<script src="tracker.js"></script>
|
||||
</body></html>
|
||||
@@ -0,0 +1,41 @@
|
||||
'use strict';
|
||||
const $ = id => document.getElementById(id);
|
||||
const labels = {missing:'Brak rejestracji',registered:'Zarejestrowana',stub_candidate:'Podejrzenie stubu',partial:'Częściowa',implemented:'Zaimplementowana',unverified:'Niezweryfikowana',asm_reviewed:'Przegląd ASM',runtime_tested:'Test zachowania',stale:'Dowód wymaga aktualizacji'};
|
||||
let data, selected = null;
|
||||
const el = (tag, text, cls) => {const n = document.createElement(tag); if(text !== undefined) n.textContent = text; if(cls) n.className = cls; return n;};
|
||||
const family = b => String(b.family || (typeof b.engine === 'object' ? b.engine?.family || b.engine?.name : b.engine) || b.library || 'Nieustalona');
|
||||
const binaryName = b => String(b.library || b.name || '?').split(/[\\/]/).pop().replace(/^.*__[0-9a-f]+__/i,'');
|
||||
const inFamily = b => !$('family').value || family(b) === $('family').value;
|
||||
function variants(){const previous=$('binary').value;$('binary').replaceChildren(new Option('Wszystkie warianty',''));for(const b of data.binaries.filter(inFamily)){const games=(b.games||[]).join(', ');const opt=el('option',`${binaryName(b)}${games ? ' · '+games : ''} · ${String(b.sha256||b.id).slice(0,10)}`);opt.value=b.id;$('binary').append(opt);}if([...$('binary').options].some(o=>o.value===previous))$('binary').value=previous;}
|
||||
const original = m => (m.original || []).filter(o => (!$('binary').value || o.binary_id === $('binary').value) && data.binaries.some(b=>b.id===o.binary_id && inFamily(b)));
|
||||
const registered = m => m.status !== 'missing';
|
||||
const reviewed = m => ['asm_reviewed','runtime_tested'].includes(m.verification);
|
||||
const typeName = name => name === '*' ? 'Zakres globalny (*)' : name;
|
||||
function matches(t,m){const q=$('search').value.trim().toUpperCase();return (!q || `${t.name} ${m.name}`.toUpperCase().includes(q)) && (!$('status').value || m.status === $('status').value) && (!$('gaps').checked || ['missing','stub_candidate'].includes(m.status));}
|
||||
function stat(number,label){const n=el('div',undefined,'stat');n.append(el('strong',number),el('span',label));return n;}
|
||||
function badge(value){return el('span',labels[value] || value,'badge '+value);}
|
||||
function render(){
|
||||
const query=$('search').value.trim().toUpperCase();
|
||||
const groups=data.types.map(t=>({...t,unresolved:t.methods.length===0,methods:t.methods.filter(m=>matches(t,m))})).filter(t=>t.methods.length || (t.unresolved && (!query || t.name.toUpperCase().includes(query)) && !$('status').value && !$('gaps').checked));
|
||||
const active=groups.filter(t=>!selected || t.name===selected), methods=active.flatMap(t=>t.methods);
|
||||
$('scope-note').textContent=($('binary').value || $('family').value) ? 'Wybrany wariant lub rodzina ogranicza dowody oryginału. Metody bez dopasowania pozostają widoczne jako „nieustalone”; liczniki obejmują widoczne pary typ / metoda. Weryfikacja jest zbiorcza i nie zmienia się po wyborze wariantu.' : 'Widok jest sumą API wariantów. Rejestracja metody nie potwierdza jej poprawności. Liczniki obejmują widoczne pary typ / metoda, w tym dziedziczone.';
|
||||
$('stats').replaceChildren(stat(methods.length,'par typ / metoda (także dziedziczone)'),stat(methods.filter(registered).length,'zarejestrowanych w emulatorze'),stat(methods.filter(m=>m.status==='implemented').length,'oznaczonych jako zaimplementowane'),stat(methods.filter(reviewed).length,'z aktualnym dowodem ASM / testu'));
|
||||
$('types').replaceChildren();
|
||||
for(const t of groups){const b=el('button',undefined,'type');b.type='button';b.setAttribute('aria-pressed',String(selected===t.name));const n=t.methods.length,r=t.methods.filter(registered).length,v=t.methods.filter(reviewed).length;b.append(el('strong',typeName(t.name)),el('small',n ? `${n} metod · ${r} rejestracji · ${v} dowodów` : 'Metody nierozpoznane — typ wykryty'));for(const [amount,cls,title] of [[r,'','Rejestracja'],[v,'review','Weryfikacja']]){const bar=el('div',undefined,'bar '+cls);bar.title=`${title}: ${amount} / ${n}`;const fill=el('i');fill.style.width=`${n ? 100*amount/n : 0}%`;bar.append(fill);b.append(bar);}b.onclick=()=>{selected=selected===t.name?null:t.name;render();};$('types').append(b);}
|
||||
const chosen=data.types.find(t=>t.name===selected);$('type-notes').hidden=!chosen?.mapping_notes?.length;$('type-notes').textContent=(chosen?.mapping_notes||[]).map(note=>typeof note==='string'?note:JSON.stringify(note)).join('\n');$('methods-heading').textContent=selected?typeName(selected):'Wszystkie metody';$('count').textContent=`${methods.length} pozycji`;$('rows').replaceChildren();
|
||||
for(const t of active)for(const m of t.methods){const tr=el('tr'),name=el('td');name.append(el('small',typeName(t.name)),el('strong',m.name));const s=el('td'),v=el('td');s.append(badge(m.status));v.append(badge(m.verification));const found=original(m);const availability=el('td',found.length?`Wykryta · ${new Set(found.map(o=>o.binary_id)).size} wariantów`:'Nieustalone');const action=el('td'),b=el('button','Dowody ↗','quiet');b.setAttribute('aria-label',`Dowody ${t.name}.${m.name}`);b.onclick=()=>detail(t,m);action.append(b);tr.append(name,s,v,availability,action);$('rows').append(tr);}
|
||||
$('empty').hidden=methods.length>0;$('empty').textContent=chosen && !chosen.methods.length ? 'Typ został wykryty, ale metody nie zostały jeszcze rozpoznane. Brak wpisów nie oznacza braku API.' : 'Brak metod dla wybranych filtrów.';
|
||||
}
|
||||
function detail(t,m){$('detail-title').textContent=`${typeName(t.name)} · ${m.name}`;const body=$('detail-body');body.replaceChildren();const statuses=el('p');statuses.append(badge(m.status),document.createTextNode(' '),badge(m.verification));body.append(statuses);
|
||||
const section=(title,text)=>{body.append(el('h3',title));if(text !== undefined)body.append(el('pre',text));};
|
||||
section('Implementacja emulatora',m.source?`${m.source.path}:${m.source.line}\nWłaściciel: ${m.source.owner || '—'}\nHash treści: ${m.source.body_sha256 || '—'}`:'Nie znaleziono rejestracji w emulatorze.');
|
||||
section('Dowody z oryginału');const entries=original(m);if(!entries.length)body.append(el('p','Nieustalone — ekstrakcja nie dostarcza dopasowania w wybranym zakresie.'));
|
||||
for(const o of entries){const b=data.binaries.find(b=>b.id===o.binary_id);const a=el('article');a.append(el('strong',b?binaryName(b):o.binary_id));const dl=el('dl');for(const [key,value] of [['SHA-256',b?.sha256],['Silnik',typeof b?.engine==='object'?JSON.stringify(b.engine):b?.engine],['Właściciel',o.owner],['Id metody',o.method_id],['Implementacja',o.impl_addr],['Gałąź dispatchera',o.case_addr],['Funkcja rejestrująca',o.registration_addr],['Wywołanie rejestracji',o.registration_call_addr],['Przekazanie ID',o.id_push_addr],['Napis',o.string_addr],['Id dowodów',(o.evidence_ids||[]).join(', ')||'—'],['Dopasowanie',o.mapping==='explicit'?'Jawne':'Wywnioskowane'],['Dziedziczenie',o.inherited?'Tak':'Nie']]){dl.append(el('dt',key),el('dd',value===null || value===undefined?'—':String(value)));}a.append(dl);if(o.registration_addr || o.evidence_ids?.length)a.append(el('p','Dowód rejestracji ASM potwierdza rozpoznanie API; nie stanowi weryfikacji zachowania implementacji.','muted'));if(b?.limitations?.length){a.append(el('strong','Ograniczenia ekstrakcji'));a.append(el('pre',b.limitations.map(item=>typeof item==='string'?item:JSON.stringify(item,null,2)).join('\n')));}if(b?.coverage !== undefined)a.append(el('pre','Zakres ekstrakcji: '+(typeof b.coverage==='string'?b.coverage:JSON.stringify(b.coverage,null,2))));body.append(a);}
|
||||
section('Przeglądy i testy');body.append(el('p','Poniżej wszystkie zapisane przeglądy metody. Zbiorczy status weryfikacji nie oznacza potwierdzenia każdego wariantu; zakres dowodu określają jego dane.'));if(!(m.reviews||[]).length)body.append(el('p','Brak zapisanych dowodów weryfikacji zachowania.'));for(const review of m.reviews||[])body.append(el('pre',typeof review==='string'?review:JSON.stringify(review,null,2)));
|
||||
const notes=[...(t.mapping_notes||[]),...(m.notes||[])];if(notes.length)section('Uwagi',notes.map(n=>typeof n==='string'?n:JSON.stringify(n,null,2)).join('\n'));
|
||||
$('details').showModal();
|
||||
}
|
||||
$('close').onclick=()=>$('details').close();$('all-types').onclick=()=>{selected=null;render();};for(const id of ['search','family','binary','status','gaps'])$(id).addEventListener(id==='search'?'input':'change',()=>{if(data){if(id==='family')variants();render();}});
|
||||
async function load(){try{const res=await fetch('/tracker');if(!res.ok){const text=await res.text();let reason=text;try{const parsed=JSON.parse(text);reason=typeof parsed.detail==='string'?parsed.detail:JSON.stringify(parsed.detail||parsed);}catch{}throw new Error(res.status===503?`Katalog nie jest jeszcze skonfigurowany. ${reason}`:`Błąd API (${res.status}): ${reason}`);}data=await res.json();if(!Array.isArray(data.types)||!Array.isArray(data.binaries))throw new Error('Odpowiedź nie zawiera katalogu typów i wariantów.');for(const name of [...new Set(data.binaries.map(family))].sort())$('family').append(new Option(name,name));variants();$('revision').textContent=`Emulator: ${data.emulator?.commit || 'rewizja nieznana'}${data.emulator?.dirty?' · lokalne zmiany':''} · ${data.binaries.length} wariantów · ${data.types.length} typów`;
|
||||
if(data.warnings?.length){$('warnings').hidden=false;$('warnings-summary').textContent=`Uwagi do katalogu (${data.warnings.length})`;$('warnings-body').textContent=[...new Set(data.warnings.map(w=>typeof w==='string'?w:JSON.stringify(w)))].join('\n');}if(data.inventory){const names={engine:'Silniki',plugin:'Biblioteki skryptowe',native_dependency:'Biblioteki natywne',native:'Biblioteki natywne',excluded_early_engine:'Wcześniejsze silniki (poza zakresem)'};$('inventory').hidden=false;$('inventory').textContent=Object.entries(data.inventory.scopes).map(([scope,c])=>scope.startsWith('excluded') ? `${names[scope]||scope}: ${c.target}` : `${names[scope]||scope}: ${c.loaded} / ${c.target} snapshotów, ${c.pending} oczekuje`).join(' · ')+'. Snapshot nie oznacza kompletnej ekstrakcji ani zgodności zachowania.';}$('message').hidden=true;$('workspace').hidden=false;render();}catch(error){$('message').replaceChildren(el('p',error.message));const retry=el('button','Spróbuj ponownie');retry.onclick=()=>{data=null;$('binary').replaceChildren(new Option('Wszystkie warianty',''));$('family').replaceChildren(new Option('Wszystkie rodziny',''));$('message').textContent='Wczytywanie…';load();};$('message').append(retry);}}
|
||||
load();
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Start the local Atlas with this repository's snapshots and review registry."""
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--emulator-root', required=True, type=Path)
|
||||
p.add_argument('--inventory', type=Path)
|
||||
p.add_argument('--port', type=int, default=8766)
|
||||
a = p.parse_args()
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
if not (a.emulator_root / 'core/src').is_dir():
|
||||
p.error('--emulator-root must point to Rex-EMoolator')
|
||||
os.environ['AMS_EMULATOR_ROOT'] = str(a.emulator_root.resolve())
|
||||
os.environ['AMS_TRACKER_SNAPSHOTS'] = str(root / 'snapshots')
|
||||
os.environ['AMS_TRACKER_REVIEWS'] = str(root / 'tracker-reviews.json')
|
||||
inventory = a.inventory or root / 'data/binary-inventory.json'
|
||||
if inventory.is_file():
|
||||
os.environ['AMS_TRACKER_INVENTORY'] = str(inventory.resolve())
|
||||
import uvicorn
|
||||
uvicorn.run('ams.api.app:create_app', factory=True, host='127.0.0.1', port=a.port)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+332
@@ -0,0 +1,332 @@
|
||||
"""Evidence-aware script API tracker. Never treats registration as correctness.
|
||||
|
||||
The Java adapter deliberately recognises the repository's MethodSpec registrations,
|
||||
not arbitrary Java. Unknown dispatch and incomplete native extraction stay explicit.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import Counter, defaultdict
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
from .snapshot import Snapshot
|
||||
|
||||
ALIASES = {"BOOL": "BOOLEAN", "INT": "INTEGER"}
|
||||
# Established factory mappings in the PIKLIB8/BlooMoo reference snapshots. When
|
||||
# used to fill an unresolved mapping in another build, provenance stays inferred.
|
||||
CLASS_ALIASES = {"CMC_CanvasObserver": "CANVAS_OBSERVER", "CMC_Groupped": "GROUP"}
|
||||
ENTRY = re.compile(r'Map\s*\.\s*entry\s*\(\s*"([A-Za-z0-9_]+)"\s*,\s*MethodSpec\s*\.\s*of\s*\(')
|
||||
JAVA_TOKENS = re.compile(r'"""[\s\S]*?"""|"(?:\\.|[^"\\])*"|\'(?:\\.|[^\'\\])*\'|//[^\n]*|/\*[\s\S]*?\*/')
|
||||
SHA = re.compile(r"[0-9a-f]{64}")
|
||||
|
||||
|
||||
def canonical(name: str) -> str:
|
||||
name = name.upper()
|
||||
return ALIASES.get(name, name)
|
||||
|
||||
|
||||
def digest(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def binary_family(binary: dict) -> str:
|
||||
library = binary.get("library", binary.get("name", "?").split("__")[-1]).upper()
|
||||
if library.startswith("BLOOMOO"):
|
||||
return "BlooMoo"
|
||||
return {"PIKLIB61.DLL": "Piklib 6.1", "PIKLIB71.DLL": "Piklib 7.1",
|
||||
"PIKLIB72.DLL": "Piklib 7.2", "PIKLIB8.DLL": "Piklib 8"}.get(library, library.removesuffix(".DLL"))
|
||||
|
||||
|
||||
def code_mask(text: str) -> str:
|
||||
"""Mask literals/comments without shifting source positions or line numbers."""
|
||||
return JAVA_TOKENS.sub(lambda m: re.sub(r"[^\n]", " ", m.group()), text)
|
||||
|
||||
|
||||
def closing(code: str, start: int) -> int:
|
||||
depth = 0
|
||||
for i in range(start, len(code)):
|
||||
if code[i] == "(":
|
||||
depth += 1
|
||||
elif code[i] == ")":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i + 1
|
||||
raise ValueError("Unbalanced MethodSpec registration")
|
||||
|
||||
|
||||
def _git(root: Path, *args: str) -> str | None:
|
||||
result = subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True)
|
||||
return result.stdout.strip() if result.returncode == 0 else None
|
||||
|
||||
|
||||
def scan_emulator(root: Path) -> dict:
|
||||
root = root.resolve()
|
||||
source_root = root / "core/src"
|
||||
variable_root = source_root / "pl/genschu/bloomooemulator/interpreter/variable"
|
||||
if not (variable_root / "Variable.java").is_file():
|
||||
raise ValueError("AMS_EMULATOR_ROOT must point to the Rex-EMoolator repository")
|
||||
tree = hashlib.sha256()
|
||||
for path in sorted(source_root.rglob("*.java")):
|
||||
tree.update(str(path.relative_to(root)).encode() + b"\0" + path.read_bytes() + b"\0")
|
||||
types = {}
|
||||
warnings = []
|
||||
for path in sorted(variable_root.glob("*Variable.java")):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
code = code_mask(text)
|
||||
match = re.search(r"return\s+VariableType\.([A-Z_]+)\s*;", code)
|
||||
if path.name == "Variable.java":
|
||||
name = "*"
|
||||
elif match:
|
||||
name = canonical(match[1])
|
||||
else:
|
||||
warnings.append("Nie rozpoznano typu Java: " + path.name)
|
||||
continue
|
||||
notes = []
|
||||
if name != "*" and re.search(r"\bMethodResult\s+callMethod\s*\(", code):
|
||||
notes.append("Własny dispatcher: lista rejestracji nie opisuje wszystkich wywołań dynamicznych.")
|
||||
methods = {}
|
||||
for entry in ENTRY.finditer(text):
|
||||
if not code[entry.start():entry.start() + 3].strip():
|
||||
continue
|
||||
end = closing(code, code.index("(", entry.start()))
|
||||
body = text[entry.start():end]
|
||||
body_code = code[entry.start():end]
|
||||
status = "registered"
|
||||
method_notes = []
|
||||
if "UnsupportedOperationException" in body_code or re.search(r"not (?:yet )?implemented", body, re.I):
|
||||
status = "stub_candidate"
|
||||
method_notes.append("Heurystyka: wyjątek lub komunikat o braku implementacji; wymaga przeglądu.")
|
||||
if re.search(r"->\s*MethodResult\.noReturn\(\)", body_code):
|
||||
method_notes.append("Pusta operacja; może odpowiadać oryginałowi. Nie oznaczono automatycznie jako stub.")
|
||||
method = entry[1].upper()
|
||||
if method in methods:
|
||||
raise ValueError(f"Duplicate registration: {name}.{method}")
|
||||
methods[method] = {"status": status, "notes": method_notes, "source": {
|
||||
"path": str(path.relative_to(root)), "line": text.count("\n", 0, entry.start()) + 1,
|
||||
"owner": name, "body_sha256": digest(body.encode()), "file_sha256": digest(text.encode()),
|
||||
}}
|
||||
types[name] = {"methods": methods, "mapping_notes": notes}
|
||||
return {"root": str(root), "commit": _git(root, "rev-parse", "HEAD"),
|
||||
"dirty": bool(_git(root, "status", "--porcelain")),
|
||||
"source_tree_sha256": tree.hexdigest(), "types": types, "warnings": warnings}
|
||||
|
||||
|
||||
def owner_name(owner: str) -> str:
|
||||
if owner in ("CMC", "CMC_Object"):
|
||||
return "*"
|
||||
if owner in CLASS_ALIASES:
|
||||
return CLASS_ALIASES[owner]
|
||||
return canonical(owner.removeprefix("CMC_"))
|
||||
|
||||
|
||||
def build_tracker(snapshots: list[Snapshot], emulator: dict, reviews: list[dict] | None = None) -> dict:
|
||||
"""Join script APIs, keeping every binary and inherited declaration as evidence."""
|
||||
warnings = list(emulator.get("warnings", []))
|
||||
rows = {}
|
||||
binaries = []
|
||||
|
||||
def type_row(name):
|
||||
return rows.setdefault(name, {"name": name, "mapping_notes": [], "methods": {}})
|
||||
|
||||
def method_row(name, method):
|
||||
return type_row(name)["methods"].setdefault(method, {
|
||||
"name": method, "status": "missing", "verification": "unverified",
|
||||
"source": None, "original": [], "reviews": [], "notes": [],
|
||||
})
|
||||
|
||||
seen = set()
|
||||
for snap in snapshots:
|
||||
sha = snap.binary.get("sha256")
|
||||
if not isinstance(sha, str) or not SHA.fullmatch(sha.lower()):
|
||||
sha = None
|
||||
else:
|
||||
sha = sha.lower()
|
||||
# A content hash of the snapshot is NOT the identity of its DLL.
|
||||
bid = sha or "snapshot:" + digest(json.dumps(snap.raw, sort_keys=True).encode())
|
||||
if bid in seen:
|
||||
warnings.append("Powtórzony snapshot pominięty: " + snap.binary.get("name", bid))
|
||||
continue
|
||||
seen.add(bid)
|
||||
binaries.append({"id": bid, "sha256": sha, "name": snap.binary.get("name", "?"),
|
||||
"engine": snap.binary.get("engine"), "identity": "binary" if sha else "snapshot_only",
|
||||
"library": snap.binary.get("library", snap.binary.get("name", "?").split("__")[-1]),
|
||||
"family": binary_family(snap.binary),
|
||||
"sources": snap.binary.get("sources", []), "games": snap.binary.get("games", []),
|
||||
"coverage": snap.raw.get("coverage", snap.raw.get("extraction", {}).get("scope",
|
||||
snap.binary.get("extraction", {}).get("coverage"))),
|
||||
"limitations": snap.raw.get("limitations", []) + snap.raw.get("extraction", {}).get("limitations", [])})
|
||||
if not sha:
|
||||
warnings.append(snap.binary.get("name", "?") + ": brak SHA-256 DLL; dowodów nie można przypisać do binarki.")
|
||||
owners = defaultdict(list)
|
||||
for m in snap.methods:
|
||||
owners[m["owner"]].append(m)
|
||||
id_names = defaultdict(set)
|
||||
name_ids = defaultdict(set)
|
||||
for m in snap.methods:
|
||||
if m.get("id") is not None:
|
||||
id_names[(m["owner"], m["id"])].add(m["name"].upper())
|
||||
name_ids[(m["owner"], m["name"].upper())].add(m["id"])
|
||||
dispatch = {(d["owner"], d.get("id")): d for d in snap.method_dispatch if d.get("id") is not None}
|
||||
parents = defaultdict(list)
|
||||
for edge in snap.method_inheritance:
|
||||
parents[edge["runner"].removesuffix("_Runner")].append(edge["base_runner"].removesuffix("_Runner"))
|
||||
mappings = {}
|
||||
used_owners = set()
|
||||
for t in snap.types:
|
||||
name = canonical(t["script_name"])
|
||||
row = type_row(name)
|
||||
owner = t.get("cpp_class")
|
||||
mapping = "explicit"
|
||||
if t.get("cpp_class_provenance"):
|
||||
row["mapping_notes"].append(t["cpp_class_provenance"])
|
||||
# Known extractor failure: unrelated factory entries all mapped to Behaviour.
|
||||
if owner == "CMC_Behaviour" and name != "BEHAVIOUR":
|
||||
owner = None
|
||||
if not owner:
|
||||
candidate = [o for o in owners if owner_name(o) == name]
|
||||
owner = candidate[0] if len(candidate) == 1 else None
|
||||
mapping = "inferred"
|
||||
if owner:
|
||||
mappings[name] = (owner, mapping)
|
||||
used_owners.add(owner)
|
||||
else:
|
||||
row["mapping_notes"].append(f"{snap.binary.get('name', '?')}: typ wykryty, klasa/metody nieustalone.")
|
||||
for owner in owners:
|
||||
if owner not in used_owners:
|
||||
# Includes singleton classes and separate base scopes. Label inference explicitly.
|
||||
name = owner_name(owner)
|
||||
if name not in mappings:
|
||||
mappings[name] = (owner, "inferred")
|
||||
for name, (owner, mapping) in mappings.items():
|
||||
visited = set()
|
||||
def add(current, inherited=False):
|
||||
if current in visited:
|
||||
return
|
||||
visited.add(current)
|
||||
for m in owners[current]:
|
||||
row = method_row(name, m["name"].upper())
|
||||
if len(name_ids[(current, m["name"].upper())]) > 1:
|
||||
note = "Oryginał rejestruje tę nazwę z różnymi ID; skuteczny wybór wymaga sprawdzenia hashtable/dispatchera."
|
||||
if note not in row["notes"]:
|
||||
row["notes"].append(note)
|
||||
if len(id_names[(current, m.get("id"))]) > 1:
|
||||
note = "ID współdzielone przez nazwy: " + ", ".join(sorted(id_names[(current, m.get("id"))]))
|
||||
if note not in row["notes"]:
|
||||
row["notes"].append(note)
|
||||
# Child declarations override names from ancestors for this binary.
|
||||
if any(o["binary_id"] == bid and o["owner"] != current for o in row["original"]):
|
||||
continue
|
||||
d = dispatch.get((current, m.get("id")), {}) if m.get("id") is not None else {}
|
||||
row["original"].append({"binary_id": bid, "owner": current, "method_id": m.get("id"),
|
||||
"impl_addr": d.get("impl_addr"), "case_addr": d.get("case_addr"),
|
||||
"mapping": mapping, "inherited": inherited,
|
||||
"registration_addr": m.get("registration_addr"), "string_addr": m.get("string_addr"),
|
||||
"registration_call_addr": m.get("registration_call_addr"),
|
||||
"id_push_addr": m.get("id_push_addr"),
|
||||
"evidence_ids": m.get("evidence_ids", [])})
|
||||
for parent in parents[current]:
|
||||
add(parent, True)
|
||||
add(owner)
|
||||
for name, t in emulator["types"].items():
|
||||
type_row(name)["mapping_notes"].extend(t["mapping_notes"])
|
||||
for method, binding in t["methods"].items():
|
||||
row = method_row(name, method)
|
||||
notes = row["notes"] + binding.get("notes", [])
|
||||
row.update(binding)
|
||||
row["notes"] = notes
|
||||
# Global fallback exists on concrete Variable implementations, not native-only types.
|
||||
globals_ = emulator["types"].get("*", {}).get("methods", {})
|
||||
for name, t in rows.items():
|
||||
if name not in emulator["types"] or name == "*":
|
||||
continue
|
||||
for method, row in t["methods"].items():
|
||||
if row["source"] is None and method in globals_:
|
||||
row.update(globals_[method])
|
||||
row["notes"] = row["notes"] + ["Obsługa przez globalny fallback Variable."]
|
||||
# Reviews are declarative, version-controlled assertions; strict anchors prevent
|
||||
# accidentally making an entire family green based on one stale binary/address.
|
||||
for review in reviews or []:
|
||||
validate_review(review)
|
||||
row = rows.get(canonical(review["type"]), {}).get("methods", {}).get(review["method"].upper())
|
||||
if row is None:
|
||||
warnings.append("Przegląd nie pasuje do API: " + review["type"] + "." + review["method"])
|
||||
continue
|
||||
source = row["source"] or {}
|
||||
current = (source.get("body_sha256") == review["body_sha256"]
|
||||
and emulator["source_tree_sha256"] == review["source_tree_sha256"]
|
||||
and emulator.get("commit") == review["emulator_commit"])
|
||||
matches = any(o["binary_id"] == review["binary_sha256"] for o in row["original"])
|
||||
row["reviews"].append({**review, "current": current and matches})
|
||||
for t in rows.values():
|
||||
for row in t["methods"].values():
|
||||
current = [r for r in row["reviews"] if r["current"]]
|
||||
targets = {o["binary_id"] for o in row["original"]}
|
||||
verified = {r["binary_sha256"] for r in current if r["verdict"] == "matches"}
|
||||
row["verified_variants"] = len(verified & targets)
|
||||
if targets and targets <= verified:
|
||||
row["verification"] = "runtime_tested" if all(any(r["binary_sha256"] == b and r["kind"] == "runtime" and r["verdict"] == "matches" for r in current) for b in targets) else "asm_reviewed"
|
||||
row["status"] = "implemented"
|
||||
elif any(r["verdict"] == "partial" for r in current):
|
||||
row["status"] = "partial"
|
||||
elif row["reviews"] and not current:
|
||||
row["verification"] = "stale"
|
||||
t["methods"] = sorted(t["methods"].values(), key=lambda m: m["name"])
|
||||
t["mapping_notes"] = sorted(set(t["mapping_notes"]))
|
||||
all_rows = [m for t in rows.values() for m in t["methods"]]
|
||||
return {"schema": "tracker/1", "emulator": {k: v for k, v in emulator.items() if k not in ("types", "warnings")},
|
||||
"binaries": sorted(binaries, key=lambda b: (b["name"].lower(), b["id"])),
|
||||
"types": sorted(rows.values(), key=lambda t: t["name"]),
|
||||
"summary": {"types": len(rows), "methods": len(all_rows), "status": dict(Counter(m["status"] for m in all_rows)),
|
||||
"verification": dict(Counter(m["verification"] for m in all_rows))},
|
||||
"warnings": warnings + ["Brak obserwacji w wariancie oznacza nieustalone, nie brak metody.",
|
||||
"Rejestracja Java nie dowodzi poprawności. Liczniki obejmują pary typ/metoda, także odziedziczone.",
|
||||
"Dopasowania klas po nazwie są wnioskowane; zakres * to metody bazowe/globalne."]}
|
||||
|
||||
|
||||
def validate_review(review: dict) -> None:
|
||||
required = ("type", "method", "binary_sha256", "rva", "emulator_commit", "body_sha256",
|
||||
"source_tree_sha256", "reviewer", "date", "notes", "evidence", "kind", "verdict")
|
||||
if any(not isinstance(review.get(k), str) or not review[k].strip() for k in required):
|
||||
raise ValueError("Review requires nonempty fields: " + ", ".join(required))
|
||||
for k in ("binary_sha256", "body_sha256", "source_tree_sha256"):
|
||||
if not SHA.fullmatch(review[k]):
|
||||
raise ValueError("Invalid review digest: " + k)
|
||||
if not re.fullmatch(r"0x[0-9a-fA-F]+", review["rva"]):
|
||||
raise ValueError("Review RVA must be hexadecimal (0x...)")
|
||||
if review["kind"] not in ("asm", "runtime") or review["verdict"] not in ("matches", "partial", "differs"):
|
||||
raise ValueError("Invalid review kind/verdict")
|
||||
|
||||
|
||||
def load_reviews(path: Path | None) -> list[dict]:
|
||||
if path is None:
|
||||
return []
|
||||
data = json.loads(path.read_text())
|
||||
if data.get("schema") != "tracker-reviews/1" or not isinstance(data.get("reviews"), list):
|
||||
raise ValueError("Expected tracker-reviews/1 with a reviews list")
|
||||
for review in data["reviews"]:
|
||||
validate_review(review)
|
||||
# A single current assertion per binary/method/source; conflicting verdicts fail closed.
|
||||
keys = [(r["type"], r["method"], r["binary_sha256"], r["source_tree_sha256"], r["emulator_commit"]) for r in data["reviews"]]
|
||||
if len(keys) != len(set(keys)):
|
||||
raise ValueError("Duplicate review for the same method, binary and source revision")
|
||||
return data["reviews"]
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("snapshots", nargs="+")
|
||||
parser.add_argument("--emulator-root", required=True, type=Path)
|
||||
parser.add_argument("--reviews", type=Path)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
args = parser.parse_args(argv)
|
||||
result = build_tracker([Snapshot.load(p) for p in args.snapshots], scan_emulator(args.emulator_root), load_reviews(args.reviews))
|
||||
args.out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(result["summary"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user