feat(acquire): resumable batch extraction queue and binary inventory

Inventory classifies the DLL repository by SHA; the batch queue runs each
engine build in a fresh PyGhidra process with a timeout, fingerprints the
extractor environment and keeps per-attempt state and logs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Patryk Gensch
2026-10-24 18:24:15 +02:00
co-authored by Claude Opus 5.5
parent bddb5949a6
commit b0013c5114
9 changed files with 1164 additions and 1 deletions
+253
View File
@@ -0,0 +1,253 @@
"""Resumable, SHA-addressed engine extraction queue (POSIX)."""
from __future__ import annotations
import argparse
from contextlib import contextmanager
from datetime import datetime, timezone
import fcntl
import hashlib
from importlib import metadata
import json
import os
from pathlib import Path
import re
import signal
import subprocess
import sys
import uuid
def digest(path):
h = hashlib.sha256()
with Path(path).open('rb') as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
h.update(chunk)
return h.hexdigest()
def fingerprint(env=None):
from . import ghidra
env = os.environ if env is None else env
root = Path(__file__).resolve().parents[2]
scripts = Path(env.get('AMS_GHIDRA_SCRIPTS', root / 'ghidra_scripts'))
paths = [Path(__file__), Path(ghidra.__file__), *sorted(scripts.glob('*.py'))]
h = hashlib.sha256()
for path in paths:
h.update(path.name.encode())
h.update(path.read_bytes())
h.update(json.dumps(provider_profile(env), sort_keys=True).encode())
return h.hexdigest()
def provider_profile(env):
provider = 'pyghidra' if env.get('AMS_USE_PYGHIDRA') else 'headless'
home = env.get('GHIDRA_INSTALL_DIR') or env.get('GHIDRA_HOME')
launcher = env.get('GHIDRA_HEADLESS')
if provider == 'headless' and launcher:
home = str(Path(launcher).resolve().parent.parent)
elif provider == 'headless' and not home:
import shutil
launcher = shutil.which('analyzeHeadless', path=env.get('PATH'))
if launcher:
home = str(Path(launcher).resolve().parent.parent)
properties = Path(home) / 'Ghidra' / 'application.properties' if home else None
try:
pyghidra_version = metadata.version('pyghidra')
except metadata.PackageNotFoundError:
pyghidra_version = None
return {'provider': provider, 'ghidra_home': str(Path(home).resolve()) if home else None,
'launcher': launcher, 'pyghidra_version': pyghidra_version,
'application_properties': properties.read_text() if properties and properties.is_file() else None,
'python': sys.version, 'java_home': env.get('JAVA_HOME')}
def atomic_json(path, data):
temp = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp')
try:
temp.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
os.replace(temp, path)
finally:
temp.unlink(missing_ok=True)
def validate_snapshot(path, sha):
data = json.loads(path.read_text())
if not isinstance(data, dict) or not isinstance(data.get('binary'), dict):
raise ValueError('Snapshot has no binary identity')
snapshot_sha = data['binary'].get('sha256')
if not isinstance(snapshot_sha, str) or snapshot_sha.lower() != sha:
raise ValueError('Snapshot SHA does not match manifest')
if not isinstance(data.get('types'), list) or not data['types']:
raise ValueError('Snapshot has no extracted types')
if not isinstance(data.get('methods'), list) or not data['methods']:
raise ValueError('Snapshot has no extracted methods')
return data
@contextmanager
def queue_lock(directory):
with (directory / '.queue.lock').open('a') as stream:
try:
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise RuntimeError('Another extraction queue is using this output directory')
try:
yield
finally:
fcntl.flock(stream, fcntl.LOCK_UN)
def execute_worker(source, output, log, timeout, env):
with log.open('wb') as stream:
process = subprocess.Popen(
[sys.executable, '-m', 'ams.acquire.batch', '--worker', str(source), str(output)],
stdout=stream, stderr=subprocess.STDOUT, env=env, start_new_session=True)
try:
code = process.wait(timeout=timeout)
except BaseException:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait()
raise
if code:
raise RuntimeError('Extractor exited with code ' + str(code))
def run_queue(manifest, out_dir, *, ghidra_home=None, timeout=1800, limit=None,
retry_failed=False, worker=execute_worker):
if timeout <= 0 or (limit is not None and limit < 0):
raise ValueError('Timeout must be positive; limit must be nonnegative')
data = json.loads(Path(manifest).read_text())
if data.get('schema') != 'binary-inventory/1':
raise ValueError('Expected binary-inventory/1 manifest')
builds = data['builds']
seen = set()
for build in builds:
sha = build['sha256']
if not isinstance(sha, str) or not re.fullmatch('[0-9a-f]{64}', sha) or sha in seen:
raise ValueError('Invalid or duplicate manifest SHA: ' + str(sha))
seen.add(sha)
out_dir = Path(out_dir).resolve()
out_dir.mkdir(parents=True, exist_ok=True)
with queue_lock(out_dir):
return _run(builds, out_dir, ghidra_home, timeout, limit, retry_failed, worker)
def _run(builds, directory, ghidra_home, timeout, limit, retry_failed, worker):
for name in ('states', 'logs'):
(directory / name).mkdir(exist_ok=True)
env = dict(os.environ)
if ghidra_home:
env.update(GHIDRA_INSTALL_DIR=str(ghidra_home), GHIDRA_HOME=str(ghidra_home), AMS_USE_PYGHIDRA='1')
extractor = fingerprint(env)
profile = provider_profile(env)
report = {'schema': 'extraction-batch/1', 'attempted': 0, 'succeeded': 0,
'pending': 0, 'skipped_success': 0, 'skipped_failed': 0, 'skipped_scope': 0, 'failures': []}
for build in builds:
if build.get('scope') != 'engine':
report['skipped_scope'] += 1
continue
sha = build['sha256']
state_path = directory / 'states' / (sha + '.json')
output = directory / (sha + '.snapshot.json')
old = json.loads(state_path.read_text()) if state_path.exists() else {}
same = old.get('extractor_fingerprint') == extractor
# Validate source even when resuming: a manifest is not proof of current contents.
source = None
for candidate in build.get('paths', []):
try:
if digest(candidate) == sha:
source = Path(candidate).resolve()
break
except OSError:
continue
if same and old.get('status') == 'success' and source:
try:
validate_snapshot(output, sha)
if digest(output) != old.get('snapshot_sha256'):
raise ValueError('Snapshot content changed since successful extraction')
except (OSError, ValueError, TypeError, AttributeError):
pass
else:
report['skipped_success'] += 1
continue
if same and old.get('status') in ('failed', 'running') and not retry_failed:
report['skipped_failed'] += 1
report['failures'].append({'sha256': sha, 'error': old.get('error', 'Interrupted attempt; use --retry-failed')})
continue
if limit is not None and report['attempted'] >= limit:
report['pending'] += 1
continue
attempt = old.get('attempts', 0) + 1
log = directory / 'logs' / (sha + '.' + str(attempt) + '.log')
temp = directory / (sha + '.' + uuid.uuid4().hex + '.pending.json')
state = {'sha256': sha, 'family': build.get('family'), 'extractor_fingerprint': extractor,
'status': 'running', 'attempts': attempt, 'source': str(source) if source else None,
'input_name': source.name if source else None, 'provider_profile': profile,
'started_at': datetime.now(timezone.utc).isoformat(), 'log': str(log)}
atomic_json(state_path, state)
report['attempted'] += 1
try:
if not source:
raise ValueError('No source path matches manifest SHA')
worker(source, temp, log, timeout, env)
snapshot = validate_snapshot(temp, sha)
if digest(source) != sha:
raise ValueError('Source changed during extraction')
snapshot['binary'].update(
library=source.name.split('__')[-1],
sources=build.get('paths', []),
games=sorted({Path(p).name.split('__')[0] for p in build.get('paths', []) if '__' in Path(p).name}),
extraction={'fingerprint': extractor, 'provider': profile,
'started_at': state['started_at'], 'coverage': 'partial'})
snapshot.setdefault('limitations', []).append(
'Automated heuristic extraction; missing entries are unknown, not proof of absence.')
atomic_json(temp, snapshot)
os.replace(temp, output)
state['status'] = 'success'
state['snapshot'] = str(output)
state['snapshot_sha256'] = digest(output)
state['coverage'] = {'types': len(snapshot['types']), 'methods': len(snapshot['methods']),
'dispatch': len(snapshot.get('method_dispatch') or [])}
report['succeeded'] += 1
except Exception as exc:
state.update(status='failed', error=str(exc))
if not log.exists():
log.write_text(str(exc) + '\n')
report['failures'].append({'sha256': sha, 'error': str(exc)})
finally:
temp.unlink(missing_ok=True)
state['finished_at'] = datetime.now(timezone.utc).isoformat()
atomic_json(state_path, state)
print(json.dumps({'sha256': sha, 'status': state['status'], 'attempt': attempt}), flush=True)
atomic_json(directory / 'failure-report.json', report)
return report
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--worker', nargs=2, metavar=('DLL', 'OUTPUT'), help=argparse.SUPPRESS)
parser.add_argument('--manifest', type=Path)
parser.add_argument('--out-dir', type=Path)
parser.add_argument('--ghidra-home', type=Path)
parser.add_argument('--timeout', type=float, default=1800)
parser.add_argument('--limit', type=int)
parser.add_argument('--retry-failed', action='store_true')
args = parser.parse_args()
if args.worker:
from .ghidra import run_extractor
run_extractor(*args.worker)
return
if not args.manifest or not args.out_dir:
parser.error('--manifest and --out-dir are required')
result = run_queue(args.manifest, args.out_dir, ghidra_home=args.ghidra_home,
timeout=args.timeout, limit=args.limit, retry_failed=args.retry_failed)
print(json.dumps(result))
if result['failures']:
raise SystemExit(1)
if __name__ == '__main__':
main()
+19 -1
View File
@@ -20,6 +20,8 @@ then `analyzeHeadless` on PATH.
from __future__ import annotations
import os
import json
import re
import shutil
import subprocess
import tempfile
@@ -44,7 +46,7 @@ def find_headless() -> str | None:
env = os.environ.get("GHIDRA_HEADLESS")
if env and os.path.isfile(env):
return env
home = os.environ.get("GHIDRA_HOME")
home = os.environ.get("GHIDRA_INSTALL_DIR") or os.environ.get("GHIDRA_HOME")
if home:
for name in ("analyzeHeadless", "analyzeHeadless.bat"):
cand = os.path.join(home, "support", name)
@@ -123,9 +125,20 @@ def run_extractor_pyghidra(dll_path: str, out_path: str, *, script_dir: str | No
proj_dir = tempfile.mkdtemp(prefix="ams_pyghidra_")
try:
# Keep headless preferences/caches separate from the user's GUI profile.
from pyghidra.launcher import PyGhidraLauncher, HeadlessPyGhidraLauncher
if not PyGhidraLauncher.has_launched():
launcher = HeadlessPyGhidraLauncher(verbose=False)
runtime_home = os.path.join(proj_dir, "runtime-home")
os.makedirs(runtime_home)
launcher.vm_args.extend(["-Duser.home=" + runtime_home, "-Xmx2G"])
launcher.start()
pyghidra.run_script(
dll_path, script_path,
project_location=proj_dir, project_name="ams_" + uuid.uuid4().hex[:8],
# Ghidra domain-file names reject Unicode combining marks and some
# Polish letters. This changes only the internal project name.
program_name=re.sub(r"[^A-Za-z0-9_.-]", "_", Path(dll_path).name),
script_args=[out_path], analyze=True, verbose=False,
)
except Exception as e: # jpype/Ghidra errors aren't a tidy hierarchy
@@ -135,4 +148,9 @@ def run_extractor_pyghidra(dll_path: str, out_path: str, *, script_dir: str | No
if not os.path.isfile(out_path):
raise GhidraRunError("extractor produced no snapshot at {0} (PyGhidra path)".format(out_path))
with open(out_path, encoding="utf-8") as stream:
snapshot = json.load(stream)
snapshot.setdefault("binary", {})["name"] = Path(dll_path).name
with open(out_path, "w", encoding="utf-8") as stream:
json.dump(snapshot, stream, indent=2, ensure_ascii=False)
return out_path
+66
View File
@@ -0,0 +1,66 @@
"""Inventory all DLL builds by content before scheduling expensive native analysis."""
import argparse
from collections import Counter
import hashlib
import json
from pathlib import Path
import re
def inventory(roots: list[Path], snapshot_dir: Path | None = None) -> dict:
extracted = set()
if snapshot_dir:
for path in snapshot_dir.glob("*.snapshot.json"):
sha = json.loads(path.read_text()).get("binary", {}).get("sha256")
if sha:
extracted.add(sha.lower())
builds = {}
for root in roots:
if not root.exists():
raise ValueError("Source does not exist: " + str(root))
paths = [root] if root.is_file() else sorted(root.rglob("*"))
for path in paths:
if path.suffix.lower() != ".dll" or not path.is_file():
continue
blob = path.read_bytes()
sha = hashlib.sha256(blob).hexdigest()
name = path.name.split("__")[-1].upper()
if re.fullmatch(r"PIKLIB(?:61|71|72|8)\.DLL", name):
family, scope = name.removesuffix(".DLL"), "engine"
elif name.startswith("BLOOMOO"):
family, scope = "BLOOMOO", "engine"
elif name in ("WORLD.DLL", "KOLOROWANKA.DLL"):
family, scope = name.removesuffix(".DLL"), "plugin"
elif name == "SEKAI.DLL":
family, scope = "SEKAI", "native_dependency"
elif "PIKLIB" in name:
family, scope = name.removesuffix(".DLL"), "excluded_early_engine"
else:
family, scope = name, "unclassified"
build = builds.setdefault(sha, {"sha256": sha, "size": len(blob), "family": family,
"scope": scope, "paths": [], "analysis": "snapshot_available" if sha in extracted else "pending",
"suggested_snapshot": sha + ".snapshot.json"})
absolute = str(path.resolve())
if absolute not in build["paths"]:
build["paths"].append(absolute)
result = sorted(builds.values(), key=lambda b: (b["family"], b["sha256"]))
return {"schema": "binary-inventory/1", "builds": result,
"summary": {"files": sum(len(b["paths"]) for b in result), "unique": len(result),
"by_scope": dict(Counter(b["scope"] for b in result))},
"limitations": ["Scope is inferred from filenames; validate renamed or unclassified DLLs.",
"snapshot_available means identity matches; it does not establish complete API extraction."]}
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("roots", nargs="+", type=Path)
p.add_argument("--snapshots", type=Path)
p.add_argument("--out", type=Path, required=True)
a = p.parse_args()
result = inventory(a.roots, a.snapshots)
a.out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(result["summary"]))
if __name__ == "__main__":
main()