Files
Aidem-Media-DLL-Analysis/ams/acquire/batch.py
T
Patryk GenschandClaude Opus 5.5 b0013c5114 feat(acquire): resumable batch extraction queue and binary inventory
Inventory classifies the DLL repository by SHA; the batch queue runs each
engine build in a fresh PyGhidra process with a timeout, fingerprints the
extractor environment and keeps per-attempt state and logs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-24 18:24:15 +02:00

254 lines
11 KiB
Python

"""Resumable, SHA-addressed engine extraction queue (POSIX)."""
from __future__ import annotations
import argparse
from contextlib import contextmanager
from datetime import datetime, timezone
import fcntl
import hashlib
from importlib import metadata
import json
import os
from pathlib import Path
import re
import signal
import subprocess
import sys
import uuid
def digest(path):
h = hashlib.sha256()
with Path(path).open('rb') as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
h.update(chunk)
return h.hexdigest()
def fingerprint(env=None):
from . import ghidra
env = os.environ if env is None else env
root = Path(__file__).resolve().parents[2]
scripts = Path(env.get('AMS_GHIDRA_SCRIPTS', root / 'ghidra_scripts'))
paths = [Path(__file__), Path(ghidra.__file__), *sorted(scripts.glob('*.py'))]
h = hashlib.sha256()
for path in paths:
h.update(path.name.encode())
h.update(path.read_bytes())
h.update(json.dumps(provider_profile(env), sort_keys=True).encode())
return h.hexdigest()
def provider_profile(env):
provider = 'pyghidra' if env.get('AMS_USE_PYGHIDRA') else 'headless'
home = env.get('GHIDRA_INSTALL_DIR') or env.get('GHIDRA_HOME')
launcher = env.get('GHIDRA_HEADLESS')
if provider == 'headless' and launcher:
home = str(Path(launcher).resolve().parent.parent)
elif provider == 'headless' and not home:
import shutil
launcher = shutil.which('analyzeHeadless', path=env.get('PATH'))
if launcher:
home = str(Path(launcher).resolve().parent.parent)
properties = Path(home) / 'Ghidra' / 'application.properties' if home else None
try:
pyghidra_version = metadata.version('pyghidra')
except metadata.PackageNotFoundError:
pyghidra_version = None
return {'provider': provider, 'ghidra_home': str(Path(home).resolve()) if home else None,
'launcher': launcher, 'pyghidra_version': pyghidra_version,
'application_properties': properties.read_text() if properties and properties.is_file() else None,
'python': sys.version, 'java_home': env.get('JAVA_HOME')}
def atomic_json(path, data):
temp = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp')
try:
temp.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
os.replace(temp, path)
finally:
temp.unlink(missing_ok=True)
def validate_snapshot(path, sha):
data = json.loads(path.read_text())
if not isinstance(data, dict) or not isinstance(data.get('binary'), dict):
raise ValueError('Snapshot has no binary identity')
snapshot_sha = data['binary'].get('sha256')
if not isinstance(snapshot_sha, str) or snapshot_sha.lower() != sha:
raise ValueError('Snapshot SHA does not match manifest')
if not isinstance(data.get('types'), list) or not data['types']:
raise ValueError('Snapshot has no extracted types')
if not isinstance(data.get('methods'), list) or not data['methods']:
raise ValueError('Snapshot has no extracted methods')
return data
@contextmanager
def queue_lock(directory):
with (directory / '.queue.lock').open('a') as stream:
try:
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise RuntimeError('Another extraction queue is using this output directory')
try:
yield
finally:
fcntl.flock(stream, fcntl.LOCK_UN)
def execute_worker(source, output, log, timeout, env):
with log.open('wb') as stream:
process = subprocess.Popen(
[sys.executable, '-m', 'ams.acquire.batch', '--worker', str(source), str(output)],
stdout=stream, stderr=subprocess.STDOUT, env=env, start_new_session=True)
try:
code = process.wait(timeout=timeout)
except BaseException:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait()
raise
if code:
raise RuntimeError('Extractor exited with code ' + str(code))
def run_queue(manifest, out_dir, *, ghidra_home=None, timeout=1800, limit=None,
retry_failed=False, worker=execute_worker):
if timeout <= 0 or (limit is not None and limit < 0):
raise ValueError('Timeout must be positive; limit must be nonnegative')
data = json.loads(Path(manifest).read_text())
if data.get('schema') != 'binary-inventory/1':
raise ValueError('Expected binary-inventory/1 manifest')
builds = data['builds']
seen = set()
for build in builds:
sha = build['sha256']
if not isinstance(sha, str) or not re.fullmatch('[0-9a-f]{64}', sha) or sha in seen:
raise ValueError('Invalid or duplicate manifest SHA: ' + str(sha))
seen.add(sha)
out_dir = Path(out_dir).resolve()
out_dir.mkdir(parents=True, exist_ok=True)
with queue_lock(out_dir):
return _run(builds, out_dir, ghidra_home, timeout, limit, retry_failed, worker)
def _run(builds, directory, ghidra_home, timeout, limit, retry_failed, worker):
for name in ('states', 'logs'):
(directory / name).mkdir(exist_ok=True)
env = dict(os.environ)
if ghidra_home:
env.update(GHIDRA_INSTALL_DIR=str(ghidra_home), GHIDRA_HOME=str(ghidra_home), AMS_USE_PYGHIDRA='1')
extractor = fingerprint(env)
profile = provider_profile(env)
report = {'schema': 'extraction-batch/1', 'attempted': 0, 'succeeded': 0,
'pending': 0, 'skipped_success': 0, 'skipped_failed': 0, 'skipped_scope': 0, 'failures': []}
for build in builds:
if build.get('scope') != 'engine':
report['skipped_scope'] += 1
continue
sha = build['sha256']
state_path = directory / 'states' / (sha + '.json')
output = directory / (sha + '.snapshot.json')
old = json.loads(state_path.read_text()) if state_path.exists() else {}
same = old.get('extractor_fingerprint') == extractor
# Validate source even when resuming: a manifest is not proof of current contents.
source = None
for candidate in build.get('paths', []):
try:
if digest(candidate) == sha:
source = Path(candidate).resolve()
break
except OSError:
continue
if same and old.get('status') == 'success' and source:
try:
validate_snapshot(output, sha)
if digest(output) != old.get('snapshot_sha256'):
raise ValueError('Snapshot content changed since successful extraction')
except (OSError, ValueError, TypeError, AttributeError):
pass
else:
report['skipped_success'] += 1
continue
if same and old.get('status') in ('failed', 'running') and not retry_failed:
report['skipped_failed'] += 1
report['failures'].append({'sha256': sha, 'error': old.get('error', 'Interrupted attempt; use --retry-failed')})
continue
if limit is not None and report['attempted'] >= limit:
report['pending'] += 1
continue
attempt = old.get('attempts', 0) + 1
log = directory / 'logs' / (sha + '.' + str(attempt) + '.log')
temp = directory / (sha + '.' + uuid.uuid4().hex + '.pending.json')
state = {'sha256': sha, 'family': build.get('family'), 'extractor_fingerprint': extractor,
'status': 'running', 'attempts': attempt, 'source': str(source) if source else None,
'input_name': source.name if source else None, 'provider_profile': profile,
'started_at': datetime.now(timezone.utc).isoformat(), 'log': str(log)}
atomic_json(state_path, state)
report['attempted'] += 1
try:
if not source:
raise ValueError('No source path matches manifest SHA')
worker(source, temp, log, timeout, env)
snapshot = validate_snapshot(temp, sha)
if digest(source) != sha:
raise ValueError('Source changed during extraction')
snapshot['binary'].update(
library=source.name.split('__')[-1],
sources=build.get('paths', []),
games=sorted({Path(p).name.split('__')[0] for p in build.get('paths', []) if '__' in Path(p).name}),
extraction={'fingerprint': extractor, 'provider': profile,
'started_at': state['started_at'], 'coverage': 'partial'})
snapshot.setdefault('limitations', []).append(
'Automated heuristic extraction; missing entries are unknown, not proof of absence.')
atomic_json(temp, snapshot)
os.replace(temp, output)
state['status'] = 'success'
state['snapshot'] = str(output)
state['snapshot_sha256'] = digest(output)
state['coverage'] = {'types': len(snapshot['types']), 'methods': len(snapshot['methods']),
'dispatch': len(snapshot.get('method_dispatch') or [])}
report['succeeded'] += 1
except Exception as exc:
state.update(status='failed', error=str(exc))
if not log.exists():
log.write_text(str(exc) + '\n')
report['failures'].append({'sha256': sha, 'error': str(exc)})
finally:
temp.unlink(missing_ok=True)
state['finished_at'] = datetime.now(timezone.utc).isoformat()
atomic_json(state_path, state)
print(json.dumps({'sha256': sha, 'status': state['status'], 'attempt': attempt}), flush=True)
atomic_json(directory / 'failure-report.json', report)
return report
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--worker', nargs=2, metavar=('DLL', 'OUTPUT'), help=argparse.SUPPRESS)
parser.add_argument('--manifest', type=Path)
parser.add_argument('--out-dir', type=Path)
parser.add_argument('--ghidra-home', type=Path)
parser.add_argument('--timeout', type=float, default=1800)
parser.add_argument('--limit', type=int)
parser.add_argument('--retry-failed', action='store_true')
args = parser.parse_args()
if args.worker:
from .ghidra import run_extractor
run_extractor(*args.worker)
return
if not args.manifest or not args.out_dir:
parser.error('--manifest and --out-dir are required')
result = run_queue(args.manifest, args.out_dir, ghidra_home=args.ghidra_home,
timeout=args.timeout, limit=args.limit, retry_failed=args.retry_failed)
print(json.dumps(result))
if result['failures']:
raise SystemExit(1)
if __name__ == '__main__':
main()