"""Resumable, SHA-addressed engine extraction queue (POSIX).""" from __future__ import annotations import argparse from contextlib import contextmanager from datetime import datetime, timezone import fcntl import hashlib from importlib import metadata import json import os from pathlib import Path import re import signal import subprocess import sys import uuid def digest(path): h = hashlib.sha256() with Path(path).open('rb') as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b''): h.update(chunk) return h.hexdigest() def fingerprint(env=None): from . import ghidra env = os.environ if env is None else env root = Path(__file__).resolve().parents[2] scripts = Path(env.get('AMS_GHIDRA_SCRIPTS', root / 'ghidra_scripts')) paths = [Path(__file__), Path(ghidra.__file__), *sorted(scripts.glob('*.py'))] h = hashlib.sha256() for path in paths: h.update(path.name.encode()) h.update(path.read_bytes()) h.update(json.dumps(provider_profile(env), sort_keys=True).encode()) return h.hexdigest() def provider_profile(env): provider = 'pyghidra' if env.get('AMS_USE_PYGHIDRA') else 'headless' home = env.get('GHIDRA_INSTALL_DIR') or env.get('GHIDRA_HOME') launcher = env.get('GHIDRA_HEADLESS') if provider == 'headless' and launcher: home = str(Path(launcher).resolve().parent.parent) elif provider == 'headless' and not home: import shutil launcher = shutil.which('analyzeHeadless', path=env.get('PATH')) if launcher: home = str(Path(launcher).resolve().parent.parent) properties = Path(home) / 'Ghidra' / 'application.properties' if home else None try: pyghidra_version = metadata.version('pyghidra') except metadata.PackageNotFoundError: pyghidra_version = None return {'provider': provider, 'ghidra_home': str(Path(home).resolve()) if home else None, 'launcher': launcher, 'pyghidra_version': pyghidra_version, 'application_properties': properties.read_text() if properties and properties.is_file() else None, 'python': sys.version, 'java_home': env.get('JAVA_HOME')} def atomic_json(path, data): temp = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp') try: temp.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n') os.replace(temp, path) finally: temp.unlink(missing_ok=True) def validate_snapshot(path, sha): data = json.loads(path.read_text()) if not isinstance(data, dict) or not isinstance(data.get('binary'), dict): raise ValueError('Snapshot has no binary identity') snapshot_sha = data['binary'].get('sha256') if not isinstance(snapshot_sha, str) or snapshot_sha.lower() != sha: raise ValueError('Snapshot SHA does not match manifest') if not isinstance(data.get('types'), list) or not data['types']: raise ValueError('Snapshot has no extracted types') if not isinstance(data.get('methods'), list) or not data['methods']: raise ValueError('Snapshot has no extracted methods') return data @contextmanager def queue_lock(directory): with (directory / '.queue.lock').open('a') as stream: try: fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: raise RuntimeError('Another extraction queue is using this output directory') try: yield finally: fcntl.flock(stream, fcntl.LOCK_UN) def execute_worker(source, output, log, timeout, env): with log.open('wb') as stream: process = subprocess.Popen( [sys.executable, '-m', 'ams.acquire.batch', '--worker', str(source), str(output)], stdout=stream, stderr=subprocess.STDOUT, env=env, start_new_session=True) try: code = process.wait(timeout=timeout) except BaseException: try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass process.wait() raise if code: raise RuntimeError('Extractor exited with code ' + str(code)) def run_queue(manifest, out_dir, *, ghidra_home=None, timeout=1800, limit=None, retry_failed=False, worker=execute_worker): if timeout <= 0 or (limit is not None and limit < 0): raise ValueError('Timeout must be positive; limit must be nonnegative') data = json.loads(Path(manifest).read_text()) if data.get('schema') != 'binary-inventory/1': raise ValueError('Expected binary-inventory/1 manifest') builds = data['builds'] seen = set() for build in builds: sha = build['sha256'] if not isinstance(sha, str) or not re.fullmatch('[0-9a-f]{64}', sha) or sha in seen: raise ValueError('Invalid or duplicate manifest SHA: ' + str(sha)) seen.add(sha) out_dir = Path(out_dir).resolve() out_dir.mkdir(parents=True, exist_ok=True) with queue_lock(out_dir): return _run(builds, out_dir, ghidra_home, timeout, limit, retry_failed, worker) def _run(builds, directory, ghidra_home, timeout, limit, retry_failed, worker): for name in ('states', 'logs'): (directory / name).mkdir(exist_ok=True) env = dict(os.environ) if ghidra_home: env.update(GHIDRA_INSTALL_DIR=str(ghidra_home), GHIDRA_HOME=str(ghidra_home), AMS_USE_PYGHIDRA='1') extractor = fingerprint(env) profile = provider_profile(env) report = {'schema': 'extraction-batch/1', 'attempted': 0, 'succeeded': 0, 'pending': 0, 'skipped_success': 0, 'skipped_failed': 0, 'skipped_scope': 0, 'failures': []} for build in builds: if build.get('scope') != 'engine': report['skipped_scope'] += 1 continue sha = build['sha256'] state_path = directory / 'states' / (sha + '.json') output = directory / (sha + '.snapshot.json') old = json.loads(state_path.read_text()) if state_path.exists() else {} same = old.get('extractor_fingerprint') == extractor # Validate source even when resuming: a manifest is not proof of current contents. source = None for candidate in build.get('paths', []): try: if digest(candidate) == sha: source = Path(candidate).resolve() break except OSError: continue if same and old.get('status') == 'success' and source: try: validate_snapshot(output, sha) if digest(output) != old.get('snapshot_sha256'): raise ValueError('Snapshot content changed since successful extraction') except (OSError, ValueError, TypeError, AttributeError): pass else: report['skipped_success'] += 1 continue if same and old.get('status') in ('failed', 'running') and not retry_failed: report['skipped_failed'] += 1 report['failures'].append({'sha256': sha, 'error': old.get('error', 'Interrupted attempt; use --retry-failed')}) continue if limit is not None and report['attempted'] >= limit: report['pending'] += 1 continue attempt = old.get('attempts', 0) + 1 log = directory / 'logs' / (sha + '.' + str(attempt) + '.log') temp = directory / (sha + '.' + uuid.uuid4().hex + '.pending.json') state = {'sha256': sha, 'family': build.get('family'), 'extractor_fingerprint': extractor, 'status': 'running', 'attempts': attempt, 'source': str(source) if source else None, 'input_name': source.name if source else None, 'provider_profile': profile, 'started_at': datetime.now(timezone.utc).isoformat(), 'log': str(log)} atomic_json(state_path, state) report['attempted'] += 1 try: if not source: raise ValueError('No source path matches manifest SHA') worker(source, temp, log, timeout, env) snapshot = validate_snapshot(temp, sha) if digest(source) != sha: raise ValueError('Source changed during extraction') snapshot['binary'].update( library=source.name.split('__')[-1], sources=build.get('paths', []), games=sorted({Path(p).name.split('__')[0] for p in build.get('paths', []) if '__' in Path(p).name}), extraction={'fingerprint': extractor, 'provider': profile, 'started_at': state['started_at'], 'coverage': 'partial'}) snapshot.setdefault('limitations', []).append( 'Automated heuristic extraction; missing entries are unknown, not proof of absence.') atomic_json(temp, snapshot) os.replace(temp, output) state['status'] = 'success' state['snapshot'] = str(output) state['snapshot_sha256'] = digest(output) state['coverage'] = {'types': len(snapshot['types']), 'methods': len(snapshot['methods']), 'dispatch': len(snapshot.get('method_dispatch') or [])} report['succeeded'] += 1 except Exception as exc: state.update(status='failed', error=str(exc)) if not log.exists(): log.write_text(str(exc) + '\n') report['failures'].append({'sha256': sha, 'error': str(exc)}) finally: temp.unlink(missing_ok=True) state['finished_at'] = datetime.now(timezone.utc).isoformat() atomic_json(state_path, state) print(json.dumps({'sha256': sha, 'status': state['status'], 'attempt': attempt}), flush=True) atomic_json(directory / 'failure-report.json', report) return report def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--worker', nargs=2, metavar=('DLL', 'OUTPUT'), help=argparse.SUPPRESS) parser.add_argument('--manifest', type=Path) parser.add_argument('--out-dir', type=Path) parser.add_argument('--ghidra-home', type=Path) parser.add_argument('--timeout', type=float, default=1800) parser.add_argument('--limit', type=int) parser.add_argument('--retry-failed', action='store_true') args = parser.parse_args() if args.worker: from .ghidra import run_extractor run_extractor(*args.worker) return if not args.manifest or not args.out_dir: parser.error('--manifest and --out-dir are required') result = run_queue(args.manifest, args.out_dir, ghidra_home=args.ghidra_home, timeout=args.timeout, limit=args.limit, retry_failed=args.retry_failed) print(json.dumps(result)) if result['failures']: raise SystemExit(1) if __name__ == '__main__': main()