Inventory classifies the DLL repository by SHA; the batch queue runs each engine build in a fresh PyGhidra process with a timeout, fingerprints the extractor environment and keeps per-attempt state and logs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
254 lines
11 KiB
Python
254 lines
11 KiB
Python
"""Resumable, SHA-addressed engine extraction queue (POSIX)."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
import fcntl
|
|
import hashlib
|
|
from importlib import metadata
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import uuid
|
|
|
|
|
|
def digest(path):
|
|
h = hashlib.sha256()
|
|
with Path(path).open('rb') as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def fingerprint(env=None):
|
|
from . import ghidra
|
|
env = os.environ if env is None else env
|
|
root = Path(__file__).resolve().parents[2]
|
|
scripts = Path(env.get('AMS_GHIDRA_SCRIPTS', root / 'ghidra_scripts'))
|
|
paths = [Path(__file__), Path(ghidra.__file__), *sorted(scripts.glob('*.py'))]
|
|
h = hashlib.sha256()
|
|
for path in paths:
|
|
h.update(path.name.encode())
|
|
h.update(path.read_bytes())
|
|
h.update(json.dumps(provider_profile(env), sort_keys=True).encode())
|
|
return h.hexdigest()
|
|
|
|
|
|
def provider_profile(env):
|
|
provider = 'pyghidra' if env.get('AMS_USE_PYGHIDRA') else 'headless'
|
|
home = env.get('GHIDRA_INSTALL_DIR') or env.get('GHIDRA_HOME')
|
|
launcher = env.get('GHIDRA_HEADLESS')
|
|
if provider == 'headless' and launcher:
|
|
home = str(Path(launcher).resolve().parent.parent)
|
|
elif provider == 'headless' and not home:
|
|
import shutil
|
|
launcher = shutil.which('analyzeHeadless', path=env.get('PATH'))
|
|
if launcher:
|
|
home = str(Path(launcher).resolve().parent.parent)
|
|
properties = Path(home) / 'Ghidra' / 'application.properties' if home else None
|
|
try:
|
|
pyghidra_version = metadata.version('pyghidra')
|
|
except metadata.PackageNotFoundError:
|
|
pyghidra_version = None
|
|
return {'provider': provider, 'ghidra_home': str(Path(home).resolve()) if home else None,
|
|
'launcher': launcher, 'pyghidra_version': pyghidra_version,
|
|
'application_properties': properties.read_text() if properties and properties.is_file() else None,
|
|
'python': sys.version, 'java_home': env.get('JAVA_HOME')}
|
|
|
|
|
|
def atomic_json(path, data):
|
|
temp = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp')
|
|
try:
|
|
temp.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
|
|
os.replace(temp, path)
|
|
finally:
|
|
temp.unlink(missing_ok=True)
|
|
|
|
|
|
def validate_snapshot(path, sha):
|
|
data = json.loads(path.read_text())
|
|
if not isinstance(data, dict) or not isinstance(data.get('binary'), dict):
|
|
raise ValueError('Snapshot has no binary identity')
|
|
snapshot_sha = data['binary'].get('sha256')
|
|
if not isinstance(snapshot_sha, str) or snapshot_sha.lower() != sha:
|
|
raise ValueError('Snapshot SHA does not match manifest')
|
|
if not isinstance(data.get('types'), list) or not data['types']:
|
|
raise ValueError('Snapshot has no extracted types')
|
|
if not isinstance(data.get('methods'), list) or not data['methods']:
|
|
raise ValueError('Snapshot has no extracted methods')
|
|
return data
|
|
|
|
|
|
@contextmanager
|
|
def queue_lock(directory):
|
|
with (directory / '.queue.lock').open('a') as stream:
|
|
try:
|
|
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
raise RuntimeError('Another extraction queue is using this output directory')
|
|
try:
|
|
yield
|
|
finally:
|
|
fcntl.flock(stream, fcntl.LOCK_UN)
|
|
|
|
|
|
def execute_worker(source, output, log, timeout, env):
|
|
with log.open('wb') as stream:
|
|
process = subprocess.Popen(
|
|
[sys.executable, '-m', 'ams.acquire.batch', '--worker', str(source), str(output)],
|
|
stdout=stream, stderr=subprocess.STDOUT, env=env, start_new_session=True)
|
|
try:
|
|
code = process.wait(timeout=timeout)
|
|
except BaseException:
|
|
try:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
process.wait()
|
|
raise
|
|
if code:
|
|
raise RuntimeError('Extractor exited with code ' + str(code))
|
|
|
|
|
|
def run_queue(manifest, out_dir, *, ghidra_home=None, timeout=1800, limit=None,
|
|
retry_failed=False, worker=execute_worker):
|
|
if timeout <= 0 or (limit is not None and limit < 0):
|
|
raise ValueError('Timeout must be positive; limit must be nonnegative')
|
|
data = json.loads(Path(manifest).read_text())
|
|
if data.get('schema') != 'binary-inventory/1':
|
|
raise ValueError('Expected binary-inventory/1 manifest')
|
|
builds = data['builds']
|
|
seen = set()
|
|
for build in builds:
|
|
sha = build['sha256']
|
|
if not isinstance(sha, str) or not re.fullmatch('[0-9a-f]{64}', sha) or sha in seen:
|
|
raise ValueError('Invalid or duplicate manifest SHA: ' + str(sha))
|
|
seen.add(sha)
|
|
out_dir = Path(out_dir).resolve()
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
with queue_lock(out_dir):
|
|
return _run(builds, out_dir, ghidra_home, timeout, limit, retry_failed, worker)
|
|
|
|
|
|
def _run(builds, directory, ghidra_home, timeout, limit, retry_failed, worker):
|
|
for name in ('states', 'logs'):
|
|
(directory / name).mkdir(exist_ok=True)
|
|
env = dict(os.environ)
|
|
if ghidra_home:
|
|
env.update(GHIDRA_INSTALL_DIR=str(ghidra_home), GHIDRA_HOME=str(ghidra_home), AMS_USE_PYGHIDRA='1')
|
|
extractor = fingerprint(env)
|
|
profile = provider_profile(env)
|
|
report = {'schema': 'extraction-batch/1', 'attempted': 0, 'succeeded': 0,
|
|
'pending': 0, 'skipped_success': 0, 'skipped_failed': 0, 'skipped_scope': 0, 'failures': []}
|
|
for build in builds:
|
|
if build.get('scope') != 'engine':
|
|
report['skipped_scope'] += 1
|
|
continue
|
|
sha = build['sha256']
|
|
state_path = directory / 'states' / (sha + '.json')
|
|
output = directory / (sha + '.snapshot.json')
|
|
old = json.loads(state_path.read_text()) if state_path.exists() else {}
|
|
same = old.get('extractor_fingerprint') == extractor
|
|
# Validate source even when resuming: a manifest is not proof of current contents.
|
|
source = None
|
|
for candidate in build.get('paths', []):
|
|
try:
|
|
if digest(candidate) == sha:
|
|
source = Path(candidate).resolve()
|
|
break
|
|
except OSError:
|
|
continue
|
|
if same and old.get('status') == 'success' and source:
|
|
try:
|
|
validate_snapshot(output, sha)
|
|
if digest(output) != old.get('snapshot_sha256'):
|
|
raise ValueError('Snapshot content changed since successful extraction')
|
|
except (OSError, ValueError, TypeError, AttributeError):
|
|
pass
|
|
else:
|
|
report['skipped_success'] += 1
|
|
continue
|
|
if same and old.get('status') in ('failed', 'running') and not retry_failed:
|
|
report['skipped_failed'] += 1
|
|
report['failures'].append({'sha256': sha, 'error': old.get('error', 'Interrupted attempt; use --retry-failed')})
|
|
continue
|
|
if limit is not None and report['attempted'] >= limit:
|
|
report['pending'] += 1
|
|
continue
|
|
attempt = old.get('attempts', 0) + 1
|
|
log = directory / 'logs' / (sha + '.' + str(attempt) + '.log')
|
|
temp = directory / (sha + '.' + uuid.uuid4().hex + '.pending.json')
|
|
state = {'sha256': sha, 'family': build.get('family'), 'extractor_fingerprint': extractor,
|
|
'status': 'running', 'attempts': attempt, 'source': str(source) if source else None,
|
|
'input_name': source.name if source else None, 'provider_profile': profile,
|
|
'started_at': datetime.now(timezone.utc).isoformat(), 'log': str(log)}
|
|
atomic_json(state_path, state)
|
|
report['attempted'] += 1
|
|
try:
|
|
if not source:
|
|
raise ValueError('No source path matches manifest SHA')
|
|
worker(source, temp, log, timeout, env)
|
|
snapshot = validate_snapshot(temp, sha)
|
|
if digest(source) != sha:
|
|
raise ValueError('Source changed during extraction')
|
|
snapshot['binary'].update(
|
|
library=source.name.split('__')[-1],
|
|
sources=build.get('paths', []),
|
|
games=sorted({Path(p).name.split('__')[0] for p in build.get('paths', []) if '__' in Path(p).name}),
|
|
extraction={'fingerprint': extractor, 'provider': profile,
|
|
'started_at': state['started_at'], 'coverage': 'partial'})
|
|
snapshot.setdefault('limitations', []).append(
|
|
'Automated heuristic extraction; missing entries are unknown, not proof of absence.')
|
|
atomic_json(temp, snapshot)
|
|
os.replace(temp, output)
|
|
state['status'] = 'success'
|
|
state['snapshot'] = str(output)
|
|
state['snapshot_sha256'] = digest(output)
|
|
state['coverage'] = {'types': len(snapshot['types']), 'methods': len(snapshot['methods']),
|
|
'dispatch': len(snapshot.get('method_dispatch') or [])}
|
|
report['succeeded'] += 1
|
|
except Exception as exc:
|
|
state.update(status='failed', error=str(exc))
|
|
if not log.exists():
|
|
log.write_text(str(exc) + '\n')
|
|
report['failures'].append({'sha256': sha, 'error': str(exc)})
|
|
finally:
|
|
temp.unlink(missing_ok=True)
|
|
state['finished_at'] = datetime.now(timezone.utc).isoformat()
|
|
atomic_json(state_path, state)
|
|
print(json.dumps({'sha256': sha, 'status': state['status'], 'attempt': attempt}), flush=True)
|
|
atomic_json(directory / 'failure-report.json', report)
|
|
return report
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--worker', nargs=2, metavar=('DLL', 'OUTPUT'), help=argparse.SUPPRESS)
|
|
parser.add_argument('--manifest', type=Path)
|
|
parser.add_argument('--out-dir', type=Path)
|
|
parser.add_argument('--ghidra-home', type=Path)
|
|
parser.add_argument('--timeout', type=float, default=1800)
|
|
parser.add_argument('--limit', type=int)
|
|
parser.add_argument('--retry-failed', action='store_true')
|
|
args = parser.parse_args()
|
|
if args.worker:
|
|
from .ghidra import run_extractor
|
|
run_extractor(*args.worker)
|
|
return
|
|
if not args.manifest or not args.out_dir:
|
|
parser.error('--manifest and --out-dir are required')
|
|
result = run_queue(args.manifest, args.out_dir, ghidra_home=args.ghidra_home,
|
|
timeout=args.timeout, limit=args.limit, retry_failed=args.retry_failed)
|
|
print(json.dumps(result))
|
|
if result['failures']:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|