A module DLL has no engine factory: its CMC_ExternObject::resolve compares TYPE (CXString::equals or operator==), allocates and constructs. The runner of each type is the vftable its constructor or a virtual installs whose prepare slot chains to CMC_Runner::prepareMthHashSet (defined when analysis left it out); registrations, dispatch, signatures, per-type events and fields then come from the engine passes, limited to the module's own runners. Snapshots record module_of (the host engine DLL). An unoptimised run() keeps the switch index in a stack slot; its bounds are followed through that slot only when the register match finds none. The batch queue extracts plugins too, the inventory knows MATRIX, QMAN, INERTIA and BOARD, and the tracker marks module-mapped types. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
258 lines
11 KiB
Python
258 lines
11 KiB
Python
"""Resumable, SHA-addressed extraction queue for engines and scene modules (POSIX)."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from contextlib import contextmanager
|
|
from datetime import datetime, timezone
|
|
import fcntl
|
|
import hashlib
|
|
from importlib import metadata
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import uuid
|
|
|
|
|
|
def digest(path):
|
|
h = hashlib.sha256()
|
|
with Path(path).open('rb') as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def fingerprint(env=None):
|
|
from . import ghidra
|
|
env = os.environ if env is None else env
|
|
root = Path(__file__).resolve().parents[2]
|
|
scripts = Path(env.get('AMS_GHIDRA_SCRIPTS', root / 'ghidra_scripts'))
|
|
paths = [Path(__file__), Path(ghidra.__file__), *sorted(scripts.glob('*.py'))]
|
|
h = hashlib.sha256()
|
|
for path in paths:
|
|
h.update(path.name.encode())
|
|
h.update(path.read_bytes())
|
|
h.update(json.dumps(provider_profile(env), sort_keys=True).encode())
|
|
return h.hexdigest()
|
|
|
|
|
|
def provider_profile(env):
|
|
provider = 'pyghidra' if env.get('AMS_USE_PYGHIDRA') else 'headless'
|
|
home = env.get('GHIDRA_INSTALL_DIR') or env.get('GHIDRA_HOME')
|
|
launcher = env.get('GHIDRA_HEADLESS')
|
|
if provider == 'headless' and launcher:
|
|
home = str(Path(launcher).resolve().parent.parent)
|
|
elif provider == 'headless' and not home:
|
|
import shutil
|
|
launcher = shutil.which('analyzeHeadless', path=env.get('PATH'))
|
|
if launcher:
|
|
home = str(Path(launcher).resolve().parent.parent)
|
|
properties = Path(home) / 'Ghidra' / 'application.properties' if home else None
|
|
try:
|
|
pyghidra_version = metadata.version('pyghidra')
|
|
except metadata.PackageNotFoundError:
|
|
pyghidra_version = None
|
|
return {'provider': provider, 'ghidra_home': str(Path(home).resolve()) if home else None,
|
|
'launcher': launcher, 'pyghidra_version': pyghidra_version,
|
|
'application_properties': properties.read_text() if properties and properties.is_file() else None,
|
|
'python': sys.version, 'java_home': env.get('JAVA_HOME')}
|
|
|
|
|
|
def atomic_json(path, data):
|
|
temp = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp')
|
|
try:
|
|
temp.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
|
|
os.replace(temp, path)
|
|
finally:
|
|
temp.unlink(missing_ok=True)
|
|
|
|
|
|
def validate_snapshot(path, sha):
|
|
data = json.loads(path.read_text())
|
|
if not isinstance(data, dict) or not isinstance(data.get('binary'), dict):
|
|
raise ValueError('Snapshot has no binary identity')
|
|
snapshot_sha = data['binary'].get('sha256')
|
|
if not isinstance(snapshot_sha, str) or snapshot_sha.lower() != sha:
|
|
raise ValueError('Snapshot SHA does not match manifest')
|
|
if not isinstance(data.get('types'), list) or not data['types']:
|
|
raise ValueError('Snapshot has no extracted types')
|
|
if not isinstance(data.get('methods'), list) or not data['methods']:
|
|
raise ValueError('Snapshot has no extracted methods')
|
|
return data
|
|
|
|
|
|
@contextmanager
|
|
def queue_lock(directory):
|
|
with (directory / '.queue.lock').open('a') as stream:
|
|
try:
|
|
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
raise RuntimeError('Another extraction queue is using this output directory')
|
|
try:
|
|
yield
|
|
finally:
|
|
fcntl.flock(stream, fcntl.LOCK_UN)
|
|
|
|
|
|
def execute_worker(source, output, log, timeout, env):
|
|
with log.open('wb') as stream:
|
|
process = subprocess.Popen(
|
|
[sys.executable, '-m', 'ams.acquire.batch', '--worker', str(source), str(output)],
|
|
stdout=stream, stderr=subprocess.STDOUT, env=env, start_new_session=True)
|
|
try:
|
|
code = process.wait(timeout=timeout)
|
|
except BaseException:
|
|
try:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
process.wait()
|
|
raise
|
|
if code:
|
|
raise RuntimeError('Extractor exited with code ' + str(code))
|
|
|
|
|
|
# Engines and the scene modules (SCENE:DLLS=) that add script types; the extractor handles both.
|
|
EXTRACTED_SCOPES = ('engine', 'plugin')
|
|
|
|
|
|
def run_queue(manifest, out_dir, *, ghidra_home=None, timeout=1800, limit=None,
|
|
retry_failed=False, worker=execute_worker):
|
|
if timeout <= 0 or (limit is not None and limit < 0):
|
|
raise ValueError('Timeout must be positive; limit must be nonnegative')
|
|
data = json.loads(Path(manifest).read_text())
|
|
if data.get('schema') != 'binary-inventory/1':
|
|
raise ValueError('Expected binary-inventory/1 manifest')
|
|
builds = data['builds']
|
|
seen = set()
|
|
for build in builds:
|
|
sha = build['sha256']
|
|
if not isinstance(sha, str) or not re.fullmatch('[0-9a-f]{64}', sha) or sha in seen:
|
|
raise ValueError('Invalid or duplicate manifest SHA: ' + str(sha))
|
|
seen.add(sha)
|
|
out_dir = Path(out_dir).resolve()
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
with queue_lock(out_dir):
|
|
return _run(builds, out_dir, ghidra_home, timeout, limit, retry_failed, worker)
|
|
|
|
|
|
def _run(builds, directory, ghidra_home, timeout, limit, retry_failed, worker):
|
|
for name in ('states', 'logs'):
|
|
(directory / name).mkdir(exist_ok=True)
|
|
env = dict(os.environ)
|
|
if ghidra_home:
|
|
env.update(GHIDRA_INSTALL_DIR=str(ghidra_home), GHIDRA_HOME=str(ghidra_home), AMS_USE_PYGHIDRA='1')
|
|
extractor = fingerprint(env)
|
|
profile = provider_profile(env)
|
|
report = {'schema': 'extraction-batch/1', 'attempted': 0, 'succeeded': 0,
|
|
'pending': 0, 'skipped_success': 0, 'skipped_failed': 0, 'skipped_scope': 0, 'failures': []}
|
|
for build in builds:
|
|
if build.get('scope') not in EXTRACTED_SCOPES:
|
|
report['skipped_scope'] += 1
|
|
continue
|
|
sha = build['sha256']
|
|
state_path = directory / 'states' / (sha + '.json')
|
|
output = directory / (sha + '.snapshot.json')
|
|
old = json.loads(state_path.read_text()) if state_path.exists() else {}
|
|
same = old.get('extractor_fingerprint') == extractor
|
|
# Validate source even when resuming: a manifest is not proof of current contents.
|
|
source = None
|
|
for candidate in build.get('paths', []):
|
|
try:
|
|
if digest(candidate) == sha:
|
|
source = Path(candidate).resolve()
|
|
break
|
|
except OSError:
|
|
continue
|
|
if same and old.get('status') == 'success' and source:
|
|
try:
|
|
validate_snapshot(output, sha)
|
|
if digest(output) != old.get('snapshot_sha256'):
|
|
raise ValueError('Snapshot content changed since successful extraction')
|
|
except (OSError, ValueError, TypeError, AttributeError):
|
|
pass
|
|
else:
|
|
report['skipped_success'] += 1
|
|
continue
|
|
if same and old.get('status') in ('failed', 'running') and not retry_failed:
|
|
report['skipped_failed'] += 1
|
|
report['failures'].append({'sha256': sha, 'error': old.get('error', 'Interrupted attempt; use --retry-failed')})
|
|
continue
|
|
if limit is not None and report['attempted'] >= limit:
|
|
report['pending'] += 1
|
|
continue
|
|
attempt = old.get('attempts', 0) + 1
|
|
log = directory / 'logs' / (sha + '.' + str(attempt) + '.log')
|
|
temp = directory / (sha + '.' + uuid.uuid4().hex + '.pending.json')
|
|
state = {'sha256': sha, 'family': build.get('family'), 'extractor_fingerprint': extractor,
|
|
'status': 'running', 'attempts': attempt, 'source': str(source) if source else None,
|
|
'input_name': source.name if source else None, 'provider_profile': profile,
|
|
'started_at': datetime.now(timezone.utc).isoformat(), 'log': str(log)}
|
|
atomic_json(state_path, state)
|
|
report['attempted'] += 1
|
|
try:
|
|
if not source:
|
|
raise ValueError('No source path matches manifest SHA')
|
|
worker(source, temp, log, timeout, env)
|
|
snapshot = validate_snapshot(temp, sha)
|
|
if digest(source) != sha:
|
|
raise ValueError('Source changed during extraction')
|
|
snapshot['binary'].update(
|
|
library=source.name.split('__')[-1],
|
|
sources=build.get('paths', []),
|
|
games=sorted({Path(p).name.split('__')[0] for p in build.get('paths', []) if '__' in Path(p).name}),
|
|
extraction={'fingerprint': extractor, 'provider': profile,
|
|
'started_at': state['started_at'], 'coverage': 'partial'})
|
|
snapshot.setdefault('limitations', []).append(
|
|
'Automated heuristic extraction; missing entries are unknown, not proof of absence.')
|
|
atomic_json(temp, snapshot)
|
|
os.replace(temp, output)
|
|
state['status'] = 'success'
|
|
state['snapshot'] = str(output)
|
|
state['snapshot_sha256'] = digest(output)
|
|
state['coverage'] = {'types': len(snapshot['types']), 'methods': len(snapshot['methods']),
|
|
'dispatch': len(snapshot.get('method_dispatch') or [])}
|
|
report['succeeded'] += 1
|
|
except Exception as exc:
|
|
state.update(status='failed', error=str(exc))
|
|
if not log.exists():
|
|
log.write_text(str(exc) + '\n')
|
|
report['failures'].append({'sha256': sha, 'error': str(exc)})
|
|
finally:
|
|
temp.unlink(missing_ok=True)
|
|
state['finished_at'] = datetime.now(timezone.utc).isoformat()
|
|
atomic_json(state_path, state)
|
|
print(json.dumps({'sha256': sha, 'status': state['status'], 'attempt': attempt}), flush=True)
|
|
atomic_json(directory / 'failure-report.json', report)
|
|
return report
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--worker', nargs=2, metavar=('DLL', 'OUTPUT'), help=argparse.SUPPRESS)
|
|
parser.add_argument('--manifest', type=Path)
|
|
parser.add_argument('--out-dir', type=Path)
|
|
parser.add_argument('--ghidra-home', type=Path)
|
|
parser.add_argument('--timeout', type=float, default=1800)
|
|
parser.add_argument('--limit', type=int)
|
|
parser.add_argument('--retry-failed', action='store_true')
|
|
args = parser.parse_args()
|
|
if args.worker:
|
|
from .ghidra import run_extractor
|
|
run_extractor(*args.worker)
|
|
return
|
|
if not args.manifest or not args.out_dir:
|
|
parser.error('--manifest and --out-dir are required')
|
|
result = run_queue(args.manifest, args.out_dir, ghidra_home=args.ghidra_home,
|
|
timeout=args.timeout, limit=args.limit, retry_failed=args.retry_failed)
|
|
print(json.dumps(result))
|
|
if result['failures']:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|