Files
Aidem-Media-DLL-Analysis/ams/acquire/batch.py
T
Patryk GenschandClaude Opus 5.5 4e65268a1e feat(extractor): extract scene modules (SCENE:DLLS=) like engines
A module DLL has no engine factory: its CMC_ExternObject::resolve compares
TYPE (CXString::equals or operator==), allocates and constructs. The
runner of each type is the vftable its constructor or a virtual installs
whose prepare slot chains to CMC_Runner::prepareMthHashSet (defined when
analysis left it out); registrations, dispatch, signatures, per-type
events and fields then come from the engine passes, limited to the
module's own runners. Snapshots record module_of (the host engine DLL).

An unoptimised run() keeps the switch index in a stack slot; its bounds
are followed through that slot only when the register match finds none.
The batch queue extracts plugins too, the inventory knows MATRIX, QMAN,
INERTIA and BOARD, and the tracker marks module-mapped types.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-28 20:58:40 +01:00

258 lines
11 KiB
Python

"""Resumable, SHA-addressed extraction queue for engines and scene modules (POSIX)."""
from __future__ import annotations
import argparse
from contextlib import contextmanager
from datetime import datetime, timezone
import fcntl
import hashlib
from importlib import metadata
import json
import os
from pathlib import Path
import re
import signal
import subprocess
import sys
import uuid
def digest(path):
h = hashlib.sha256()
with Path(path).open('rb') as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
h.update(chunk)
return h.hexdigest()
def fingerprint(env=None):
from . import ghidra
env = os.environ if env is None else env
root = Path(__file__).resolve().parents[2]
scripts = Path(env.get('AMS_GHIDRA_SCRIPTS', root / 'ghidra_scripts'))
paths = [Path(__file__), Path(ghidra.__file__), *sorted(scripts.glob('*.py'))]
h = hashlib.sha256()
for path in paths:
h.update(path.name.encode())
h.update(path.read_bytes())
h.update(json.dumps(provider_profile(env), sort_keys=True).encode())
return h.hexdigest()
def provider_profile(env):
provider = 'pyghidra' if env.get('AMS_USE_PYGHIDRA') else 'headless'
home = env.get('GHIDRA_INSTALL_DIR') or env.get('GHIDRA_HOME')
launcher = env.get('GHIDRA_HEADLESS')
if provider == 'headless' and launcher:
home = str(Path(launcher).resolve().parent.parent)
elif provider == 'headless' and not home:
import shutil
launcher = shutil.which('analyzeHeadless', path=env.get('PATH'))
if launcher:
home = str(Path(launcher).resolve().parent.parent)
properties = Path(home) / 'Ghidra' / 'application.properties' if home else None
try:
pyghidra_version = metadata.version('pyghidra')
except metadata.PackageNotFoundError:
pyghidra_version = None
return {'provider': provider, 'ghidra_home': str(Path(home).resolve()) if home else None,
'launcher': launcher, 'pyghidra_version': pyghidra_version,
'application_properties': properties.read_text() if properties and properties.is_file() else None,
'python': sys.version, 'java_home': env.get('JAVA_HOME')}
def atomic_json(path, data):
temp = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp')
try:
temp.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
os.replace(temp, path)
finally:
temp.unlink(missing_ok=True)
def validate_snapshot(path, sha):
data = json.loads(path.read_text())
if not isinstance(data, dict) or not isinstance(data.get('binary'), dict):
raise ValueError('Snapshot has no binary identity')
snapshot_sha = data['binary'].get('sha256')
if not isinstance(snapshot_sha, str) or snapshot_sha.lower() != sha:
raise ValueError('Snapshot SHA does not match manifest')
if not isinstance(data.get('types'), list) or not data['types']:
raise ValueError('Snapshot has no extracted types')
if not isinstance(data.get('methods'), list) or not data['methods']:
raise ValueError('Snapshot has no extracted methods')
return data
@contextmanager
def queue_lock(directory):
with (directory / '.queue.lock').open('a') as stream:
try:
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise RuntimeError('Another extraction queue is using this output directory')
try:
yield
finally:
fcntl.flock(stream, fcntl.LOCK_UN)
def execute_worker(source, output, log, timeout, env):
with log.open('wb') as stream:
process = subprocess.Popen(
[sys.executable, '-m', 'ams.acquire.batch', '--worker', str(source), str(output)],
stdout=stream, stderr=subprocess.STDOUT, env=env, start_new_session=True)
try:
code = process.wait(timeout=timeout)
except BaseException:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait()
raise
if code:
raise RuntimeError('Extractor exited with code ' + str(code))
# Engines and the scene modules (SCENE:DLLS=) that add script types; the extractor handles both.
EXTRACTED_SCOPES = ('engine', 'plugin')
def run_queue(manifest, out_dir, *, ghidra_home=None, timeout=1800, limit=None,
retry_failed=False, worker=execute_worker):
if timeout <= 0 or (limit is not None and limit < 0):
raise ValueError('Timeout must be positive; limit must be nonnegative')
data = json.loads(Path(manifest).read_text())
if data.get('schema') != 'binary-inventory/1':
raise ValueError('Expected binary-inventory/1 manifest')
builds = data['builds']
seen = set()
for build in builds:
sha = build['sha256']
if not isinstance(sha, str) or not re.fullmatch('[0-9a-f]{64}', sha) or sha in seen:
raise ValueError('Invalid or duplicate manifest SHA: ' + str(sha))
seen.add(sha)
out_dir = Path(out_dir).resolve()
out_dir.mkdir(parents=True, exist_ok=True)
with queue_lock(out_dir):
return _run(builds, out_dir, ghidra_home, timeout, limit, retry_failed, worker)
def _run(builds, directory, ghidra_home, timeout, limit, retry_failed, worker):
for name in ('states', 'logs'):
(directory / name).mkdir(exist_ok=True)
env = dict(os.environ)
if ghidra_home:
env.update(GHIDRA_INSTALL_DIR=str(ghidra_home), GHIDRA_HOME=str(ghidra_home), AMS_USE_PYGHIDRA='1')
extractor = fingerprint(env)
profile = provider_profile(env)
report = {'schema': 'extraction-batch/1', 'attempted': 0, 'succeeded': 0,
'pending': 0, 'skipped_success': 0, 'skipped_failed': 0, 'skipped_scope': 0, 'failures': []}
for build in builds:
if build.get('scope') not in EXTRACTED_SCOPES:
report['skipped_scope'] += 1
continue
sha = build['sha256']
state_path = directory / 'states' / (sha + '.json')
output = directory / (sha + '.snapshot.json')
old = json.loads(state_path.read_text()) if state_path.exists() else {}
same = old.get('extractor_fingerprint') == extractor
# Validate source even when resuming: a manifest is not proof of current contents.
source = None
for candidate in build.get('paths', []):
try:
if digest(candidate) == sha:
source = Path(candidate).resolve()
break
except OSError:
continue
if same and old.get('status') == 'success' and source:
try:
validate_snapshot(output, sha)
if digest(output) != old.get('snapshot_sha256'):
raise ValueError('Snapshot content changed since successful extraction')
except (OSError, ValueError, TypeError, AttributeError):
pass
else:
report['skipped_success'] += 1
continue
if same and old.get('status') in ('failed', 'running') and not retry_failed:
report['skipped_failed'] += 1
report['failures'].append({'sha256': sha, 'error': old.get('error', 'Interrupted attempt; use --retry-failed')})
continue
if limit is not None and report['attempted'] >= limit:
report['pending'] += 1
continue
attempt = old.get('attempts', 0) + 1
log = directory / 'logs' / (sha + '.' + str(attempt) + '.log')
temp = directory / (sha + '.' + uuid.uuid4().hex + '.pending.json')
state = {'sha256': sha, 'family': build.get('family'), 'extractor_fingerprint': extractor,
'status': 'running', 'attempts': attempt, 'source': str(source) if source else None,
'input_name': source.name if source else None, 'provider_profile': profile,
'started_at': datetime.now(timezone.utc).isoformat(), 'log': str(log)}
atomic_json(state_path, state)
report['attempted'] += 1
try:
if not source:
raise ValueError('No source path matches manifest SHA')
worker(source, temp, log, timeout, env)
snapshot = validate_snapshot(temp, sha)
if digest(source) != sha:
raise ValueError('Source changed during extraction')
snapshot['binary'].update(
library=source.name.split('__')[-1],
sources=build.get('paths', []),
games=sorted({Path(p).name.split('__')[0] for p in build.get('paths', []) if '__' in Path(p).name}),
extraction={'fingerprint': extractor, 'provider': profile,
'started_at': state['started_at'], 'coverage': 'partial'})
snapshot.setdefault('limitations', []).append(
'Automated heuristic extraction; missing entries are unknown, not proof of absence.')
atomic_json(temp, snapshot)
os.replace(temp, output)
state['status'] = 'success'
state['snapshot'] = str(output)
state['snapshot_sha256'] = digest(output)
state['coverage'] = {'types': len(snapshot['types']), 'methods': len(snapshot['methods']),
'dispatch': len(snapshot.get('method_dispatch') or [])}
report['succeeded'] += 1
except Exception as exc:
state.update(status='failed', error=str(exc))
if not log.exists():
log.write_text(str(exc) + '\n')
report['failures'].append({'sha256': sha, 'error': str(exc)})
finally:
temp.unlink(missing_ok=True)
state['finished_at'] = datetime.now(timezone.utc).isoformat()
atomic_json(state_path, state)
print(json.dumps({'sha256': sha, 'status': state['status'], 'attempt': attempt}), flush=True)
atomic_json(directory / 'failure-report.json', report)
return report
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--worker', nargs=2, metavar=('DLL', 'OUTPUT'), help=argparse.SUPPRESS)
parser.add_argument('--manifest', type=Path)
parser.add_argument('--out-dir', type=Path)
parser.add_argument('--ghidra-home', type=Path)
parser.add_argument('--timeout', type=float, default=1800)
parser.add_argument('--limit', type=int)
parser.add_argument('--retry-failed', action='store_true')
args = parser.parse_args()
if args.worker:
from .ghidra import run_extractor
run_extractor(*args.worker)
return
if not args.manifest or not args.out_dir:
parser.error('--manifest and --out-dir are required')
result = run_queue(args.manifest, args.out_dir, ghidra_home=args.ghidra_home,
timeout=args.timeout, limit=args.limit, retry_failed=args.retry_failed)
print(json.dumps(result))
if result['failures']:
raise SystemExit(1)
if __name__ == '__main__':
main()