feat(acquire): resumable batch extraction queue and binary inventory
Inventory classifies the DLL repository by SHA; the batch queue runs each engine build in a fresh PyGhidra process with a timeout, fingerprints the extractor environment and keeps per-attempt state and logs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
bddb5949a6
commit
b0013c5114
@@ -0,0 +1,253 @@
|
||||
"""Resumable, SHA-addressed engine extraction queue (POSIX)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
import fcntl
|
||||
import hashlib
|
||||
from importlib import metadata
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
|
||||
def digest(path):
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open('rb') as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b''):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def fingerprint(env=None):
|
||||
from . import ghidra
|
||||
env = os.environ if env is None else env
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
scripts = Path(env.get('AMS_GHIDRA_SCRIPTS', root / 'ghidra_scripts'))
|
||||
paths = [Path(__file__), Path(ghidra.__file__), *sorted(scripts.glob('*.py'))]
|
||||
h = hashlib.sha256()
|
||||
for path in paths:
|
||||
h.update(path.name.encode())
|
||||
h.update(path.read_bytes())
|
||||
h.update(json.dumps(provider_profile(env), sort_keys=True).encode())
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def provider_profile(env):
|
||||
provider = 'pyghidra' if env.get('AMS_USE_PYGHIDRA') else 'headless'
|
||||
home = env.get('GHIDRA_INSTALL_DIR') or env.get('GHIDRA_HOME')
|
||||
launcher = env.get('GHIDRA_HEADLESS')
|
||||
if provider == 'headless' and launcher:
|
||||
home = str(Path(launcher).resolve().parent.parent)
|
||||
elif provider == 'headless' and not home:
|
||||
import shutil
|
||||
launcher = shutil.which('analyzeHeadless', path=env.get('PATH'))
|
||||
if launcher:
|
||||
home = str(Path(launcher).resolve().parent.parent)
|
||||
properties = Path(home) / 'Ghidra' / 'application.properties' if home else None
|
||||
try:
|
||||
pyghidra_version = metadata.version('pyghidra')
|
||||
except metadata.PackageNotFoundError:
|
||||
pyghidra_version = None
|
||||
return {'provider': provider, 'ghidra_home': str(Path(home).resolve()) if home else None,
|
||||
'launcher': launcher, 'pyghidra_version': pyghidra_version,
|
||||
'application_properties': properties.read_text() if properties and properties.is_file() else None,
|
||||
'python': sys.version, 'java_home': env.get('JAVA_HOME')}
|
||||
|
||||
|
||||
def atomic_json(path, data):
|
||||
temp = path.with_name(path.name + '.' + uuid.uuid4().hex + '.tmp')
|
||||
try:
|
||||
temp.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
|
||||
os.replace(temp, path)
|
||||
finally:
|
||||
temp.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def validate_snapshot(path, sha):
|
||||
data = json.loads(path.read_text())
|
||||
if not isinstance(data, dict) or not isinstance(data.get('binary'), dict):
|
||||
raise ValueError('Snapshot has no binary identity')
|
||||
snapshot_sha = data['binary'].get('sha256')
|
||||
if not isinstance(snapshot_sha, str) or snapshot_sha.lower() != sha:
|
||||
raise ValueError('Snapshot SHA does not match manifest')
|
||||
if not isinstance(data.get('types'), list) or not data['types']:
|
||||
raise ValueError('Snapshot has no extracted types')
|
||||
if not isinstance(data.get('methods'), list) or not data['methods']:
|
||||
raise ValueError('Snapshot has no extracted methods')
|
||||
return data
|
||||
|
||||
|
||||
@contextmanager
|
||||
def queue_lock(directory):
|
||||
with (directory / '.queue.lock').open('a') as stream:
|
||||
try:
|
||||
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
raise RuntimeError('Another extraction queue is using this output directory')
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(stream, fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def execute_worker(source, output, log, timeout, env):
|
||||
with log.open('wb') as stream:
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, '-m', 'ams.acquire.batch', '--worker', str(source), str(output)],
|
||||
stdout=stream, stderr=subprocess.STDOUT, env=env, start_new_session=True)
|
||||
try:
|
||||
code = process.wait(timeout=timeout)
|
||||
except BaseException:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
process.wait()
|
||||
raise
|
||||
if code:
|
||||
raise RuntimeError('Extractor exited with code ' + str(code))
|
||||
|
||||
|
||||
def run_queue(manifest, out_dir, *, ghidra_home=None, timeout=1800, limit=None,
|
||||
retry_failed=False, worker=execute_worker):
|
||||
if timeout <= 0 or (limit is not None and limit < 0):
|
||||
raise ValueError('Timeout must be positive; limit must be nonnegative')
|
||||
data = json.loads(Path(manifest).read_text())
|
||||
if data.get('schema') != 'binary-inventory/1':
|
||||
raise ValueError('Expected binary-inventory/1 manifest')
|
||||
builds = data['builds']
|
||||
seen = set()
|
||||
for build in builds:
|
||||
sha = build['sha256']
|
||||
if not isinstance(sha, str) or not re.fullmatch('[0-9a-f]{64}', sha) or sha in seen:
|
||||
raise ValueError('Invalid or duplicate manifest SHA: ' + str(sha))
|
||||
seen.add(sha)
|
||||
out_dir = Path(out_dir).resolve()
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
with queue_lock(out_dir):
|
||||
return _run(builds, out_dir, ghidra_home, timeout, limit, retry_failed, worker)
|
||||
|
||||
|
||||
def _run(builds, directory, ghidra_home, timeout, limit, retry_failed, worker):
|
||||
for name in ('states', 'logs'):
|
||||
(directory / name).mkdir(exist_ok=True)
|
||||
env = dict(os.environ)
|
||||
if ghidra_home:
|
||||
env.update(GHIDRA_INSTALL_DIR=str(ghidra_home), GHIDRA_HOME=str(ghidra_home), AMS_USE_PYGHIDRA='1')
|
||||
extractor = fingerprint(env)
|
||||
profile = provider_profile(env)
|
||||
report = {'schema': 'extraction-batch/1', 'attempted': 0, 'succeeded': 0,
|
||||
'pending': 0, 'skipped_success': 0, 'skipped_failed': 0, 'skipped_scope': 0, 'failures': []}
|
||||
for build in builds:
|
||||
if build.get('scope') != 'engine':
|
||||
report['skipped_scope'] += 1
|
||||
continue
|
||||
sha = build['sha256']
|
||||
state_path = directory / 'states' / (sha + '.json')
|
||||
output = directory / (sha + '.snapshot.json')
|
||||
old = json.loads(state_path.read_text()) if state_path.exists() else {}
|
||||
same = old.get('extractor_fingerprint') == extractor
|
||||
# Validate source even when resuming: a manifest is not proof of current contents.
|
||||
source = None
|
||||
for candidate in build.get('paths', []):
|
||||
try:
|
||||
if digest(candidate) == sha:
|
||||
source = Path(candidate).resolve()
|
||||
break
|
||||
except OSError:
|
||||
continue
|
||||
if same and old.get('status') == 'success' and source:
|
||||
try:
|
||||
validate_snapshot(output, sha)
|
||||
if digest(output) != old.get('snapshot_sha256'):
|
||||
raise ValueError('Snapshot content changed since successful extraction')
|
||||
except (OSError, ValueError, TypeError, AttributeError):
|
||||
pass
|
||||
else:
|
||||
report['skipped_success'] += 1
|
||||
continue
|
||||
if same and old.get('status') in ('failed', 'running') and not retry_failed:
|
||||
report['skipped_failed'] += 1
|
||||
report['failures'].append({'sha256': sha, 'error': old.get('error', 'Interrupted attempt; use --retry-failed')})
|
||||
continue
|
||||
if limit is not None and report['attempted'] >= limit:
|
||||
report['pending'] += 1
|
||||
continue
|
||||
attempt = old.get('attempts', 0) + 1
|
||||
log = directory / 'logs' / (sha + '.' + str(attempt) + '.log')
|
||||
temp = directory / (sha + '.' + uuid.uuid4().hex + '.pending.json')
|
||||
state = {'sha256': sha, 'family': build.get('family'), 'extractor_fingerprint': extractor,
|
||||
'status': 'running', 'attempts': attempt, 'source': str(source) if source else None,
|
||||
'input_name': source.name if source else None, 'provider_profile': profile,
|
||||
'started_at': datetime.now(timezone.utc).isoformat(), 'log': str(log)}
|
||||
atomic_json(state_path, state)
|
||||
report['attempted'] += 1
|
||||
try:
|
||||
if not source:
|
||||
raise ValueError('No source path matches manifest SHA')
|
||||
worker(source, temp, log, timeout, env)
|
||||
snapshot = validate_snapshot(temp, sha)
|
||||
if digest(source) != sha:
|
||||
raise ValueError('Source changed during extraction')
|
||||
snapshot['binary'].update(
|
||||
library=source.name.split('__')[-1],
|
||||
sources=build.get('paths', []),
|
||||
games=sorted({Path(p).name.split('__')[0] for p in build.get('paths', []) if '__' in Path(p).name}),
|
||||
extraction={'fingerprint': extractor, 'provider': profile,
|
||||
'started_at': state['started_at'], 'coverage': 'partial'})
|
||||
snapshot.setdefault('limitations', []).append(
|
||||
'Automated heuristic extraction; missing entries are unknown, not proof of absence.')
|
||||
atomic_json(temp, snapshot)
|
||||
os.replace(temp, output)
|
||||
state['status'] = 'success'
|
||||
state['snapshot'] = str(output)
|
||||
state['snapshot_sha256'] = digest(output)
|
||||
state['coverage'] = {'types': len(snapshot['types']), 'methods': len(snapshot['methods']),
|
||||
'dispatch': len(snapshot.get('method_dispatch') or [])}
|
||||
report['succeeded'] += 1
|
||||
except Exception as exc:
|
||||
state.update(status='failed', error=str(exc))
|
||||
if not log.exists():
|
||||
log.write_text(str(exc) + '\n')
|
||||
report['failures'].append({'sha256': sha, 'error': str(exc)})
|
||||
finally:
|
||||
temp.unlink(missing_ok=True)
|
||||
state['finished_at'] = datetime.now(timezone.utc).isoformat()
|
||||
atomic_json(state_path, state)
|
||||
print(json.dumps({'sha256': sha, 'status': state['status'], 'attempt': attempt}), flush=True)
|
||||
atomic_json(directory / 'failure-report.json', report)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--worker', nargs=2, metavar=('DLL', 'OUTPUT'), help=argparse.SUPPRESS)
|
||||
parser.add_argument('--manifest', type=Path)
|
||||
parser.add_argument('--out-dir', type=Path)
|
||||
parser.add_argument('--ghidra-home', type=Path)
|
||||
parser.add_argument('--timeout', type=float, default=1800)
|
||||
parser.add_argument('--limit', type=int)
|
||||
parser.add_argument('--retry-failed', action='store_true')
|
||||
args = parser.parse_args()
|
||||
if args.worker:
|
||||
from .ghidra import run_extractor
|
||||
run_extractor(*args.worker)
|
||||
return
|
||||
if not args.manifest or not args.out_dir:
|
||||
parser.error('--manifest and --out-dir are required')
|
||||
result = run_queue(args.manifest, args.out_dir, ghidra_home=args.ghidra_home,
|
||||
timeout=args.timeout, limit=args.limit, retry_failed=args.retry_failed)
|
||||
print(json.dumps(result))
|
||||
if result['failures']:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+19
-1
@@ -20,6 +20,8 @@ then `analyzeHeadless` on PATH.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -44,7 +46,7 @@ def find_headless() -> str | None:
|
||||
env = os.environ.get("GHIDRA_HEADLESS")
|
||||
if env and os.path.isfile(env):
|
||||
return env
|
||||
home = os.environ.get("GHIDRA_HOME")
|
||||
home = os.environ.get("GHIDRA_INSTALL_DIR") or os.environ.get("GHIDRA_HOME")
|
||||
if home:
|
||||
for name in ("analyzeHeadless", "analyzeHeadless.bat"):
|
||||
cand = os.path.join(home, "support", name)
|
||||
@@ -123,9 +125,20 @@ def run_extractor_pyghidra(dll_path: str, out_path: str, *, script_dir: str | No
|
||||
|
||||
proj_dir = tempfile.mkdtemp(prefix="ams_pyghidra_")
|
||||
try:
|
||||
# Keep headless preferences/caches separate from the user's GUI profile.
|
||||
from pyghidra.launcher import PyGhidraLauncher, HeadlessPyGhidraLauncher
|
||||
if not PyGhidraLauncher.has_launched():
|
||||
launcher = HeadlessPyGhidraLauncher(verbose=False)
|
||||
runtime_home = os.path.join(proj_dir, "runtime-home")
|
||||
os.makedirs(runtime_home)
|
||||
launcher.vm_args.extend(["-Duser.home=" + runtime_home, "-Xmx2G"])
|
||||
launcher.start()
|
||||
pyghidra.run_script(
|
||||
dll_path, script_path,
|
||||
project_location=proj_dir, project_name="ams_" + uuid.uuid4().hex[:8],
|
||||
# Ghidra domain-file names reject Unicode combining marks and some
|
||||
# Polish letters. This changes only the internal project name.
|
||||
program_name=re.sub(r"[^A-Za-z0-9_.-]", "_", Path(dll_path).name),
|
||||
script_args=[out_path], analyze=True, verbose=False,
|
||||
)
|
||||
except Exception as e: # jpype/Ghidra errors aren't a tidy hierarchy
|
||||
@@ -135,4 +148,9 @@ def run_extractor_pyghidra(dll_path: str, out_path: str, *, script_dir: str | No
|
||||
|
||||
if not os.path.isfile(out_path):
|
||||
raise GhidraRunError("extractor produced no snapshot at {0} (PyGhidra path)".format(out_path))
|
||||
with open(out_path, encoding="utf-8") as stream:
|
||||
snapshot = json.load(stream)
|
||||
snapshot.setdefault("binary", {})["name"] = Path(dll_path).name
|
||||
with open(out_path, "w", encoding="utf-8") as stream:
|
||||
json.dump(snapshot, stream, indent=2, ensure_ascii=False)
|
||||
return out_path
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Inventory all DLL builds by content before scheduling expensive native analysis."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
|
||||
def inventory(roots: list[Path], snapshot_dir: Path | None = None) -> dict:
|
||||
extracted = set()
|
||||
if snapshot_dir:
|
||||
for path in snapshot_dir.glob("*.snapshot.json"):
|
||||
sha = json.loads(path.read_text()).get("binary", {}).get("sha256")
|
||||
if sha:
|
||||
extracted.add(sha.lower())
|
||||
builds = {}
|
||||
for root in roots:
|
||||
if not root.exists():
|
||||
raise ValueError("Source does not exist: " + str(root))
|
||||
paths = [root] if root.is_file() else sorted(root.rglob("*"))
|
||||
for path in paths:
|
||||
if path.suffix.lower() != ".dll" or not path.is_file():
|
||||
continue
|
||||
blob = path.read_bytes()
|
||||
sha = hashlib.sha256(blob).hexdigest()
|
||||
name = path.name.split("__")[-1].upper()
|
||||
if re.fullmatch(r"PIKLIB(?:61|71|72|8)\.DLL", name):
|
||||
family, scope = name.removesuffix(".DLL"), "engine"
|
||||
elif name.startswith("BLOOMOO"):
|
||||
family, scope = "BLOOMOO", "engine"
|
||||
elif name in ("WORLD.DLL", "KOLOROWANKA.DLL"):
|
||||
family, scope = name.removesuffix(".DLL"), "plugin"
|
||||
elif name == "SEKAI.DLL":
|
||||
family, scope = "SEKAI", "native_dependency"
|
||||
elif "PIKLIB" in name:
|
||||
family, scope = name.removesuffix(".DLL"), "excluded_early_engine"
|
||||
else:
|
||||
family, scope = name, "unclassified"
|
||||
build = builds.setdefault(sha, {"sha256": sha, "size": len(blob), "family": family,
|
||||
"scope": scope, "paths": [], "analysis": "snapshot_available" if sha in extracted else "pending",
|
||||
"suggested_snapshot": sha + ".snapshot.json"})
|
||||
absolute = str(path.resolve())
|
||||
if absolute not in build["paths"]:
|
||||
build["paths"].append(absolute)
|
||||
result = sorted(builds.values(), key=lambda b: (b["family"], b["sha256"]))
|
||||
return {"schema": "binary-inventory/1", "builds": result,
|
||||
"summary": {"files": sum(len(b["paths"]) for b in result), "unique": len(result),
|
||||
"by_scope": dict(Counter(b["scope"] for b in result))},
|
||||
"limitations": ["Scope is inferred from filenames; validate renamed or unclassified DLLs.",
|
||||
"snapshot_available means identity matches; it does not establish complete API extraction."]}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("roots", nargs="+", type=Path)
|
||||
p.add_argument("--snapshots", type=Path)
|
||||
p.add_argument("--out", type=Path, required=True)
|
||||
a = p.parse_args()
|
||||
result = inventory(a.roots, a.snapshots)
|
||||
a.out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(result["summary"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user