feat(acquire): resumable batch extraction queue and binary inventory

Inventory classifies the DLL repository by SHA; the batch queue runs each
engine build in a fresh PyGhidra process with a timeout, fingerprints the
extractor environment and keeps per-attempt state and logs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Patryk Gensch
2026-10-24 18:24:15 +02:00
co-authored by Claude Opus 5.5
parent bddb5949a6
commit b0013c5114
9 changed files with 1164 additions and 1 deletions
+66
View File
@@ -0,0 +1,66 @@
"""Inventory all DLL builds by content before scheduling expensive native analysis."""
import argparse
from collections import Counter
import hashlib
import json
from pathlib import Path
import re
def inventory(roots: list[Path], snapshot_dir: Path | None = None) -> dict:
extracted = set()
if snapshot_dir:
for path in snapshot_dir.glob("*.snapshot.json"):
sha = json.loads(path.read_text()).get("binary", {}).get("sha256")
if sha:
extracted.add(sha.lower())
builds = {}
for root in roots:
if not root.exists():
raise ValueError("Source does not exist: " + str(root))
paths = [root] if root.is_file() else sorted(root.rglob("*"))
for path in paths:
if path.suffix.lower() != ".dll" or not path.is_file():
continue
blob = path.read_bytes()
sha = hashlib.sha256(blob).hexdigest()
name = path.name.split("__")[-1].upper()
if re.fullmatch(r"PIKLIB(?:61|71|72|8)\.DLL", name):
family, scope = name.removesuffix(".DLL"), "engine"
elif name.startswith("BLOOMOO"):
family, scope = "BLOOMOO", "engine"
elif name in ("WORLD.DLL", "KOLOROWANKA.DLL"):
family, scope = name.removesuffix(".DLL"), "plugin"
elif name == "SEKAI.DLL":
family, scope = "SEKAI", "native_dependency"
elif "PIKLIB" in name:
family, scope = name.removesuffix(".DLL"), "excluded_early_engine"
else:
family, scope = name, "unclassified"
build = builds.setdefault(sha, {"sha256": sha, "size": len(blob), "family": family,
"scope": scope, "paths": [], "analysis": "snapshot_available" if sha in extracted else "pending",
"suggested_snapshot": sha + ".snapshot.json"})
absolute = str(path.resolve())
if absolute not in build["paths"]:
build["paths"].append(absolute)
result = sorted(builds.values(), key=lambda b: (b["family"], b["sha256"]))
return {"schema": "binary-inventory/1", "builds": result,
"summary": {"files": sum(len(b["paths"]) for b in result), "unique": len(result),
"by_scope": dict(Counter(b["scope"] for b in result))},
"limitations": ["Scope is inferred from filenames; validate renamed or unclassified DLLs.",
"snapshot_available means identity matches; it does not establish complete API extraction."]}
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("roots", nargs="+", type=Path)
p.add_argument("--snapshots", type=Path)
p.add_argument("--out", type=Path, required=True)
a = p.parse_args()
result = inventory(a.roots, a.snapshots)
a.out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(result["summary"]))
if __name__ == "__main__":
main()