feat(acquire): resumable batch extraction queue and binary inventory
Inventory classifies the DLL repository by SHA; the batch queue runs each engine build in a fresh PyGhidra process with a timeout, fingerprints the extractor environment and keeps per-attempt state and logs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
bddb5949a6
commit
b0013c5114
@@ -0,0 +1,66 @@
|
||||
"""Inventory all DLL builds by content before scheduling expensive native analysis."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
|
||||
def inventory(roots: list[Path], snapshot_dir: Path | None = None) -> dict:
|
||||
extracted = set()
|
||||
if snapshot_dir:
|
||||
for path in snapshot_dir.glob("*.snapshot.json"):
|
||||
sha = json.loads(path.read_text()).get("binary", {}).get("sha256")
|
||||
if sha:
|
||||
extracted.add(sha.lower())
|
||||
builds = {}
|
||||
for root in roots:
|
||||
if not root.exists():
|
||||
raise ValueError("Source does not exist: " + str(root))
|
||||
paths = [root] if root.is_file() else sorted(root.rglob("*"))
|
||||
for path in paths:
|
||||
if path.suffix.lower() != ".dll" or not path.is_file():
|
||||
continue
|
||||
blob = path.read_bytes()
|
||||
sha = hashlib.sha256(blob).hexdigest()
|
||||
name = path.name.split("__")[-1].upper()
|
||||
if re.fullmatch(r"PIKLIB(?:61|71|72|8)\.DLL", name):
|
||||
family, scope = name.removesuffix(".DLL"), "engine"
|
||||
elif name.startswith("BLOOMOO"):
|
||||
family, scope = "BLOOMOO", "engine"
|
||||
elif name in ("WORLD.DLL", "KOLOROWANKA.DLL"):
|
||||
family, scope = name.removesuffix(".DLL"), "plugin"
|
||||
elif name == "SEKAI.DLL":
|
||||
family, scope = "SEKAI", "native_dependency"
|
||||
elif "PIKLIB" in name:
|
||||
family, scope = name.removesuffix(".DLL"), "excluded_early_engine"
|
||||
else:
|
||||
family, scope = name, "unclassified"
|
||||
build = builds.setdefault(sha, {"sha256": sha, "size": len(blob), "family": family,
|
||||
"scope": scope, "paths": [], "analysis": "snapshot_available" if sha in extracted else "pending",
|
||||
"suggested_snapshot": sha + ".snapshot.json"})
|
||||
absolute = str(path.resolve())
|
||||
if absolute not in build["paths"]:
|
||||
build["paths"].append(absolute)
|
||||
result = sorted(builds.values(), key=lambda b: (b["family"], b["sha256"]))
|
||||
return {"schema": "binary-inventory/1", "builds": result,
|
||||
"summary": {"files": sum(len(b["paths"]) for b in result), "unique": len(result),
|
||||
"by_scope": dict(Counter(b["scope"] for b in result))},
|
||||
"limitations": ["Scope is inferred from filenames; validate renamed or unclassified DLLs.",
|
||||
"snapshot_available means identity matches; it does not establish complete API extraction."]}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("roots", nargs="+", type=Path)
|
||||
p.add_argument("--snapshots", type=Path)
|
||||
p.add_argument("--out", type=Path, required=True)
|
||||
a = p.parse_args()
|
||||
result = inventory(a.roots, a.snapshots)
|
||||
a.out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps(result["summary"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user