Initialized project with game ingesting

This commit is contained in:
Patryk Gensch
2026-08-11 00:31:56 +02:00
commit fd21d8c9a6
16 changed files with 1574 additions and 0 deletions
@@ -0,0 +1,54 @@
package pl.genschu.rexcatalog;
import com.badlogic.gdx.Application;
import com.badlogic.gdx.ApplicationAdapter;
import com.badlogic.gdx.Gdx;
import com.badlogic.gdx.backends.headless.HeadlessApplication;
import com.badlogic.gdx.backends.headless.HeadlessApplicationConfiguration;
/**
* Podnosi headless aplikację libGDX, żeby statyki {@code Gdx.app} / {@code Gdx.files}
* były zainicjalizowane.
*
* <p>Klasy z {@code :core}, których używa katalog, sięgają po {@code Gdx.app} wyłącznie
* do logowania (np. {@code CNVParser}, {@code GameIniResolver}, {@code ImageLoader}),
* ale bez tego wywołania dostalibyśmy NPE na pierwszym logu. Prawdziwy kontekst GL
* pozostaje niedostępny — {@code Gdx.graphics} jest {@code null} — więc katalog nie może
* dotykać {@code Texture} ani {@code FontLoader}; do rasteryzacji służy {@code Pixmap}.
*/
public final class Headless {
private static boolean started;
private Headless() {
}
public static synchronized void boot() {
if (started) {
return;
}
HeadlessApplicationConfiguration config = new HeadlessApplicationConfiguration();
new HeadlessApplication(new ApplicationAdapter() {
}, config);
// core loguje bardzo gadatliwie (CNVParser: log na każdy obiekt) — przy
// indeksowaniu interesują nas tylko błędy.
Gdx.app.setLogLevel(Application.LOG_ERROR);
started = true;
}
/**
* Zatrzymuje headless aplikację.
*
* <p>Bezwzględnie konieczne w CLI: {@code HeadlessApplication} tworzy wątek
* <b>nie-daemon</b>, więc bez tego wywołania JVM nie kończy pracy nawet po
* powrocie z {@code main()} — proces wisi w {@code DestroyJavaVM}, czekając
* w nieskończoność na pętlę libGDX.
*/
public static synchronized void shutdown() {
if (!started) {
return;
}
Gdx.app.exit();
started = false;
}
}
@@ -0,0 +1,185 @@
package pl.genschu.rexcatalog;
import pl.genschu.rexcatalog.db.Database;
import pl.genschu.rexcatalog.ingest.Ingestor;
import java.io.File;
import java.nio.file.Path;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
/**
* CLI katalogu. Krok 1: indeksowanie obrazów do SQLite.
* REST + MCP siadają później na tej samej bazie.
*/
public final class Main {
private static final String CORE_VERSION =
System.getProperty("catalog.coreVersion", "nieznana");
public static void main(String[] args) throws Exception {
if (args.length == 0) {
usage();
return;
}
Path dataDir = Path.of(System.getProperty("catalog.data", "data"));
try {
switch (args[0]) {
case "ingest" -> ingest(dataDir, args);
case "list" -> list(dataDir);
case "stats" -> stats(dataDir);
default -> usage();
}
} finally {
// bez tego proces wisi po zakończeniu pracy — patrz Headless.shutdown()
Headless.shutdown();
}
}
private static void ingest(Path dataDir, String[] args) throws Exception {
if (args.length < 2) {
System.err.println("ingest wymaga co najmniej jednej ścieżki");
return;
}
Headless.boot();
List<File> sources = new ArrayList<>();
for (int i = 1; i < args.length; i++) {
sources.addAll(Ingestor.expand(new File(args[i])));
}
if (sources.isEmpty()) {
System.err.println("Nie znaleziono nic do zaindeksowania.");
return;
}
System.out.printf("Indeksuję %d kopii (core %s)%n%n", sources.size(), CORE_VERSION);
try (Database db = Database.open(dataDir)) {
Ingestor ingestor = new Ingestor(db, CORE_VERSION);
int ok = 0;
int failed = 0;
for (File source : sources) {
long started = System.currentTimeMillis();
System.out.printf(" %-46s ", truncate(source.getName(), 46));
System.out.flush();
try {
Ingestor.Result result = ingestor.ingest(source);
long seconds = (System.currentTimeMillis() - started) / 1000;
if ("unreadable".equals(result.status())) {
System.out.printf("NIECZYTELNY: %s%n", result.statusNote());
failed++;
continue;
}
System.out.printf("%5d plików %2ds%n", result.fileCount(), seconds);
if (result.roots().isEmpty()) {
System.out.printf(" └─ %s%n", "installer".equals(result.status())
? "płyta instalacyjna — silnik nierozpoznawalny bez rozpakowania"
: "silnik nierozpoznany");
}
for (Ingestor.RootSummary root : result.roots()) {
System.out.printf(" └─ %-12s %-13s %s%n",
root.rootPath().isEmpty() ? "(korzeń)" : root.rootPath() + "/",
root.engine(),
root.edition() == null ? "" : truncate(root.edition(), 44));
}
ok++;
} catch (Exception e) {
System.out.printf("BŁĄD: %s%n", e.getMessage());
failed++;
}
}
System.out.printf("%nGotowe: %d zaindeksowanych, %d nieudanych.%n", ok, failed);
}
}
private static void list(Path dataDir) throws Exception {
try (Database db = Database.open(dataDir);
Statement st = db.connection().createStatement();
ResultSet rs = st.executeQuery("""
SELECT c.display_name, c.container, c.status, c.file_count,
g.root_path,
COALESCE(g.engine_override, g.engine_detected) AS engine,
MAX(CASE WHEN d.field = 'edition' THEN d.value END) AS edition,
MAX(CASE WHEN d.field = 'family' THEN d.value END) AS family
FROM copy c
LEFT JOIN game_root g ON g.copy_id = c.id
LEFT JOIN detection d ON d.copy_id = c.id
AND d.root_path = COALESCE(g.root_path, '')
GROUP BY c.id, g.id
ORDER BY family, edition, c.display_name, g.root_path
""")) {
System.out.printf("%-40s %-10s %-10s %-13s %-22s %s%n",
"KOPIA", "STATUS", "KORZEŃ", "SILNIK", "RODZINA", "WYDANIE");
while (rs.next()) {
String root = rs.getString("root_path");
System.out.printf("%-40s %-10s %-10s %-13s %-22s %s%n",
truncate(rs.getString("display_name"), 40),
rs.getString("status"),
root == null ? "" : (root.isEmpty() ? "/" : root + "/"),
nvl(rs.getString("engine")),
nvl(rs.getString("family")),
nvl(rs.getString("edition")));
}
}
}
private static void stats(Path dataDir) throws Exception {
try (Database db = Database.open(dataDir);
Statement st = db.connection().createStatement()) {
query(st, "kopie", "SELECT COUNT(*) FROM copy");
query(st, "pliki (wystąpienia)", "SELECT COUNT(*) FROM file");
query(st, "bloby (unikalne)", "SELECT COUNT(*) FROM blob");
query(st, "bloby współdzielone przez >1 kopię",
"SELECT COUNT(*) FROM (SELECT blob_sha1 FROM file GROUP BY blob_sha1 "
+ "HAVING COUNT(DISTINCT copy_id) > 1)");
query(st, "zaszyfrowane skrypty", "SELECT COUNT(*) FROM blob WHERE encrypted = 1");
System.out.println("\nFormaty:");
try (ResultSet rs = st.executeQuery(
"SELECT format, COUNT(*) n, SUM(size) bytes FROM blob "
+ "GROUP BY format ORDER BY n DESC LIMIT 15")) {
while (rs.next()) {
System.out.printf(" %-8s %6d %8.1f MB%n",
nvl(rs.getString("format")), rs.getInt("n"),
rs.getLong("bytes") / 1024.0 / 1024.0);
}
}
}
}
private static void query(Statement st, String label, String sql) throws Exception {
try (ResultSet rs = st.executeQuery(sql)) {
System.out.printf("%-38s %d%n", label + ":", rs.getLong(1));
}
}
private static String nvl(String value) {
return value == null ? "" : value;
}
private static String truncate(String value, int max) {
if (value == null) {
return "";
}
return value.length() <= max ? value : value.substring(0, max - 1) + "";
}
private static void usage() {
System.out.println("""
rex-catalog — katalog obrazów gier Aidem Media
ingest <ścieżka...> zaindeksuj obraz, katalog gry lub katalog kolekcji
list wypisz zaindeksowane kopie
stats statystyki bazy
Baza: -Dcatalog.data=<katalog> (domyślnie ./data)
""");
}
}
@@ -0,0 +1,194 @@
package pl.genschu.rexcatalog.db;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
/**
* Połączenie z bazą katalogu (SQLite) wraz z aplikacją schematu.
*
* <p>SQLite, nie Postgres: cały katalog to jeden plik obok cache'u pochodnych,
* więc przeniesienie kolekcji na inną maszynę to skopiowanie katalogu {@code data/}.
*/
public final class Database implements AutoCloseable {
private final Connection connection;
private Database(Connection connection) {
this.connection = connection;
}
public static Database open(Path dataDir) throws Exception {
Files.createDirectories(dataDir);
Path dbFile = dataDir.resolve("catalog.sqlite");
Connection conn = DriverManager.getConnection("jdbc:sqlite:" + dbFile.toAbsolutePath());
try (Statement st = conn.createStatement()) {
st.execute("PRAGMA journal_mode = WAL");
st.execute("PRAGMA synchronous = NORMAL");
st.execute("PRAGMA foreign_keys = ON");
}
Database db = new Database(conn);
db.applySchema();
return db;
}
public Connection connection() {
return connection;
}
private void applySchema() throws SQLException {
try (Statement st = connection.createStatement()) {
for (String ddl : SCHEMA) {
st.executeUpdate(ddl);
}
}
}
@Override
public void close() throws SQLException {
connection.close();
}
/**
* Model trójpoziomowy: {@code title} (dzieło) → {@code edition} (wydanie)
* → {@code copy} (konkretny plik na dysku).
*
* <p>Rozdział {@code file} / {@code blob} sprawia, że porównanie dwóch wydań
* jest operacją na zbiorach {@code blob_sha1}, a cache pochodnych (odszyfrowany
* CNV, PNG z IMG) można kluczować treścią — identyczny plik w trzech obrazach
* dekodujemy raz.
*/
private static final String[] SCHEMA = {
"""
CREATE TABLE IF NOT EXISTS title (
id INTEGER PRIMARY KEY,
slug TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
series TEXT,
publisher TEXT DEFAULT 'Aidem Media'
)
""",
"""
CREATE TABLE IF NOT EXISTS edition (
id INTEGER PRIMARY KEY,
title_id INTEGER REFERENCES title(id),
label TEXT,
dll_sha1 TEXT UNIQUE,
engine_detected TEXT,
engine_override TEXT,
compiler_detected TEXT,
compiler_override TEXT,
release_date TEXT,
distributor TEXT,
notes TEXT
)
""",
"""
CREATE TABLE IF NOT EXISTS edition_language (
edition_id INTEGER NOT NULL REFERENCES edition(id),
lang TEXT NOT NULL,
role TEXT NOT NULL CHECK (role IN ('audio', 'text', 'ui')),
PRIMARY KEY (edition_id, lang, role)
)
""",
// Kopia to nośnik, nie gra: jedna płyta bywa dwupackiem z dwoma drzewami
// gier, więc przypisanie do wydania wisi przy game_root, nie tutaj.
// status mówi, czy zawartość w ogóle dało się odczytać.
"""
CREATE TABLE IF NOT EXISTS copy (
id INTEGER PRIMARY KEY,
container TEXT NOT NULL CHECK (container IN ('iso', 'dir', 'zip')),
status TEXT NOT NULL DEFAULT 'ok'
CHECK (status IN ('ok', 'installer', 'unreadable')),
status_note TEXT,
path TEXT NOT NULL UNIQUE,
display_name TEXT,
size INTEGER,
sha256 TEXT,
file_count INTEGER,
ingested_at TEXT,
core_version TEXT,
source_kind TEXT NOT NULL DEFAULT 'nieznane'
CHECK (source_kind IN ('wlasny_zgraj', 'archive_org',
'redump', 'inne', 'nieznane')),
source_url TEXT,
source_ref TEXT,
acquired_at TEXT,
rip_tool TEXT,
media_note TEXT,
verified INTEGER NOT NULL DEFAULT 0,
notes TEXT
)
""",
// Poziom pośredni copy → game_root → edition. root_path to '' dla zwykłego
// obrazu, a np. 'herkules' / 'odyseusz' dla dwupacka — dwa drzewa gier na
// jednej płycie, każde z własnym silnikiem i własnym wydaniem (a więc i tytułem).
"""
CREATE TABLE IF NOT EXISTS game_root (
id INTEGER PRIMARY KEY,
copy_id INTEGER NOT NULL REFERENCES copy(id) ON DELETE CASCADE,
root_path TEXT NOT NULL,
edition_id INTEGER REFERENCES edition(id),
engine_detected TEXT,
engine_override TEXT,
UNIQUE (copy_id, root_path)
)
""",
"""
CREATE TABLE IF NOT EXISTS blob (
sha1 TEXT PRIMARY KEY,
size INTEGER NOT NULL,
format TEXT,
encrypted INTEGER NOT NULL DEFAULT 0
)
""",
// path jest kanoniczny (małe litery) — IsoFileSystem normalizuje ścieżki,
// a LocalFileSystem zachowuje wielkość liter z dysku. Bez wspólnej postaci
// porównanie obrazu z rozpakowanym katalogiem rozjeżdża się na ścieżkach.
// path_raw zachowuje oryginał, o ile kontener go zna.
"""
CREATE TABLE IF NOT EXISTS file (
copy_id INTEGER NOT NULL REFERENCES copy(id) ON DELETE CASCADE,
path TEXT NOT NULL,
path_raw TEXT,
blob_sha1 TEXT NOT NULL REFERENCES blob(sha1),
PRIMARY KEY (copy_id, path)
)
""",
"CREATE INDEX IF NOT EXISTS idx_file_blob ON file(blob_sha1)",
"CREATE INDEX IF NOT EXISTS idx_blob_format ON blob(format)",
// Każdy detektor dopisuje wiersz z dowodem, zamiast nadpisywać pole.
// Dorzucenie detektora kompilera to nowy wiersz, nie migracja schematu.
"""
CREATE TABLE IF NOT EXISTS detection (
id INTEGER PRIMARY KEY,
copy_id INTEGER NOT NULL REFERENCES copy(id) ON DELETE CASCADE,
root_path TEXT NOT NULL DEFAULT '',
field TEXT NOT NULL,
value TEXT,
confidence REAL,
evidence TEXT,
detector TEXT,
core_version TEXT,
detected_at TEXT
)
""",
"CREATE INDEX IF NOT EXISTS idx_detection_copy ON detection(copy_id, field)",
// tool_version pozwala unieważnić tylko te artefakty, których parser się zmienił
"""
CREATE TABLE IF NOT EXISTS artifact (
blob_sha1 TEXT NOT NULL REFERENCES blob(sha1),
kind TEXT NOT NULL,
path TEXT NOT NULL,
tool_version TEXT NOT NULL,
created_at TEXT,
PRIMARY KEY (blob_sha1, kind)
)
"""
};
}
@@ -0,0 +1,208 @@
package pl.genschu.rexcatalog.ingest;
import pl.genschu.bloomooemulator.logic.GameFamilies;
import pl.genschu.bloomooemulator.logic.KnownHashes;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
/**
* Rozpoznaje silnik w każdym korzeniu gry znalezionym w kopii.
*
* <p>Kolekcja zawiera co najmniej trzy różne silniki (BlooMoo, Piklib, autorski
* KI Engine z <i>Tajemnicy Trzeciego Wymiaru</i>), a płyty bywają dwupackami
* z osobnym drzewem gry na katalog. Dlatego detekcja jest łańcuchem sygnatur
* uruchamianym per korzeń, a nie pojedynczym sprawdzeniem nazwy biblioteki.
*
* <p>Tablice hashy pochodzą z {@code :core} ({@link KnownHashes}, {@link GameFamilies}),
* więc wydania Piklib/BlooMoo aktualizują się razem z emulatorem. Silniki spoza tej
* rodziny dostają samą nazwę — konkretne wydanie ustala człowiek przez override.
*/
public final class EngineDetector {
private EngineDetector() {
}
/** Ustalenie detektora wraz z dowodem, na którym się opiera. */
public record Fact(String field, String value, double confidence, String evidence) {
}
/** Jeden korzeń gry w kopii — {@code ""} dla zwykłego obrazu, {@code "herkules"} dla dwupacka. */
public record Root(String rootPath, String engine, List<Fact> facts) {
}
/**
* Znajduje korzenie gier i rozpoznaje ich silniki.
*
* @param blobSha1ByPath ścieżki w kontenerze (oryginalna wielkość liter) → SHA-1
*/
public static List<Root> detect(Map<String, String> blobSha1ByPath) {
// ścieżki w postaci kanonicznej, żeby sygnatury nie zależały od kontenera
Map<String, String> byCanonicalPath = new java.util.LinkedHashMap<>();
for (Map.Entry<String, String> e : blobSha1ByPath.entrySet()) {
byCanonicalPath.put(e.getKey().toLowerCase(Locale.ROOT), e.getValue());
}
List<Root> roots = new ArrayList<>();
for (String candidate : candidateRoots(byCanonicalPath)) {
Root root = matchAt(byCanonicalPath, candidate);
if (root != null) {
roots.add(root);
}
}
return roots;
}
/** Korzeń kontenera plus pierwszy poziom katalogów — tam mieszkają drzewa dwupacków. */
private static Set<String> candidateRoots(Map<String, String> files) {
Set<String> roots = new LinkedHashSet<>();
roots.add("");
for (String path : files.keySet()) {
int slash = path.indexOf('/');
if (slash > 0) {
roots.add(path.substring(0, slash));
}
}
return roots;
}
private static Root matchAt(Map<String, String> files, String root) {
String prefix = root.isEmpty() ? "" : root + "/";
// nazwy plików leżących bezpośrednio w tym korzeniu
Map<String, String> direct = new java.util.LinkedHashMap<>();
for (Map.Entry<String, String> e : files.entrySet()) {
String path = e.getKey();
if (!path.startsWith(prefix)) {
continue;
}
String rest = path.substring(prefix.length());
if (rest.indexOf('/') < 0) {
direct.put(rest, e.getValue());
}
}
if (direct.isEmpty()) {
return null;
}
Root bloomooOrPiklib = matchAidem(direct, root, prefix);
if (bloomooOrPiklib != null) {
return bloomooOrPiklib;
}
Root ki = matchKiEngine(direct, files, root, prefix);
if (ki != null) {
return ki;
}
return matchUnity(direct, files, root, prefix);
}
/** BlooMoo / Piklib — jedyne silniki, dla których znamy tablice hashy wydań. */
private static Root matchAidem(Map<String, String> direct, String root, String prefix) {
for (Map.Entry<String, String> e : direct.entrySet()) {
String name = e.getKey();
if (!name.matches("bloomoodll\\.dll|piklib\\d+\\.dll")) {
continue;
}
String hash = e.getValue().toUpperCase(Locale.ROOT);
String evidence = prefix + name + " sha1=" + hash;
String engine = name.equals("bloomoodll.dll") ? "BlooMoo" : piklibVersion(name);
String gameName = KnownHashes.checkHash(hash);
String family = GameFamilies.familyFor(hash, gameName);
boolean known = !"Nieznana gra".equals(gameName);
List<Fact> facts = new ArrayList<>();
facts.add(new Fact("engine", engine, 1.0, evidence));
facts.add(new Fact("engine_dll_sha1", hash, 1.0, evidence));
facts.add(new Fact("edition", gameName, known ? 1.0 : 0.0, evidence));
if (family != null) {
facts.add(new Fact("family", family, known ? 1.0 : 0.5, evidence));
}
return new Root(root, engine, facts);
}
return null;
}
private static String piklibVersion(String dllName) {
String version = dllName.substring("piklib".length(), dllName.length() - ".dll".length());
if (version.length() > 1) {
version = version.charAt(0) + "." + version.substring(1);
}
return "Piklib v" + version;
}
/**
* Autorski silnik 3D Aidem Media. Nazwa „KI Engine" pochodzi z analizy {@code ki.exe}
* w Ghidrze; niezależnie potwierdzają ją własne formaty assetów o prefiksie KI
* ({@code .kimodel}, {@code .kianim}).
*/
private static Root matchKiEngine(Map<String, String> direct, Map<String, String> files,
String root, String prefix) {
boolean hasExe = direct.containsKey("ki.exe");
boolean hasRenderer = direct.containsKey("rex3d.dll");
if (!hasExe && !hasRenderer) {
return null;
}
List<String> markers = new ArrayList<>();
if (hasExe) {
markers.add(prefix + "ki.exe sha1=" + direct.get("ki.exe"));
}
if (hasRenderer) {
markers.add(prefix + "rex3d.dll sha1=" + direct.get("rex3d.dll"));
}
long kiAssets = files.keySet().stream()
.filter(p -> p.startsWith(prefix))
.filter(p -> p.endsWith(".kimodel") || p.endsWith(".kianim"))
.count();
if (kiAssets > 0) {
markers.add(kiAssets + " assetów .kimodel/.kianim");
}
double confidence = (hasExe && hasRenderer) ? 0.9 : 0.6;
if (kiAssets > 0) {
confidence = Math.min(1.0, confidence + 0.1);
}
String evidence = String.join(", ", markers);
return new Root(root, "KI Engine", List.of(
new Fact("engine", "KI Engine", confidence, evidence)));
}
/** Unity — rozpoznawane po runtime'u albo po katalogu {@code *_Data/}. */
private static Root matchUnity(Map<String, String> direct, Map<String, String> files,
String root, String prefix) {
if (direct.containsKey("unityplayer.dll")) {
return new Root(root, "Unity", List.of(new Fact(
"engine", "Unity", 0.9, prefix + "unityplayer.dll")));
}
String managed = files.keySet().stream()
.filter(p -> p.startsWith(prefix))
.filter(p -> p.contains("_data/managed/assembly-csharp.dll"))
.findFirst().orElse(null);
if (managed != null) {
return new Root(root, "Unity", List.of(new Fact("engine", "Unity", 0.9, managed)));
}
return null;
}
/**
* Rozpoznaje płytę instalacyjną: instalator i archiwa zamiast plików gry.
* Nie da się z niej nic wyczytać bez rozpakowania, więc katalog powinien to
* powiedzieć wprost, zamiast raportować „nieznana gra".
*/
public static boolean looksLikeInstallerDisc(Map<String, String> blobSha1ByPath) {
Set<String> names = new LinkedHashSet<>();
for (String path : blobSha1ByPath.keySet()) {
names.add(path.toLowerCase(Locale.ROOT));
}
boolean installer = names.stream().anyMatch(n ->
n.equals("setup.exe") || n.startsWith("instmsi") || n.endsWith(".msi"));
boolean archives = names.stream().anyMatch(n -> n.endsWith(".cab"));
return installer && (archives || names.size() < 64);
}
}
@@ -0,0 +1,77 @@
package pl.genschu.rexcatalog.ingest;
import java.util.Locale;
/**
* Rozpoznaje format pliku po nagłówku, z odwrotem do rozszerzenia.
*
* <p>Magic bytes biorą się z dokumentacji formatów w repo emulatora
* ({@code docs/pl/formats/}) i z podglądu realnych plików.
*/
public final class FormatSniffer {
private FormatSniffer() {
}
/** Ile bajtów początku pliku wystarczy do rozpoznania. */
public static final int PROBE_BYTES = 16;
public record Result(String format, boolean encrypted) {
}
public static Result sniff(String path, byte[] head, int headLength) {
boolean encrypted = startsWith(head, headLength, "{<");
String byMagic = byMagic(head, headLength);
if (byMagic != null) {
return new Result(byMagic, encrypted);
}
return new Result(byExtension(path, encrypted), encrypted);
}
private static String byMagic(byte[] head, int len) {
if (startsWith(head, len, "PIK\0")) {
return "IMG";
}
if (startsWith(head, len, "NVP\0")) {
return "ANN";
}
if (startsWith(head, len, "RIFF")) {
return "WAV";
}
if (startsWith(head, len, "MZ")) {
return "PE";
}
if (startsWith(head, len, "{<")) {
// zaszyfrowany skrypt — konkretny rodzaj rozstrzyga rozszerzenie
return null;
}
return null;
}
private static String byExtension(String path, boolean encrypted) {
int dot = path.lastIndexOf('.');
if (dot < 0 || dot == path.length() - 1) {
return encrypted ? "SCRIPT" : "BIN";
}
String ext = path.substring(dot + 1).toUpperCase(Locale.ROOT);
return switch (ext) {
case "CNV", "DEF", "SEQ", "CLASS", "INI" -> ext;
case "DTA", "ARR", "IMG", "ANN", "FNT", "SEK", "INE", "MAR" -> ext;
case "WAV", "MID", "AVI", "BMP", "TXT", "DLL", "EXE" -> ext;
default -> encrypted ? "SCRIPT" : ext;
};
}
private static boolean startsWith(byte[] data, int len, String prefix) {
if (len < prefix.length()) {
return false;
}
for (int i = 0; i < prefix.length(); i++) {
if ((data[i] & 0xFF) != prefix.charAt(i)) {
return false;
}
}
return true;
}
}
@@ -0,0 +1,375 @@
package pl.genschu.rexcatalog.ingest;
import pl.genschu.bloomooemulator.engine.filesystem.AssetSourceDispatcher;
import pl.genschu.bloomooemulator.engine.filesystem.IFileSystem;
import pl.genschu.rexcatalog.db.Database;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.security.MessageDigest;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
/**
* Indeksuje pojedynczą kopię (ISO / ZIP / rozpakowany katalog) do bazy katalogu.
*
* <p>Nic nie rozpakowuje na dysk — czyta przez {@link IFileSystem} z {@code :core},
* czyli tę samą implementację ISO9660/Joliet, której używa emulator przy uruchamianiu gry.
* Kopia, której nie da się odczytać, i tak trafia do bazy — ze statusem i powodem,
* bo „mam ten obraz, ale go nie otwieram" to też fakt wart skatalogowania.
*/
public final class Ingestor {
private final Database db;
private final String coreVersion;
public Ingestor(Database db, String coreVersion) {
this.db = db;
this.coreVersion = coreVersion;
}
public record RootSummary(String rootPath, String engine, String edition) {
}
public record Result(String path, String container, String status, String statusNote,
int fileCount, long bytes, List<RootSummary> roots) {
}
public Result ingest(File source) throws Exception {
String container = containerType(source);
String containerSha256 = source.isDirectory() ? null : sha256OfFile(source);
Map<String, String> sha1ByPath = new LinkedHashMap<>();
Map<String, Long> sizeByPath = new LinkedHashMap<>();
Map<String, FormatSniffer.Result> formatByPath = new LinkedHashMap<>();
long totalBytes;
try {
IFileSystem fs = AssetSourceDispatcher.openAssets(source);
totalBytes = walk(fs, "", sha1ByPath, sizeByPath, formatByPath);
} catch (Exception e) {
// np. Wojna Trojańska.iso — obraz UDF, którego IsoFileSystem nie parsuje
String note = e.getMessage() == null ? e.toString() : e.getMessage();
persistUnreadable(source, container, containerSha256, note);
return new Result(source.getPath(), container, "unreadable", note, 0, 0, List.of());
}
List<EngineDetector.Root> roots = EngineDetector.detect(sha1ByPath);
String status = "ok";
String statusNote = null;
if (roots.isEmpty() && EngineDetector.looksLikeInstallerDisc(sha1ByPath)) {
status = "installer";
statusNote = "płyta instalacyjna — pliki gry spakowane w instalatorze";
}
long containerSize = source.isDirectory() ? totalBytes : source.length();
persist(source, container, status, statusNote, containerSize, containerSha256,
sha1ByPath, sizeByPath, formatByPath, roots);
List<RootSummary> summaries = new ArrayList<>();
for (EngineDetector.Root root : roots) {
summaries.add(new RootSummary(root.rootPath(), root.engine(), editionOf(root)));
}
return new Result(source.getPath(), container, status, statusNote,
sha1ByPath.size(), totalBytes, summaries);
}
private static String editionOf(EngineDetector.Root root) {
return root.facts().stream()
.filter(f -> f.field().equals("edition"))
.map(EngineDetector.Fact::value)
.findFirst()
.orElse(null);
}
private static String containerType(File source) throws IOException {
if (source.isDirectory()) {
return "dir";
}
String name = source.getName().toLowerCase(Locale.ROOT);
if (name.endsWith(".iso")) {
return "iso";
}
if (name.endsWith(".zip")) {
return "zip";
}
throw new IOException("Nieobsługiwane źródło: " + source);
}
/** Rekurencyjnie liczy SHA-1 każdego pliku i wykrywa format w jednym przebiegu odczytu. */
private long walk(IFileSystem fs, String dir,
Map<String, String> sha1ByPath,
Map<String, Long> sizeByPath,
Map<String, FormatSniffer.Result> formatByPath) throws Exception {
String[] entries = fs.list(dir);
if (entries == null) {
return 0;
}
long bytes = 0;
for (String entry : entries) {
String path = dir.isEmpty() ? entry : dir + "/" + entry;
if (fs.isDirectory(path)) {
bytes += walk(fs, path, sha1ByPath, sizeByPath, formatByPath);
continue;
}
MessageDigest sha1 = MessageDigest.getInstance("SHA-1");
byte[] head = new byte[FormatSniffer.PROBE_BYTES];
int headLength = 0;
long size = 0;
try (InputStream in = fs.open(path)) {
byte[] buffer = new byte[64 * 1024];
int read;
while ((read = in.read(buffer)) != -1) {
sha1.update(buffer, 0, read);
size += read;
if (headLength < head.length) {
int copy = Math.min(head.length - headLength, read);
System.arraycopy(buffer, 0, head, headLength, copy);
headLength += copy;
}
}
} catch (IOException e) {
System.err.println(" ! nie udało się odczytać " + path + ": " + e.getMessage());
continue;
}
sha1ByPath.put(path, hex(sha1.digest()));
sizeByPath.put(path, size);
formatByPath.put(path, FormatSniffer.sniff(path, head, headLength));
bytes += size;
}
return bytes;
}
private void persistUnreadable(File source, String container, String sha256, String note)
throws Exception {
Connection conn = db.connection();
boolean auto = conn.getAutoCommit();
conn.setAutoCommit(false);
try {
upsertCopy(conn, source, container, "unreadable", note,
source.length(), sha256, 0);
conn.commit();
} catch (Exception e) {
conn.rollback();
throw e;
} finally {
conn.setAutoCommit(auto);
}
}
private void persist(File source, String container, String status, String statusNote,
long size, String sha256,
Map<String, String> sha1ByPath,
Map<String, Long> sizeByPath,
Map<String, FormatSniffer.Result> formatByPath,
List<EngineDetector.Root> roots) throws Exception {
Connection conn = db.connection();
boolean previousAutoCommit = conn.getAutoCommit();
conn.setAutoCommit(false);
try {
long copyId = upsertCopy(conn, source, container, status, statusNote,
size, sha256, sha1ByPath.size());
try (Statement st = conn.createStatement()) {
st.executeUpdate("DELETE FROM file WHERE copy_id = " + copyId);
st.executeUpdate("DELETE FROM detection WHERE copy_id = " + copyId);
st.executeUpdate("DELETE FROM game_root WHERE copy_id = " + copyId);
}
try (PreparedStatement blobStmt = conn.prepareStatement("""
INSERT INTO blob (sha1, size, format, encrypted) VALUES (?, ?, ?, ?)
ON CONFLICT(sha1) DO UPDATE SET
format = excluded.format, encrypted = excluded.encrypted
""");
PreparedStatement fileStmt = conn.prepareStatement(
"INSERT INTO file (copy_id, path, path_raw, blob_sha1) VALUES (?, ?, ?, ?)")) {
for (Map.Entry<String, String> entry : sha1ByPath.entrySet()) {
String path = entry.getKey();
String sha1 = entry.getValue();
FormatSniffer.Result format = formatByPath.get(path);
blobStmt.setString(1, sha1);
blobStmt.setLong(2, sizeByPath.get(path));
blobStmt.setString(3, format.format());
blobStmt.setInt(4, format.encrypted() ? 1 : 0);
blobStmt.addBatch();
fileStmt.setLong(1, copyId);
fileStmt.setString(2, canonical(path));
fileStmt.setString(3, path);
fileStmt.setString(4, sha1);
fileStmt.addBatch();
}
blobStmt.executeBatch();
fileStmt.executeBatch();
}
String now = Instant.now().toString();
try (PreparedStatement rootStmt = conn.prepareStatement(
"INSERT INTO game_root (copy_id, root_path, engine_detected) VALUES (?, ?, ?)");
PreparedStatement detStmt = conn.prepareStatement("""
INSERT INTO detection
(copy_id, root_path, field, value, confidence, evidence,
detector, core_version, detected_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""")) {
for (EngineDetector.Root root : roots) {
rootStmt.setLong(1, copyId);
rootStmt.setString(2, root.rootPath());
rootStmt.setString(3, root.engine());
rootStmt.addBatch();
for (EngineDetector.Fact fact : root.facts()) {
detStmt.setLong(1, copyId);
detStmt.setString(2, root.rootPath());
detStmt.setString(3, fact.field());
detStmt.setString(4, fact.value());
detStmt.setDouble(5, fact.confidence());
detStmt.setString(6, fact.evidence());
detStmt.setString(7, "EngineDetector");
detStmt.setString(8, coreVersion);
detStmt.setString(9, now);
detStmt.addBatch();
}
}
rootStmt.executeBatch();
detStmt.executeBatch();
}
conn.commit();
} catch (Exception e) {
conn.rollback();
throw e;
} finally {
conn.setAutoCommit(previousAutoCommit);
}
}
private long upsertCopy(Connection conn, File source, String container,
String status, String statusNote,
long size, String sha256, int fileCount) throws Exception {
String path = source.getAbsolutePath();
try (PreparedStatement st = conn.prepareStatement("""
INSERT INTO copy (container, status, status_note, path, display_name,
size, sha256, file_count, ingested_at, core_version)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(path) DO UPDATE SET
container = excluded.container, status = excluded.status,
status_note = excluded.status_note, size = excluded.size,
sha256 = excluded.sha256, file_count = excluded.file_count,
ingested_at = excluded.ingested_at, core_version = excluded.core_version
""")) {
st.setString(1, container);
st.setString(2, status);
st.setString(3, statusNote);
st.setString(4, path);
st.setString(5, source.getName());
st.setLong(6, size);
st.setString(7, sha256);
st.setInt(8, fileCount);
st.setString(9, Instant.now().toString());
st.setString(10, coreVersion);
st.executeUpdate();
}
try (PreparedStatement st = conn.prepareStatement("SELECT id FROM copy WHERE path = ?")) {
st.setString(1, path);
try (ResultSet rs = st.executeQuery()) {
rs.next();
return rs.getLong(1);
}
}
}
private static String sha256OfFile(File file) throws Exception {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
try (InputStream in = Files.newInputStream(file.toPath())) {
byte[] buffer = new byte[1 << 20];
int read;
while ((read = in.read(buffer)) != -1) {
digest.update(buffer, 0, read);
}
}
return hex(digest.digest());
}
/**
* Kanoniczna postać ścieżki w kontenerze — małe litery i separator '/'.
* Zgodna z {@code IsoFileSystem.normalize()}, więc obraz i rozpakowany katalog
* tej samej gry dają porównywalne ścieżki.
*/
private static String canonical(String path) {
return path.replace('\\', '/').toLowerCase(Locale.ROOT);
}
private static String hex(byte[] bytes) {
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
}
/**
* Rozwija wskazaną ścieżkę na listę kopii do zaindeksowania: pojedynczy obraz,
* katalog gry (rozpoznany po bibliotece silnika w korzeniu) albo katalog kolekcji,
* którego zawartość trzeba przejrzeć.
*/
public static List<File> expand(File path) {
List<File> out = new ArrayList<>();
if (!path.isDirectory()) {
out.add(path);
return out;
}
if (looksLikeGameDirectory(path)) {
out.add(path);
return out;
}
File[] children = path.listFiles();
if (children == null) {
return out;
}
Arrays.sort(children);
for (File child : children) {
String name = child.getName().toLowerCase(Locale.ROOT);
if (child.isDirectory() && looksLikeGameDirectory(child)) {
out.add(child);
} else if (child.isFile() && (name.endsWith(".iso") || name.endsWith(".zip"))) {
out.add(child);
}
}
return out;
}
private static boolean looksLikeGameDirectory(File dir) {
File[] children = dir.listFiles();
if (children == null) {
return false;
}
for (File child : children) {
if (!child.isFile()) {
continue;
}
String name = child.getName().toLowerCase(Locale.ROOT);
if (name.matches("bloomoodll\\.dll|piklib\\d+\\.dll") || name.equals("ki.exe")) {
return true;
}
}
return false;
}
}