diff --git a/src/main/java/pl/genschu/rexcatalog/Main.java b/src/main/java/pl/genschu/rexcatalog/Main.java index 45c15f0..1729606 100644 --- a/src/main/java/pl/genschu/rexcatalog/Main.java +++ b/src/main/java/pl/genschu/rexcatalog/Main.java @@ -1,13 +1,20 @@ package pl.genschu.rexcatalog; +import pl.genschu.rexcatalog.catalog.Editor; +import pl.genschu.rexcatalog.catalog.Promoter; import pl.genschu.rexcatalog.db.Database; import pl.genschu.rexcatalog.ingest.Ingestor; +import pl.genschu.rexcatalog.ingest.MetadataDetector; +import pl.genschu.rexcatalog.script.ScriptDecoder; +import pl.genschu.rexcatalog.script.ScriptSearch; import java.io.File; import java.nio.file.Path; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** @@ -31,6 +38,14 @@ public final class Main { case "ingest" -> ingest(dataDir, args); case "list" -> list(dataDir); case "stats" -> stats(dataDir); + case "decode" -> decode(dataDir, args); + case "analyze" -> analyze(dataDir); + case "promote" -> promote(dataDir); + case "titles" -> titles(dataDir); + case "set" -> set(dataDir, args); + case "lang" -> lang(dataDir, args); + case "find" -> find(dataDir, args); + case "cat" -> cat(dataDir, args); default -> usage(); } } finally { @@ -97,6 +112,255 @@ public final class Main { } } + private static void decode(Path dataDir, String[] args) throws Exception { + boolean force = args.length > 1 && args[1].equals("--force"); + Headless.boot(); + + try (Database db = Database.open(dataDir)) { + ScriptDecoder decoder = new ScriptDecoder(db, dataDir, CORE_VERSION); + int pending = force ? -1 : decoder.pendingCount(); + if (pending == 0) { + System.out.println("Cache skryptów aktualny (core " + CORE_VERSION + ")."); + return; + } + System.out.printf("Odszyfrowuję skrypty (core %s)%s%n", CORE_VERSION, + force ? ", wymuszone od nowa" : ""); + + long started = System.currentTimeMillis(); + ScriptDecoder.Stats stats = decoder.run(force); + long seconds = (System.currentTimeMillis() - started) / 1000; + + System.out.printf("%nOdszyfrowanych: %d powtórzeń pominiętych: %d błędów: %d (%ds)%n", + stats.decoded(), stats.skipped(), stats.failed(), seconds); + System.out.printf("W indeksie: %d skryptów%n", + new ScriptSearch(db, dataDir).indexedCount()); + } + } + + private static void analyze(Path dataDir) throws Exception { + try (Database db = Database.open(dataDir)) { + List results = + new MetadataDetector(db, dataDir, CORE_VERSION).run(); + if (results.isEmpty()) { + System.out.println("Brak plików application.def — uruchom najpierw: ingest, decode"); + return; + } + System.out.printf("Analizuję application.def w %d korzeniach%n%n", results.size()); + + for (MetadataDetector.RootResult result : results) { + String where = result.rootPath().isEmpty() + ? result.copyName() + : result.copyName() + " / " + result.rootPath(); + System.out.println(" " + truncate(where, 60)); + if (result.note() != null) { + System.out.println(" ! " + result.note()); + continue; + } + for (MetadataDetector.Fact fact : result.facts()) { + if (fact.field().equals("app_def_sha1")) { + continue; + } + System.out.printf(" %-16s %-28s (pewność %.2f)%n", + fact.field(), truncate(fact.value(), 28), fact.confidence()); + } + System.out.println(); + } + } + } + + private static void promote(Path dataDir) throws Exception { + try (Database db = Database.open(dataDir)) { + List assignments = new Promoter(db).run(); + if (assignments.isEmpty()) { + System.out.println("Brak korzeni gier — uruchom najpierw: ingest"); + return; + } + System.out.printf("Podpinam %d korzeni do tytułów i wydań%n%n", assignments.size()); + + int review = 0; + for (Promoter.Assignment a : assignments) { + String where = a.rootPath().isEmpty() ? a.copyName() + : a.copyName() + " / " + a.rootPath(); + System.out.printf(" %-46s → %-32s %s%s%n", truncate(where, 46), + truncate(a.title(), 32), + a.label() == null ? "" : "[" + truncate(a.label(), 30) + "]", + a.ambiguous() ? " ⚠ do przejrzenia" : ""); + if (a.ambiguous()) { + review++; + } + } + if (review > 0) { + System.out.printf("%n%d tytułów wymaga potwierdzenia nazwy — popraw przez:%n" + + " set title name=\"...\"%n", review); + } + } + } + + private static void titles(Path dataDir) throws Exception { + try (Database db = Database.open(dataDir); + Statement st = db.connection().createStatement(); + ResultSet rs = st.executeQuery(""" + SELECT t.id AS title_id, t.name AS title, t.series, t.publisher, + e.id AS edition_id, e.label, + COALESCE(e.release_date_override, ( + SELECT d.value FROM game_root g + JOIN detection d ON d.copy_id = g.copy_id + AND d.root_path = g.root_path + WHERE g.edition_id = e.id AND d.field = 'release_date' LIMIT 1 + )) AS release_date, + COALESCE(e.engine_override, ( + SELECT g.engine_detected FROM game_root g + WHERE g.edition_id = e.id LIMIT 1 + )) AS engine, + COALESCE(e.engine_version_override, ( + SELECT d.value FROM game_root g + JOIN detection d ON d.copy_id = g.copy_id + AND d.root_path = g.root_path + WHERE g.edition_id = e.id AND d.field = 'engine_version' LIMIT 1 + )) AS engine_version, + (SELECT GROUP_CONCAT(DISTINCT lang) FROM edition_language + WHERE edition_id = e.id) AS langs, + (SELECT COUNT(*) FROM game_root WHERE edition_id = e.id) AS roots + FROM edition e + LEFT JOIN title t ON t.id = e.title_id + ORDER BY t.series IS NULL, t.series, t.name, release_date + """)) { + + System.out.printf("%-4s %-30s %-26s %-11s %-13s %-9s %s%n", + "ID", "TYTUŁ", "WYDANIE", "DATA", "SILNIK", "JĘZYKI", "KOPIE"); + while (rs.next()) { + System.out.printf("%-4d %-30s %-26s %-11s %-13s %-9s %d%n", + rs.getInt("edition_id"), + truncate(rs.getString("title"), 30), + truncate(nvl(rs.getString("label")), 26), + nvl(rs.getString("release_date")), + truncate(nvl(rs.getString("engine")), 13), + truncate(nvl(rs.getString("langs")), 9), + rs.getInt("roots")); + } + System.out.println("\nID to identyfikator wydania — używaj go w set edition i lang."); + } + } + + private static void set(Path dataDir, String[] args) throws Exception { + if (args.length < 4) { + System.err.println(""" + set pole=wartość [pole=wartość...] + + set copy "Wojna Troj" source_kind=wlasny_zgraj rip_tool=dd + set edition 3 release_date_override=2004-11-15 distributor="Aidem Media" + set title 2 name="Poznaj Mity: Herkules" + """); + System.err.println("Encje: " + String.join(", ", Editor.entities())); + return; + } + try (Database db = Database.open(dataDir)) { + try { + int changed = new Editor(db).set(args[1], args[2], + Arrays.asList(args).subList(3, args.length)); + System.out.printf("Zmienionych wierszy: %d%n", changed); + } catch (IllegalArgumentException e) { + System.err.println(e.getMessage()); + } + } + } + + private static void lang(Path dataDir, String[] args) throws Exception { + if (args.length < 5 || !(args[1].equals("add") || args[1].equals("rm"))) { + System.err.println(""" + lang add|rm + + lang add 3 pl audio + lang add 3 cs text + lang rm 3 hu ui + + Rola: audio, text, ui. Wydanie: id albo fragment nazwy tytułu. + """); + return; + } + try (Database db = Database.open(dataDir)) { + Editor editor = new Editor(db); + try { + if (args[1].equals("add")) { + editor.addLanguage(args[2], args[3], args[4]); + System.out.println("Dodano."); + } else { + int removed = editor.removeLanguage(args[2], args[3], args[4]); + System.out.println(removed > 0 ? "Usunięto." : "Nie było takiego wpisu."); + } + } catch (IllegalArgumentException e) { + System.err.println(e.getMessage()); + } + } + } + + private static void find(Path dataDir, String[] args) throws Exception { + if (args.length < 2) { + System.err.println("find wymaga zapytania, np. find SHOWCURSOR"); + return; + } + String query = String.join(" ", Arrays.copyOfRange(args, 1, args.length)); + + try (Database db = Database.open(dataDir)) { + ScriptSearch search = new ScriptSearch(db, dataDir); + ScriptSearch.Result result; + try { + result = search.find(query, 25); + } catch (SQLException e) { + System.err.println("Nieprawidłowe zapytanie FTS5: " + e.getMessage()); + return; + } + List hits = result.hits(); + if (hits.isEmpty()) { + System.out.println("Brak trafień dla: " + result.effectiveQuery()); + return; + } + System.out.printf("Trafienia dla %s (%d)%s:%n%n", + result.effectiveQuery(), hits.size(), + result.quoted() ? " — potraktowane jako fraza" : ""); + for (ScriptSearch.Hit hit : hits) { + System.out.println(" " + String.join("\n ", hit.paths())); + System.out.println(" w: " + String.join(", ", hit.games())); + System.out.println(" " + hit.snippet().replaceAll("\\s+", " ").trim()); + System.out.println(" sha1: " + hit.sha1()); + System.out.println(); + } + } + } + + private static void cat(Path dataDir, String[] args) throws Exception { + if (args.length < 2) { + System.err.println("cat wymaga ścieżki albo SHA-1, np. cat arcade.cnv"); + return; + } + try (Database db = Database.open(dataDir)) { + ScriptSearch search = new ScriptSearch(db, dataDir); + List candidates = search.resolve(args[1]); + + if (candidates.isEmpty()) { + System.err.println("Nic nie pasuje do: " + args[1]); + return; + } + if (candidates.size() > 1) { + System.err.printf("Niejednoznaczne — %d różnych wersji tego pliku:%n", + candidates.size()); + for (String sha1 : candidates) { + System.err.printf(" %s %s%n", sha1, + String.join(", ", search.gamesFor(sha1))); + } + System.err.println("Podaj SHA-1, żeby wskazać konkretną."); + return; + } + String sha1 = candidates.get(0); + String body = search.body(sha1); + if (body == null) { + System.err.println("Ten blob nie jest w cache'u — uruchom najpierw: decode"); + return; + } + System.out.println(body); + } + } + private static void list(Path dataDir) throws Exception { try (Database db = Database.open(dataDir); Statement st = db.connection().createStatement(); @@ -180,6 +444,15 @@ public final class Main { list wypisz zaindeksowane kopie stats statystyki bazy + decode [--force] odszyfruj skrypty do cache'u i indeksu + analyze wyciągnij metadane z application.def + promote utwórz tytuły i wydania z wykrytych faktów + titles lista tytułów z metadanymi + set ... edytuj metadane ręczne + lang add|rm ... wersje językowe wydania + find przeszukaj skrypty (składnia FTS5) + cat <ścieżka|sha1> wypisz odszyfrowany skrypt + Baza: -Dcatalog.data= (domyślnie ./data) """); } diff --git a/src/main/java/pl/genschu/rexcatalog/catalog/Editor.java b/src/main/java/pl/genschu/rexcatalog/catalog/Editor.java new file mode 100644 index 0000000..21b4e48 --- /dev/null +++ b/src/main/java/pl/genschu/rexcatalog/catalog/Editor.java @@ -0,0 +1,232 @@ +package pl.genschu.rexcatalog.catalog; + +import pl.genschu.rexcatalog.db.Database; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** + * Ręczna edycja metadanych, których żaden detektor nie wymyśli: skąd pochodzi plik, + * jakie ma wersje językowe, czym to skompilowano. + * + *

Pola są na białej liście per encja — literówka w nazwie kolumny ma się skończyć + * komunikatem, a nie zapytaniem SQL sklejonym z wejścia. + */ +public final class Editor { + + private static final Map> EDITABLE = Map.of( + "copy", new LinkedHashSet<>(List.of( + "source_kind", "source_url", "source_ref", "acquired_at", + "rip_tool", "media_note", "verified", "notes", "display_name")), + "edition", new LinkedHashSet<>(List.of( + "label", "engine_override", "engine_version_override", + "compiler_override", "release_date_override", "distributor", "notes")), + "title", new LinkedHashSet<>(List.of("name", "series", "publisher")) + ); + + private static final Set SOURCE_KINDS = + Set.of("wlasny_zgraj", "archive_org", "redump", "inne", "nieznane"); + + private static final Set LANGUAGE_ROLES = Set.of("audio", "text", "ui"); + + private final Database db; + + public Editor(Database db) { + this.db = db; + } + + public static Set editableFields(String entity) { + return EDITABLE.getOrDefault(entity, Set.of()); + } + + public static Set entities() { + return EDITABLE.keySet(); + } + + /** Zwraca liczbę zmienionych wierszy albo rzuca z opisem, co jest nie tak. */ + public int set(String entity, String id, List assignments) throws Exception { + Set allowed = EDITABLE.get(entity); + if (allowed == null) { + throw new IllegalArgumentException("Nieznana encja: " + entity + + " (dostępne: " + String.join(", ", EDITABLE.keySet()) + ")"); + } + if (assignments.isEmpty()) { + throw new IllegalArgumentException("Podaj co najmniej jedno przypisanie pole=wartość"); + } + + List columns = new ArrayList<>(); + List values = new ArrayList<>(); + for (String assignment : merge(assignments)) { + int eq = assignment.indexOf('='); + if (eq <= 0) { + throw new IllegalArgumentException("Oczekiwano pole=wartość, dostałem: " + assignment); + } + String column = assignment.substring(0, eq).trim().toLowerCase(Locale.ROOT); + String value = unquote(assignment.substring(eq + 1).trim()); + + if (!allowed.contains(column)) { + throw new IllegalArgumentException("Pole '" + column + "' nie jest edytowalne w " + + entity + " (dostępne: " + String.join(", ", allowed) + ")"); + } + if (column.equals("source_kind") && !SOURCE_KINDS.contains(value)) { + throw new IllegalArgumentException("source_kind musi być jednym z: " + + String.join(", ", SOURCE_KINDS)); + } + columns.add(column); + values.add(value.isEmpty() ? null : value); + } + + long rowId = resolveId(entity, id); + String sql = "UPDATE " + entity + " SET " + + String.join(", ", columns.stream().map(c -> c + " = ?").toList()) + + " WHERE id = ?"; + try (PreparedStatement st = db.connection().prepareStatement(sql)) { + int index = 1; + for (String value : values) { + st.setString(index++, value); + } + st.setLong(index, rowId); + return st.executeUpdate(); + } + } + + /** + * Skleja wartości wielowyrazowe. Token bez znaku równości nie może być osobnym + * przypisaniem, więc dopisujemy go do poprzedniej wartości — dzięki temu + * {@code distributor=Komputer Świat Gry} działa bez walki z cudzysłowami, + * które i tak gubi {@code gradle run --args}. + */ + private static List merge(List tokens) { + List merged = new ArrayList<>(); + for (String token : tokens) { + if (token.contains("=") || merged.isEmpty()) { + merged.add(token); + } else { + merged.set(merged.size() - 1, merged.get(merged.size() - 1) + " " + token); + } + } + return merged; + } + + private static String unquote(String value) { + if (value.length() >= 2 && value.startsWith("\"") && value.endsWith("\"")) { + return value.substring(1, value.length() - 1); + } + return value; + } + + /** + * Przyjmuje albo liczbowy identyfikator, albo fragment nazwy — przy pracy z CLI + * nikt nie pamięta, że Wojna Trojańska ma copy_id 13. + */ + private long resolveId(String entity, String id) throws Exception { + if (id.matches("\\d+")) { + return Long.parseLong(id); + } + String column = switch (entity) { + case "copy" -> "display_name"; + case "title" -> "name"; + case "edition" -> "label"; + default -> throw new IllegalArgumentException("Encja " + entity + + " wymaga liczbowego id"); + }; + List ids = new ArrayList<>(); + List names = new ArrayList<>(); + try (PreparedStatement st = db.connection().prepareStatement( + "SELECT id, " + column + " FROM " + entity + + " WHERE lower(" + column + ") LIKE ? ORDER BY id")) { + st.setString(1, "%" + id.toLowerCase(Locale.ROOT) + "%"); + try (ResultSet rs = st.executeQuery()) { + while (rs.next()) { + ids.add(rs.getLong(1)); + names.add(rs.getString(2)); + } + } + } + if (ids.isEmpty()) { + throw new IllegalArgumentException("Nic nie pasuje do: " + id); + } + if (ids.size() > 1) { + StringBuilder sb = new StringBuilder("Niejednoznaczne — pasuje " + ids.size() + ":"); + for (int i = 0; i < ids.size(); i++) { + sb.append("\n ").append(ids.get(i)).append(" ").append(names.get(i)); + } + throw new IllegalArgumentException(sb.toString()); + } + return ids.get(0); + } + + /** + * Wersje językowe są listą, nie polem — wydanie bywa miksem (polski + czeski + + * węgierski), a rola rozdziela dubbing od napisów i interfejsu. + */ + public void addLanguage(String editionId, String lang, String role) throws Exception { + if (!LANGUAGE_ROLES.contains(role)) { + throw new IllegalArgumentException("Rola musi być jedną z: " + + String.join(", ", LANGUAGE_ROLES)); + } + long id = resolveEdition(editionId); + try (PreparedStatement st = db.connection().prepareStatement(""" + INSERT INTO edition_language (edition_id, lang, role) VALUES (?, ?, ?) + ON CONFLICT(edition_id, lang, role) DO NOTHING + """)) { + st.setLong(1, id); + st.setString(2, lang.toLowerCase(Locale.ROOT)); + st.setString(3, role); + st.executeUpdate(); + } + } + + public int removeLanguage(String editionId, String lang, String role) throws Exception { + long id = resolveEdition(editionId); + try (PreparedStatement st = db.connection().prepareStatement( + "DELETE FROM edition_language WHERE edition_id = ? AND lang = ? AND role = ?")) { + st.setLong(1, id); + st.setString(2, lang.toLowerCase(Locale.ROOT)); + st.setString(3, role); + return st.executeUpdate(); + } + } + + /** Wydanie wskazujemy po id albo po nazwie tytułu, do którego należy. */ + private long resolveEdition(String identifier) throws Exception { + if (identifier.matches("\\d+")) { + return Long.parseLong(identifier); + } + List ids = new ArrayList<>(); + List labels = new ArrayList<>(); + try (PreparedStatement st = db.connection().prepareStatement(""" + SELECT e.id, t.name || COALESCE(' — ' || e.label, '') + FROM edition e LEFT JOIN title t ON t.id = e.title_id + WHERE lower(t.name) LIKE ? OR lower(e.label) LIKE ? + ORDER BY e.id + """)) { + String pattern = "%" + identifier.toLowerCase(Locale.ROOT) + "%"; + st.setString(1, pattern); + st.setString(2, pattern); + try (ResultSet rs = st.executeQuery()) { + while (rs.next()) { + ids.add(rs.getLong(1)); + labels.add(rs.getString(2)); + } + } + } + if (ids.isEmpty()) { + throw new IllegalArgumentException("Żadne wydanie nie pasuje do: " + identifier); + } + if (ids.size() > 1) { + StringBuilder sb = new StringBuilder("Niejednoznaczne — pasuje " + ids.size() + ":"); + for (int i = 0; i < ids.size(); i++) { + sb.append("\n ").append(ids.get(i)).append(" ").append(labels.get(i)); + } + throw new IllegalArgumentException(sb.toString()); + } + return ids.get(0); + } +} diff --git a/src/main/java/pl/genschu/rexcatalog/catalog/Promoter.java b/src/main/java/pl/genschu/rexcatalog/catalog/Promoter.java new file mode 100644 index 0000000..974483e --- /dev/null +++ b/src/main/java/pl/genschu/rexcatalog/catalog/Promoter.java @@ -0,0 +1,262 @@ +package pl.genschu.rexcatalog.catalog; + +import pl.genschu.rexcatalog.db.Database; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.text.Normalizer; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Materializuje encje katalogu z faktów zebranych przez detektory: tworzy tytuły + * i wydania, po czym podpina do nich korzenie gier. + * + *

Operacja jest powtarzalna i nie nadpisuje pracy człowieka — pola {@code *_override}, + * {@code label}, {@code distributor} i {@code notes} zostają nietknięte, a tytuł raz + * nazwany zachowuje nazwę, bo kluczem jest slug, nie nazwa. + */ +public final class Promoter { + + private final Database db; + + public Promoter(Database db) { + this.db = db; + } + + public record Assignment(String copyName, String rootPath, String title, String label, + String fingerprint, boolean ambiguous) { + } + + private record RootFacts(long copyId, String copyName, String rootPath, String copySha256, + String dllSha1, String appDefSha1, String edition, String family, + int rootsOnCopy) { + } + + public List run() throws Exception { + List roots = collect(); + List assignments = new ArrayList<>(); + + Connection conn = db.connection(); + boolean previousAutoCommit = conn.getAutoCommit(); + conn.setAutoCommit(false); + try { + for (RootFacts root : roots) { + String fingerprint = fingerprint(root); + Naming naming = naming(root); + + long titleId = upsertTitle(conn, naming); + long editionId = upsertEdition(conn, fingerprint, titleId, naming, root); + linkRoot(conn, root.copyId(), root.rootPath(), editionId); + + assignments.add(new Assignment(root.copyName(), root.rootPath(), + naming.title(), naming.label(), fingerprint, naming.needsReview())); + } + // po zmianie reguły nazewniczej stare tytuły zostają bez wydań — sprzątamy je, + // żeby `titles` nie pokazywał duchów po poprzednim przebiegu + try (PreparedStatement st = conn.prepareStatement(""" + DELETE FROM title WHERE id NOT IN + (SELECT title_id FROM edition WHERE title_id IS NOT NULL) + """)) { + st.executeUpdate(); + } + conn.commit(); + } catch (Exception e) { + conn.rollback(); + throw e; + } finally { + conn.setAutoCommit(previousAutoCommit); + } + return assignments; + } + + /** + * Klucz scalania. Hash biblioteki plus odcisk application.def — dwie kopie tego + * samego wydania (obraz i katalog rozpakowany) dają ten sam odcisk i jedno wydanie, + * a dwie gry z jednej płyty rozchodzą się na application.def. + * + *

Gdy nie ma ani biblioteki, ani opisu projektu (np. silnik KI), zostaje + * tożsamość nośnika — świadomie ostrożna, bo nie mamy czym scalać. + */ + private static String fingerprint(RootFacts root) { + if (root.dllSha1() != null || root.appDefSha1() != null) { + return "engine:" + nvl(root.dllSha1()) + "|app:" + nvl(root.appDefSha1()); + } + return "copy:" + nvl(root.copySha256()) + "|root:" + root.rootPath(); + } + + /** Rozbiór nazwy z tablicy hashy na trzy poziomy katalogu. */ + private record Naming(String title, String label, String series, boolean needsReview) { + } + + /** + * Wpis w {@code KnownHashes} miesza dzieło z wydaniem: „Reksio i UFO (pierwsza wersja)" + * to tytuł plus etykieta w nawiasie. Rozdzielenie ich sprawia, że dwa wydania tej samej + * gry trafiają pod jeden tytuł, zamiast udawać dwie różne gry. + * + *

Dwupack wymaga dodatkowo rozbicia po ukośniku: jeden wpis opisuje obie gry + * („Poznaj Mity: Herkules/Odyseusz"), bo obie stoją na tej samej bibliotece. + * Wybór członu opieramy na nazwie katalogu korzenia. + */ + private static Naming naming(RootFacts root) { + String edition = root.edition(); + if (edition == null || edition.isBlank()) { + // brak wpisu w tablicy hashy — z nazwy pliku wyjdzie co najwyżej przybliżenie + return new Naming(root.copyName().replaceAll("(?i)\\.(iso|zip)$", "").trim(), + null, null, true); + } + + String base = edition; + String label = null; + int paren = edition.indexOf(" ("); + if (paren > 0) { + base = edition.substring(0, paren).trim(); + // w danych trafia się nawias bez domknięcia, stąd replaceAll zamiast substring + label = edition.substring(paren + 2).replaceAll("\\)\\s*$", "").trim(); + } + + boolean needsReview = false; + if (root.rootsOnCopy() > 1 && !root.rootPath().isEmpty()) { + int colon = base.lastIndexOf(':'); + String prefix = colon >= 0 ? base.substring(0, colon + 1).trim() + " " : ""; + String rest = colon >= 0 ? base.substring(colon + 1).trim() : base; + + String matched = null; + for (String part : rest.split("/")) { + if (part.trim().equalsIgnoreCase(root.rootPath().trim())) { + matched = prefix + part.trim(); + } + } + if (matched != null) { + base = matched; + } else { + base = base + " (" + root.rootPath() + ")"; + needsReview = true; + } + } + return new Naming(base, label, series(base), needsReview); + } + + /** „Poznaj Mity: Herkules" → seria „Poznaj Mity". Bez dwukropka nie zgadujemy. */ + private static String series(String title) { + int colon = title.indexOf(": "); + return colon > 0 ? title.substring(0, colon).trim() : null; + } + + private long upsertTitle(Connection conn, Naming naming) throws Exception { + String slug = slug(naming.title()); + // nazwa nie jest nadpisywana: klucz to slug, więc ręczna zmiana nazwy przetrwa + try (PreparedStatement st = conn.prepareStatement(""" + INSERT INTO title (slug, name, series) VALUES (?, ?, ?) + ON CONFLICT(slug) DO UPDATE SET series = COALESCE(title.series, excluded.series) + """)) { + st.setString(1, slug); + st.setString(2, naming.title()); + st.setString(3, naming.series()); + st.executeUpdate(); + } + try (PreparedStatement st = conn.prepareStatement("SELECT id FROM title WHERE slug = ?")) { + st.setString(1, slug); + try (ResultSet rs = st.executeQuery()) { + rs.next(); + return rs.getLong(1); + } + } + } + + private long upsertEdition(Connection conn, String fingerprint, long titleId, + Naming naming, RootFacts root) throws Exception { + // aktualizujemy wyłącznie pola mechaniczne — poprawki człowieka zostają, + // dlatego label tylko uzupełniamy, gdy jest pusty + try (PreparedStatement st = conn.prepareStatement(""" + INSERT INTO edition (fingerprint, title_id, label, dll_sha1, app_def_sha1) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(fingerprint) DO UPDATE SET + title_id = excluded.title_id, + label = COALESCE(edition.label, excluded.label), + dll_sha1 = excluded.dll_sha1, + app_def_sha1 = excluded.app_def_sha1 + """)) { + st.setString(1, fingerprint); + st.setLong(2, titleId); + st.setString(3, naming.label()); + st.setString(4, root.dllSha1()); + st.setString(5, root.appDefSha1()); + st.executeUpdate(); + } + try (PreparedStatement st = conn.prepareStatement( + "SELECT id FROM edition WHERE fingerprint = ?")) { + st.setString(1, fingerprint); + try (ResultSet rs = st.executeQuery()) { + rs.next(); + return rs.getLong(1); + } + } + } + + private void linkRoot(Connection conn, long copyId, String rootPath, long editionId) + throws Exception { + try (PreparedStatement st = conn.prepareStatement( + "UPDATE game_root SET edition_id = ? WHERE copy_id = ? AND root_path = ?")) { + st.setLong(1, editionId); + st.setLong(2, copyId); + st.setString(3, rootPath); + st.executeUpdate(); + } + } + + private List collect() throws Exception { + Map rootsPerCopy = new LinkedHashMap<>(); + try (PreparedStatement st = db.connection().prepareStatement( + "SELECT copy_id, COUNT(*) n FROM game_root GROUP BY copy_id"); + ResultSet rs = st.executeQuery()) { + while (rs.next()) { + rootsPerCopy.put(rs.getLong("copy_id"), rs.getInt("n")); + } + } + + List out = new ArrayList<>(); + try (PreparedStatement st = db.connection().prepareStatement(""" + SELECT g.copy_id, g.root_path, c.display_name, c.sha256, + MAX(CASE WHEN d.field = 'engine_dll_sha1' THEN d.value END) AS dll_sha1, + MAX(CASE WHEN d.field = 'app_def_sha1' THEN d.value END) AS app_def_sha1, + MAX(CASE WHEN d.field = 'edition' THEN d.value END) AS edition, + MAX(CASE WHEN d.field = 'family' THEN d.value END) AS family + FROM game_root g + JOIN copy c ON c.id = g.copy_id + LEFT JOIN detection d ON d.copy_id = g.copy_id AND d.root_path = g.root_path + GROUP BY g.copy_id, g.root_path + ORDER BY c.display_name, g.root_path + """); + ResultSet rs = st.executeQuery()) { + + while (rs.next()) { + long copyId = rs.getLong("copy_id"); + out.add(new RootFacts(copyId, rs.getString("display_name"), + rs.getString("root_path"), rs.getString("sha256"), + rs.getString("dll_sha1"), rs.getString("app_def_sha1"), + rs.getString("edition"), rs.getString("family"), + rootsPerCopy.getOrDefault(copyId, 1))); + } + } + return out; + } + + /** Slug bez ogonków i znaków spoza [a-z0-9-], żeby był stabilnym kluczem tytułu. */ + static String slug(String name) { + String ascii = Normalizer.normalize(name.replace('ł', 'l').replace('Ł', 'L'), + Normalizer.Form.NFD) + .replaceAll("\\p{M}", ""); + return ascii.toLowerCase(Locale.ROOT) + .replaceAll("[^a-z0-9]+", "-") + .replaceAll("^-+|-+$", ""); + } + + private static String nvl(String value) { + return value == null ? "" : value; + } +} diff --git a/src/main/java/pl/genschu/rexcatalog/db/Database.java b/src/main/java/pl/genschu/rexcatalog/db/Database.java index c82b177..d0dd87d 100644 --- a/src/main/java/pl/genschu/rexcatalog/db/Database.java +++ b/src/main/java/pl/genschu/rexcatalog/db/Database.java @@ -4,6 +4,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -41,6 +43,7 @@ public final class Database implements AutoCloseable { } private void applySchema() throws SQLException { + dropLegacyEdition(); try (Statement st = connection.createStatement()) { for (String ddl : SCHEMA) { st.executeUpdate(ddl); @@ -49,6 +52,51 @@ public final class Database implements AutoCloseable { migrate(); } + /** + * Stara wersja {@code edition} miała {@code dll_sha1 UNIQUE}, co jest nie do + * pogodzenia z dwupackiem — dwie gry na jednej płycie dzielą bibliotekę silnika. + * Tabela jest przebudowywana tylko wtedy, gdy jest pusta; jeśli zdążyły w niej + * osiąść dane kuratorskie, wołamy o ręczną migrację zamiast je skasować. + */ + private void dropLegacyEdition() throws SQLException { + if (!tableExists("edition") || hasColumn("edition", "fingerprint")) { + return; + } + try (Statement st = connection.createStatement(); + ResultSet rs = st.executeQuery("SELECT COUNT(*) FROM edition")) { + if (rs.next() && rs.getLong(1) > 0) { + throw new SQLException("Tabela edition ma stary układ i zawiera dane. " + + "Przenieś je ręcznie albo usuń data/catalog.sqlite i przebuduj " + + "(ingest, decode, analyze, promote)."); + } + } + try (Statement st = connection.createStatement()) { + st.executeUpdate("DROP TABLE edition"); + } + } + + private boolean tableExists(String table) throws SQLException { + try (PreparedStatement st = connection.prepareStatement( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?")) { + st.setString(1, table); + try (ResultSet rs = st.executeQuery()) { + return rs.next(); + } + } + } + + private boolean hasColumn(String table, String column) throws SQLException { + try (Statement st = connection.createStatement(); + ResultSet rs = st.executeQuery("PRAGMA table_info(" + table + ")")) { + while (rs.next()) { + if (column.equalsIgnoreCase(rs.getString("name"))) { + return true; + } + } + return false; + } + } + /** * Dokłada kolumny, których {@code CREATE TABLE IF NOT EXISTS} nie doda do już * istniejącej tabeli. Tylko {@code ADD COLUMN} — bez przebudowy tabel, bo na @@ -59,14 +107,10 @@ public final class Database implements AutoCloseable { } private void addColumnIfMissing(String table, String column, String type) throws SQLException { + if (hasColumn(table, column)) { + return; + } try (Statement st = connection.createStatement()) { - try (java.sql.ResultSet rs = st.executeQuery("PRAGMA table_info(" + table + ")")) { - while (rs.next()) { - if (column.equalsIgnoreCase(rs.getString("name"))) { - return; - } - } - } st.executeUpdate("ALTER TABLE " + table + " ADD COLUMN " + column + " " + type); } } @@ -95,19 +139,27 @@ public final class Database implements AutoCloseable { publisher TEXT DEFAULT 'Aidem Media' ) """, + // Wydanie jest tożsamością kuratorską: co wykryte, siedzi w `detection` + // z dowodem, a tutaj trzymamy tylko poprawki człowieka (*_override) + // i pola, których nie da się wykryć. + // + // fingerprint to klucz scalania kopii w jedno wydanie. Sam hash biblioteki + // nie wystarcza — Herkules i Odyseusz mają identyczny Piklib 8, a różnią się + // dopiero odciskiem application.def. """ CREATE TABLE IF NOT EXISTS edition ( - id INTEGER PRIMARY KEY, - title_id INTEGER REFERENCES title(id), - label TEXT, - dll_sha1 TEXT UNIQUE, - engine_detected TEXT, - engine_override TEXT, - compiler_detected TEXT, - compiler_override TEXT, - release_date TEXT, - distributor TEXT, - notes TEXT + id INTEGER PRIMARY KEY, + title_id INTEGER REFERENCES title(id), + fingerprint TEXT NOT NULL UNIQUE, + label TEXT, + dll_sha1 TEXT, + app_def_sha1 TEXT, + engine_override TEXT, + engine_version_override TEXT, + compiler_override TEXT, + release_date_override TEXT, + distributor TEXT, + notes TEXT ) """, """ @@ -216,6 +268,19 @@ public final class Database implements AutoCloseable { created_at TEXT, PRIMARY KEY (blob_sha1, kind) ) + """, + // Indeks pełnotekstowy odszyfrowanych skryptów. Kluczem jest blob, nie plik — + // ten sam skrypt bywa w kilku obrazach, a wtedy szukamy raz i pokazujemy, + // w których kopiach występuje. + // + // tokenchars '_' trzyma identyfikatory w całości (SHOW_CURSOR to jeden token, + // nie dwa), a remove_diacritics pozwala znaleźć polski tekst bez ogonków. + """ + CREATE VIRTUAL TABLE IF NOT EXISTS script_fts USING fts5( + body, + blob_sha1 UNINDEXED, + tokenize = "unicode61 remove_diacritics 2 tokenchars '_'" + ) """ }; } diff --git a/src/main/java/pl/genschu/rexcatalog/ingest/MetadataDetector.java b/src/main/java/pl/genschu/rexcatalog/ingest/MetadataDetector.java new file mode 100644 index 0000000..0e5e687 --- /dev/null +++ b/src/main/java/pl/genschu/rexcatalog/ingest/MetadataDetector.java @@ -0,0 +1,232 @@ +package pl.genschu.rexcatalog.ingest; + +import pl.genschu.rexcatalog.db.Database; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Wyciąga metadane wydania z {@code dane/application.def} — pliku opisu projektu, + * który Piklib i BlooMoo trzymają razem z grą. + * + *

Źródło jest lepsze niż tablice hashy, bo pochodzi z samych danych gry, a nie + * z listy rozpoznanych bibliotek. Działa na odszyfrowanym cache'u, więc wymaga + * wcześniejszego {@code decode}. + */ +public final class MetadataDetector { + + private static final String DETECTOR = "MetadataDetector"; + + /** Obiekt typu APPLICATION nie zawsze nazywa się GAME — bywa UFO, PIRACI. */ + private static final Pattern APPLICATION = + Pattern.compile("^(\\w+):TYPE=APPLICATION\\s*$", Pattern.MULTILINE); + + private static final Pattern ISO_DATE = Pattern.compile("^(\\d{4}-\\d{2}-\\d{2})"); + + private final Database db; + private final Path dataDir; + private final String coreVersion; + + public MetadataDetector(Database db, Path dataDir, String coreVersion) { + this.db = db; + this.dataDir = dataDir; + this.coreVersion = coreVersion; + } + + public record Fact(String field, String value, double confidence, String evidence) { + } + + public record RootResult(String copyName, String rootPath, String note, List facts) { + } + + private record Target(long copyId, String copyName, String rootPath, + String filePath, String blobSha1, String artifactPath) { + } + + public List run() throws Exception { + List targets = targets(); + List results = new ArrayList<>(); + + Connection conn = db.connection(); + boolean previousAutoCommit = conn.getAutoCommit(); + conn.setAutoCommit(false); + try (PreparedStatement delete = conn.prepareStatement( + "DELETE FROM detection WHERE copy_id = ? AND root_path = ? AND detector = ?"); + PreparedStatement insert = conn.prepareStatement(""" + INSERT INTO detection + (copy_id, root_path, field, value, confidence, evidence, + detector, core_version, detected_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """)) { + + String now = Instant.now().toString(); + for (Target target : targets) { + delete.setLong(1, target.copyId()); + delete.setString(2, target.rootPath()); + delete.setString(3, DETECTOR); + delete.executeUpdate(); + + if (target.artifactPath() == null) { + results.add(new RootResult(target.copyName(), target.rootPath(), + "brak application.def w cache'u — uruchom decode", List.of())); + continue; + } + + String body = Files.readString(dataDir.resolve(target.artifactPath()), + StandardCharsets.UTF_8); + List facts = parse(body, target.filePath(), target.blobSha1()); + if (facts.isEmpty()) { + results.add(new RootResult(target.copyName(), target.rootPath(), + "nie znaleziono obiektu TYPE=APPLICATION", List.of())); + continue; + } + + for (Fact fact : facts) { + insert.setLong(1, target.copyId()); + insert.setString(2, target.rootPath()); + insert.setString(3, fact.field()); + insert.setString(4, fact.value()); + insert.setDouble(5, fact.confidence()); + insert.setString(6, fact.evidence()); + insert.setString(7, DETECTOR); + insert.setString(8, coreVersion); + insert.setString(9, now); + insert.executeUpdate(); + } + results.add(new RootResult(target.copyName(), target.rootPath(), null, facts)); + } + conn.commit(); + } catch (Exception e) { + conn.rollback(); + throw e; + } finally { + conn.setAutoCommit(previousAutoCommit); + } + return results; + } + + private List parse(String body, String filePath, String blobSha1) { + Matcher application = APPLICATION.matcher(body); + if (!application.find()) { + return List.of(); + } + String object = application.group(1); + List facts = new ArrayList<>(); + + // odcisk pliku opisu — w dwupacku to jedyne, co odróżnia dwa korzenie, + // bo bibliotekę Piklib obie gry mają bit w bit tę samą + facts.add(new Fact("app_def_sha1", blobSha1, 1.0, filePath)); + facts.add(new Fact("app_object", object, 1.0, filePath + ": OBJECT=" + object)); + + field(body, object, "LASTMODIFYTIME").ifPresent(raw -> { + Matcher iso = ISO_DATE.matcher(raw); + // data ostatniej modyfikacji projektu przybliża datę builda, nie wydania — + // stąd pewność poniżej jedynki nawet przy czytelnym formacie + facts.add(iso.find() + ? new Fact("release_date", iso.group(1), 0.7, + evidence(filePath, object, "LASTMODIFYTIME", raw)) + : new Fact("release_date", raw, 0.2, + evidence(filePath, object, "LASTMODIFYTIME", raw) + + " (nierozpoznany format daty)")); + }); + + field(body, object, "CREATIONTIME").ifPresent(raw -> + // wspólna dla całej serii — to data założenia projektu, nie wydania + facts.add(new Fact("project_created", raw, 1.0, + evidence(filePath, object, "CREATIONTIME", raw)))); + + field(body, object, "VERSION").ifPresent(raw -> + facts.add(new Fact("game_version", raw, 1.0, + evidence(filePath, object, "VERSION", raw)))); + + field(body, object, "BLOOMOO_VERSION").ifPresent(raw -> + facts.add(new Fact("engine_version", raw, 0.95, + evidence(filePath, object, "BLOOMOO_VERSION", raw)))); + + field(body, object, "EPISODES").ifPresent(raw -> + facts.add(new Fact("episodes", raw, 1.0, + evidence(filePath, object, "EPISODES", raw)))); + + field(body, object, "AUTHOR") + .map(MetadataDetector::unquote) + .filter(value -> !value.isBlank()) + .ifPresent(value -> facts.add(new Fact("author", value, 1.0, + evidence(filePath, object, "AUTHOR", value)))); + + return facts; + } + + private static java.util.Optional field(String body, String object, String key) { + Matcher m = Pattern.compile("^" + Pattern.quote(object) + ":" + key + "=(.*)$", + Pattern.MULTILINE).matcher(body); + if (!m.find()) { + return java.util.Optional.empty(); + } + String value = m.group(1).trim(); + return value.isEmpty() ? java.util.Optional.empty() : java.util.Optional.of(value); + } + + /** Skrypty bywają cytowane zarówno ASCII, jak i typograficznymi cudzysłowami. */ + private static String unquote(String value) { + return value.replaceAll("^[\"”„“]+|[\"”„“]+$", "").trim(); + } + + private static String evidence(String filePath, String object, String key, String value) { + return filePath + ": " + object + ":" + key + "=" + value; + } + + /** + * Dopasowuje application.def do korzenia gry. W dwupacku każdy korzeń ma własny, + * więc wybieramy plik leżący najpłycej pod danym prefiksem. + */ + private List targets() throws Exception { + Map best = new LinkedHashMap<>(); + try (PreparedStatement st = db.connection().prepareStatement(""" + SELECT g.copy_id, c.display_name, g.root_path, + COALESCE(f.path_raw, f.path) AS file_path, f.path AS canonical, + f.blob_sha1, a.path AS artifact_path + FROM game_root g + JOIN copy c ON c.id = g.copy_id + JOIN file f ON f.copy_id = g.copy_id + LEFT JOIN artifact a ON a.blob_sha1 = f.blob_sha1 AND a.kind = 'script' + WHERE f.path LIKE '%application.def' + ORDER BY g.copy_id, g.root_path + """); + ResultSet rs = st.executeQuery()) { + + while (rs.next()) { + String rootPath = rs.getString("root_path"); + String canonical = rs.getString("canonical"); + String prefix = rootPath.isEmpty() ? "" : rootPath + "/"; + if (!canonical.startsWith(prefix)) { + continue; + } + String key = rs.getLong("copy_id") + "" + rootPath; + Target candidate = new Target(rs.getLong("copy_id"), rs.getString("display_name"), + rootPath, rs.getString("file_path"), rs.getString("blob_sha1"), + rs.getString("artifact_path")); + + Target current = best.get(key); + if (current == null || depth(canonical) < depth(current.filePath())) { + best.put(key, candidate); + } + } + } + return new ArrayList<>(best.values()); + } + + private static int depth(String path) { + return (int) path.chars().filter(c -> c == '/').count(); + } +} diff --git a/src/main/java/pl/genschu/rexcatalog/script/ScriptDecoder.java b/src/main/java/pl/genschu/rexcatalog/script/ScriptDecoder.java new file mode 100644 index 0000000..94be067 --- /dev/null +++ b/src/main/java/pl/genschu/rexcatalog/script/ScriptDecoder.java @@ -0,0 +1,213 @@ +package pl.genschu.rexcatalog.script; + +import pl.genschu.bloomooemulator.encoding.ScriptDecypher; +import pl.genschu.bloomooemulator.engine.filesystem.AssetSourceDispatcher; +import pl.genschu.bloomooemulator.engine.filesystem.IFileSystem; +import pl.genschu.rexcatalog.db.Database; + +import java.io.File; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Statement; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Odszyfrowuje skrypty przez {@link ScriptDecypher} z {@code :core} i buduje z nich + * cache adresowany treścią plus indeks pełnotekstowy. + * + *

Jednostką pracy jest blob, nie plik: identyczny skrypt leżący w pięciu obrazach + * odszyfrowujemy raz. Powtórne uruchomienie przerabia tylko to, czego brakuje albo + * co zostało zrobione inną wersją {@code :core} — {@code artifact.tool_version} + * jest kluczem unieważnienia. + */ +public final class ScriptDecoder { + + /** Formaty, które {@code ScriptDecypher} potrafi rozwinąć w tekst. */ + private static final Set SCRIPT_FORMATS = Set.of("CNV", "CLASS", "DEF", "SCRIPT"); + + private static final String KIND = "script"; + + private final Database db; + private final Path dataDir; + private final String coreVersion; + + public ScriptDecoder(Database db, Path dataDir, String coreVersion) { + this.db = db; + this.dataDir = dataDir; + this.coreVersion = coreVersion; + } + + public record Stats(int decoded, int skipped, int failed) { + } + + /** Jeden skrypt do odszyfrowania: gdzie leży w kontenerze i jaka jest jego treść. */ + private record Pending(String pathInContainer, String sha1) { + } + + public Stats run(boolean force) throws Exception { + Map> byContainer = pendingWork(force); + if (byContainer.isEmpty()) { + return new Stats(0, 0, 0); + } + + Connection conn = db.connection(); + boolean previousAutoCommit = conn.getAutoCommit(); + conn.setAutoCommit(false); + + int decoded = 0; + int failed = 0; + int skipped = 0; + Set done = new HashSet<>(); + + try (PreparedStatement artifactStmt = conn.prepareStatement(""" + INSERT INTO artifact (blob_sha1, kind, path, tool_version, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(blob_sha1, kind) DO UPDATE SET + path = excluded.path, tool_version = excluded.tool_version, + created_at = excluded.created_at + """); + PreparedStatement ftsDelete = conn.prepareStatement( + "DELETE FROM script_fts WHERE blob_sha1 = ?"); + PreparedStatement ftsInsert = conn.prepareStatement( + "INSERT INTO script_fts (body, blob_sha1) VALUES (?, ?)")) { + + for (Map.Entry> entry : byContainer.entrySet()) { + File container = new File(entry.getKey()); + IFileSystem fs; + try { + fs = AssetSourceDispatcher.openAssets(container); + } catch (Exception e) { + System.err.printf(" ! nie otwieram %s: %s%n", + container.getName(), e.getMessage()); + failed += entry.getValue().size(); + continue; + } + + for (Pending pending : entry.getValue()) { + // ten sam blob w kolejnym obrazie — treść jest ta sama z definicji + if (!done.add(pending.sha1())) { + skipped++; + continue; + } + try { + byte[] raw = readAll(fs, pending.pathInContainer()); + String text = ScriptDecypher.decypherIfNeeded(raw); + + String relative = cachePath(pending.sha1()); + Path target = dataDir.resolve(relative); + Files.createDirectories(target.getParent()); + Files.writeString(target, text, StandardCharsets.UTF_8); + + artifactStmt.setString(1, pending.sha1()); + artifactStmt.setString(2, KIND); + artifactStmt.setString(3, relative); + artifactStmt.setString(4, coreVersion); + artifactStmt.setString(5, Instant.now().toString()); + artifactStmt.executeUpdate(); + + ftsDelete.setString(1, pending.sha1()); + ftsDelete.executeUpdate(); + ftsInsert.setString(1, text); + ftsInsert.setString(2, pending.sha1()); + ftsInsert.executeUpdate(); + + decoded++; + } catch (Exception e) { + System.err.printf(" ! %s (%s): %s%n", pending.pathInContainer(), + container.getName(), e.getMessage()); + failed++; + } + } + } + conn.commit(); + } catch (Exception e) { + conn.rollback(); + throw e; + } finally { + conn.setAutoCommit(previousAutoCommit); + } + + return new Stats(decoded, skipped, failed); + } + + /** + * Zbiera skrypty wymagające przerobienia, pogrupowane po kontenerze — żeby + * każdy obraz otwierać raz, a nie raz na plik. + */ + private Map> pendingWork(boolean force) throws Exception { + String formats = String.join(",", java.util.Collections.nCopies(SCRIPT_FORMATS.size(), "?")); + String sql = """ + SELECT c.path AS container, COALESCE(f.path_raw, f.path) AS in_container, + f.blob_sha1 + FROM copy c + JOIN file f ON f.copy_id = c.id + JOIN blob b ON b.sha1 = f.blob_sha1 + LEFT JOIN artifact a ON a.blob_sha1 = b.sha1 AND a.kind = 'script' + WHERE c.status <> 'unreadable' + AND b.format IN (%s) + """.formatted(formats) + + (force ? "" : " AND (a.blob_sha1 IS NULL OR a.tool_version <> ?)\n") + + "ORDER BY c.id"; + + Map> byContainer = new LinkedHashMap<>(); + try (PreparedStatement st = db.connection().prepareStatement(sql)) { + int index = 1; + for (String format : SCRIPT_FORMATS) { + st.setString(index++, format); + } + if (!force) { + st.setString(index, coreVersion); + } + try (ResultSet rs = st.executeQuery()) { + while (rs.next()) { + byContainer.computeIfAbsent(rs.getString("container"), k -> new ArrayList<>()) + .add(new Pending(rs.getString("in_container"), + rs.getString("blob_sha1"))); + } + } + } + return byContainer; + } + + /** Rozgałęzienie po dwóch pierwszych znakach — bez tego jeden katalog z tysiącami plików. */ + private static String cachePath(String sha1) { + return "artifacts/" + KIND + "/" + sha1.substring(0, 2) + "/" + sha1 + ".txt"; + } + + private static byte[] readAll(IFileSystem fs, String path) throws Exception { + try (InputStream in = fs.open(path)) { + return in.readAllBytes(); + } + } + + /** Ile skryptów czeka na przerobienie — do komunikatu przed startem. */ + public int pendingCount() throws Exception { + int total = 0; + for (List list : pendingWork(false).values()) { + total += list.size(); + } + return total; + } + + /** Usuwa z cache'u i indeksu wszystko, czego nie ma już w bazie blobów. */ + public int prune() throws Exception { + try (Statement st = db.connection().createStatement()) { + return st.executeUpdate(""" + DELETE FROM artifact + WHERE kind = 'script' + AND blob_sha1 NOT IN (SELECT blob_sha1 FROM file) + """); + } + } +} diff --git a/src/main/java/pl/genschu/rexcatalog/script/ScriptSearch.java b/src/main/java/pl/genschu/rexcatalog/script/ScriptSearch.java new file mode 100644 index 0000000..05d59cb --- /dev/null +++ b/src/main/java/pl/genschu/rexcatalog/script/ScriptSearch.java @@ -0,0 +1,136 @@ +package pl.genschu.rexcatalog.script; + +import pl.genschu.rexcatalog.db.Database; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/** + * Odpytywanie indeksu skryptów. Wynik jest zawsze wiązany z kopiami, w których + * dany skrypt występuje — inaczej trafienie w blob nic nie mówi o tym, której gry dotyczy. + */ +public final class ScriptSearch { + + private final Database db; + private final Path dataDir; + + public ScriptSearch(Database db, Path dataDir) { + this.db = db; + this.dataDir = dataDir; + } + + public record Hit(String sha1, String snippet, List paths, List games) { + } + + public record Result(String effectiveQuery, boolean quoted, List hits) { + } + + /** + * Szuka najpierw dosłownie, żeby nie odbierać składni FTS5 (operatory, {@code *}, + * {@code NEAR}). Dopiero gdy zapytanie się nie parsuje — a tak kończy każde + * wyrażenie ze skryptu, choćby {@code TYPE=EPISODE} — powtarza je jako frazę. + */ + public Result find(String query, int limit) throws Exception { + try { + return new Result(query, false, search(query, limit)); + } catch (java.sql.SQLException e) { + String phrase = '"' + query.replace("\"", "\"\"") + '"'; + return new Result(phrase, true, search(phrase, limit)); + } + } + + private List search(String query, int limit) throws Exception { + List hits = new ArrayList<>(); + try (PreparedStatement st = db.connection().prepareStatement(""" + SELECT blob_sha1, snippet(script_fts, 0, '»', '«', ' … ', 12) AS snip + FROM script_fts + WHERE script_fts MATCH ? + ORDER BY rank + LIMIT ? + """)) { + st.setString(1, query); + st.setInt(2, limit); + try (ResultSet rs = st.executeQuery()) { + while (rs.next()) { + String sha1 = rs.getString("blob_sha1"); + hits.add(new Hit(sha1, rs.getString("snip"), + pathsOf(sha1), gamesOf(sha1))); + } + } + } + return hits; + } + + public long indexedCount() throws Exception { + try (PreparedStatement st = db.connection().prepareStatement( + "SELECT COUNT(*) FROM script_fts"); + ResultSet rs = st.executeQuery()) { + return rs.next() ? rs.getLong(1) : 0; + } + } + + private List pathsOf(String sha1) throws Exception { + return oneColumn("SELECT DISTINCT COALESCE(path_raw, path) FROM file " + + "WHERE blob_sha1 = ? ORDER BY 1", sha1); + } + + private List gamesOf(String sha1) throws Exception { + return oneColumn("SELECT DISTINCT c.display_name FROM file f " + + "JOIN copy c ON c.id = f.copy_id WHERE f.blob_sha1 = ? ORDER BY 1", sha1); + } + + private List oneColumn(String sql, String argument) throws Exception { + List out = new ArrayList<>(); + try (PreparedStatement st = db.connection().prepareStatement(sql)) { + st.setString(1, argument); + try (ResultSet rs = st.executeQuery()) { + while (rs.next()) { + out.add(rs.getString(1)); + } + } + } + return out; + } + + /** + * Zamienia to, co podał użytkownik, na blob: albo pełny SHA-1, albo fragment + * ścieżki. Niejednoznaczny fragment nie jest zgadywany — wołający dostaje + * wszystkich kandydatów i decyduje sam. + */ + public List resolve(String identifier) throws Exception { + if (identifier.matches("[0-9a-fA-F]{40}")) { + return List.of(identifier.toLowerCase(Locale.ROOT)); + } + return oneColumn("SELECT DISTINCT blob_sha1 FROM file WHERE path LIKE ? ORDER BY 1", + "%" + identifier.toLowerCase(Locale.ROOT).replace('\\', '/') + "%"); + } + + /** Zwraca odszyfrowaną treść z cache'u albo {@code null}, jeśli blob nie był dekodowany. */ + public String body(String sha1) throws Exception { + try (PreparedStatement st = db.connection().prepareStatement( + "SELECT path FROM artifact WHERE blob_sha1 = ? AND kind = 'script'")) { + st.setString(1, sha1); + try (ResultSet rs = st.executeQuery()) { + if (!rs.next()) { + return null; + } + Path file = dataDir.resolve(rs.getString("path")); + return Files.exists(file) ? Files.readString(file, StandardCharsets.UTF_8) : null; + } + } + } + + public List pathsFor(String sha1) throws Exception { + return pathsOf(sha1); + } + + public List gamesFor(String sha1) throws Exception { + return gamesOf(sha1); + } +}