import { test } from 'node:test'; import assert from 'node:assert/strict'; import { parseSrt, serializeSrt, decodeSubtitleBuffer, protectFormatting, rewrap, hasTranslatableText, cleanModelText, songKind, stripSongs } from '../src/srt.js'; import { parseNumberedLines, jsonSchemaFor, pickJsonValue, detectProfile, mtMessages } from '../src/prompts.js'; import { detectLanguage } from '../src/langdetect.js'; import { estimateMemory, parseWindowsRegistryGpus } from '../src/vram.js'; import { normalizeLmsModelRef } from '../src/engines/lmstudio.js'; import { explainPullError } from '../src/engines/ollama.js'; import { PullJob } from '../src/pulls.js'; import { looksLikeReasoning, repairMerges } from '../src/translator.js'; test('parseSrt keeps timing lines verbatim and handles messy input', () => { const src = '1\r\n00:00:01,000 --> 00:00:02,500\r\nHello\r\nthere\r\n\r\n\r\n2\r\n00:00:03,000 --> 00:00:04,000 X1:10 X2:20\r\nBye\r\n3\r\n00:00:05.5 --> 00:00:06,000\r\n42\r\n'; const { cues, eol } = parseSrt(src); assert.equal(eol, '\r\n'); assert.equal(cues.length, 3); assert.equal(cues[0].text, 'Hello\nthere'); assert.equal(cues[1].timing, '00:00:03,000 --> 00:00:04,000 X1:10 X2:20'); // index "3" glued to cue 2 without a blank line is recognised as the next index assert.equal(cues[1].text, 'Bye'); assert.equal(cues[2].text, '42'); // numeric dialogue line is kept assert.equal(cues[2].startMs, 5500); }); test('serializeSrt renumbers and round-trips timings', () => { const { cues } = parseSrt('5\n00:00:01,000 --> 00:00:02,000\nA\n\n9\n00:00:02,000 --> 00:00:03,000\nB\n'); const out = serializeSrt(cues, { getText: (c) => c.text.toLowerCase() }); assert.equal(out, '1\n00:00:01,000 --> 00:00:02,000\na\n\n2\n00:00:02,000 --> 00:00:03,000\nb\n'); }); test('decodeSubtitleBuffer detects windows-1250 Polish', () => { const text = 'Nie wiem, czy już tu był.\nMożesz mi pomóc? Żółć gęślą jaźń.\nTo się nie uda, że też musiałeś.'; // encode to cp1250 manually for the Polish letters used const map = { 'ą': 0xb9, 'ć': 0xe6, 'ę': 0xea, 'ł': 0xb3, 'ń': 0xf1, 'ó': 0xf3, 'ś': 0x9c, 'ź': 0x9f, 'ż': 0xbf, 'Ż': 0xaf }; const bytes = Buffer.from([...text].map((ch) => map[ch] ?? ch.charCodeAt(0))); const r = decodeSubtitleBuffer(bytes); assert.equal(r.encoding, 'windows-1250'); assert.equal(r.text, text); assert.equal(decodeSubtitleBuffer(Buffer.from('zażółć', 'utf8')).encoding, 'utf-8'); }); test('protectFormatting strips and restores wrapping tags', () => { let p = protectFormatting('{\\an8}Hello\nworld'); assert.equal(p.text, 'Hello\nworld'); assert.equal(p.restore('Cześć\nświecie'), '{\\an8}Cześć\nświecie'); p = protectFormatting('One\nTwo'); assert.equal(p.text, 'One\nTwo'); assert.equal(p.restore('Jeden\nDwa'), 'Jeden\nDwa'); p = protectFormatting('Say this now'); assert.equal(p.text, 'Say this now'); assert.equal(hasTranslatableText('♪ ♪'), false); assert.equal(hasTranslatableText('♪ la la ♪'), true); }); test('rewrap balances long lines and keeps dialogue', () => { assert.equal(rewrap('Krótki tekst.', 42), 'Krótki tekst.'); const r = rewrap('Słuchaj, nieważne co myślisz, że się stało, to nie jest tak, jak wygląda.', 42); const lines = r.split('\n'); assert.equal(lines.length, 2); assert.ok(lines.every((l) => l.length <= 42), r); assert.equal(rewrap('- Nie wierzysz mi, naprawdę, nawet odrobinę?\n- A powinienem?', 30), '- Nie wierzysz mi, naprawdę, nawet odrobinę?\n- A powinienem?'); // no one-letter word left at a line end for (const t of ['Detektyw Morgan chce spotkać się z Tobą w swoim gabinecie. Prosimy.', 'Detektyw Morgan chce spotkać się z Tobą w swoim gabinecie. Prosimy, jak najszybciej.']) { const s = rewrap(t, 42); assert.ok(!/(^|\s)\p{L}$/u.test(s.split('\n')[0]), s); } }); test('cleanModelText removes think blocks and fences', () => { assert.equal(cleanModelText('hmm\n```json\n{"1":"a"}\n```'), '{"1":"a"}'); }); test('parseNumberedLines takes the final answer block, ignores drafts', () => { const t = 'The user wants...\n1. Produce natural Polish.\n[1] Where? -> Gdzie?\n[2] Told -> Mówiłem\nFinal check.[1] Gdzie byłaś?\n[2] Mówiłam ci,
że u siostry.\n[3] Nigdy.'; assert.deepEqual([...parseNumberedLines(t, [1, 2, 3])], [[1, 'Gdzie byłaś?'], [2, 'Mówiłam ci,\nże u siostry.'], [3, 'Nigdy.']]); assert.deepEqual([...parseNumberedLines('[1] a\n[3] c', [1, 2, 3])], [[1, 'a'], [3, 'c']]); }); test('jsonSchemaFor anchors keys with the start of each source line', () => { const s = jsonSchemaFor([{ id: 4, text: 'The partner\'s office was on the twelfth floor' }, { id: 7, text: '"Quoted" \\ text' }]); assert.deepEqual(s.required, ["4|The partner's office…", '7|Quoted text']); assert.equal(s.additionalProperties, false); assert.equal(pickJsonValue({ '4|The partner': 'x' }, { id: 4, text: 'The partner left' }), 'x'); assert.equal(pickJsonValue({ 4: 'y' }, { id: 4, text: 'Anything' }), 'y'); }); test('profiles and MT templates', () => { assert.equal(detectProfile('translategemma:12b'), 'translategemma'); assert.equal(detectProfile('hf.co/tencent/HY-MT1.5-7B-GGUF'), 'hunyuan'); assert.equal(detectProfile('qwen3:8b'), 'generic'); const [m] = mtMessages('translategemma', { text: 'Hello', source: 'en', target: 'pl' }); assert.match(m.content, /^You are a professional English \(en\) to Polish \(pl\) translator\./); assert.match(m.content, /Polish:\n\n\nHello$/); }); test('detectLanguage', () => { assert.equal(detectLanguage("Where were you last night? I told you, I was at my sister's place. You don't know what it is."), 'en'); assert.equal(detectLanguage('Nie wiem, co się stało. To jest tylko żart, ale już nie mogę.'), 'pl'); assert.equal(detectLanguage('これは日本語のテキストです。'), 'ja'); }); test('repairMerges splits a sentence merged into one entry', () => { const batch = [ { id: 17, text: "The partner's office was on the twelfth floor" }, { id: 18, text: 'of a building that had seen better days.' }, { id: 19, text: 'Mr. Callahan?' }, ]; const map = new Map([[17, 'Biuro partnera było na dwunastym piętrze budynku, który widział lepsze czasy.'], [18, ''], [19, 'Panie Callahan?']]); assert.deepEqual(repairMerges(batch, map), [[17, 18]]); assert.equal(map.get(17), 'Biuro partnera było na dwunastym piętrze'); assert.equal(map.get(18), 'budynku, który widział lepsze czasy.'); // A legitimately short translation next to an empty one is not "repaired". const m2 = new Map([[1, 'Tak.'], [2, '']]); assert.deepEqual(repairMerges([{ id: 1, text: 'Yes.' }, { id: 2, text: 'Something else entirely here.' }], m2), []); }); test('looksLikeReasoning', () => { assert.equal(looksLikeReasoning('The user wants me to translate...', 'Hi'), true); assert.equal(looksLikeReasoning('Cześć, jak się masz?', 'Hi, how are you?'), false); }); const GEMMA4 = { 'general.architecture': 'gemma4', 'gemma4.block_count': 42, 'gemma4.embedding_length': 2560, 'gemma4.attention.head_count': 8, 'gemma4.attention.head_count_kv': 2, 'gemma4.attention.key_length': 512, 'gemma4.attention.value_length': 512, 'gemma4.attention.key_length_swa': 256, 'gemma4.attention.value_length_swa': 256, 'gemma4.attention.sliding_window': 512, 'gemma4.attention.shared_kv_layers': 18, 'gemma4.attention.sliding_window_pattern': Array.from({ length: 42 }, (_, i) => i % 6 !== 5), 'tokenizer.ggml.tokens': { length: 262144 }, }; test('estimateMemory: SWA, shared KV, KV quantization', () => { const f16 = estimateMemory(GEMMA4, { modelBytes: 5e9, nCtx: 32768, kType: 'f16', vType: 'f16', flashAttention: true }); assert.deepEqual(f16.layers, { full: 4, swa: 20, recurrent: 0, shared: 18 }); // 4 global layers × 32768 × 2 heads × (512+512) × 2 B + 20 SWA layers × 1024 × 2 × 512 × 2 B assert.equal(f16.kv, 4 * 32768 * 2 * 1024 * 2 + 20 * 1024 * 2 * 512 * 2); const q8 = estimateMemory(GEMMA4, { modelBytes: 5e9, nCtx: 32768, kType: 'q8_0', vType: 'q8_0', flashAttention: true }); assert.ok(Math.abs(q8.kv / f16.kv - 34 / 64) < 1e-9); const hybrid = estimateMemory({ 'general.architecture': 'qwen35', 'qwen35.block_count': 32, 'qwen35.embedding_length': 4096, 'qwen35.attention.head_count': 16, 'qwen35.attention.head_count_kv': 4, 'qwen35.attention.key_length': 256, 'qwen35.attention.value_length': 256, 'qwen35.full_attention_interval': 4, 'qwen35.ssm.inner_size': 4096, 'qwen35.ssm.state_size': 128, 'qwen35.ssm.conv_kernel': 4, 'qwen35.ssm.group_count': 16, }, { modelBytes: 5e9, nCtx: 8192 }); assert.equal(hybrid.layers.full, 8); assert.equal(hybrid.layers.recurrent, 24); assert.equal(hybrid.kv, 8 * 8192 * 4 * 512 * 2); }); test('song detection and removal', () => { assert.equal(songKind('♪ ♪'), 'full'); assert.equal(songKind("♪ I've been waiting\nall my life\nfor you ♪"), 'full'); assert.equal(songKind('# Jingle bells #'), 'full'); assert.equal(songKind('- ♪ Happy birthday ♪\n- Stop singing!'), 'partial'); assert.equal(songKind('[music playing]'), null); assert.equal(songKind('I paid #1 price'), null); // An unclosed note doesn't swallow the following dialogue line. assert.equal(stripSongs('♪ Happy birthday\n- Thanks, guys!'), 'Thanks, guys!'); assert.equal(stripSongs('- ♪ La la ♪\n- Stop!'), 'Stop!'); assert.equal(stripSongs('- ♪ La la ♪\n- Stop!'), 'Stop!'); assert.equal(stripSongs('♪ only lyrics ♪'), ''); assert.equal(stripSongs('Plain line'), 'Plain line'); }); test('parseWindowsRegistryGpus reads VRAM and names from reg query output', () => { const qw = [ 'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0000', ' HardwareInformation.qwMemorySize REG_QWORD 0x400000000', '', 'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0001', ' HardwareInformation.qwMemorySize REG_QWORD 0x8000000', '', 'End of search: 2 match(es) found.', ].join('\r\n'); const desc = [ 'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0000', ' DriverDesc REG_SZ AMD Radeon RX 7800 XT', '', 'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0001', ' DriverDesc REG_SZ Intel(R) UHD Graphics', ].join('\r\n'); assert.deepEqual(parseWindowsRegistryGpus(qw, desc), [{ name: 'AMD Radeon RX 7800 XT', total: 16 * 2 ** 30 }]); }); test('normalizeLmsModelRef accepts the usual ways of naming a model', () => { assert.deepEqual(normalizeLmsModelRef('google/gemma-3-12b'), { model: 'google/gemma-3-12b', quantization: undefined }); assert.deepEqual(normalizeLmsModelRef('hf.co/speakleash/Bielik-11B-v3.0-Instruct-GGUF:Q4_K_M'), { model: 'https://huggingface.co/speakleash/Bielik-11B-v3.0-Instruct-GGUF', quantization: 'Q4_K_M' }); assert.deepEqual(normalizeLmsModelRef('bartowski/EuroLLM-9B-Instruct-GGUF', 'Q5_K_M'), { model: 'https://huggingface.co/bartowski/EuroLLM-9B-Instruct-GGUF', quantization: 'Q5_K_M' }); assert.deepEqual(normalizeLmsModelRef('https://huggingface.co/tencent/Hy-MT2-7B-GGUF/'), { model: 'https://huggingface.co/tencent/Hy-MT2-7B-GGUF', quantization: undefined }); // quantization only applies to Hugging Face links assert.deepEqual(normalizeLmsModelRef('qwen/qwen3-8b', 'Q4_K_M'), { model: 'qwen/qwen3-8b', quantization: undefined }); }); function fakePullEngine(steps = 10, delay = 60) { return { kind: 'ollama', capabilities: () => ({}), calls: [], async pull(model, { onProgress, signal }) { // Resumes from wherever the previous attempt stopped, like Ollama does. let i = this.resumeAt || 0; this.calls.push(i); for (; i <= steps; i++) { if (signal.aborted) { this.resumeAt = i; throw new Error('aborted'); } onProgress({ phase: 'downloading', total: steps * 100, completed: i * 100, digests: ['sha256:' + 'a'.repeat(64)] }); await new Promise((r) => setTimeout(r, delay)); } return { ok: true }; }, }; } test('PullJob: one job through pause → resume → done, with speed and ETA', async () => { const eng = fakePullEngine(10, 150); const job = new PullJob({ engine: eng, engineKind: 'ollama', model: 'x' }); const seen = []; job.on('progress', (s) => seen.push(s.progress)); job.start(); await new Promise((r) => setTimeout(r, 250)); assert.equal(await job.pause(), true); assert.equal(job.state, 'paused'); assert.match(job.note, /Wznów/); const pausedAt = job.progress.completed; assert.equal(job.resume(), true); await job.runPromise; assert.equal(job.state, 'done'); assert.ok(eng.calls[1] > 0, 'second attempt resumed, not restarted'); assert.ok(job.progress.completed >= pausedAt); assert.ok(seen.some((p) => p.bytesPerSec > 0 && p.etaSec != null), 'speed/ETA computed'); }); test('PullJob: cancel deletes only this download\'s partial files', async () => { const { mkdtemp, writeFile, readdir } = await import('node:fs/promises'); const os = await import('node:os'); const path = await import('node:path'); const { OllamaEngine } = await import('../src/engines/ollama.js'); const dir = await mkdtemp(path.join(os.tmpdir(), 'blobs-')); const mine = 'a'.repeat(64), other = 'b'.repeat(64); for (const f of [`sha256-${mine}-partial`, `sha256-${mine}-partial-0`, `sha256-${mine}-partial-1`, `sha256-${other}-partial`, `sha256-${mine}`, `sha256-${other}`]) await writeFile(path.join(dir, f), 'xxxx'); const ollama = new OllamaEngine({}); const eng = { ...fakePullEngine(50, 30), discardPartial: (d) => ollama.discardPartial(d, dir) }; const job = new PullJob({ engine: eng, engineKind: 'ollama', model: 'x' }); job.start(); await new Promise((r) => setTimeout(r, 150)); assert.equal(await job.cancel(), true); assert.equal(job.state, 'cancelled'); assert.match(job.note, /usunięto/); // the finished blob with the same digest and another download's partial stay untouched assert.deepEqual((await readdir(dir)).sort(), [`sha256-${mine}`, `sha256-${other}`, `sha256-${other}-partial`].sort()); assert.equal(job.resume(), false, 'a cancelled download cannot be resumed'); }); test('explainPullError turns Ollama pull errors into hints', () => { assert.match(explainPullError('pull model manifest: file does not exist', 'foo:1b'), /bibliotece Ollamy/); assert.match(explainPullError('pull model manifest: realm host "huggingface.co" does not match original host "hf.co"', 'hf.co/x/y-GGUF:Q4_K_M'), /Hugging Face/); assert.equal(explainPullError('something else', 'foo'), 'Ollama: something else'); });