Files
2026-11-15 15:39:55 +01:00

256 lines
15 KiB
JavaScript
Raw Permalink Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { parseSrt, serializeSrt, decodeSubtitleBuffer, protectFormatting, rewrap, hasTranslatableText, cleanModelText, songKind, stripSongs } from '../src/srt.js';
import { parseNumberedLines, jsonSchemaFor, pickJsonValue, detectProfile, mtMessages } from '../src/prompts.js';
import { detectLanguage } from '../src/langdetect.js';
import { estimateMemory, parseWindowsRegistryGpus } from '../src/vram.js';
import { normalizeLmsModelRef } from '../src/engines/lmstudio.js';
import { explainPullError } from '../src/engines/ollama.js';
import { PullJob } from '../src/pulls.js';
import { looksLikeReasoning, repairMerges } from '../src/translator.js';
test('parseSrt keeps timing lines verbatim and handles messy input', () => {
const src = '1\r\n00:00:01,000 --> 00:00:02,500\r\nHello\r\nthere\r\n\r\n\r\n2\r\n00:00:03,000 --> 00:00:04,000 X1:10 X2:20\r\n<i>Bye</i>\r\n3\r\n00:00:05.5 --> 00:00:06,000\r\n42\r\n';
const { cues, eol } = parseSrt(src);
assert.equal(eol, '\r\n');
assert.equal(cues.length, 3);
assert.equal(cues[0].text, 'Hello\nthere');
assert.equal(cues[1].timing, '00:00:03,000 --> 00:00:04,000 X1:10 X2:20');
// index "3" glued to cue 2 without a blank line is recognised as the next index
assert.equal(cues[1].text, '<i>Bye</i>');
assert.equal(cues[2].text, '42'); // numeric dialogue line is kept
assert.equal(cues[2].startMs, 5500);
});
test('serializeSrt renumbers and round-trips timings', () => {
const { cues } = parseSrt('5\n00:00:01,000 --> 00:00:02,000\nA\n\n9\n00:00:02,000 --> 00:00:03,000\nB\n');
const out = serializeSrt(cues, { getText: (c) => c.text.toLowerCase() });
assert.equal(out, '1\n00:00:01,000 --> 00:00:02,000\na\n\n2\n00:00:02,000 --> 00:00:03,000\nb\n');
});
test('decodeSubtitleBuffer detects windows-1250 Polish', () => {
const text = 'Nie wiem, czy już tu był.\nMożesz mi pomóc? Żółć gęślą jaźń.\nTo się nie uda, że też musiałeś.';
// encode to cp1250 manually for the Polish letters used
const map = { 'ą': 0xb9, 'ć': 0xe6, 'ę': 0xea, 'ł': 0xb3, 'ń': 0xf1, 'ó': 0xf3, 'ś': 0x9c, 'ź': 0x9f, 'ż': 0xbf, 'Ż': 0xaf };
const bytes = Buffer.from([...text].map((ch) => map[ch] ?? ch.charCodeAt(0)));
const r = decodeSubtitleBuffer(bytes);
assert.equal(r.encoding, 'windows-1250');
assert.equal(r.text, text);
assert.equal(decodeSubtitleBuffer(Buffer.from('zażółć', 'utf8')).encoding, 'utf-8');
});
test('protectFormatting strips and restores wrapping tags', () => {
let p = protectFormatting('{\\an8}<i>Hello\nworld</i>');
assert.equal(p.text, 'Hello\nworld');
assert.equal(p.restore('Cześć\nświecie'), '{\\an8}<i>Cześć\nświecie</i>');
p = protectFormatting('<i>One</i>\n<i>Two</i>');
assert.equal(p.text, 'One\nTwo');
assert.equal(p.restore('Jeden\nDwa'), '<i>Jeden</i>\n<i>Dwa</i>');
p = protectFormatting('Say <i>this</i> now');
assert.equal(p.text, 'Say <i>this</i> now');
assert.equal(hasTranslatableText('♪ ♪'), false);
assert.equal(hasTranslatableText('<i>♪ la la ♪</i>'), true);
});
test('rewrap balances long lines and keeps dialogue', () => {
assert.equal(rewrap('Krótki tekst.', 42), 'Krótki tekst.');
const r = rewrap('Słuchaj, nieważne co myślisz, że się stało, to nie jest tak, jak wygląda.', 42);
const lines = r.split('\n');
assert.equal(lines.length, 2);
assert.ok(lines.every((l) => l.length <= 42), r);
assert.equal(rewrap('- Nie wierzysz mi, naprawdę, nawet odrobinę?\n- A powinienem?', 30), '- Nie wierzysz mi, naprawdę, nawet odrobinę?\n- A powinienem?');
// no one-letter word left at a line end
for (const t of ['Detektyw Morgan chce spotkać się z Tobą w swoim gabinecie. Prosimy.', 'Detektyw Morgan chce spotkać się z Tobą w swoim gabinecie. Prosimy, jak najszybciej.']) {
const s = rewrap(t, 42);
assert.ok(!/(^|\s)\p{L}$/u.test(s.split('\n')[0]), s);
}
});
test('cleanModelText removes think blocks and fences', () => {
assert.equal(cleanModelText('<think>hmm</think>\n```json\n{"1":"a"}\n```'), '{"1":"a"}');
});
test('parseNumberedLines takes the final answer block, ignores drafts', () => {
const t = 'The user wants...\n1. Produce natural Polish.\n[1] Where? -> Gdzie?\n[2] Told -> Mówiłem\nFinal check.[1] Gdzie byłaś?\n[2] Mówiłam ci, <br> że u siostry.\n[3] Nigdy.';
assert.deepEqual([...parseNumberedLines(t, [1, 2, 3])], [[1, 'Gdzie byłaś?'], [2, 'Mówiłam ci,\nże u siostry.'], [3, 'Nigdy.']]);
assert.deepEqual([...parseNumberedLines('[1] a\n[3] c', [1, 2, 3])], [[1, 'a'], [3, 'c']]);
});
test('jsonSchemaFor anchors keys with the start of each source line', () => {
const s = jsonSchemaFor([{ id: 4, text: '<i>The partner\'s office was on the twelfth floor</i>' }, { id: 7, text: '"Quoted" \\ text' }]);
assert.deepEqual(s.required, ["4|The partner's office…", '7|Quoted text']);
assert.equal(s.additionalProperties, false);
assert.equal(pickJsonValue({ '4|The partner': 'x' }, { id: 4, text: 'The partner left' }), 'x');
assert.equal(pickJsonValue({ 4: 'y' }, { id: 4, text: 'Anything' }), 'y');
});
test('profiles and MT templates', () => {
assert.equal(detectProfile('translategemma:12b'), 'translategemma');
assert.equal(detectProfile('hf.co/tencent/HY-MT1.5-7B-GGUF'), 'hunyuan');
assert.equal(detectProfile('qwen3:8b'), 'generic');
const [m] = mtMessages('translategemma', { text: 'Hello', source: 'en', target: 'pl' });
assert.match(m.content, /^You are a professional English \(en\) to Polish \(pl\) translator\./);
assert.match(m.content, /Polish:\n\n\nHello$/);
});
test('detectLanguage', () => {
assert.equal(detectLanguage("Where were you last night? I told you, I was at my sister's place. You don't know what it is."), 'en');
assert.equal(detectLanguage('Nie wiem, co się stało. To jest tylko żart, ale już nie mogę.'), 'pl');
assert.equal(detectLanguage('これは日本語のテキストです。'), 'ja');
});
test('repairMerges splits a sentence merged into one entry', () => {
const batch = [
{ id: 17, text: "The partner's office was on the twelfth floor" },
{ id: 18, text: 'of a building that had seen better days.' },
{ id: 19, text: 'Mr. Callahan?' },
];
const map = new Map([[17, 'Biuro partnera było na dwunastym piętrze budynku, który widział lepsze czasy.'], [18, ''], [19, 'Panie Callahan?']]);
assert.deepEqual(repairMerges(batch, map), [[17, 18]]);
assert.equal(map.get(17), 'Biuro partnera było na dwunastym piętrze');
assert.equal(map.get(18), 'budynku, który widział lepsze czasy.');
// A legitimately short translation next to an empty one is not "repaired".
const m2 = new Map([[1, 'Tak.'], [2, '']]);
assert.deepEqual(repairMerges([{ id: 1, text: 'Yes.' }, { id: 2, text: 'Something else entirely here.' }], m2), []);
});
test('looksLikeReasoning', () => {
assert.equal(looksLikeReasoning('The user wants me to translate...', 'Hi'), true);
assert.equal(looksLikeReasoning('Cześć, jak się masz?', 'Hi, how are you?'), false);
});
const GEMMA4 = {
'general.architecture': 'gemma4', 'gemma4.block_count': 42, 'gemma4.embedding_length': 2560,
'gemma4.attention.head_count': 8, 'gemma4.attention.head_count_kv': 2, 'gemma4.attention.key_length': 512,
'gemma4.attention.value_length': 512, 'gemma4.attention.key_length_swa': 256, 'gemma4.attention.value_length_swa': 256,
'gemma4.attention.sliding_window': 512, 'gemma4.attention.shared_kv_layers': 18,
'gemma4.attention.sliding_window_pattern': Array.from({ length: 42 }, (_, i) => i % 6 !== 5),
'tokenizer.ggml.tokens': { length: 262144 },
};
test('estimateMemory: SWA, shared KV, KV quantization', () => {
const f16 = estimateMemory(GEMMA4, { modelBytes: 5e9, nCtx: 32768, kType: 'f16', vType: 'f16', flashAttention: true });
assert.deepEqual(f16.layers, { full: 4, swa: 20, recurrent: 0, shared: 18 });
// 4 global layers × 32768 × 2 heads × (512+512) × 2 B + 20 SWA layers × 1024 × 2 × 512 × 2 B
assert.equal(f16.kv, 4 * 32768 * 2 * 1024 * 2 + 20 * 1024 * 2 * 512 * 2);
const q8 = estimateMemory(GEMMA4, { modelBytes: 5e9, nCtx: 32768, kType: 'q8_0', vType: 'q8_0', flashAttention: true });
assert.ok(Math.abs(q8.kv / f16.kv - 34 / 64) < 1e-9);
const hybrid = estimateMemory({
'general.architecture': 'qwen35', 'qwen35.block_count': 32, 'qwen35.embedding_length': 4096, 'qwen35.attention.head_count': 16,
'qwen35.attention.head_count_kv': 4, 'qwen35.attention.key_length': 256, 'qwen35.attention.value_length': 256,
'qwen35.full_attention_interval': 4, 'qwen35.ssm.inner_size': 4096, 'qwen35.ssm.state_size': 128, 'qwen35.ssm.conv_kernel': 4, 'qwen35.ssm.group_count': 16,
}, { modelBytes: 5e9, nCtx: 8192 });
assert.equal(hybrid.layers.full, 8);
assert.equal(hybrid.layers.recurrent, 24);
assert.equal(hybrid.kv, 8 * 8192 * 4 * 512 * 2);
});
test('song detection and removal', () => {
assert.equal(songKind('♪ ♪'), 'full');
assert.equal(songKind("<i>♪ I've been waiting\nall my life\nfor you ♪</i>"), 'full');
assert.equal(songKind('# Jingle bells #'), 'full');
assert.equal(songKind('- ♪ Happy birthday ♪\n- Stop singing!'), 'partial');
assert.equal(songKind('[music playing]'), null);
assert.equal(songKind('I paid #1 price'), null);
// An unclosed note doesn't swallow the following dialogue line.
assert.equal(stripSongs('♪ Happy birthday\n- Thanks, guys!'), 'Thanks, guys!');
assert.equal(stripSongs('- ♪ La la ♪\n- Stop!'), 'Stop!');
assert.equal(stripSongs('<i>- ♪ La la ♪\n- Stop!</i>'), '<i>Stop!</i>');
assert.equal(stripSongs('♪ only lyrics ♪'), '');
assert.equal(stripSongs('Plain line'), 'Plain line');
});
test('parseWindowsRegistryGpus reads VRAM and names from reg query output', () => {
const qw = [
'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0000',
' HardwareInformation.qwMemorySize REG_QWORD 0x400000000',
'',
'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0001',
' HardwareInformation.qwMemorySize REG_QWORD 0x8000000',
'',
'End of search: 2 match(es) found.',
].join('\r\n');
const desc = [
'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0000',
' DriverDesc REG_SZ AMD Radeon RX 7800 XT',
'',
'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0001',
' DriverDesc REG_SZ Intel(R) UHD Graphics',
].join('\r\n');
assert.deepEqual(parseWindowsRegistryGpus(qw, desc), [{ name: 'AMD Radeon RX 7800 XT', total: 16 * 2 ** 30 }]);
});
test('normalizeLmsModelRef accepts the usual ways of naming a model', () => {
assert.deepEqual(normalizeLmsModelRef('google/gemma-3-12b'), { model: 'google/gemma-3-12b', quantization: undefined });
assert.deepEqual(normalizeLmsModelRef('hf.co/speakleash/Bielik-11B-v3.0-Instruct-GGUF:Q4_K_M'), { model: 'https://huggingface.co/speakleash/Bielik-11B-v3.0-Instruct-GGUF', quantization: 'Q4_K_M' });
assert.deepEqual(normalizeLmsModelRef('bartowski/EuroLLM-9B-Instruct-GGUF', 'Q5_K_M'), { model: 'https://huggingface.co/bartowski/EuroLLM-9B-Instruct-GGUF', quantization: 'Q5_K_M' });
assert.deepEqual(normalizeLmsModelRef('https://huggingface.co/tencent/Hy-MT2-7B-GGUF/'), { model: 'https://huggingface.co/tencent/Hy-MT2-7B-GGUF', quantization: undefined });
// quantization only applies to Hugging Face links
assert.deepEqual(normalizeLmsModelRef('qwen/qwen3-8b', 'Q4_K_M'), { model: 'qwen/qwen3-8b', quantization: undefined });
});
function fakePullEngine(steps = 10, delay = 60) {
return {
kind: 'ollama',
capabilities: () => ({}),
calls: [],
async pull(model, { onProgress, signal }) {
// Resumes from wherever the previous attempt stopped, like Ollama does.
let i = this.resumeAt || 0;
this.calls.push(i);
for (; i <= steps; i++) {
if (signal.aborted) { this.resumeAt = i; throw new Error('aborted'); }
onProgress({ phase: 'downloading', total: steps * 100, completed: i * 100, digests: ['sha256:' + 'a'.repeat(64)] });
await new Promise((r) => setTimeout(r, delay));
}
return { ok: true };
},
};
}
test('PullJob: one job through pause → resume → done, with speed and ETA', async () => {
const eng = fakePullEngine(10, 150);
const job = new PullJob({ engine: eng, engineKind: 'ollama', model: 'x' });
const seen = [];
job.on('progress', (s) => seen.push(s.progress));
job.start();
await new Promise((r) => setTimeout(r, 250));
assert.equal(await job.pause(), true);
assert.equal(job.state, 'paused');
assert.match(job.note, /Wznów/);
const pausedAt = job.progress.completed;
assert.equal(job.resume(), true);
await job.runPromise;
assert.equal(job.state, 'done');
assert.ok(eng.calls[1] > 0, 'second attempt resumed, not restarted');
assert.ok(job.progress.completed >= pausedAt);
assert.ok(seen.some((p) => p.bytesPerSec > 0 && p.etaSec != null), 'speed/ETA computed');
});
test('PullJob: cancel deletes only this download\'s partial files', async () => {
const { mkdtemp, writeFile, readdir } = await import('node:fs/promises');
const os = await import('node:os');
const path = await import('node:path');
const { OllamaEngine } = await import('../src/engines/ollama.js');
const dir = await mkdtemp(path.join(os.tmpdir(), 'blobs-'));
const mine = 'a'.repeat(64), other = 'b'.repeat(64);
for (const f of [`sha256-${mine}-partial`, `sha256-${mine}-partial-0`, `sha256-${mine}-partial-1`, `sha256-${other}-partial`, `sha256-${mine}`, `sha256-${other}`]) await writeFile(path.join(dir, f), 'xxxx');
const ollama = new OllamaEngine({});
const eng = { ...fakePullEngine(50, 30), discardPartial: (d) => ollama.discardPartial(d, dir) };
const job = new PullJob({ engine: eng, engineKind: 'ollama', model: 'x' });
job.start();
await new Promise((r) => setTimeout(r, 150));
assert.equal(await job.cancel(), true);
assert.equal(job.state, 'cancelled');
assert.match(job.note, /usunięto/);
// the finished blob with the same digest and another download's partial stay untouched
assert.deepEqual((await readdir(dir)).sort(), [`sha256-${mine}`, `sha256-${other}`, `sha256-${other}-partial`].sort());
assert.equal(job.resume(), false, 'a cancelled download cannot be resumed');
});
test('explainPullError turns Ollama pull errors into hints', () => {
assert.match(explainPullError('pull model manifest: file does not exist', 'foo:1b'), /bibliotece Ollamy/);
assert.match(explainPullError('pull model manifest: realm host "huggingface.co" does not match original host "hf.co"', 'hf.co/x/y-GGUF:Q4_K_M'), /Hugging Face/);
assert.equal(explainPullError('something else', 'foo'), 'Ollama: something else');
});