Initialize repo

This commit is contained in:
Patryk Gensch
2026-11-15 15:39:55 +01:00
commit af37f7966d
27 changed files with 5678 additions and 0 deletions
+255
View File
@@ -0,0 +1,255 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { parseSrt, serializeSrt, decodeSubtitleBuffer, protectFormatting, rewrap, hasTranslatableText, cleanModelText, songKind, stripSongs } from '../src/srt.js';
import { parseNumberedLines, jsonSchemaFor, pickJsonValue, detectProfile, mtMessages } from '../src/prompts.js';
import { detectLanguage } from '../src/langdetect.js';
import { estimateMemory, parseWindowsRegistryGpus } from '../src/vram.js';
import { normalizeLmsModelRef } from '../src/engines/lmstudio.js';
import { explainPullError } from '../src/engines/ollama.js';
import { PullJob } from '../src/pulls.js';
import { looksLikeReasoning, repairMerges } from '../src/translator.js';
test('parseSrt keeps timing lines verbatim and handles messy input', () => {
const src = '1\r\n00:00:01,000 --> 00:00:02,500\r\nHello\r\nthere\r\n\r\n\r\n2\r\n00:00:03,000 --> 00:00:04,000 X1:10 X2:20\r\n<i>Bye</i>\r\n3\r\n00:00:05.5 --> 00:00:06,000\r\n42\r\n';
const { cues, eol } = parseSrt(src);
assert.equal(eol, '\r\n');
assert.equal(cues.length, 3);
assert.equal(cues[0].text, 'Hello\nthere');
assert.equal(cues[1].timing, '00:00:03,000 --> 00:00:04,000 X1:10 X2:20');
// index "3" glued to cue 2 without a blank line is recognised as the next index
assert.equal(cues[1].text, '<i>Bye</i>');
assert.equal(cues[2].text, '42'); // numeric dialogue line is kept
assert.equal(cues[2].startMs, 5500);
});
test('serializeSrt renumbers and round-trips timings', () => {
const { cues } = parseSrt('5\n00:00:01,000 --> 00:00:02,000\nA\n\n9\n00:00:02,000 --> 00:00:03,000\nB\n');
const out = serializeSrt(cues, { getText: (c) => c.text.toLowerCase() });
assert.equal(out, '1\n00:00:01,000 --> 00:00:02,000\na\n\n2\n00:00:02,000 --> 00:00:03,000\nb\n');
});
test('decodeSubtitleBuffer detects windows-1250 Polish', () => {
const text = 'Nie wiem, czy już tu był.\nMożesz mi pomóc? Żółć gęślą jaźń.\nTo się nie uda, że też musiałeś.';
// encode to cp1250 manually for the Polish letters used
const map = { 'ą': 0xb9, 'ć': 0xe6, 'ę': 0xea, 'ł': 0xb3, 'ń': 0xf1, 'ó': 0xf3, 'ś': 0x9c, 'ź': 0x9f, 'ż': 0xbf, 'Ż': 0xaf };
const bytes = Buffer.from([...text].map((ch) => map[ch] ?? ch.charCodeAt(0)));
const r = decodeSubtitleBuffer(bytes);
assert.equal(r.encoding, 'windows-1250');
assert.equal(r.text, text);
assert.equal(decodeSubtitleBuffer(Buffer.from('zażółć', 'utf8')).encoding, 'utf-8');
});
test('protectFormatting strips and restores wrapping tags', () => {
let p = protectFormatting('{\\an8}<i>Hello\nworld</i>');
assert.equal(p.text, 'Hello\nworld');
assert.equal(p.restore('Cześć\nświecie'), '{\\an8}<i>Cześć\nświecie</i>');
p = protectFormatting('<i>One</i>\n<i>Two</i>');
assert.equal(p.text, 'One\nTwo');
assert.equal(p.restore('Jeden\nDwa'), '<i>Jeden</i>\n<i>Dwa</i>');
p = protectFormatting('Say <i>this</i> now');
assert.equal(p.text, 'Say <i>this</i> now');
assert.equal(hasTranslatableText('♪ ♪'), false);
assert.equal(hasTranslatableText('<i>♪ la la ♪</i>'), true);
});
test('rewrap balances long lines and keeps dialogue', () => {
assert.equal(rewrap('Krótki tekst.', 42), 'Krótki tekst.');
const r = rewrap('Słuchaj, nieważne co myślisz, że się stało, to nie jest tak, jak wygląda.', 42);
const lines = r.split('\n');
assert.equal(lines.length, 2);
assert.ok(lines.every((l) => l.length <= 42), r);
assert.equal(rewrap('- Nie wierzysz mi, naprawdę, nawet odrobinę?\n- A powinienem?', 30), '- Nie wierzysz mi, naprawdę, nawet odrobinę?\n- A powinienem?');
// no one-letter word left at a line end
for (const t of ['Detektyw Morgan chce spotkać się z Tobą w swoim gabinecie. Prosimy.', 'Detektyw Morgan chce spotkać się z Tobą w swoim gabinecie. Prosimy, jak najszybciej.']) {
const s = rewrap(t, 42);
assert.ok(!/(^|\s)\p{L}$/u.test(s.split('\n')[0]), s);
}
});
test('cleanModelText removes think blocks and fences', () => {
assert.equal(cleanModelText('<think>hmm</think>\n```json\n{"1":"a"}\n```'), '{"1":"a"}');
});
test('parseNumberedLines takes the final answer block, ignores drafts', () => {
const t = 'The user wants...\n1. Produce natural Polish.\n[1] Where? -> Gdzie?\n[2] Told -> Mówiłem\nFinal check.[1] Gdzie byłaś?\n[2] Mówiłam ci, <br> że u siostry.\n[3] Nigdy.';
assert.deepEqual([...parseNumberedLines(t, [1, 2, 3])], [[1, 'Gdzie byłaś?'], [2, 'Mówiłam ci,\nże u siostry.'], [3, 'Nigdy.']]);
assert.deepEqual([...parseNumberedLines('[1] a\n[3] c', [1, 2, 3])], [[1, 'a'], [3, 'c']]);
});
test('jsonSchemaFor anchors keys with the start of each source line', () => {
const s = jsonSchemaFor([{ id: 4, text: '<i>The partner\'s office was on the twelfth floor</i>' }, { id: 7, text: '"Quoted" \\ text' }]);
assert.deepEqual(s.required, ["4|The partner's office…", '7|Quoted text']);
assert.equal(s.additionalProperties, false);
assert.equal(pickJsonValue({ '4|The partner': 'x' }, { id: 4, text: 'The partner left' }), 'x');
assert.equal(pickJsonValue({ 4: 'y' }, { id: 4, text: 'Anything' }), 'y');
});
test('profiles and MT templates', () => {
assert.equal(detectProfile('translategemma:12b'), 'translategemma');
assert.equal(detectProfile('hf.co/tencent/HY-MT1.5-7B-GGUF'), 'hunyuan');
assert.equal(detectProfile('qwen3:8b'), 'generic');
const [m] = mtMessages('translategemma', { text: 'Hello', source: 'en', target: 'pl' });
assert.match(m.content, /^You are a professional English \(en\) to Polish \(pl\) translator\./);
assert.match(m.content, /Polish:\n\n\nHello$/);
});
test('detectLanguage', () => {
assert.equal(detectLanguage("Where were you last night? I told you, I was at my sister's place. You don't know what it is."), 'en');
assert.equal(detectLanguage('Nie wiem, co się stało. To jest tylko żart, ale już nie mogę.'), 'pl');
assert.equal(detectLanguage('これは日本語のテキストです。'), 'ja');
});
test('repairMerges splits a sentence merged into one entry', () => {
const batch = [
{ id: 17, text: "The partner's office was on the twelfth floor" },
{ id: 18, text: 'of a building that had seen better days.' },
{ id: 19, text: 'Mr. Callahan?' },
];
const map = new Map([[17, 'Biuro partnera było na dwunastym piętrze budynku, który widział lepsze czasy.'], [18, ''], [19, 'Panie Callahan?']]);
assert.deepEqual(repairMerges(batch, map), [[17, 18]]);
assert.equal(map.get(17), 'Biuro partnera było na dwunastym piętrze');
assert.equal(map.get(18), 'budynku, który widział lepsze czasy.');
// A legitimately short translation next to an empty one is not "repaired".
const m2 = new Map([[1, 'Tak.'], [2, '']]);
assert.deepEqual(repairMerges([{ id: 1, text: 'Yes.' }, { id: 2, text: 'Something else entirely here.' }], m2), []);
});
test('looksLikeReasoning', () => {
assert.equal(looksLikeReasoning('The user wants me to translate...', 'Hi'), true);
assert.equal(looksLikeReasoning('Cześć, jak się masz?', 'Hi, how are you?'), false);
});
const GEMMA4 = {
'general.architecture': 'gemma4', 'gemma4.block_count': 42, 'gemma4.embedding_length': 2560,
'gemma4.attention.head_count': 8, 'gemma4.attention.head_count_kv': 2, 'gemma4.attention.key_length': 512,
'gemma4.attention.value_length': 512, 'gemma4.attention.key_length_swa': 256, 'gemma4.attention.value_length_swa': 256,
'gemma4.attention.sliding_window': 512, 'gemma4.attention.shared_kv_layers': 18,
'gemma4.attention.sliding_window_pattern': Array.from({ length: 42 }, (_, i) => i % 6 !== 5),
'tokenizer.ggml.tokens': { length: 262144 },
};
test('estimateMemory: SWA, shared KV, KV quantization', () => {
const f16 = estimateMemory(GEMMA4, { modelBytes: 5e9, nCtx: 32768, kType: 'f16', vType: 'f16', flashAttention: true });
assert.deepEqual(f16.layers, { full: 4, swa: 20, recurrent: 0, shared: 18 });
// 4 global layers × 32768 × 2 heads × (512+512) × 2 B + 20 SWA layers × 1024 × 2 × 512 × 2 B
assert.equal(f16.kv, 4 * 32768 * 2 * 1024 * 2 + 20 * 1024 * 2 * 512 * 2);
const q8 = estimateMemory(GEMMA4, { modelBytes: 5e9, nCtx: 32768, kType: 'q8_0', vType: 'q8_0', flashAttention: true });
assert.ok(Math.abs(q8.kv / f16.kv - 34 / 64) < 1e-9);
const hybrid = estimateMemory({
'general.architecture': 'qwen35', 'qwen35.block_count': 32, 'qwen35.embedding_length': 4096, 'qwen35.attention.head_count': 16,
'qwen35.attention.head_count_kv': 4, 'qwen35.attention.key_length': 256, 'qwen35.attention.value_length': 256,
'qwen35.full_attention_interval': 4, 'qwen35.ssm.inner_size': 4096, 'qwen35.ssm.state_size': 128, 'qwen35.ssm.conv_kernel': 4, 'qwen35.ssm.group_count': 16,
}, { modelBytes: 5e9, nCtx: 8192 });
assert.equal(hybrid.layers.full, 8);
assert.equal(hybrid.layers.recurrent, 24);
assert.equal(hybrid.kv, 8 * 8192 * 4 * 512 * 2);
});
test('song detection and removal', () => {
assert.equal(songKind('♪ ♪'), 'full');
assert.equal(songKind("<i>♪ I've been waiting\nall my life\nfor you ♪</i>"), 'full');
assert.equal(songKind('# Jingle bells #'), 'full');
assert.equal(songKind('- ♪ Happy birthday ♪\n- Stop singing!'), 'partial');
assert.equal(songKind('[music playing]'), null);
assert.equal(songKind('I paid #1 price'), null);
// An unclosed note doesn't swallow the following dialogue line.
assert.equal(stripSongs('♪ Happy birthday\n- Thanks, guys!'), 'Thanks, guys!');
assert.equal(stripSongs('- ♪ La la ♪\n- Stop!'), 'Stop!');
assert.equal(stripSongs('<i>- ♪ La la ♪\n- Stop!</i>'), '<i>Stop!</i>');
assert.equal(stripSongs('♪ only lyrics ♪'), '');
assert.equal(stripSongs('Plain line'), 'Plain line');
});
test('parseWindowsRegistryGpus reads VRAM and names from reg query output', () => {
const qw = [
'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0000',
' HardwareInformation.qwMemorySize REG_QWORD 0x400000000',
'',
'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0001',
' HardwareInformation.qwMemorySize REG_QWORD 0x8000000',
'',
'End of search: 2 match(es) found.',
].join('\r\n');
const desc = [
'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0000',
' DriverDesc REG_SZ AMD Radeon RX 7800 XT',
'',
'HKEY_LOCAL_MACHINE\\SYSTEM\\CurrentControlSet\\Control\\Class\\{4d36e968-e325-11ce-bfc1-08002be10318}\\0001',
' DriverDesc REG_SZ Intel(R) UHD Graphics',
].join('\r\n');
assert.deepEqual(parseWindowsRegistryGpus(qw, desc), [{ name: 'AMD Radeon RX 7800 XT', total: 16 * 2 ** 30 }]);
});
test('normalizeLmsModelRef accepts the usual ways of naming a model', () => {
assert.deepEqual(normalizeLmsModelRef('google/gemma-3-12b'), { model: 'google/gemma-3-12b', quantization: undefined });
assert.deepEqual(normalizeLmsModelRef('hf.co/speakleash/Bielik-11B-v3.0-Instruct-GGUF:Q4_K_M'), { model: 'https://huggingface.co/speakleash/Bielik-11B-v3.0-Instruct-GGUF', quantization: 'Q4_K_M' });
assert.deepEqual(normalizeLmsModelRef('bartowski/EuroLLM-9B-Instruct-GGUF', 'Q5_K_M'), { model: 'https://huggingface.co/bartowski/EuroLLM-9B-Instruct-GGUF', quantization: 'Q5_K_M' });
assert.deepEqual(normalizeLmsModelRef('https://huggingface.co/tencent/Hy-MT2-7B-GGUF/'), { model: 'https://huggingface.co/tencent/Hy-MT2-7B-GGUF', quantization: undefined });
// quantization only applies to Hugging Face links
assert.deepEqual(normalizeLmsModelRef('qwen/qwen3-8b', 'Q4_K_M'), { model: 'qwen/qwen3-8b', quantization: undefined });
});
function fakePullEngine(steps = 10, delay = 60) {
return {
kind: 'ollama',
capabilities: () => ({}),
calls: [],
async pull(model, { onProgress, signal }) {
// Resumes from wherever the previous attempt stopped, like Ollama does.
let i = this.resumeAt || 0;
this.calls.push(i);
for (; i <= steps; i++) {
if (signal.aborted) { this.resumeAt = i; throw new Error('aborted'); }
onProgress({ phase: 'downloading', total: steps * 100, completed: i * 100, digests: ['sha256:' + 'a'.repeat(64)] });
await new Promise((r) => setTimeout(r, delay));
}
return { ok: true };
},
};
}
test('PullJob: one job through pause → resume → done, with speed and ETA', async () => {
const eng = fakePullEngine(10, 150);
const job = new PullJob({ engine: eng, engineKind: 'ollama', model: 'x' });
const seen = [];
job.on('progress', (s) => seen.push(s.progress));
job.start();
await new Promise((r) => setTimeout(r, 250));
assert.equal(await job.pause(), true);
assert.equal(job.state, 'paused');
assert.match(job.note, /Wznów/);
const pausedAt = job.progress.completed;
assert.equal(job.resume(), true);
await job.runPromise;
assert.equal(job.state, 'done');
assert.ok(eng.calls[1] > 0, 'second attempt resumed, not restarted');
assert.ok(job.progress.completed >= pausedAt);
assert.ok(seen.some((p) => p.bytesPerSec > 0 && p.etaSec != null), 'speed/ETA computed');
});
test('PullJob: cancel deletes only this download\'s partial files', async () => {
const { mkdtemp, writeFile, readdir } = await import('node:fs/promises');
const os = await import('node:os');
const path = await import('node:path');
const { OllamaEngine } = await import('../src/engines/ollama.js');
const dir = await mkdtemp(path.join(os.tmpdir(), 'blobs-'));
const mine = 'a'.repeat(64), other = 'b'.repeat(64);
for (const f of [`sha256-${mine}-partial`, `sha256-${mine}-partial-0`, `sha256-${mine}-partial-1`, `sha256-${other}-partial`, `sha256-${mine}`, `sha256-${other}`]) await writeFile(path.join(dir, f), 'xxxx');
const ollama = new OllamaEngine({});
const eng = { ...fakePullEngine(50, 30), discardPartial: (d) => ollama.discardPartial(d, dir) };
const job = new PullJob({ engine: eng, engineKind: 'ollama', model: 'x' });
job.start();
await new Promise((r) => setTimeout(r, 150));
assert.equal(await job.cancel(), true);
assert.equal(job.state, 'cancelled');
assert.match(job.note, /usunięto/);
// the finished blob with the same digest and another download's partial stay untouched
assert.deepEqual((await readdir(dir)).sort(), [`sha256-${mine}`, `sha256-${other}`, `sha256-${other}-partial`].sort());
assert.equal(job.resume(), false, 'a cancelled download cannot be resumed');
});
test('explainPullError turns Ollama pull errors into hints', () => {
assert.match(explainPullError('pull model manifest: file does not exist', 'foo:1b'), /bibliotece Ollamy/);
assert.match(explainPullError('pull model manifest: realm host "huggingface.co" does not match original host "hf.co"', 'hf.co/x/y-GGUF:Q4_K_M'), /Hugging Face/);
assert.equal(explainPullError('something else', 'foo'), 'Ollama: something else');
});