commit d6ce9c9e61990039a2863ed2394c64e3b8b6b424 equwal <truex@equwal.com> 2026-08-26 03:20:24 -0700 Switch language and model live, without restarting Streams change language mid-session, and restarting to do so drops captions for the duration of a model load. Language and model are now controlled at runtime over the existing WebSocket, with a control panel at /control.html. Language changes are instant (no reload) and apply to the next sentence; 1-9 hotkeys cover the configured set, defaulting to fi,ru,ja,es,pt,en,auto. Model changes are queued onto the worker thread that owns the model, so the websocket thread never loads one, and audio queued for the outgoing model is dropped rather than transcribed by the wrong one. A failed load keeps the previous model rather than leaving the process without one. Also fixes two problems this surfaced: - Non-Latin captions were mangled in logs, because Windows encodes redirected stdout with the console codepage. Forced to UTF-8, so ru/ja output survives. - soundcard warns on every loopback gap, flooding the console when the capture device is idle. Silenced; gaps are irrelevant to speech recognition. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
README.md | 44 ++++++++++ control.html | 233 ++++++++++++++++++++++++++++++++++++++++++++++++++++ livecap.py | 177 +++++++++++++++++++++++++++++++++++++-- tests/test_smoke.py | 65 +++++++++++++++ 4 files changed, 511 insertions(+), 8 deletions(-)
diff --git a/README.md b/README.md index 77c0775..8f78e18 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,50 @@ Check your devices first if needed: .\run.ps1 --list-devices ``` +## Changing language and model while streaming + +`run.ps1` also prints a control panel URL. Open it in any browser (a second +monitor, a phone on the same machine, or an OBS dock via +**Docks → Custom Browser Docks**): + +``` +http://127.0.0.1:8777/control.html?ws=8765 +``` + +- **Language** — one click, or press `1`–`9`. Applies to the next sentence. + No restart and no model reload, so it is effectively instant. +- **Model** — a dropdown. Swapping reloads the model, which pauses captions for + a few seconds; the panel shows a *loading* state while it happens. +- **Toggles** — English translation, live partial text, and clear captions. +- A live feed of what is being recognised, so you can sanity-check without + looking at the stream. + +Choose which languages get buttons: + +```bash +.\run.ps1 --langs fi,ru,ja,es,pt,en,auto --lang fi +``` + +That default set covers Finnish, Russian, Japanese, Spanish, Portuguese, +English and auto-detect. Any [Whisper language code](https://github.com/openai/whisper#available-models-and-languages) +works. + +Two things worth knowing: + +- **Whisper has no regional variants.** Argentine Spanish is `es` — there is no + `es-AR`. The model handles Rioplatense pronunciation and *voseo* as part of + `es`, but it normalises toward standard orthography and will not reliably + reproduce regional slang. The same applies to Brazilian vs. European + Portuguese: both are `pt`. +- **`auto` re-detects per segment**, which sounds ideal for multilingual streams + but flips on short or noisy utterances and can mislabel mid-sentence. For a + stream that switches language deliberately, the `1`–`9` keys are far more + reliable than `auto`. + +Wider models are better at non-English audio. If you mostly caption Russian or +Japanese, `medium` is worth the latency if your CPU can take it — check with +`bench.py` first. + ## Choosing what gets captioned **Default (`--loopback`)** captures everything your speakers play — guests, diff --git a/control.html b/control.html new file mode 100644 index 0000000..bf7f083 --- /dev/null +++ b/control.html @@ -0,0 +1,233 @@ +<!doctype html> +<html lang="en"> +<head> +<meta charset="utf-8"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<title>livecap control</title> +<style> + :root { + --bg: #0f1115; --panel: #171a21; --line: #262b36; + --fg: #e8ecf3; --dim: #98a2b3; --accent: #7dd3fc; --ok: #4ade80; --warn: #fbbf24; + } + * { box-sizing: border-box; } + body { + margin: 0; padding: 24px; background: var(--bg); color: var(--fg); + font: 15px/1.5 "Inter", "Segoe UI", system-ui, sans-serif; + } + .wrap { max-width: 760px; margin: 0 auto; display: grid; gap: 18px; } + h1 { font-size: 18px; margin: 0; font-weight: 650; letter-spacing: .01em; } + h1 span { color: var(--dim); font-weight: 400; font-size: 13px; margin-left: 8px; } + .card { background: var(--panel); border: 1px solid var(--line); border-radius: 10px; padding: 16px; } + .card h2 { + font-size: 11px; text-transform: uppercase; letter-spacing: .09em; + color: var(--dim); margin: 0 0 12px; font-weight: 600; + } + .row { display: flex; flex-wrap: wrap; gap: 8px; align-items: center; } + button { + font: inherit; font-weight: 550; color: var(--fg); cursor: pointer; + background: #1e2430; border: 1px solid var(--line); border-radius: 7px; + padding: 8px 14px; transition: background .12s, border-color .12s; + } + button:hover:not(:disabled) { background: #273040; border-color: #3a4256; } + button.on { background: var(--accent); border-color: var(--accent); color: #06202c; } + button:disabled { opacity: .45; cursor: not-allowed; } + button .k { + font-size: 11px; color: var(--dim); margin-left: 7px; + font-variant-numeric: tabular-nums; + } + button.on .k { color: #06202c; opacity: .65; } + select { + font: inherit; color: var(--fg); background: #1e2430; + border: 1px solid var(--line); border-radius: 7px; padding: 8px 12px; + } + .status { display: flex; gap: 18px; flex-wrap: wrap; align-items: baseline; } + .status b { font-weight: 600; } + .dot { width: 9px; height: 9px; border-radius: 50%; display: inline-block; margin-right: 7px; } + .dot.up { background: var(--ok); box-shadow: 0 0 8px var(--ok); } + .dot.down { background: #ef4444; box-shadow: 0 0 8px #ef4444; } + .dot.busy { background: var(--warn); box-shadow: 0 0 8px var(--warn); } + #feed { display: grid; gap: 7px; min-height: 76px; } + .cap { padding: 8px 11px; background: #12161d; border-left: 2px solid var(--line); border-radius: 4px; } + .cap.partial { color: var(--dim); font-style: italic; border-left-color: var(--warn); } + .cap .tr { display: block; color: var(--accent); font-size: 13px; margin-top: 3px; } + .empty { color: var(--dim); font-size: 13px; } + .hint { color: var(--dim); font-size: 12.5px; margin-top: 10px; } + code { background: #12161d; padding: 2px 6px; border-radius: 4px; font-size: 12.5px; } +</style> +</head> +<body> +<div class="wrap"> + <h1>livecap<span id="conn">connecting…</span></h1> + + <div class="card"> + <h2>Language</h2> + <div class="row" id="langs"></div> + <div class="hint">Press <code>1</code>–<code>9</code> to switch instantly. Takes + effect on the next sentence; no restart, no model reload.</div> + </div> + + <div class="card"> + <h2>Model</h2> + <div class="row"> + <select id="model"></select> + <span id="loading" class="empty"></span> + </div> + <div class="hint">Switching reloads the model and pauses captions for a few + seconds. Bigger is more accurate but slower.</div> + </div> + + <div class="card"> + <h2>Options</h2> + <div class="row"> + <button id="translate">English translation</button> + <button id="partials">Live partial text</button> + <button id="clear">Clear captions</button> + </div> + </div> + + <div class="card"> + <h2>Live output</h2> + <div id="feed"><div class="empty">Waiting for speech…</div></div> + </div> +</div> + +<script> +(function () { + const q = new URLSearchParams(location.search); + const WS_PORT = parseInt(q.get("ws") || "8765", 10); + + // Whisper has no regional variants: Argentine Spanish is served by "es". + const NAMES = { + fi: "Finnish", ru: "Russian", ja: "Japanese", es: "Spanish", + pt: "Portuguese", en: "English", sv: "Swedish", de: "German", + fr: "French", it: "Italian", pl: "Polish", uk: "Ukrainian", + et: "Estonian", no: "Norwegian", da: "Danish", nl: "Dutch", + zh: "Chinese", ko: "Korean", auto: "Auto-detect" + }; + + let ws = null, state = null; + const feed = []; + + const el = (id) => document.getElementById(id); + const send = (cmd, value) => ws && ws.readyState === 1 && + ws.send(JSON.stringify({ cmd: cmd, value: value })); + + function renderControls() { + if (!state) return; + + const langs = el("langs"); + langs.innerHTML = ""; + state.langs.forEach((code, i) => { + const b = document.createElement("button"); + b.className = code === state.lang ? "on" : ""; + b.innerHTML = ""; + b.appendChild(document.createTextNode(NAMES[code] || code)); + if (i < 9) { + const k = document.createElement("span"); + k.className = "k"; + k.textContent = i + 1; + b.appendChild(k); + } + b.onclick = () => send("set_lang", code); + langs.appendChild(b); + }); + + const sel = el("model"); + if (sel.dataset.key !== state.models.join(",")) { + sel.dataset.key = state.models.join(","); + sel.innerHTML = ""; + state.models.forEach((m) => { + const o = document.createElement("option"); + o.value = m; o.textContent = m; + sel.appendChild(o); + }); + } + sel.value = state.model; + sel.disabled = state.loading; + el("loading").textContent = state.loading ? "loading model…" : ""; + + el("translate").className = state.translate ? "on" : ""; + el("partials").className = state.partials ? "on" : ""; + + el("conn").innerHTML = ""; + const dot = document.createElement("span"); + dot.className = "dot " + (state.loading ? "busy" : "up"); + el("conn").appendChild(dot); + el("conn").appendChild(document.createTextNode( + (NAMES[state.lang] || state.lang) + " · " + state.model)); + } + + function renderFeed() { + const box = el("feed"); + box.innerHTML = ""; + if (!feed.length) { + box.innerHTML = '<div class="empty">Waiting for speech…</div>'; + return; + } + feed.slice(-6).forEach((m) => { + const d = document.createElement("div"); + d.className = "cap" + (m.type === "partial" ? " partial" : ""); + d.textContent = m.text; + if (m.tr) { + const s = document.createElement("span"); + s.className = "tr"; + s.textContent = m.tr; + d.appendChild(s); + } + box.appendChild(d); + }); + box.scrollTop = box.scrollHeight; + } + + function connect() { + ws = new WebSocket("ws://127.0.0.1:" + WS_PORT); + + ws.onopen = () => send("status"); + + ws.onclose = () => { + el("conn").innerHTML = '<span class="dot down"></span>disconnected — is run.ps1 still going?'; + setTimeout(connect, 1500); + }; + ws.onerror = () => ws.close(); + + ws.onmessage = (ev) => { + let m; + try { m = JSON.parse(ev.data); } catch (e) { return; } + + if (m.type === "status") { + state = m; + renderControls(); + } else if (m.type === "clear") { + feed.length = 0; + renderFeed(); + } else if (m.type === "partial") { + if (feed.length && feed[feed.length - 1].type === "partial") feed.pop(); + feed.push(m); + renderFeed(); + } else if (m.type === "final") { + if (feed.length && feed[feed.length - 1].type === "partial") feed.pop(); + feed.push(m); + if (feed.length > 40) feed.shift(); + renderFeed(); + } + }; + } + + el("model").onchange = (e) => send("set_model", e.target.value); + el("translate").onclick = () => state && send("set_translate", !state.translate); + el("partials").onclick = () => state && send("set_partials", !state.partials); + el("clear").onclick = () => send("clear"); + + document.addEventListener("keydown", (e) => { + if (e.target.tagName === "SELECT") return; + const n = parseInt(e.key, 10); + if (state && n >= 1 && n <= 9 && state.langs[n - 1]) { + send("set_lang", state.langs[n - 1]); + } + }); + + connect(); +})(); +</script> +</body> +</html> diff --git a/livecap.py b/livecap.py index 930bb9f..0693dce 100644 --- a/livecap.py +++ b/livecap.py @@ -26,10 +26,19 @@ from functools import partial from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +import warnings + import numpy as np import soundcard as sc from scipy.signal import resample_poly +# Loopback capture reports gaps whenever the device goes idle. Harmless for +# speech recognition, but it otherwise floods the console during quiet moments. +try: + warnings.filterwarnings("ignore", category=sc.SoundcardRuntimeWarning) +except AttributeError: + warnings.filterwarnings("ignore", message="data discontinuity in recording") + HERE = Path(__file__).resolve().parent TARGET_SR = 16000 CAPTURE_SR = 48000 # WASAPI shared-mode mix rate on virtually every Windows box @@ -50,6 +59,15 @@ HALLUCINATIONS = [ HALLUCINATION_RE = [re.compile(p, re.I) for p in HALLUCINATIONS] +# Captions are frequently non-Latin (ru, ja, ...). Windows otherwise encodes +# stdout with the console codepage, which mangles them once output is redirected. +for _s in (sys.stdout, sys.stderr): + try: + _s.reconfigure(encoding="utf-8", errors="replace") + except (AttributeError, ValueError): + pass + + def log(*a): print("[" + time.strftime("%H:%M:%S") + "]", *a, flush=True) @@ -225,19 +243,23 @@ class Bus: log("file write failed:", e) -async def ws_server(bus, args, stop): +async def ws_server(bus, args, ctl, stop): import websockets async def handler(ws): bus.clients.add(ws) - log("overlay connected (%d client(s))" % len(bus.clients)) + log("client connected (%d total)" % len(bus.clients)) try: + await ws.send(json.dumps(ctl.status(), ensure_ascii=False)) for m in list(bus.history)[-4:]: await ws.send(json.dumps(m, ensure_ascii=False)) - await ws.wait_closed() + async for raw in ws: + ctl.handle(raw) + except Exception: + pass finally: bus.clients.discard(ws) - log("overlay disconnected (%d client(s))" % len(bus.clients)) + log("client disconnected (%d total)" % len(bus.clients)) async with websockets.serve(handler, "127.0.0.1", args.ws_port, ping_interval=20): log("websocket ws://127.0.0.1:%d" % args.ws_port) @@ -259,11 +281,92 @@ def http_server(args): url += "&tr=1" log("OBS Browser Source URL (copy this):") log(" " + url) + log("Control panel (open in any browser):") + log(" http://127.0.0.1:%d/control.html?ws=%d" % (args.http_port, args.ws_port)) return srv # ----------------------------------------------------------- transcription -- +class Controller: + """Applies live language/model changes coming from the control panel.""" + + def __init__(self, args, bus): + self.args = args + self.bus = bus + self.tr = None # set once the Transcriber exists + self.cmds = queue.Queue() # model swaps run on the worker thread + self.loading = False + + def status(self): + return { + "type": "status", + "lang": self.args.lang, + "model": self.args.model, + "loading": self.loading, + "translate": bool(self.args.translate), + "partials": bool(self.args.partials), + "langs": self.args.langs, + "models": self.args.model_choices, + } + + def push_status(self): + self.bus.publish(self.status()) + + def handle(self, raw): + try: + m = json.loads(raw) + except (ValueError, TypeError): + return + cmd, val = m.get("cmd"), m.get("value") + + if cmd == "set_lang": + if not valid_language(val): + log("ignoring unknown language %r" % (val,)) + return + if val != self.args.lang: + self.args.lang = val + if self.tr is not None: + self.tr.context = "" # old-language prompt would poison it + log("language -> %s" % val) + self.bus.publish({"type": "clear"}) + self.push_status() + + elif cmd == "set_model": + if val and val != self.args.model: + self.cmds.put(("model", val)) + else: + self.push_status() + + elif cmd == "set_translate": + self.args.translate = bool(val) + log("translate -> %s" % self.args.translate) + self.push_status() + + elif cmd == "set_partials": + self.args.partials = bool(val) + log("partials -> %s" % self.args.partials) + self.push_status() + + elif cmd == "clear": + self.bus.publish({"type": "clear"}) + + elif cmd == "status": + self.push_status() + + +def valid_language(code): + if code == "auto": + return True + if not isinstance(code, str) or not code: + return False + try: + from faster_whisper.tokenizer import _LANGUAGE_CODES + return code in _LANGUAGE_CODES + except Exception: + return bool(re.fullmatch(r"[a-z]{2,3}", code)) + + def looks_hallucinated(text): t = text.strip().lower() if len(t) > 40: @@ -287,6 +390,31 @@ class Transcriber: self.context = "" self.busy = threading.Event() + def reload(self, name, ctl): + """Swap the model in place. Runs on the worker thread.""" + from faster_whisper import WhisperModel + + a = self.args + ctl.loading = True + ctl.push_status() + log("loading model %r ..." % name) + t0 = time.time() + try: + new = WhisperModel(name, device=a.compute_device, compute_type=a.compute, + cpu_threads=a.threads, num_workers=1) + except Exception as e: + log("model %r failed to load (%s); staying on %r" % (name, e, a.model)) + ctl.loading = False + ctl.push_status() + return + old, self.model = self.model, new + del old + a.model = name + self.context = "" + ctl.loading = False + ctl.push_status() + log("model -> %s (%.1fs)" % (name, time.time() - t0)) + def run(self, audio, final): a = self.args segs, info = self.model.transcribe( @@ -316,8 +444,28 @@ class Transcriber: condition_on_previous_text=False, vad_filter=True, without_timestamps=True) return re.sub(r"\s+", " ", " ".join(s.text.strip() for s in segs)).strip() - def worker(self, jobs, stop): + def worker(self, jobs, stop, ctl): while not stop.is_set(): + # Model swaps take priority and must happen on this thread. + try: + what, value = ctl.cmds.get_nowait() + except queue.Empty: + pass + else: + if what == "model": + self.busy.set() + try: + while True: # drop audio queued for the old model + jobs.get_nowait() + jobs.task_done() + except queue.Empty: + pass + try: + self.reload(value, ctl) + finally: + self.busy.clear() + continue + try: kind, audio, sr = jobs.get(timeout=0.25) except queue.Empty: @@ -490,7 +638,12 @@ def build_parser(): g = p.add_argument_group("model") g.add_argument("--model", default="small", help="tiny|base|small|medium|large-v3|large-v3-turbo, or a local CT2 dir") - g.add_argument("--lang", default="fi", help="fi, en, sv, ... or 'auto'") + g.add_argument("--lang", default="fi", help="fi, ru, ja, es, pt, ... or 'auto'") + g.add_argument("--langs", default="fi,ru,ja,es,pt,en,auto", + help="languages offered as one-click buttons in the control panel") + g.add_argument("--model-choices", dest="model_choices", + default="tiny,base,small,medium", + help="models offered in the control panel dropdown") g.add_argument("--compute-device", default="cpu", choices=["cpu", "cuda"]) g.add_argument("--compute", default="int8", help="int8|int8_float32|float32|float16") g.add_argument("--threads", type=int, default=max(2, (os.cpu_count() or 8) - 2)) @@ -527,6 +680,12 @@ def build_parser(): def main(): args = build_parser().parse_args() + args.langs = [s.strip() for s in args.langs.split(",") if s.strip()] + args.model_choices = [s.strip() for s in args.model_choices.split(",") if s.strip()] + if args.model not in args.model_choices: + args.model_choices.insert(0, args.model) + if args.lang not in args.langs: + args.langs.insert(0, args.lang) if args.list_devices: list_devices() @@ -538,7 +697,9 @@ def main(): stop_ev = threading.Event() jobs = queue.Queue(maxsize=3) + ctl = Controller(args, bus) tr = Transcriber(args, bus) + ctl.tr = tr dev = resolve_device(args.audio_device, args.loopback) srv = http_server(args) @@ -548,10 +709,10 @@ def main(): def run_loop(): asyncio.set_event_loop(loop) - loop.run_until_complete(ws_server(bus, args, ws_stop)) + loop.run_until_complete(ws_server(bus, args, ctl, ws_stop)) threading.Thread(target=run_loop, daemon=True).start() - threading.Thread(target=tr.worker, args=(jobs, stop_ev), daemon=True).start() + threading.Thread(target=tr.worker, args=(jobs, stop_ev, ctl), daemon=True).start() try: with Capture(dev, args.loopback) as cap: diff --git a/tests/test_smoke.py b/tests/test_smoke.py index e09f76a..d7ed8fe 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -76,6 +76,71 @@ def test_captions_file_keeps_last_n_lines(tmp_path=None): os.chdir(cwd) +def _controller(): + import argparse + + args = argparse.Namespace( + lang="fi", model="small", translate=False, partials=True, + langs=["fi", "ru", "ja", "es", "pt", "auto"], + model_choices=["tiny", "base", "small"], + txt="captions.txt", txt_lines=2) + bus = livecap.Bus(args) + bus.publish = lambda msg: bus.history.append(msg) # no websocket in tests + return livecap.Controller(args, bus), args + + +def test_language_codes_validated(): + for good in ("fi", "ru", "ja", "es", "pt", "en", "auto"): + assert livecap.valid_language(good), good + for bad in ("klingon", "", None, "e", 42): + assert not livecap.valid_language(bad), bad + + +def test_controller_switches_language(): + ctl, args = _controller() + ctl.handle('{"cmd":"set_lang","value":"ja"}') + assert args.lang == "ja" + ctl.handle('{"cmd":"set_lang","value":"ru"}') + assert args.lang == "ru" + + +def test_controller_rejects_bad_language_and_bad_json(): + ctl, args = _controller() + ctl.handle('{"cmd":"set_lang","value":"klingon"}') + assert args.lang == "fi" + ctl.handle("not json at all") + ctl.handle('{"cmd":"nonsense"}') + assert args.lang == "fi" + + +def test_model_change_is_queued_for_the_worker_thread(): + """Model loading must not happen on the websocket thread.""" + ctl, args = _controller() + ctl.handle('{"cmd":"set_model","value":"base"}') + assert ctl.cmds.get_nowait() == ("model", "base") + assert args.model == "small", "model must not change until the worker reloads it" + + ctl.handle('{"cmd":"set_model","value":"small"}') # already current + assert ctl.cmds.empty() + + +def test_toggles(): + ctl, args = _controller() + ctl.handle('{"cmd":"set_translate","value":true}') + assert args.translate is True + ctl.handle('{"cmd":"set_partials","value":false}') + assert args.partials is False + + +def test_status_payload_is_complete(): + ctl, _ = _controller() + s = ctl.status() + for key in ("type", "lang", "model", "loading", "translate", + "partials", "langs", "models"): + assert key in s, key + assert s["type"] == "status" + + def main(): tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")] failed = 0