tests/test_smoke.py (11902 bytes)
1 """Dependency-free smoke tests. Run with: python tests\\test_smoke.py""" 2 3 import sys 4 from pathlib import Path 5 6 sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) 7 8 import numpy as np 9 10 import livecap 11 12 13 def test_hallucination_filter(): 14 """Stock Whisper-on-silence phrases are dropped, real speech is kept.""" 15 for bad in ("Tekstitys: YLE 2021", "Kiitos kun katsoit!", "Kiitos.", 16 "Thanks for watching", "Subtitles by someone", " "): 17 assert livecap.looks_hallucinated(bad), bad 18 for good in ("Moi, mita kuuluu tanaan?", 19 "Nyt ollaan taas sen verran syrjaisilla seuduilla", 20 "Kiitos kun tulit mukaan, puhutaan seuraavaksi saasta ja " 21 "siita mita ensi viikolla tapahtuu"): 22 assert not livecap.looks_hallucinated(good), good 23 24 25 def test_resample_to_whisper_rate(): 26 """48 kHz in, 16 kHz out, still in range, tone preserved.""" 27 sr = 48000 28 t = np.arange(sr) / sr 29 sig = (0.2 * np.sin(2 * np.pi * 440 * t)).astype(np.float32) 30 31 out = livecap.to_whisper(sig, sr) 32 33 assert out.dtype == np.float32, out.dtype 34 assert abs(len(out) - livecap.TARGET_SR) <= 2, len(out) 35 assert np.max(np.abs(out)) <= 1.0 36 37 spec = np.abs(np.fft.rfft(out)) 38 freq = np.fft.rfftfreq(len(out), 1 / livecap.TARGET_SR)[int(np.argmax(spec))] 39 assert 430 < freq < 450, freq 40 41 42 def test_quiet_audio_is_gained_up(): 43 """Very quiet input is normalised toward a level Whisper can use.""" 44 quiet = (0.01 * np.sin(np.linspace(0, 100, 16000))).astype(np.float32) 45 out = livecap.to_whisper(quiet, livecap.TARGET_SR) 46 assert np.max(np.abs(out)) > 0.3, np.max(np.abs(out)) 47 48 49 def test_silence_does_not_divide_by_zero(): 50 out = livecap.to_whisper(np.zeros(16000, dtype=np.float32), livecap.TARGET_SR) 51 assert np.all(out == 0) 52 assert len(out) == 16000 53 54 55 def test_captions_file_keeps_last_n_lines(tmp_path=None): 56 """captions.txt is a rolling window; captions.log keeps everything.""" 57 import argparse 58 import os 59 import tempfile 60 61 cwd = os.getcwd() 62 with tempfile.TemporaryDirectory() as d: 63 os.chdir(d) 64 try: 65 livecap.HERE = Path(d) 66 bus = livecap.Bus(argparse.Namespace( 67 txt="captions.txt", txt_lines=2, translate=False)) 68 for line in ("eka", "toka", "kolmas"): 69 bus.publish({"type": "final", "text": line, "tr": "", "ts": 0}) 70 71 assert (Path(d) / "captions.txt").read_text(encoding="utf-8").split() \ 72 == ["toka", "kolmas"] 73 log = (Path(d) / "captions.log").read_text(encoding="utf-8") 74 assert log.count("\n") == 3 and "eka" in log 75 finally: 76 os.chdir(cwd) 77 78 79 def test_collapse_repeated_chunks(): 80 """Consecutive segments repeating the same sentence collapse to one.""" 81 one = "私はそれを見つけたことがありました" 82 assert livecap.collapse_repeats(" ".join([one] * 4)) == one 83 assert livecap.collapse_repeats(one) == one 84 85 86 def test_collapse_repeated_unit_inside_a_segment(): 87 """The loop need not start at the beginning of the string.""" 88 assert livecap.collapse_repeats("この日の日の日の日") == "この日" 89 assert livecap.collapse_repeats("abababab") == "ab" 90 assert livecap.collapse_repeats("xabababab") == "xab" 91 92 93 def test_short_words_may_repeat_twice(): 94 """Doubling is normal speech; a longer run is Whisper looping.""" 95 assert livecap.collapse_repeats("very very good") == "very very good" 96 assert livecap.collapse_repeats("no no no no no") == "no no" 97 98 99 def test_collapse_leaves_normal_text_alone(): 100 for text in ["今日はとてもいい天気ですね。", 101 "Kyllä se tästä, kyllä se tästä.", 102 "Привет, как дела сегодня?", 103 "the cat sat on the mat"]: 104 assert livecap.collapse_repeats(text) == " ".join(text.split()), text 105 106 107 def test_is_repetitive(): 108 one = "私はそれを見つけたことがありました" 109 assert livecap.is_repetitive(" ".join([one] * 4)) 110 assert not livecap.is_repetitive(one) 111 assert not livecap.is_repetitive("Kyllä se tästä, kyllä se tästä.") 112 assert not livecap.is_repetitive("hi") 113 114 115 LANGS = ["fi", "ru", "ja", "es", "pt", "en"] 116 117 118 def _det(**kw): 119 return livecap.LanguageDetector(LANGS + ["auto"], **kw) 120 121 122 def test_detector_ignores_languages_not_in_use(): 123 """An outsider may win outright and still not be the answer. 124 125 But only when enough evidence lands inside the set - if the audio is 126 overwhelmingly a language not in use, the honest answer is "no idea", 127 not the best of the leftovers. 128 """ 129 d = _det() 130 # de leads, yet fi holds the majority of the mass that is actually usable 131 assert d.observe([("de", 0.3), ("fi", 0.55), ("ja", 0.05)], duration=4) == "fi" 132 133 d2 = _det() 134 # here de dominates and almost nothing is in the set: refuse to guess 135 assert d2.observe([("de", 0.7), ("fi", 0.2), ("ja", 0.05)], duration=4) is None 136 137 138 def test_detector_folds_close_relatives(): 139 """Estonian mass belongs to Finnish; Ukrainian to Russian; Galician to Portuguese.""" 140 d = _det() 141 folded, mass = d.fold([("et", 0.5), ("fi", 0.3), ("ru", 0.1)]) 142 assert folded["fi"] == 0.8, folded 143 assert 0.89 < mass < 0.91, mass 144 145 folded, _ = d.fold([("uk", 0.4), ("bg", 0.2), ("ru", 0.1)]) 146 assert abs(folded["ru"] - 0.7) < 1e-9, folded 147 148 folded, _ = d.fold([("gl", 0.6), ("es", 0.2)]) 149 assert abs(folded["pt"] - 0.6) < 1e-9 and abs(folded["es"] - 0.2) < 1e-9, folded 150 151 152 def test_detector_splits_do_not_lose_to_an_outsider(): 153 """fi+et together beat ja, even though ja beats each individually.""" 154 d = _det() 155 assert d.observe([("fi", 0.3), ("et", 0.3), ("ja", 0.35)], duration=4) == "fi" 156 157 158 def test_detector_does_not_flap_on_one_odd_segment(): 159 d = _det(hold=2) 160 for _ in range(4): 161 d.observe([("ja", 0.95)], duration=4) 162 assert d.current == "ja" 163 # a single confident Spanish reading must not switch it 164 assert d.observe([("es", 0.95)], duration=4) == "ja" 165 166 167 def test_detector_switches_when_change_is_sustained(): 168 d = _det(hold=2) 169 for _ in range(4): 170 d.observe([("ja", 0.95)], duration=4) 171 assert d.current == "ja" 172 d.observe([("ru", 0.95)], duration=4) 173 d.observe([("ru", 0.95)], duration=4) 174 for _ in range(3): 175 d.observe([("ru", 0.95)], duration=4) 176 assert d.current == "ru", d.scores 177 178 179 def test_detector_weights_short_audio_less(): 180 d = _det() 181 for _ in range(3): 182 d.observe([("en", 0.9)], duration=5) 183 strong = dict(d.scores) 184 d2 = _det() 185 for _ in range(3): 186 d2.observe([("en", 0.9)], duration=0.5) 187 assert strong["en"] > d2.scores["en"] 188 189 190 def test_detector_reset_and_confidence(): 191 d = _det() 192 d.observe([("ja", 0.99)], duration=4) 193 assert d.current == "ja" and d.confidence() > 0.9 194 d.reset() 195 assert d.current is None and d.confidence() == 0.0 196 assert all(v == 0.0 for v in d.scores.values()) 197 198 199 def test_detector_defers_on_an_ambiguous_first_reading(): 200 """Restricting the set inflates confidence; mass outside it is the tell. 201 202 "en 0.38, ko 0.25, nn 0.10" looks like a commanding en once ko and nn are 203 dropped, but only 0.38 of the mass was ever inside the allowed set. 204 """ 205 d = _det() 206 assert d.observe([("en", 0.38), ("ko", 0.25), ("nn", 0.10)], duration=2) is None 207 assert d.current is None 208 # a genuinely confident reading is adopted immediately afterwards 209 assert d.observe([("fi", 0.96), ("nn", 0.02), ("en", 0.01)], duration=4) == "fi" 210 211 212 def test_detector_adopts_a_confident_first_reading(): 213 d = _det() 214 assert d.observe([("ja", 0.93), ("zh", 0.04)], duration=4) == "ja" 215 216 217 def test_detector_low_mass_never_locks_in(): 218 d = _det() 219 for _ in range(5): 220 d.observe([("de", 0.6), ("nl", 0.3), ("en", 0.05)], duration=4) 221 assert d.current is None, d.scores 222 223 224 def test_detector_handles_empty_and_unusable_input(): 225 d = _det() 226 assert d.observe([], duration=4) is None 227 assert d.observe([("de", 0.9), ("zh", 0.1)], duration=4) is None 228 d.observe([("fi", 0.9)], duration=4) 229 assert d.observe([("de", 1.0)], duration=4) == "fi" # keeps the last good one 230 231 232 def test_model_resolution(): 233 """An explicit --model must win, even when it equals the default.""" 234 import tempfile 235 236 with tempfile.TemporaryDirectory() as d: 237 here = Path(d) 238 239 # nothing built yet 240 assert livecap.resolve_model(None, "fi", here)[0] == "small" 241 assert livecap.resolve_model(None, "ru", here)[0] == "small" 242 243 built = here / livecap.FI_MODEL_DIR 244 built.mkdir(parents=True) 245 (built / "model.bin").write_bytes(b"x") 246 247 name, why = livecap.resolve_model(None, "fi", here) 248 assert name == str(built), name 249 assert why and "fine-tune" in why 250 251 # the fine-tune is Finnish-only 252 assert livecap.resolve_model(None, "ru", here)[0] == "small" 253 assert livecap.resolve_model(None, "ja", here)[0] == "small" 254 255 # explicit wins, including the string that happens to be the default 256 assert livecap.resolve_model("small", "fi", here) == ("small", None) 257 assert livecap.resolve_model("large-v3", "fi", here)[0] == "large-v3" 258 259 260 def _controller(): 261 import argparse 262 263 args = argparse.Namespace( 264 lang="fi", model="small", translate=False, partials=True, 265 langs=["fi", "ru", "ja", "es", "pt", "auto"], 266 model_choices=["tiny", "base", "small"], 267 txt="captions.txt", txt_lines=2) 268 bus = livecap.Bus(args) 269 bus.publish = lambda msg: bus.history.append(msg) # no websocket in tests 270 return livecap.Controller(args, bus), args 271 272 273 def test_language_codes_validated(): 274 for good in ("fi", "ru", "ja", "es", "pt", "en", "auto"): 275 assert livecap.valid_language(good), good 276 for bad in ("klingon", "", None, "e", 42): 277 assert not livecap.valid_language(bad), bad 278 279 280 def test_controller_switches_language(): 281 ctl, args = _controller() 282 ctl.handle('{"cmd":"set_lang","value":"ja"}') 283 assert args.lang == "ja" 284 ctl.handle('{"cmd":"set_lang","value":"ru"}') 285 assert args.lang == "ru" 286 287 288 def test_controller_rejects_bad_language_and_bad_json(): 289 ctl, args = _controller() 290 ctl.handle('{"cmd":"set_lang","value":"klingon"}') 291 assert args.lang == "fi" 292 ctl.handle("not json at all") 293 ctl.handle('{"cmd":"nonsense"}') 294 assert args.lang == "fi" 295 296 297 def test_model_change_is_queued_for_the_worker_thread(): 298 """Model loading must not happen on the websocket thread.""" 299 ctl, args = _controller() 300 ctl.handle('{"cmd":"set_model","value":"base"}') 301 assert ctl.cmds.get_nowait() == ("model", "base") 302 assert args.model == "small", "model must not change until the worker reloads it" 303 304 ctl.handle('{"cmd":"set_model","value":"small"}') # already current 305 assert ctl.cmds.empty() 306 307 308 def test_toggles(): 309 ctl, args = _controller() 310 ctl.handle('{"cmd":"set_translate","value":true}') 311 assert args.translate is True 312 ctl.handle('{"cmd":"set_partials","value":false}') 313 assert args.partials is False 314 315 316 def test_status_payload_is_complete(): 317 ctl, _ = _controller() 318 s = ctl.status() 319 for key in ("type", "lang", "model", "loading", "translate", 320 "partials", "langs", "models"): 321 assert key in s, key 322 assert s["type"] == "status" 323 324 325 def main(): 326 tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")] 327 failed = 0 328 for fn in tests: 329 try: 330 fn() 331 print(" PASS %s" % fn.__name__) 332 except Exception as e: 333 failed += 1 334 print(" FAIL %s: %r" % (fn.__name__, e)) 335 print("\n%d passed, %d failed" % (len(tests) - failed, failed)) 336 return 1 if failed else 0 337 338 339 if __name__ == "__main__": 340 sys.exit(main())