test/cue-pages.test.ts (8136 bytes)
1 import fc from 'fast-check'; 2 import { describe, expect, it } from 'vitest'; 3 import { normalizeText, type OcrToken } from '../src/align'; 4 import { 5 audioPage, 6 cueParts, 7 cueProgress, 8 LOOKBACK, 9 pageStartTime, 10 SEEK_INTO, 11 type CueParts, 12 } from '../src/cue-pages'; 13 import { lastCueAt } from '../src/subtitles'; 14 15 /** Tokens of one text line, one per character; a space gets no token. */ 16 function lineTokens(text: string, page: number): OcrToken[] { 17 return [...text] 18 .filter((ch) => ch !== ' ') 19 .map((ch, i) => ({ text: ch, page, line: page, bbox: { x0: i, y0: 0, x1: i + 1, y1: 1 } })); 20 } 21 22 // Cue 8 of sample-eng-2p.srt, 17.5 s to 20 s: the last line of page 1 and 23 // the first line of page 2. 24 const PAGE_1_END = 'with nothing more than light and sound.'; 25 const PAGE_2_START = 'It rolled across the fields and woke the geese,'; 26 const crossing = [...lineTokens(PAGE_1_END, 0), ...lineTokens(PAGE_2_START, 1)]; 27 /** The alignment drops the comma at the end, so the span ends before it. */ 28 const crossingSpan = { start: 0, end: crossing.length - 1, matched: true }; 29 const CUE_8 = { start: 17.5, end: 20 }; 30 31 describe('cueParts', () => { 32 it('splits a cue at the page break by its normalized characters', () => { 33 const [parts] = cueParts(crossing, [crossingSpan]); 34 expect(parts).toEqual([ 35 { page: 0, share: 32 / 70 }, 36 { page: 1, share: 38 / 70 }, 37 ]); 38 }); 39 40 it('gives no parts for an unmatched cue', () => { 41 expect(cueParts(crossing, [{ start: 0, end: 0, matched: false }])).toEqual([[]]); 42 }); 43 44 it('gives shares that add up to 1, one part per page in page order', () => { 45 const token = fc.record({ 46 text: fc.constantFrom('a', 'B', '猫', ',', '。', ' '), 47 page: fc.nat({ max: 3 }), 48 }); 49 const input = fc 50 .array(token, { minLength: 1, maxLength: 60 }) 51 .map((ts) => ts.sort((x, y) => x.page - y.page)) 52 .chain((ts) => 53 fc.tuple(fc.constant(ts), fc.nat({ max: ts.length - 1 }), fc.nat({ max: ts.length - 1 })), 54 ); 55 fc.assert( 56 fc.property(input, ([ts, a, b]) => { 57 const tokens: OcrToken[] = ts.map((t, i) => ({ 58 ...t, 59 line: t.page, 60 bbox: { x0: i, y0: 0, x1: i + 1, y1: 1 }, 61 })); 62 const span = { start: Math.min(a, b), end: Math.max(a, b) + 1, matched: true }; 63 const [parts] = cueParts(tokens, [span]); 64 // A slow reference: count the normalized characters of each page. 65 const counts = new Map<number, number>(); 66 for (const t of tokens.slice(span.start, span.end)) { 67 const n = normalizeText(t.text).length; 68 if (n > 0) counts.set(t.page, (counts.get(t.page) ?? 0) + n); 69 } 70 const total = [...counts.values()].reduce((x, y) => x + y, 0); 71 expect(parts!.map((p) => p.page)).toEqual([...counts.keys()]); 72 parts!.forEach((p) => expect(p.share).toBeCloseTo(counts.get(p.page)! / total, 12)); 73 if (parts!.length > 0) { 74 expect(parts!.reduce((x, p) => x + p.share, 0)).toBeCloseTo(1, 12); 75 } 76 }), 77 ); 78 }); 79 }); 80 81 describe('cueProgress', () => { 82 it('goes from 0 at the start of the cue to 1 at its end', () => { 83 expect(cueProgress(CUE_8, 17)).toBe(0); 84 expect(cueProgress(CUE_8, 17.5)).toBe(0); 85 expect(cueProgress(CUE_8, 18.75)).toBe(0.5); 86 expect(cueProgress(CUE_8, 20)).toBe(1); 87 expect(cueProgress(CUE_8, 21)).toBe(1); 88 expect(cueProgress({ start: 3, end: 3 }, 3)).toBe(1); 89 }); 90 }); 91 92 describe('audioPage', () => { 93 const [parts] = cueParts(crossing, [crossingSpan]); 94 const split = CUE_8.start + (CUE_8.end - CUE_8.start) * (32 / 70); 95 96 it('turns inside a crossing cue at the share of its text on the first page', () => { 97 expect(split).toBeCloseTo(18.643, 3); 98 expect(audioPage([parts!], 0, cueProgress(CUE_8, 18.5))).toBe(0); 99 expect(audioPage([parts!], 0, cueProgress(CUE_8, split - 0.01))).toBe(0); 100 expect(audioPage([parts!], 0, cueProgress(CUE_8, split + 0.01))).toBe(1); 101 expect(audioPage([parts!], 0, 1)).toBe(1); 102 }); 103 104 it('lets an unmatched cue borrow the last page of the cue before it', () => { 105 // After the crossing cue the audio is on page 2, not back on page 1. 106 expect(audioPage([parts!, []], 1, 0)).toBe(1); 107 }); 108 109 it('gives null when no matched cue is near enough', () => { 110 const far: CueParts[] = [ 111 [{ page: 0, share: 1 }], 112 ...new Array<CueParts>(LOOKBACK + 1).fill([]), 113 ]; 114 expect(audioPage(far, far.length - 1, 0)).toBeNull(); 115 expect(audioPage(far, LOOKBACK, 0)).toBe(0); 116 }); 117 118 /** Parts of cues: each cue is unmatched, or on one to three pages that rise. */ 119 const cueList = fc.array( 120 fc.oneof( 121 fc.constant<CueParts>([]), 122 fc 123 .array(fc.integer({ min: 1, max: 20 }), { minLength: 1, maxLength: 3 }) 124 .chain((counts) => 125 fc.tuple( 126 fc.constant(counts), 127 fc.array(fc.integer({ min: 1, max: 2 }), { 128 minLength: counts.length, 129 maxLength: counts.length, 130 }), 131 fc.nat({ max: 5 }), 132 ), 133 ) 134 .map(([counts, gaps, first]) => { 135 const total = counts.reduce((x, y) => x + y, 0); 136 let page = first; 137 return counts.map((c, k) => { 138 if (k > 0) page += gaps[k]!; 139 return { page, share: c / total }; 140 }); 141 }), 142 ), 143 { minLength: 1, maxLength: 12 }, 144 ); 145 146 it('moves forward through the pages of a cue as the audio goes on', () => { 147 fc.assert( 148 fc.property( 149 cueList, 150 fc.nat(), 151 fc.double({ min: 0, max: 1, noNaN: true }), 152 (list, pick, p) => { 153 const cue = pick % list.length; 154 const own = list[cue]!; 155 if (own.length === 0) return; 156 const page = audioPage(list, cue, p); 157 expect(own.map((x) => x.page)).toContain(page); 158 expect(audioPage(list, cue, 0)).toBe(own[0]!.page); 159 expect(audioPage(list, cue, 1)).toBe(own[own.length - 1]!.page); 160 expect(audioPage(list, cue, Math.min(1, p + 0.1))!).toBeGreaterThanOrEqual(page!); 161 }, 162 ), 163 ); 164 }); 165 166 it('finds the page again at the time where the page starts', () => { 167 // Cues one after the other, 0.5 s to 5 s long, with gaps. 168 const input = cueList.chain((list) => 169 fc.tuple( 170 fc.constant(list), 171 fc.array(fc.tuple(fc.integer({ min: 0, max: 30 }), fc.integer({ min: 50, max: 500 })), { 172 minLength: list.length, 173 maxLength: list.length, 174 }), 175 ), 176 ); 177 fc.assert( 178 fc.property(input, ([list, times]) => { 179 let clock = 0; 180 const cues = times.map(([gap, length]) => { 181 const start = clock + gap / 10; 182 clock = start + length / 100; 183 return { start, end: clock }; 184 }); 185 const pages = new Set(list.flatMap((parts) => parts.map((p) => p.page))); 186 for (const page of pages) { 187 const t = pageStartTime(list, cues, page); 188 expect(t).not.toBeNull(); 189 // The clock counts whole milliseconds. 190 const ms = Math.round(t! * 1000) / 1000; 191 const cue = lastCueAt(cues, ms); 192 expect(audioPage(list, cue, cueProgress(cues[cue]!, ms))).toBe(page); 193 // No cue before has text on the page. 194 for (let i = 0; i < cue; i++) expect(list[i]!.some((p) => p.page === page)).toBe(false); 195 } 196 }), 197 ); 198 }); 199 }); 200 201 describe('pageStartTime', () => { 202 const [parts] = cueParts(crossing, [crossingSpan]); 203 204 it('gives the time where the text of page 2 starts inside the crossing cue', () => { 205 const t = pageStartTime([parts!], [CUE_8], 1); 206 expect(t).toBeCloseTo(17.5 + 2.5 * (32 / 70) + SEEK_INTO, 6); 207 }); 208 209 it('gives the start of the first cue on a page, and null for a page with no cue', () => { 210 const list: CueParts[] = [[], [{ page: 0, share: 1 }], [{ page: 0, share: 1 }]]; 211 const cues = [ 212 { start: 0, end: 2 }, 213 { start: 2, end: 4 }, 214 { start: 4, end: 6 }, 215 ]; 216 expect(pageStartTime(list, cues, 0)).toBeCloseTo(2 + SEEK_INTO, 9); 217 expect(pageStartTime(list, cues, 1)).toBeNull(); 218 }); 219 });