commit 56a9a470a72916d2357f21ff8d4a9b9d754878da truex <truex@equwal.com> 2026-09-26 14:41:41 -0700 Turn the page inside a cue that crosses a page break A cue with text on two pages had the page of its first token. So the page turned only at the next cue, and "Audio to this page" on the second page went to that next cue. The new pure module cue-pages.ts gives each cue its parts on the pages, with the share of its normalized characters on each page. The page of the audio follows the time: the first page until the start of the cue plus its length times the share on the first page, then the second page. An unmatched cue borrows the last page of the matched cue before it, not the first. "Move the audio to this page" now goes to the time where the text of the page starts: in the first cue with text on the page, and inside that cue when it starts on the page before. The time is up to 50 ms later, so a clock that rounds to milliseconds shows the same page. A property test finds the page again at that time. The button is off while the player has no position or has an error.
src/cue-pages.ts | 131 +++++++++++++++++++++++++++++ src/follower.ts | 25 +----- src/main.ts | 47 +++++++---- test/cue-pages.test.ts | 219 +++++++++++++++++++++++++++++++++++++++++++++++++ test/follower.test.ts | 33 ++------ 5 files changed, 390 insertions(+), 65 deletions(-)
diff --git a/src/cue-pages.ts b/src/cue-pages.ts new file mode 100644 index 0000000..d5992df --- /dev/null +++ b/src/cue-pages.ts @@ -0,0 +1,131 @@ +// Where each cue is on the pages, and the page of the audio at a time. Pure. +// +// Most cues are on one page. A cue at a page break has text on two pages. +// The narrator reads the text in order, so the page follows the time: the +// first page for the share of the text of the cue on it, then the next page. + +import { normalizeText, type OcrToken, type TokenSpan } from './align'; + +/** The part of a cue on one page. */ +export interface PagePart { + page: number; + /** The share of the text of the cue on this page: more than 0, at most 1. The shares of a cue add up to 1. */ + share: number; +} + +/** The parts of a cue in page order. Empty for an unmatched cue. */ +export type CueParts = readonly PagePart[]; + +/** A cue in time, in seconds. */ +export interface CueTime { + start: number; + end: number; +} + +/** How many cues back an unmatched cue borrows the page of a matched one. */ +export const LOOKBACK = 5; + +/** + * A seek to the start of a page lands this many seconds into its text, at + * most. A clock that rounds the time then does not show the page before. + */ +export const SEEK_INTO = 0.05; + +/** The count of normalized characters of each token text that was seen. */ +const weights = new Map<string, number>(); + +/** The count of normalized characters of a token: 0 for punctuation, as in the alignment. */ +function weight(text: string): number { + let n = weights.get(text); + if (n === undefined) { + n = normalizeText(text).length; + weights.set(text, n); + } + return n; +} + +/** + * The parts of each cue on the pages, from the spans of the alignment. + * The share of a page is the share of the normalized characters of the + * span on that page. The tokens must be in page order. + */ +export function cueParts(tokens: readonly OcrToken[], spans: readonly TokenSpan[]): CueParts[] { + return spans.map((span) => { + if (!span.matched) return []; + const counts: { page: number; chars: number }[] = []; + let total = 0; + for (let t = span.start; t < span.end; t++) { + const token = tokens[t]; + if (!token) continue; + const n = weight(token.text); + if (n === 0) continue; + const last = counts[counts.length - 1]; + if (last && last.page === token.page) last.chars += n; + else counts.push({ page: token.page, chars: n }); + total += n; + } + return counts.map((c) => ({ page: c.page, share: c.chars / total })); + }); +} + +/** How far time `t` is into `cue`: 0 at its start, 1 at its end and after it. */ +export function cueProgress(cue: CueTime, t: number): number { + const length = cue.end - cue.start; + if (length <= 0) return 1; + return Math.min(1, Math.max(0, (t - cue.start) / length)); +} + +/** + * The page of the audio at `progress` into cue `cue`. For a matched cue it + * is the page of the part that holds the progress: the first page until + * the share of the text on it is read, then the next page. An unmatched + * cue borrows the last page of the nearest matched cue before it, within + * LOOKBACK cues. Null when there is none. + */ +export function audioPage( + parts: readonly CueParts[], + cue: number, + progress: number, +): number | null { + const own = parts[cue]; + if (own && own.length > 0) { + let end = 0; + for (const part of own) { + end += part.share; + if (progress < end) return part.page; + } + return own[own.length - 1]!.page; + } + for (let i = cue - 1; i >= 0 && i >= cue - LOOKBACK; i--) { + const before = parts[i]; + if (before && before.length > 0) return before[before.length - 1]!.page; + } + return null; +} + +/** + * The time where the text of `page` starts, in seconds: in the first cue + * with text on the page. When that cue starts on a page before, the time + * is inside the cue, after the share of its text on the pages before. The + * time is up to SEEK_INTO later, but inside the part of the page. Null + * when no cue has text on the page. + */ +export function pageStartTime( + parts: readonly CueParts[], + cues: readonly CueTime[], + page: number, +): number | null { + for (let i = 0; i < parts.length; i++) { + const cue = cues[i]; + if (!cue) continue; + let before = 0; + for (const part of parts[i]!) { + if (part.page === page) { + const length = Math.max(0, cue.end - cue.start); + return cue.start + length * before + Math.min(SEEK_INTO, (length * part.share) / 2); + } + before += part.share; + } + } + return null; +} diff --git a/src/follower.ts b/src/follower.ts index ec2dd85..8e525ae 100644 --- a/src/follower.ts +++ b/src/follower.ts @@ -1,10 +1,9 @@ // The reader follows the audio: which cue to mark, which page to show. Pure. // -// The alignment gives each matched cue a page. At each step the follow -// compares the page of the audio with the page shown, and turns the page -// when they differ. A page turn by the user holds the follow: the page -// stays until the audio reaches it, the audio jumps, or the user asks to -// follow again. +// At each step the follow compares the page of the audio (see audioPage +// in cue-pages.ts) with the page shown, and turns the page when they +// differ. A page turn by the user holds the follow: the page stays until +// the audio reaches it, the audio jumps, or the user asks to follow again. export type FollowMode = 'highlight' | 'pages' | 'off'; @@ -43,22 +42,6 @@ export interface FollowOutput { held: boolean; } -/** How many cues back an unmatched cue borrows the page of a matched one. */ -export const LOOKBACK = 5; - -/** - * The page that stands for `cue`: its own page when the cue is matched, - * else the page of the nearest matched cue before it, within LOOKBACK. - * Null when there is none. - */ -export function pageForCue(cuePages: readonly (number | null)[], cue: number): number | null { - for (let i = cue; i >= 0 && i >= cue - LOOKBACK; i--) { - const page = cuePages[i]; - if (page !== null && page !== undefined) return page; - } - return null; -} - /** * One step of the follow. * diff --git a/src/main.ts b/src/main.ts index c501a6d..e850d4d 100644 --- a/src/main.ts +++ b/src/main.ts @@ -5,7 +5,8 @@ import { createAligner, type Aligner, type OcrToken, type TokenSpan } from './align'; import { bookText } from './book-text'; import { LocalAudioClock, OverlayClock, type ClockSource, type ClockState } from './clock-source'; -import { follow, pageForCue, type FollowEvent, type FollowMode } from './follower'; +import { audioPage, cueParts, cueProgress, pageStartTime, type CueParts } from './cue-pages'; +import { follow, type FollowEvent, type FollowMode } from './follower'; import { lookupText, tapTolerance, tokenAt } from './hit-test'; import { lineBoxes } from './line-boxes'; import { @@ -102,8 +103,8 @@ const state = { /** The aligner of the loaded subtitles. It keeps the result of each page. */ aligner: null as Aligner | null, spans: [] as TokenSpan[], - /** The page of each cue, or null for an unmatched cue. */ - cuePages: [] as (number | null)[], + /** The parts of each cue on the pages. Empty for an unmatched cue. */ + cueParts: [] as CueParts[], currentPage: -1, /** The cue of the last clock state, or -1. */ activeCue: -1, @@ -237,7 +238,7 @@ ui.pdfFile.addEventListener('change', async () => { state.pages = []; state.tokens = []; state.spans = []; - state.cuePages = []; + state.cueParts = []; state.activeCue = -1; state.markedCue = -1; state.held = false; @@ -298,7 +299,7 @@ async function readBook(): Promise<void> { state.tokens = []; // The spans index the old tokens, so they go too. state.spans = []; - state.cuePages = []; + state.cueParts = []; state.markedCue = -1; const total = pdf.numPages; const pending = new Set(Array.from({ length: total }, (_, i) => i)); @@ -365,10 +366,10 @@ function loadSubtitles(text: string, source: string): void { state.aligner = createAligner(state.cues.map((c) => c.text)); state.srt = text; state.spans = []; - state.cuePages = []; + state.cueParts = []; state.activeCue = -1; state.markedCue = -1; - ui.syncPage.disabled = state.cues.length === 0; + updateSyncPage(); renderCueList(); const matched = align(); say( @@ -458,13 +459,13 @@ function align(): number | null { state.alignTimer = null; if (!state.aligner || state.cues.length === 0 || state.tokens.length === 0) return null; state.spans = state.aligner.spans(state.pages.map((p) => p?.tokens)); - state.cuePages = state.spans.map((s) => (s.matched ? state.tokens[s.start]!.page : null)); + state.cueParts = cueParts(state.tokens, state.spans); renderCueList(); view.mark(); // The cue of now can have a new page. A new alignment is no seek, so a // page that the user turned to stays. runFollow('realign'); - return state.cuePages.filter((p) => p !== null).length; + return state.cueParts.filter((parts) => parts.length > 0).length; } // --- Cue list --- @@ -506,27 +507,37 @@ function showPlayerStatus(s: ClockState): void { } // Before a PDF is open, the start card tells what to do first. showPlayer(state.pdf ? playerMessage(s, isAndroid) : null); + updateSyncPage(); +} + +/** "Move the audio to this page" needs cues and a player with a position. */ +function updateSyncPage(): void { + const s = state.clock; + ui.syncPage.disabled = state.cues.length === 0 || !s || s.positionMs === null || s.error !== null; } /** * The follow for the last state of the clock: mark the cue of now, and - * turn to its page unless the follow is held. In the silence between two - * cues, the cue before stays the cue of now. + * turn to the page of the audio unless the follow is held. In the silence + * between two cues, the cue before stays the cue of now. In a cue on two + * pages, the page follows the time. */ function runFollow(event: FollowEvent): void { const s = state.clock; - const cue = s?.positionMs == null ? -1 : lastCueAt(state.cues, s.positionMs / 1000); + const t = s?.positionMs == null ? null : s.positionMs / 1000; + const cue = t === null ? -1 : lastCueAt(state.cues, t); if (cue !== state.activeCue) { markCueInList(cue); state.activeCue = cue; } const mode = followMode(); - const page = cue < 0 ? null : pageForCue(state.cuePages, cue); + const page = + t === null || cue < 0 ? null : audioPage(state.cueParts, cue, cueProgress(state.cues[cue]!, t)); const out = follow({ mode, currentPage: state.currentPage, cue, - matched: cue >= 0 && state.cuePages[cue] != null, + matched: (state.cueParts[cue]?.length ?? 0) > 0, page, held: state.held, event, @@ -575,13 +586,15 @@ ui.play.addEventListener('click', () => { ui.followNow.addEventListener('click', () => runFollow('follow')); +// The audio goes to where the text of the page starts, also inside a cue +// that starts on the page before. ui.syncPage.addEventListener('click', () => { - const i = state.cuePages.indexOf(state.currentPage); - if (i < 0) { + const t = pageStartTime(state.cueParts, state.cues, state.currentPage); + if (t === null) { say('No cue of the subtitles is on this page.'); return; } - seek(state.cues[i]!.start * 1000); + seek(t * 1000); }); ui.audioFile.addEventListener('change', () => { diff --git a/test/cue-pages.test.ts b/test/cue-pages.test.ts new file mode 100644 index 0000000..01a8557 --- /dev/null +++ b/test/cue-pages.test.ts @@ -0,0 +1,219 @@ +import fc from 'fast-check'; +import { describe, expect, it } from 'vitest'; +import { normalizeText, type OcrToken } from '../src/align'; +import { + audioPage, + cueParts, + cueProgress, + LOOKBACK, + pageStartTime, + SEEK_INTO, + type CueParts, +} from '../src/cue-pages'; +import { lastCueAt } from '../src/subtitles'; + +/** Tokens of one text line, one per character; a space gets no token. */ +function lineTokens(text: string, page: number): OcrToken[] { + return [...text] + .filter((ch) => ch !== ' ') + .map((ch, i) => ({ text: ch, page, line: page, bbox: { x0: i, y0: 0, x1: i + 1, y1: 1 } })); +} + +// Cue 8 of sample-eng-2p.srt, 17.5 s to 20 s: the last line of page 1 and +// the first line of page 2. +const PAGE_1_END = 'with nothing more than light and sound.'; +const PAGE_2_START = 'It rolled across the fields and woke the geese,'; +const crossing = [...lineTokens(PAGE_1_END, 0), ...lineTokens(PAGE_2_START, 1)]; +/** The alignment drops the comma at the end, so the span ends before it. */ +const crossingSpan = { start: 0, end: crossing.length - 1, matched: true }; +const CUE_8 = { start: 17.5, end: 20 }; + +describe('cueParts', () => { + it('splits a cue at the page break by its normalized characters', () => { + const [parts] = cueParts(crossing, [crossingSpan]); + expect(parts).toEqual([ + { page: 0, share: 32 / 70 }, + { page: 1, share: 38 / 70 }, + ]); + }); + + it('gives no parts for an unmatched cue', () => { + expect(cueParts(crossing, [{ start: 0, end: 0, matched: false }])).toEqual([[]]); + }); + + it('gives shares that add up to 1, one part per page in page order', () => { + const token = fc.record({ + text: fc.constantFrom('a', 'B', '猫', ',', '。', ' '), + page: fc.nat({ max: 3 }), + }); + const input = fc + .array(token, { minLength: 1, maxLength: 60 }) + .map((ts) => ts.sort((x, y) => x.page - y.page)) + .chain((ts) => + fc.tuple(fc.constant(ts), fc.nat({ max: ts.length - 1 }), fc.nat({ max: ts.length - 1 })), + ); + fc.assert( + fc.property(input, ([ts, a, b]) => { + const tokens: OcrToken[] = ts.map((t, i) => ({ + ...t, + line: t.page, + bbox: { x0: i, y0: 0, x1: i + 1, y1: 1 }, + })); + const span = { start: Math.min(a, b), end: Math.max(a, b) + 1, matched: true }; + const [parts] = cueParts(tokens, [span]); + // A slow reference: count the normalized characters of each page. + const counts = new Map<number, number>(); + for (const t of tokens.slice(span.start, span.end)) { + const n = normalizeText(t.text).length; + if (n > 0) counts.set(t.page, (counts.get(t.page) ?? 0) + n); + } + const total = [...counts.values()].reduce((x, y) => x + y, 0); + expect(parts!.map((p) => p.page)).toEqual([...counts.keys()]); + parts!.forEach((p) => expect(p.share).toBeCloseTo(counts.get(p.page)! / total, 12)); + if (parts!.length > 0) { + expect(parts!.reduce((x, p) => x + p.share, 0)).toBeCloseTo(1, 12); + } + }), + ); + }); +}); + +describe('cueProgress', () => { + it('goes from 0 at the start of the cue to 1 at its end', () => { + expect(cueProgress(CUE_8, 17)).toBe(0); + expect(cueProgress(CUE_8, 17.5)).toBe(0); + expect(cueProgress(CUE_8, 18.75)).toBe(0.5); + expect(cueProgress(CUE_8, 20)).toBe(1); + expect(cueProgress(CUE_8, 21)).toBe(1); + expect(cueProgress({ start: 3, end: 3 }, 3)).toBe(1); + }); +}); + +describe('audioPage', () => { + const [parts] = cueParts(crossing, [crossingSpan]); + const split = CUE_8.start + (CUE_8.end - CUE_8.start) * (32 / 70); + + it('turns inside a crossing cue at the share of its text on the first page', () => { + expect(split).toBeCloseTo(18.643, 3); + expect(audioPage([parts!], 0, cueProgress(CUE_8, 18.5))).toBe(0); + expect(audioPage([parts!], 0, cueProgress(CUE_8, split - 0.01))).toBe(0); + expect(audioPage([parts!], 0, cueProgress(CUE_8, split + 0.01))).toBe(1); + expect(audioPage([parts!], 0, 1)).toBe(1); + }); + + it('lets an unmatched cue borrow the last page of the cue before it', () => { + // After the crossing cue the audio is on page 2, not back on page 1. + expect(audioPage([parts!, []], 1, 0)).toBe(1); + }); + + it('gives null when no matched cue is near enough', () => { + const far: CueParts[] = [ + [{ page: 0, share: 1 }], + ...new Array<CueParts>(LOOKBACK + 1).fill([]), + ]; + expect(audioPage(far, far.length - 1, 0)).toBeNull(); + expect(audioPage(far, LOOKBACK, 0)).toBe(0); + }); + + /** Parts of cues: each cue is unmatched, or on one to three pages that rise. */ + const cueList = fc.array( + fc.oneof( + fc.constant<CueParts>([]), + fc + .array(fc.integer({ min: 1, max: 20 }), { minLength: 1, maxLength: 3 }) + .chain((counts) => + fc.tuple( + fc.constant(counts), + fc.array(fc.integer({ min: 1, max: 2 }), { + minLength: counts.length, + maxLength: counts.length, + }), + fc.nat({ max: 5 }), + ), + ) + .map(([counts, gaps, first]) => { + const total = counts.reduce((x, y) => x + y, 0); + let page = first; + return counts.map((c, k) => { + if (k > 0) page += gaps[k]!; + return { page, share: c / total }; + }); + }), + ), + { minLength: 1, maxLength: 12 }, + ); + + it('moves forward through the pages of a cue as the audio goes on', () => { + fc.assert( + fc.property( + cueList, + fc.nat(), + fc.double({ min: 0, max: 1, noNaN: true }), + (list, pick, p) => { + const cue = pick % list.length; + const own = list[cue]!; + if (own.length === 0) return; + const page = audioPage(list, cue, p); + expect(own.map((x) => x.page)).toContain(page); + expect(audioPage(list, cue, 0)).toBe(own[0]!.page); + expect(audioPage(list, cue, 1)).toBe(own[own.length - 1]!.page); + expect(audioPage(list, cue, Math.min(1, p + 0.1))!).toBeGreaterThanOrEqual(page!); + }, + ), + ); + }); + + it('finds the page again at the time where the page starts', () => { + // Cues one after the other, 0.5 s to 5 s long, with gaps. + const input = cueList.chain((list) => + fc.tuple( + fc.constant(list), + fc.array(fc.tuple(fc.integer({ min: 0, max: 30 }), fc.integer({ min: 50, max: 500 })), { + minLength: list.length, + maxLength: list.length, + }), + ), + ); + fc.assert( + fc.property(input, ([list, times]) => { + let clock = 0; + const cues = times.map(([gap, length]) => { + const start = clock + gap / 10; + clock = start + length / 100; + return { start, end: clock }; + }); + const pages = new Set(list.flatMap((parts) => parts.map((p) => p.page))); + for (const page of pages) { + const t = pageStartTime(list, cues, page); + expect(t).not.toBeNull(); + // The clock counts whole milliseconds. + const ms = Math.round(t! * 1000) / 1000; + const cue = lastCueAt(cues, ms); + expect(audioPage(list, cue, cueProgress(cues[cue]!, ms))).toBe(page); + // No cue before has text on the page. + for (let i = 0; i < cue; i++) expect(list[i]!.some((p) => p.page === page)).toBe(false); + } + }), + ); + }); +}); + +describe('pageStartTime', () => { + const [parts] = cueParts(crossing, [crossingSpan]); + + it('gives the time where the text of page 2 starts inside the crossing cue', () => { + const t = pageStartTime([parts!], [CUE_8], 1); + expect(t).toBeCloseTo(17.5 + 2.5 * (32 / 70) + SEEK_INTO, 6); + }); + + it('gives the start of the first cue on a page, and null for a page with no cue', () => { + const list: CueParts[] = [[], [{ page: 0, share: 1 }], [{ page: 0, share: 1 }]]; + const cues = [ + { start: 0, end: 2 }, + { start: 2, end: 4 }, + { start: 4, end: 6 }, + ]; + expect(pageStartTime(list, cues, 0)).toBeCloseTo(2 + SEEK_INTO, 9); + expect(pageStartTime(list, cues, 1)).toBeNull(); + }); +}); diff --git a/test/follower.test.ts b/test/follower.test.ts index 44d3216..a2e8011 100644 --- a/test/follower.test.ts +++ b/test/follower.test.ts @@ -1,15 +1,11 @@ import fc from 'fast-check'; import { describe, expect, it } from 'vitest'; -import { - follow, - LOOKBACK, - pageForCue, - type FollowEvent, - type FollowInput, - type FollowMode, -} from '../src/follower'; +import { audioPage, type CueParts } from '../src/cue-pages'; +import { follow, type FollowEvent, type FollowInput, type FollowMode } from '../src/follower'; +/** The page of each cue. Null for an unmatched cue. */ const pages: (number | null)[] = [0, 0, null, 1, 1, null, null, 2]; +const parts: CueParts[] = pages.map((p) => (p === null ? [] : [{ page: p, share: 1 }])); function input(over: Partial<FollowInput>): FollowInput { return { @@ -24,33 +20,16 @@ function input(over: Partial<FollowInput>): FollowInput { }; } -/** The input of the follow for cue `cue` of `cuePages`. */ +/** The input of the follow at the start of cue `cue` of `pages`. */ function at(cue: number, over: Partial<FollowInput> = {}): FollowInput { return input({ cue, matched: pages[cue] !== null && pages[cue] !== undefined, - page: pageForCue(pages, cue), + page: audioPage(parts, cue, 0), ...over, }); } -describe('pageForCue', () => { - it('gives the page of a matched cue', () => { - expect(pageForCue(pages, 3)).toBe(1); - }); - - it('borrows the page of the nearest matched cue before an unmatched one', () => { - expect(pageForCue(pages, 2)).toBe(0); - expect(pageForCue(pages, 6)).toBe(1); - }); - - it('gives null when no matched cue is near enough', () => { - const far = [0, ...new Array<null>(LOOKBACK + 1).fill(null)]; - expect(pageForCue(far, far.length - 1)).toBeNull(); - expect(pageForCue(far, LOOKBACK)).toBe(0); - }); -}); - describe('follow', () => { it('marks a matched cue and stays on its page', () => { expect(follow(at(1))).toEqual({ highlightCue: 1, turnToPage: null, held: false });