commit 2847df0cbbcae427e874b6710ec36abcfe0d8cfc truex <truex@equwal.com> 2026-09-26 14:08:59 -0700 Extend the cue box to the edges of its words The alignment drops punctuation and can drop a noisy letter at the edge of a cue. The box of a line then stopped in a word, and the comma after "hills" was outside. lineBoxes now grows the box of each line to the edges of its first and its last word. In a CJK word only punctuation joins, because Japanese OCR can give one word for a whole line. Each box also gets 2 CSS pixels of padding, and the page clips it.
src/line-boxes.ts | 76 +++++++++++++++++++++++++++----- src/main.ts | 9 +++- src/style.css | 2 + test/line-boxes.test.ts | 115 +++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 186 insertions(+), 16 deletions(-)
diff --git a/src/line-boxes.ts b/src/line-boxes.ts index 19b6f35..34640fa 100644 --- a/src/line-boxes.ts +++ b/src/line-boxes.ts @@ -1,28 +1,80 @@ // The boxes that mark a cue on the page. Pure: no DOM. import type { OcrToken, TokenSpan } from './align'; +import { isCjk } from './book-text'; export type Box = OcrToken['bbox']; +/** A token of punctuation or symbols only. */ +const PUNCTUATION = /^[\p{P}\p{S}]+$/u; + +/** A token of opening punctuation only: a bracket or a quote that opens. */ +const OPENING = /^[\p{Ps}\p{Pi}]+$/u; + +/** + * True when the box of a line can grow from the edge token of the span + * over `next`, the token before it (`back`) or after it. Both must have the + * same page, line and word id. + * + * In a word that is not CJK, all its tokens join: the spaces show where + * the word ends. Japanese OCR can give one word for a whole line, so in a + * CJK word only punctuation joins: an opening mark before the span, and + * another mark after it. A 。 before the span ends the cue before. + */ +function joins(edge: OcrToken, next: OcrToken | undefined, back: boolean): next is OcrToken { + if ( + next === undefined || + edge.word === undefined || + next.page !== edge.page || + next.line !== edge.line || + next.word !== edge.word + ) { + return false; + } + if (!isCjk(edge.text)) return !isCjk(next.text) || PUNCTUATION.test(next.text); + if (back) return OPENING.test(next.text); + return PUNCTUATION.test(next.text) && !OPENING.test(next.text); +} + +/** Grows `box` so that it holds `bbox`. */ +function grow(box: Box, bbox: Box): void { + box.x0 = Math.min(box.x0, bbox.x0); + box.y0 = Math.min(box.y0, bbox.y0); + box.x1 = Math.max(box.x1, bbox.x1); + box.y1 = Math.max(box.y1, bbox.y1); +} + /** * One box for each text line of `span` on `page`: the union of the boxes - * of its tokens on that line. Tokens on other pages do not count. A span - * of an older alignment can go past the end of `tokens`: those indices - * do not count either. + * of its tokens on that line, plus `pad` pixels on each side. Tokens on + * other pages do not count. A span of an older alignment can go past the + * end of `tokens`: those indices do not count either. + * + * The alignment drops punctuation, and it can drop a noisy letter at the + * edge of a cue. So the box of a line grows to the edges of its first and + * its last word (see `joins`). */ -export function lineBoxes(tokens: readonly OcrToken[], span: TokenSpan, page: number): Box[] { - const lines = new Map<number, Box>(); +export function lineBoxes( + tokens: readonly OcrToken[], + span: TokenSpan, + page: number, + pad = 0, +): Box[] { + /** For each line: its box, and the first and the last token of the span on it. */ + const lines = new Map<number, { box: Box; first: number; last: number }>(); for (let t = span.start; t < span.end; t++) { const token = tokens[t]; if (!token || token.page !== page) continue; - const box = lines.get(token.line); - if (!box) lines.set(token.line, { ...token.bbox }); + const line = lines.get(token.line); + if (!line) lines.set(token.line, { box: { ...token.bbox }, first: t, last: t }); else { - box.x0 = Math.min(box.x0, token.bbox.x0); - box.y0 = Math.min(box.y0, token.bbox.y0); - box.x1 = Math.max(box.x1, token.bbox.x1); - box.y1 = Math.max(box.y1, token.bbox.y1); + grow(line.box, token.bbox); + line.last = t; } } - return [...lines.values()]; + return [...lines.values()].map(({ box, first, last }) => { + for (let t = first - 1; joins(tokens[first]!, tokens[t], true); t--) grow(box, tokens[t]!.bbox); + for (let t = last + 1; joins(tokens[last]!, tokens[t], false); t++) grow(box, tokens[t]!.bbox); + return { x0: box.x0 - pad, y0: box.y0 - pad, x1: box.x1 + pad, y1: box.y1 + pad }; + }); } diff --git a/src/main.ts b/src/main.ts index 3940b08..2bfe1d8 100644 --- a/src/main.ts +++ b/src/main.ts @@ -30,6 +30,9 @@ const OCR_WIDTH = 1600; /** How far from a character a tap may land, as a share of the page width. */ const TAP_TOLERANCE = 0.015; +/** The space around a box of the mark, in CSS pixels. */ +const BOX_PAD = 2; + /** Milliseconds to wait after a page is read before the cues are aligned again. */ const ALIGN_DEBOUNCE = 300; @@ -553,8 +556,10 @@ function drawBoxes(): void { ui.overlay.replaceChildren(); const span = state.spans[state.markedCue]; const size = state.pages[state.currentPage]; - if (!span || !size) return; - for (const box of lineBoxes(state.tokens, span, state.currentPage)) { + const shown = ui.page.querySelector('canvas')?.getBoundingClientRect().width; + if (!span || !size || !shown) return; + const pad = (BOX_PAD * size.width) / shown; + for (const box of lineBoxes(state.tokens, span, state.currentPage, pad)) { const div = document.createElement('div'); div.className = 'box'; div.style.left = `${(100 * box.x0) / size.width}%`; diff --git a/src/style.css b/src/style.css index 3c0fc31..65824ac 100644 --- a/src/style.css +++ b/src/style.css @@ -215,10 +215,12 @@ body.menu-open #panel { } } +/* The padding of a box can go past the edge of the page. The page clips it. */ #overlay { position: absolute; inset: 0; pointer-events: none; + overflow: hidden; } .box { diff --git a/test/line-boxes.test.ts b/test/line-boxes.test.ts index 1171dc5..03ef14e 100644 --- a/test/line-boxes.test.ts +++ b/test/line-boxes.test.ts @@ -1,6 +1,7 @@ +import fc from 'fast-check'; import { describe, expect, it } from 'vitest'; -import type { OcrToken } from '../src/align'; -import { lineBoxes } from '../src/line-boxes'; +import type { OcrToken, TokenSpan } from '../src/align'; +import { lineBoxes, type Box } from '../src/line-boxes'; function token(page: number, line: number, x0: number, y0: number): OcrToken { return { text: 'a', page, line, bbox: { x0, y0, x1: x0 + 10, y1: y0 + 20 } }; @@ -14,6 +15,33 @@ const tokens = [ token(1, 100000, 10, 0), ]; +/** + * Tokens of text lines, one token per character, 10 px per character and + * 20 px per line. A space separates two words and gets no token. + */ +function textTokens(lines: string[], page = 0): OcrToken[] { + const out: OcrToken[] = []; + let word = 0; + lines.forEach((text, row) => { + [...text].forEach((ch, col) => { + if (ch === ' ') { + word++; + return; + } + const x0 = col * 10; + const y0 = row * 20; + out.push({ text: ch, page, line: row, word, bbox: { x0, y0, x1: x0 + 10, y1: y0 + 20 } }); + }); + word++; + }); + return out; +} + +/** A matched span of the tokens `from` to `to`, `to` not included. */ +function spanOf(from: number, to: number): TokenSpan { + return { start: from, end: to, matched: true }; +} + describe('lineBoxes', () => { it('gives one box for each line, the union of its token boxes', () => { expect(lineBoxes(tokens, { start: 0, end: 3, matched: true }, 0)).toEqual([ @@ -35,4 +63,87 @@ describe('lineBoxes', () => { { x0: 0, y0: 0, x1: 20, y1: 22 }, ]); }); + + it('grows to the comma and the last letter that the alignment dropped', () => { + // "quiet hills," with the span to "hill": the alignment dropped "s,". + const line = textTokens(['quiet hills,']); + expect(line.map((t) => t.text).join('')).toBe('quiethills,'); + const box = lineBoxes(line, spanOf(0, 9), 0); + expect(box).toEqual([{ x0: 0, y0: 0, x1: 120, y1: 20 }]); + }); + + it('grows to the start of the first word', () => { + // The span starts at "ills", in the middle of "hills". + const line = textTokens(['the hills rose']); + expect(lineBoxes(line, spanOf(4, 8), 0)).toEqual([{ x0: 40, y0: 0, x1: 90, y1: 20 }]); + }); + + it('does not grow over the next word or the next line', () => { + const page = textTokens(['one two', 'three']); + // The span is "two": the box stays on "two". + expect(lineBoxes(page, spanOf(3, 6), 0)).toEqual([{ x0: 40, y0: 0, x1: 70, y1: 20 }]); + }); + + it('grows over CJK punctuation, but not over a CJK letter of the same OCR word', () => { + // Japanese OCR gave one word for the whole line. + const line = textTokens(['吾輩は猫である。名前は']); + // The span is 猫である: 。 joins the box, 名前は does not. + expect(lineBoxes(line, spanOf(3, 7), 0)).toEqual([{ x0: 30, y0: 0, x1: 80, y1: 20 }]); + // The span is 名前: the 。 before it ends the cue before, and は is a letter. + expect(lineBoxes(line, spanOf(8, 10), 0)).toEqual([{ x0: 80, y0: 0, x1: 100, y1: 20 }]); + // The span is 猫だ: the brackets around it join. + const quote = textTokens(['彼は「猫だ」と言った']); + expect(lineBoxes(quote, spanOf(3, 5), 0)).toEqual([{ x0: 20, y0: 0, x1: 60, y1: 20 }]); + }); + + it('pads each box on each side', () => { + const line = textTokens(['quiet hills,']); + expect(lineBoxes(line, spanOf(0, 9), 0, 3)).toEqual([{ x0: -3, y0: -3, x1: 123, y1: 23 }]); + }); + + it('gives the box of the whole words of the span on each line', () => { + // Latin words and punctuation: a whole word of each span token is in the + // box. A slow reference: the union of the tokens of the span and of the + // tokens that share a line and a word id with a span token. + const word = fc.stringMatching(/^[a-z,.;!?"]{1,6}$/); + const text = fc.array(fc.array(word, { minLength: 1, maxLength: 5 }), { + minLength: 1, + maxLength: 4, + }); + const input = text.chain((lines) => { + const page = textTokens(lines.map((words) => words.join(' '))); + return fc.tuple( + fc.constant(page), + fc.nat({ max: page.length - 1 }), + fc.nat({ max: page.length - 1 }), + fc.nat({ max: 5 }), + ); + }); + fc.assert( + fc.property(input, ([page, a, b, pad]) => { + const span = spanOf(Math.min(a, b), Math.max(a, b) + 1); + const inSpan = page.slice(span.start, span.end); + const expected = new Map<number, Box>(); + for (const t of page) { + const keep = inSpan.some((s) => s === t || (s.line === t.line && s.word === t.word)); + if (!keep) continue; + const box = expected.get(t.line); + if (!box) expected.set(t.line, { ...t.bbox }); + else { + box.x0 = Math.min(box.x0, t.bbox.x0); + box.y0 = Math.min(box.y0, t.bbox.y0); + box.x1 = Math.max(box.x1, t.bbox.x1); + box.y1 = Math.max(box.y1, t.bbox.y1); + } + } + const padded = [...expected.values()].map((b) => ({ + x0: b.x0 - pad, + y0: b.y0 - pad, + x1: b.x1 + pad, + y1: b.y1 + pad, + })); + expect(lineBoxes(page, span, 0, pad)).toEqual(padded); + }), + ); + }); });