commit 3dcee1fbefa231c13aecb011b1161cd367ca9392 truex <truex@equwal.com> 2026-09-25 01:59:38 -0700 Read the text layer of a PDF page as tokens text-layer.ts turns pdf.js text items into the same tokens as OCR: one box per character, line and word ids, in content-stream order. A vertical font and a rotated run lay the characters along y. pdf.ts reads a page's text layer and reports none when the page has fewer than 10 characters. New fixtures: a two-page scan with a cue that crosses the page boundary, and text-layer PDFs of the same text (Helvetica, and Yu Mincho through @pdf-lib/fontkit). The e2e tests read the text-layer PDFs with pdf.js in Node and check every character in order; the OCR e2e test covers the two-page scan.
fixtures/sample-eng-2p-1.png | Bin 0 -> 293264 bytes fixtures/sample-eng-2p-2.png | Bin 0 -> 309730 bytes fixtures/sample-eng-2p-text.pdf | Bin 0 -> 1898 bytes fixtures/sample-eng-2p.pdf | Bin 0 -> 538498 bytes fixtures/sample-eng-2p.srt | 59 +++++++++++++ fixtures/sample-eng-text.pdf | Bin 0 -> 1272 bytes fixtures/sample-eng.pdf | Bin 260250 -> 260251 bytes fixtures/sample-jpn-text.pdf | Bin 0 -> 22235 bytes fixtures/sample-jpn.pdf | Bin 212495 -> 212495 bytes fixtures/sample-text.ts | 108 +++++++++++++++++------- fixtures/silence.wav | Bin 168044 -> 308044 bytes package-lock.json | 11 +++ package.json | 1 + scripts/make-fixtures.ts | 97 ++++++++++++++++----- src/ocr-tokens.ts | 2 +- src/pdf.ts | 33 +++++++- src/text-layer.ts | 182 ++++++++++++++++++++++++++++++++++++++++ test/e2e/ocr-align.test.ts | 55 +++++++++--- test/e2e/text-layer.test.ts | 99 ++++++++++++++++++++++ test/play-clock.test.ts | 3 +- test/text-layer.test.ts | 112 +++++++++++++++++++++++++ 21 files changed, 698 insertions(+), 64 deletions(-)
diff --git a/fixtures/sample-eng-2p-1.png b/fixtures/sample-eng-2p-1.png new file mode 100644 index 0000000..4e86dc2 Binary files /dev/null and b/fixtures/sample-eng-2p-1.png differ diff --git a/fixtures/sample-eng-2p-2.png b/fixtures/sample-eng-2p-2.png new file mode 100644 index 0000000..27c312f Binary files /dev/null and b/fixtures/sample-eng-2p-2.png differ diff --git a/fixtures/sample-eng-2p-text.pdf b/fixtures/sample-eng-2p-text.pdf new file mode 100644 index 0000000..186b187 --- /dev/null +++ b/fixtures/sample-eng-2p-text.pdf @@ -0,0 +1,17 @@ +The morning sun rose over the quiet hills, +and the village below began to stir. +A baker opened his shutters with a yawn. +Two children raced along the river path, +their laughter carried on the cool wind. +Far away, a bell rang nine slow times. +The day had started, as days always do, +with nothing more than light and sound. + It rolled across the fields and woke the geese, +who answered with a chorus of their own. +By noon the market square was full of carts, +and every stall had something bright to sell. +An old man sat beneath the chestnut tree +and told the children stories of the sea. +When evening came the lamps were lit again, +and the hills grew quiet, as they always do. + diff --git a/fixtures/sample-eng-2p.pdf b/fixtures/sample-eng-2p.pdf new file mode 100644 index 0000000..c49220d --- /dev/null +++ b/fixtures/sample-eng-2p.pdf @@ -0,0 +1 @@ + diff --git a/fixtures/sample-eng-2p.srt b/fixtures/sample-eng-2p.srt new file mode 100644 index 0000000..e4e3d56 --- /dev/null +++ b/fixtures/sample-eng-2p.srt @@ -0,0 +1,59 @@ +1 +00:00:00,000 --> 00:00:02,500 +The morning sun rose over the quiet hills, + +2 +00:00:02,500 --> 00:00:05,000 +and the village below began to stir. + +3 +00:00:05,000 --> 00:00:07,500 +A baker opened his shutters with a yawn. + +4 +00:00:07,500 --> 00:00:10,000 +Two children raced along the river path, + +5 +00:00:10,000 --> 00:00:12,500 +their laughter carried on the cool wind. + +6 +00:00:12,500 --> 00:00:15,000 +Far away, a bell rang nine slow times. + +7 +00:00:15,000 --> 00:00:17,500 +The day had started, as days always do, + +8 +00:00:17,500 --> 00:00:20,000 +with nothing more than light and sound. It rolled across the fields and woke the geese, + +9 +00:00:20,000 --> 00:00:22,500 +who answered with a chorus of their own. + +10 +00:00:22,500 --> 00:00:25,000 +By noon the market square was full of carts, + +11 +00:00:25,000 --> 00:00:27,500 +and every stall had something bright to sell. + +12 +00:00:27,500 --> 00:00:30,000 +An old man sat beneath the chestnut tree + +13 +00:00:30,000 --> 00:00:32,500 +and told the children stories of the sea. + +14 +00:00:32,500 --> 00:00:35,000 +When evening came the lamps were lit again, + +15 +00:00:35,000 --> 00:00:37,500 +and the hills grew quiet, as they always do. diff --git a/fixtures/sample-eng-text.pdf b/fixtures/sample-eng-text.pdf new file mode 100644 index 0000000..5080b40 --- /dev/null +++ b/fixtures/sample-eng-text.pdf @@ -0,0 +1,9 @@ +The morning sun rose over the quiet hills, +and the village below began to stir. +A baker opened his shutters with a yawn. +Two children raced along the river path, +their laughter carried on the cool wind. +Far away, a bell rang nine slow times. +The day had started, as days always do, +with nothing more than light and sound. + diff --git a/fixtures/sample-jpn-text.pdf b/fixtures/sample-jpn-text.pdf new file mode 100644 index 0000000..fd79759 --- /dev/null +++ b/fixtures/sample-jpn-text.pdf @@ -0,0 +1,8 @@ +吾輩は猫である。名前はまだ無い。 +どこで生れたかとんと見当がつかぬ。 +何でも薄暗いじめじめした所で +ニャーニャー泣いていた事だけは記憶している。 +吾輩はここで始めて人間というものを見た。 +しかもあとで聞くとそれは書生という +人間中で一番獰悪な種族であったそうだ。 + diff --git a/fixtures/sample-text.ts b/fixtures/sample-text.ts index 075d638..399e22b 100644 --- a/fixtures/sample-text.ts +++ b/fixtures/sample-text.ts @@ -1,6 +1,7 @@ -// Ground truth for the synthetic "scanned page" fixtures. -// scripts/make-fixtures.ts renders these lines to an image. -// test/e2e/ocr-align.test.ts checks OCR + alignment against this layout. +// Ground truth for the synthetic fixtures. +// scripts/make-fixtures.ts renders these lines to page images and to +// text-layer PDFs. test/e2e checks OCR, the text layer and the alignment +// against this layout. export interface Fixture { name: string; @@ -8,8 +9,12 @@ export interface Fixture { lang: string; /** Font family list for the SVG renderer. */ font: string; - /** One text line per entry. One subtitle cue per line. */ - lines: string[]; + /** A TrueType file for the text-layer PDF. Absent: Helvetica. */ + ttf?: string; + /** Pages, each a list of text lines. */ + pages: string[][]; + /** The subtitle cues, in order. */ + cues: string[]; } /** Page geometry in image pixels (A4 at 150 dpi). */ @@ -25,40 +30,87 @@ export const PAGE = { /** Each cue lasts this long, one after the other. */ export const CUE_SECONDS = 2.5; +/** One cue per line. */ +function lineCues(pages: string[][]): string[] { + return pages.flat(); +} + +/** + * One cue per line, except that the last line of each page and the first + * line of the next page form one cue. That cue crosses the page boundary. + */ +function crossingCues(pages: string[][]): string[] { + const cues: string[] = []; + pages.forEach((lines, p) => { + const first = p > 0 ? 1 : 0; + const last = p < pages.length - 1 ? lines.length - 1 : lines.length; + cues.push(...lines.slice(first, last)); + if (p < pages.length - 1) cues.push(`${lines[lines.length - 1]} ${pages[p + 1]![0]}`); + }); + return cues; +} + +const ENG_PAGE_1 = [ + 'The morning sun rose over the quiet hills,', + 'and the village below began to stir.', + 'A baker opened his shutters with a yawn.', + 'Two children raced along the river path,', + 'their laughter carried on the cool wind.', + 'Far away, a bell rang nine slow times.', + 'The day had started, as days always do,', + 'with nothing more than light and sound.', +]; + +const ENG_PAGE_2 = [ + 'It rolled across the fields and woke the geese,', + 'who answered with a chorus of their own.', + 'By noon the market square was full of carts,', + 'and every stall had something bright to sell.', + 'An old man sat beneath the chestnut tree', + 'and told the children stories of the sea.', + 'When evening came the lamps were lit again,', + 'and the hills grew quiet, as they always do.', +]; + +const JPN_PAGE = [ + '吾輩は猫である。名前はまだ無い。', + 'どこで生れたかとんと見当がつかぬ。', + '何でも薄暗いじめじめした所で', + 'ニャーニャー泣いていた事だけは記憶している。', + '吾輩はここで始めて人間というものを見た。', + 'しかもあとで聞くとそれは書生という', + '人間中で一番獰悪な種族であったそうだ。', +]; + export const ENG: Fixture = { name: 'sample-eng', lang: 'eng', font: 'Georgia, Times New Roman, serif', - lines: [ - 'The morning sun rose over the quiet hills,', - 'and the village below began to stir.', - 'A baker opened his shutters with a yawn.', - 'Two children raced along the river path,', - 'their laughter carried on the cool wind.', - 'Far away, a bell rang nine slow times.', - 'The day had started, as days always do,', - 'with nothing more than light and sound.', - ], + pages: [ENG_PAGE_1], + cues: lineCues([ENG_PAGE_1]), }; export const JPN: Fixture = { name: 'sample-jpn', lang: 'jpn', font: 'Yu Mincho, MS Mincho, Meiryo, sans-serif', - lines: [ - '吾輩は猫である。名前はまだ無い。', - 'どこで生れたかとんと見当がつかぬ。', - '何でも薄暗いじめじめした所で', - 'ニャーニャー泣いていた事だけは記憶している。', - '吾輩はここで始めて人間というものを見た。', - 'しかもあとで聞くとそれは書生という', - '人間中で一番獰悪な種族であったそうだ。', - ], + ttf: 'C:\\Windows\\Fonts\\yumin.ttf', + pages: [JPN_PAGE], + cues: lineCues([JPN_PAGE]), +}; + +/** Two pages. One cue crosses from page 1 to page 2. */ +export const ENG2: Fixture = { + name: 'sample-eng-2p', + lang: 'eng', + font: ENG.font, + pages: [ENG_PAGE_1, ENG_PAGE_2], + cues: crossingCues([ENG_PAGE_1, ENG_PAGE_2]), }; -export const FIXTURES: Fixture[] = [ENG, JPN]; +export const FIXTURES: Fixture[] = [ENG, JPN, ENG2]; -/** Vertical pixel band that line `i` occupies. */ +/** Vertical pixel band that line `i` of a page occupies. */ export function lineBand(i: number): { top: number; bottom: number } { const top = PAGE.firstBaseline + i * PAGE.lineHeight - PAGE.lineHeight * 0.75; return { top, bottom: top + PAGE.lineHeight }; @@ -74,9 +126,9 @@ function srtTime(seconds: number): string { return `${pad(h, 2)}:${pad(m, 2)}:${pad(s, 2)},${pad(frac, 3)}`; } -/** SRT text with one cue per line, CUE_SECONDS each, starting at 0. */ +/** SRT text with one cue per entry of `cues`, CUE_SECONDS each, starting at 0. */ export function toSrt(fixture: Fixture): string { - return fixture.lines + return fixture.cues .map((line, i) => { const start = i * CUE_SECONDS; return `${i + 1}\n${srtTime(start)} --> ${srtTime(start + CUE_SECONDS)}\n${line}\n`; diff --git a/fixtures/silence.wav b/fixtures/silence.wav index 5dd99b2..1ae6161 100644 Binary files a/fixtures/silence.wav and b/fixtures/silence.wav differ diff --git a/package-lock.json b/package-lock.json index 3def233..16c75ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ }, "devDependencies": { "@capacitor/cli": "^8.5.2", + "@pdf-lib/fontkit": "^1.1.1", "@types/node": "^26.6.2", "fast-check": "^4.10.2", "pdf-lib": "^1.17.1", @@ -1628,6 +1629,16 @@ "url": "https://github.com/sponsors/oxc-project" } }, + "node_modules/@pdf-lib/fontkit": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@pdf-lib/fontkit/-/fontkit-1.1.1.tgz", + "integrity": "sha512-KjMd7grNapIWS/Dm0gvfHEilSyAmeLvrEGVcqLGi0VYebuqqzTbgF29efCx7tvx+IEbG3zQciRSWl3GkUSvjZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "pako": "^1.0.6" + } + }, "node_modules/@pdf-lib/standard-fonts": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@pdf-lib/standard-fonts/-/standard-fonts-1.0.0.tgz", diff --git a/package.json b/package.json index f08051c..660a4bd 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ }, "devDependencies": { "@capacitor/cli": "^8.5.2", + "@pdf-lib/fontkit": "^1.1.1", "@types/node": "^26.6.2", "fast-check": "^4.10.2", "pdf-lib": "^1.17.1", diff --git a/scripts/make-fixtures.ts b/scripts/make-fixtures.ts index 040358a..1d00996 100644 --- a/scripts/make-fixtures.ts +++ b/scripts/make-fixtures.ts @@ -1,24 +1,36 @@ // Build the synthetic fixtures in fixtures/: -// <name>.png scanned-like page image (SVG text rendered with sharp) -// <name>.pdf one-page image-only PDF, as a scanner would produce -// <name>.srt one cue per text line -// silence.wav silent audio long enough for all cues +// <name>.png, <name>-<n>.png scanned-like page images (SVG text rendered with sharp) +// <name>.pdf image-only PDF, one page per image, as a scanner would produce +// <name>-text.pdf the same text as a text layer (pdf-lib, Helvetica or a TrueType file) +// <name>.srt the subtitle cues +// silence.wav silent audio long enough for all cues // Run: npm run fixtures -import { mkdir, writeFile } from 'node:fs/promises'; +import fontkit from '@pdf-lib/fontkit'; +import { access, mkdir, readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; -import { PDFDocument } from 'pdf-lib'; +import { PDFDocument, StandardFonts, type PDFFont } from 'pdf-lib'; import sharp from 'sharp'; import { CUE_SECONDS, FIXTURES, PAGE, toSrt, type Fixture } from '../fixtures/sample-text'; const OUT = path.resolve('fixtures'); +/** Text-layer page geometry in points (A4). */ +const TEXT_PAGE = { + width: 595, + height: 842, + marginLeft: 60, + firstBaseline: 100, + lineHeight: 30, + fontSize: 14, +}; + function escapeXml(s: string): string { return s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>'); } -function pageSvg(fixture: Fixture): string { - const text = fixture.lines +function pageSvg(fixture: Fixture, lines: string[]): string { + const text = lines .map( (line, i) => `<text x="${PAGE.marginLeft}" y="${PAGE.firstBaseline + i * PAGE.lineHeight}">${escapeXml(line)}</text>`, @@ -32,9 +44,9 @@ ${text} </svg>`; } -async function renderPng(fixture: Fixture): Promise<Buffer> { +async function renderPng(fixture: Fixture, lines: string[]): Promise<Buffer> { // A light blur and a small rotation make the page look scanned. - return sharp(Buffer.from(pageSvg(fixture))) + return sharp(Buffer.from(pageSvg(fixture, lines))) .rotate(0.4, { background: '#f4f1ea' }) .resize(PAGE.width, PAGE.height, { fit: 'cover' }) .blur(0.6) @@ -42,12 +54,43 @@ async function renderPng(fixture: Fixture): Promise<Buffer> { .toBuffer(); } -async function toPdf(png: Buffer): Promise<Uint8Array> { +async function toPdf(pngs: Buffer[]): Promise<Uint8Array> { + const doc = await PDFDocument.create(); + for (const png of pngs) { + const image = await doc.embedPng(png); + // 150 dpi image on a 72 dpi page. + const page = doc.addPage([PAGE.width * 0.48, PAGE.height * 0.48]); + page.drawImage(image, { x: 0, y: 0, width: page.getWidth(), height: page.getHeight() }); + } + return doc.save(); +} + +/** The same text as a text layer. Returns null when the TrueType file is not on this machine. */ +async function toTextPdf(fixture: Fixture): Promise<Uint8Array | null> { const doc = await PDFDocument.create(); - const image = await doc.embedPng(png); - // 150 dpi image on a 72 dpi page. - const page = doc.addPage([PAGE.width * 0.48, PAGE.height * 0.48]); - page.drawImage(image, { x: 0, y: 0, width: page.getWidth(), height: page.getHeight() }); + let font: PDFFont; + if (fixture.ttf) { + try { + await access(fixture.ttf); + } catch { + return null; + } + doc.registerFontkit(fontkit); + font = await doc.embedFont(await readFile(fixture.ttf), { subset: true }); + } else { + font = await doc.embedFont(StandardFonts.Helvetica); + } + for (const lines of fixture.pages) { + const page = doc.addPage([TEXT_PAGE.width, TEXT_PAGE.height]); + lines.forEach((line, i) => { + page.drawText(line, { + x: TEXT_PAGE.marginLeft, + y: TEXT_PAGE.height - TEXT_PAGE.firstBaseline - i * TEXT_PAGE.lineHeight, + size: TEXT_PAGE.fontSize, + font, + }); + }); + } return doc.save(); } @@ -73,16 +116,30 @@ function silentWav(seconds: number): Buffer { return buf; } +/** The image file of page `p` of a fixture: `<name>.png` for one page, `<name>-<p+1>.png` for more. */ +export function pngName(fixture: Fixture, p: number): string { + return fixture.pages.length === 1 ? `${fixture.name}.png` : `${fixture.name}-${p + 1}.png`; +} + async function main(): Promise<void> { await mkdir(OUT, { recursive: true }); let maxSeconds = 0; for (const fixture of FIXTURES) { - const png = await renderPng(fixture); - await writeFile(path.join(OUT, `${fixture.name}.png`), png); - await writeFile(path.join(OUT, `${fixture.name}.pdf`), await toPdf(png)); + const pngs: Buffer[] = []; + for (const [p, lines] of fixture.pages.entries()) { + const png = await renderPng(fixture, lines); + pngs.push(png); + await writeFile(path.join(OUT, pngName(fixture, p)), png); + } + await writeFile(path.join(OUT, `${fixture.name}.pdf`), await toPdf(pngs)); await writeFile(path.join(OUT, `${fixture.name}.srt`), toSrt(fixture)); - maxSeconds = Math.max(maxSeconds, fixture.lines.length * CUE_SECONDS); - console.log(`wrote ${fixture.name}.png/.pdf/.srt (${fixture.lines.length} lines)`); + const text = await toTextPdf(fixture); + if (text) await writeFile(path.join(OUT, `${fixture.name}-text.pdf`), text); + else console.log(`skipped ${fixture.name}-text.pdf: ${fixture.ttf} is not on this machine`); + maxSeconds = Math.max(maxSeconds, fixture.cues.length * CUE_SECONDS); + console.log( + `wrote ${fixture.name}: ${fixture.pages.length} page(s), ${fixture.cues.length} cues${text ? ', text layer' : ''}`, + ); } await writeFile(path.join(OUT, 'silence.wav'), silentWav(maxSeconds + 1)); console.log(`wrote silence.wav (${maxSeconds + 1} s)`); diff --git a/src/ocr-tokens.ts b/src/ocr-tokens.ts index a20a9f6..ed534bd 100644 --- a/src/ocr-tokens.ts +++ b/src/ocr-tokens.ts @@ -4,7 +4,7 @@ import type Tesseract from 'tesseract.js'; import type { OcrToken } from './align'; /** Line ids are unique across pages: page * LINES_PER_PAGE + line. */ -const LINES_PER_PAGE = 100000; +export const LINES_PER_PAGE = 100000; /** * One token per recognized symbol (character). Symbol level works for diff --git a/src/pdf.ts b/src/pdf.ts index 9330643..22cf881 100644 --- a/src/pdf.ts +++ b/src/pdf.ts @@ -1,26 +1,41 @@ -// pdf.js wrapper: open a PDF and render one page to a canvas. +// pdf.js wrapper: open a PDF, render one page to a canvas, read its text layer. import * as pdfjs from 'pdfjs-dist'; import workerUrl from 'pdfjs-dist/build/pdf.worker.min.mjs?url'; +import type { OcrToken } from './align'; +import { MIN_TEXT_CHARS, textChars, textItems, textItemsToTokens } from './text-layer'; pdfjs.GlobalWorkerOptions.workerSrc = workerUrl; +export interface PageText { + /** Null when the page has no useful text layer. */ + tokens: OcrToken[] | null; + /** Size of the page in pixels at `targetWidth`. */ + width: number; + height: number; +} + export interface PdfDoc { numPages: number; /** Render page `index` (0-based) so that its width is `targetWidth` pixels. */ renderPage(index: number, targetWidth: number): Promise<HTMLCanvasElement>; + /** The text layer of page `index` as tokens in the pixel space of `targetWidth`. */ + pageText(index: number, targetWidth: number): Promise<PageText>; destroy(): Promise<void>; } export async function loadPdf(data: ArrayBuffer): Promise<PdfDoc> { const task = pdfjs.getDocument({ data }); const doc = await task.promise; + const viewportOf = async (index: number, targetWidth: number) => { + const page = await doc.getPage(index + 1); + const base = page.getViewport({ scale: 1 }); + return { page, viewport: page.getViewport({ scale: targetWidth / base.width }) }; + }; return { numPages: doc.numPages, async renderPage(index, targetWidth) { - const page = await doc.getPage(index + 1); - const base = page.getViewport({ scale: 1 }); - const viewport = page.getViewport({ scale: targetWidth / base.width }); + const { page, viewport } = await viewportOf(index, targetWidth); const canvas = document.createElement('canvas'); canvas.width = Math.ceil(viewport.width); canvas.height = Math.ceil(viewport.height); @@ -30,6 +45,16 @@ export async function loadPdf(data: ArrayBuffer): Promise<PdfDoc> { await page.render({ canvas, viewport, intent: 'print' }).promise; return canvas; }, + async pageText(index, targetWidth) { + const { page, viewport } = await viewportOf(index, targetWidth); + const content = await page.getTextContent(); + const items = textItems(content.items); + const size = { width: Math.ceil(viewport.width), height: Math.ceil(viewport.height) }; + if (textChars(items) < MIN_TEXT_CHARS) return { tokens: null, ...size }; + const toPixel = (x: number, y: number) => + viewport.convertToViewportPoint(x, y) as [number, number]; + return { tokens: textItemsToTokens(items, index, toPixel), ...size }; + }, destroy: () => task.destroy(), }; } diff --git a/src/text-layer.ts b/src/text-layer.ts new file mode 100644 index 0000000..def7d6c --- /dev/null +++ b/src/text-layer.ts @@ -0,0 +1,182 @@ +// The text layer of a PDF page as tokens, the same shape as OCR. Pure. +// +// pdf.js `getTextContent()` gives one item per run of text: the string, +// the transform of its origin, its length along the run and its font +// size. Each character gets an even share of the run. The boxes go +// through `toPixel`, so they land in the pixel space of the rendered page +// like the OCR boxes do. + +import type { OcrToken } from './align'; +import { LINES_PER_PAGE } from './ocr-tokens'; + +/** The fields of a pdf.js TextItem that the conversion uses. */ +export interface TextItemLike { + str: string; + /** 'ltr', 'rtl' or 'ttb'. 'ttb' is a vertical font: the run goes down. */ + dir: string; + /** [a, b, c, d, e, f]: the x axis (a, b), the y axis (c, d), the origin (e, f). */ + transform: readonly number[]; + /** Length of the run along the x axis of the font, or the font size for 'ttb'. */ + width: number; + /** The font size, or the length of the run down the page for 'ttb'. */ + height: number; + /** True when a line break follows the item. */ + hasEOL: boolean; +} + +/** From PDF user space to page pixels. */ +export type ToPixel = (x: number, y: number) => readonly [number, number]; + +/** Part of the font size above the baseline, and below it. */ +const ASCENT = 0.8; +const DESCENT = 0.25; + +/** Fewer non-space characters than this means the page has no useful text layer. */ +export const MIN_TEXT_CHARS = 10; + +interface Vec { + x: number; + y: number; +} + +function unit(x: number, y: number): Vec { + const n = Math.hypot(x, y) || 1; + return { x: x / n, y: y / n }; +} + +/** The geometry of one run: where it starts, which way it goes, how big the glyphs are. */ +interface Run { + origin: Vec; + /** Unit vector along the run. */ + along: Vec; + /** Unit vector across the run, toward the top of the glyphs. */ + up: Vec; + /** Length of the run. */ + length: number; + /** Size of the glyphs across the run. */ + size: number; + /** True for a vertical font: the glyphs are centered on the run. */ + centered: boolean; +} + +function runOf(item: TextItemLike): Run { + const [a = 1, b = 0, c = 0, d = 1, e = 0, f = 0] = item.transform; + const origin = { x: e, y: f }; + if (item.dir === 'ttb') { + // A vertical font advances down the y axis of the font. + return { + origin, + along: unit(-c, -d), + up: unit(a, b), + length: item.height, + size: item.width, + centered: true, + }; + } + // A horizontal font advances along the x axis of the font. A rotated + // transform (|b| > |a|) lays the run along the page's y. + return { + origin, + along: unit(a, b), + up: unit(c, d), + length: item.width, + size: item.height, + centered: false, + }; +} + +/** The axis-aligned pixel box of a parallelogram given by its four corners. */ +function pixelBox(corners: readonly Vec[], toPixel: ToPixel): OcrToken['bbox'] { + const box = { x0: Infinity, y0: Infinity, x1: -Infinity, y1: -Infinity }; + for (const p of corners) { + const [x, y] = toPixel(p.x, p.y); + box.x0 = Math.min(box.x0, x); + box.y0 = Math.min(box.y0, y); + box.x1 = Math.max(box.x1, x); + box.y1 = Math.max(box.y1, y); + } + return box; +} + +/** Box of character `i` of `n` in the run. */ +function charBox(run: Run, i: number, n: number, toPixel: ToPixel): OcrToken['bbox'] { + const step = run.length / n; + const from = i * step; + const to = from + step; + const [lo, hi] = run.centered + ? [-run.size / 2, run.size / 2] + : [-run.size * DESCENT, run.size * ASCENT]; + const at = (t: number, s: number): Vec => ({ + x: run.origin.x + run.along.x * t + run.up.x * s, + y: run.origin.y + run.along.y * t + run.up.y * s, + }); + return pixelBox([at(from, lo), at(to, lo), at(to, hi), at(from, hi)], toPixel); +} + +/** True for a character that separates words and gets no token. */ +function isSpace(ch: string): boolean { + return /\s/u.test(ch); +} + +/** + * One token per character, with line and word ids, in content-stream + * order. Runs on one baseline (within half a font size) share a line. + * Spaces separate words and get no token. + */ +export function textItemsToTokens( + items: readonly TextItemLike[], + pageIndex: number, + toPixel: ToPixel, +): OcrToken[] { + const tokens: OcrToken[] = []; + const lines: { key: number; tolerance: number; id: number }[] = []; + let lineId = -1; + let word = 0; + let breakLine = true; + for (const item of items) { + const chars = [...item.str]; + if (chars.length === 0) { + if (item.hasEOL) breakLine = true; + continue; + } + const run = runOf(item); + // The baseline: the origin projected on the "up" axis. + const key = run.origin.x * run.up.x + run.origin.y * run.up.y; + const tolerance = run.size / 2; + let line = breakLine ? undefined : lines.find((l) => Math.abs(l.key - key) <= l.tolerance); + if (!line) { + line = { key, tolerance, id: pageIndex * LINES_PER_PAGE + lines.length }; + lines.push(line); + word++; + } + if (line.id !== lineId) word++; + lineId = line.id; + breakLine = item.hasEOL; + chars.forEach((ch, i) => { + if (isSpace(ch)) { + word++; + return; + } + tokens.push({ + text: ch, + page: pageIndex, + line: line.id, + word: pageIndex * LINES_PER_PAGE + word, + bbox: charBox(run, i, chars.length, toPixel), + }); + }); + } + return tokens; +} + +/** The text items of a pdf.js text content. Marked-content items have no `str` and are dropped. */ +export function textItems(items: readonly unknown[]): TextItemLike[] { + return items.filter((i): i is TextItemLike => typeof i === 'object' && i !== null && 'str' in i); +} + +/** The number of non-space characters in the items. */ +export function textChars(items: readonly TextItemLike[]): number { + let n = 0; + for (const item of items) for (const ch of item.str) if (!isSpace(ch)) n++; + return n; +} diff --git a/test/e2e/ocr-align.test.ts b/test/e2e/ocr-align.test.ts index 1feb1ad..c97229f 100644 --- a/test/e2e/ocr-align.test.ts +++ b/test/e2e/ocr-align.test.ts @@ -12,22 +12,46 @@ import { pageToTokens } from '../../src/ocr-tokens'; import { parseSubtitles } from '../../src/subtitles'; import { FIXTURES, lineBand, type Fixture } from '../../fixtures/sample-text'; -const TIMEOUT = 180_000; +const TIMEOUT = 300_000; const CACHE = '.tessdata'; +/** The image file of page `p`: `<name>.png` for one page, `<name>-<p+1>.png` for more. */ +function pngName(fixture: Fixture, p: number): string { + return fixture.pages.length === 1 ? `${fixture.name}.png` : `${fixture.name}-${p + 1}.png`; +} + async function ocrFixture(fixture: Fixture): Promise<OcrToken[]> { // tesseract.js writes the cache file only when the directory exists. await mkdir(CACHE, { recursive: true }); const worker = await createWorker(fixture.lang, 1, { cachePath: CACHE }); try { - const image = await readFile(path.resolve('fixtures', `${fixture.name}.png`)); - const { data } = await worker.recognize(image, {}, { blocks: true }); - return pageToTokens(data, 0); + const tokens: OcrToken[] = []; + for (let p = 0; p < fixture.pages.length; p++) { + const image = await readFile(path.resolve('fixtures', pngName(fixture, p))); + const { data } = await worker.recognize(image, {}, { blocks: true }); + tokens.push(...pageToTokens(data, p)); + } + return tokens; } finally { await worker.terminate(); } } +/** The page and the line on that page of cue `i`, or null for a cue that crosses pages. */ +function placeOf(fixture: Fixture, i: number): { page: number; line: number } | null { + let cue = 0; + for (const [page, lines] of fixture.pages.entries()) { + const first = page > 0 ? 1 : 0; + const last = page < fixture.pages.length - 1 ? lines.length - 1 : lines.length; + for (let line = first; line < last; line++, cue++) if (cue === i) return { page, line }; + if (page < fixture.pages.length - 1) { + if (cue === i) return null; + cue++; + } + } + throw new Error(`no cue ${i}`); +} + describe.each(FIXTURES)('OCR + alignment on $name', (fixture) => { let tokens: OcrToken[] = []; @@ -43,7 +67,7 @@ describe.each(FIXTURES)('OCR + alignment on $name', (fixture) => { it('highlights the line of each cue', async () => { const srt = await readFile(path.resolve('fixtures', `${fixture.name}.srt`), 'utf8'); const cues = parseSubtitles(srt); - expect(cues).toHaveLength(fixture.lines.length); + expect(cues).toHaveLength(fixture.cues.length); expect(tokens.length).toBeGreaterThan(0); const spans = alignCuesToTokens( @@ -56,14 +80,25 @@ describe.each(FIXTURES)('OCR + alignment on $name', (fixture) => { expect( span.end - span.start, `cue ${i} covers most of its characters`, - ).toBeGreaterThanOrEqual(Math.floor(0.6 * fixture.lines[i]!.length)); - const band = lineBand(i); + ).toBeGreaterThanOrEqual(Math.floor(0.6 * fixture.cues[i]!.length)); + const place = placeOf(fixture, i); + if (!place) { + // The cue that crosses the page boundary: its tokens lie on both pages. + const pages = new Set<number>(); + for (let t = span.start; t < span.end; t++) pages.add(tokens[t]!.page); + expect([...pages].sort(), `cue ${i} crosses the pages`).toEqual([0, 1]); + return; + } + const band = lineBand(place.line); for (let t = span.start; t < span.end; t++) { - const box = tokens[t]!.bbox; - const yMid = (box.y0 + box.y1) / 2; + const token = tokens[t]!; + expect(token.page, `cue ${i} token ${t} "${token.text}" on page ${place.page}`).toBe( + place.page, + ); + const yMid = (token.bbox.y0 + token.bbox.y1) / 2; expect( yMid, - `cue ${i} token ${t} "${tokens[t]!.text}" on line ${i}`, + `cue ${i} token ${t} "${token.text}" on line ${place.line}`, ).toBeGreaterThanOrEqual(band.top); expect(yMid).toBeLessThanOrEqual(band.bottom); } diff --git a/test/e2e/text-layer.test.ts b/test/e2e/text-layer.test.ts new file mode 100644 index 0000000..232e269 --- /dev/null +++ b/test/e2e/text-layer.test.ts @@ -0,0 +1,99 @@ +// End-to-end check of the text layer: pdf.js in Node (the legacy build) +// reads the text-layer fixture PDFs, and text-layer.ts must give one +// token per character, in order, with the lines of the fixture. +// Run: npm run test:e2e + +import { access, readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs'; +import { describe, expect, it } from 'vitest'; +import { alignCuesToTokens, normalizeText, type OcrToken } from '../../src/align'; +import { bookText } from '../../src/book-text'; +import { parseSubtitles } from '../../src/subtitles'; +import { textItems, textItemsToTokens } from '../../src/text-layer'; +import { FIXTURES, type Fixture } from '../../fixtures/sample-text'; + +const WIDTH = 1600; + +async function exists(file: string): Promise<boolean> { + try { + await access(file); + return true; + } catch { + return false; + } +} + +async function textTokens(file: string): Promise<{ tokens: OcrToken[]; height: number }> { + const data = new Uint8Array(await readFile(file)); + const task = getDocument({ data }); + const doc = await task.promise; + const tokens: OcrToken[] = []; + let height = 0; + for (let p = 0; p < doc.numPages; p++) { + const page = await doc.getPage(p + 1); + const base = page.getViewport({ scale: 1 }); + const viewport = page.getViewport({ scale: WIDTH / base.width }); + height = viewport.height; + const content = await page.getTextContent(); + const items = textItems(content.items); + const toPixel = (x: number, y: number) => + viewport.convertToViewportPoint(x, y) as [number, number]; + tokens.push(...textItemsToTokens(items, p, toPixel)); + } + await task.destroy(); + return { tokens, height }; +} + +const withText: Fixture[] = []; +for (const fixture of FIXTURES) { + if (await exists(path.resolve('fixtures', `${fixture.name}-text.pdf`))) withText.push(fixture); +} + +describe.each(withText)('text layer of $name-text.pdf', (fixture) => { + it('carries every character in order, line by line', async () => { + const { tokens, height } = await textTokens( + path.resolve('fixtures', `${fixture.name}-text.pdf`), + ); + const expected = fixture.pages.map((lines) => lines.join('\n')).join('\n\n') + '\n'; + expect(bookText(tokens)).toBe(expected); + expect(tokens.map((t) => t.text).join('')).toBe(expected.replace(/\s/g, '')); + // Each page: every box inside the page, lines from top to bottom. + let lastLine = -1; + let lastY = -Infinity; + for (const t of tokens) { + expect(t.bbox.x0).toBeGreaterThanOrEqual(0); + expect(t.bbox.x1).toBeLessThanOrEqual(WIDTH); + expect(t.bbox.y0).toBeGreaterThanOrEqual(0); + expect(t.bbox.y1).toBeLessThanOrEqual(height); + if (t.line !== lastLine) { + if (t.line % 100000 !== 0) expect(t.bbox.y0).toBeGreaterThan(lastY); + lastLine = t.line; + lastY = t.bbox.y0; + } + } + }); + + it('aligns every cue to the text layer', async () => { + const { tokens } = await textTokens(path.resolve('fixtures', `${fixture.name}-text.pdf`)); + const srt = await readFile(path.resolve('fixtures', `${fixture.name}.srt`), 'utf8'); + const spans = alignCuesToTokens( + parseSubtitles(srt).map((c) => c.text), + tokens, + ); + expect(spans).toHaveLength(fixture.cues.length); + spans.forEach((span, i) => { + expect(span.matched, `cue ${i} matched`).toBe(true); + const text = tokens + .slice(span.start, span.end) + .map((t) => t.text) + .join(''); + // The alignment drops punctuation, so a span can leave out a comma at its edge. + expect(normalizeText(text)).toBe(normalizeText(fixture.cues[i]!)); + }); + }); +}); + +it('has at least the English text-layer fixture', () => { + expect(withText.map((f) => f.name)).toContain('sample-eng-2p'); +}); diff --git a/test/play-clock.test.ts b/test/play-clock.test.ts index fdc8fd7..cbfb9be 100644 --- a/test/play-clock.test.ts +++ b/test/play-clock.test.ts @@ -70,7 +70,8 @@ describe('waitUntil', () => { expect(positionAt(playing, now)).toBeGreaterThanOrEqual(target); } else { expect(wait).toBeGreaterThan(0); - expect(positionAt(playing, now + wait)).toBeGreaterThanOrEqual(target); + // The wait is whole milliseconds; the float product can fall a hair short. + expect(positionAt(playing, now + wait)).toBeGreaterThanOrEqual(target - 1e-6); } }), ); diff --git a/test/text-layer.test.ts b/test/text-layer.test.ts new file mode 100644 index 0000000..663ca7f --- /dev/null +++ b/test/text-layer.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from 'vitest'; +import { bookText } from '../src/book-text'; +import { textChars, textItemsToTokens, type TextItemLike } from '../src/text-layer'; + +/** A page 200 pt wide and 300 pt high, rendered 2 px per pt, y flipped. */ +const toPixel = (x: number, y: number): [number, number] => [2 * x, 2 * (300 - y)]; + +function item(over: Partial<TextItemLike> & Pick<TextItemLike, 'str'>): TextItemLike { + return { + dir: 'ltr', + transform: [12, 0, 0, 12, 10, 100], + width: 60, + height: 12, + hasEOL: false, + ...over, + }; +} + +type Box = { x0: number; y0: number; x1: number; y1: number }; + +function expectBox(box: Box, expected: Box): void { + for (const k of ['x0', 'y0', 'x1', 'y1'] as const) expect(box[k], k).toBeCloseTo(expected[k], 6); +} + +describe('textItemsToTokens', () => { + it('splits a horizontal run into one box per character, left to right', () => { + const tokens = textItemsToTokens([item({ str: 'abc', width: 30 })], 0, toPixel); + expect(tokens.map((t) => t.text)).toEqual(['a', 'b', 'c']); + expect(tokens.map((t) => t.line)).toEqual([0, 0, 0]); + // 10 pt each, from x = 10; the box spans 0.25 below to 0.8 above the baseline y = 100. + expectBox(tokens[0]!.bbox, { x0: 20, y0: 2 * (300 - 109.6), x1: 40, y1: 2 * (300 - 97) }); + expect(tokens[1]!.bbox.x0).toBeCloseTo(40, 6); + expect(tokens[2]!.bbox.x1).toBeCloseTo(80, 6); + }); + + it('stacks the characters of a vertical font down the page', () => { + // A vertical run at x = 150 from y = 250 down, 14 pt per glyph. + const tokens = textItemsToTokens( + [ + item({ + str: '吾輩は', + dir: 'ttb', + transform: [14, 0, 0, 14, 150, 250], + width: 14, + height: 42, + }), + ], + 0, + toPixel, + ); + expect(tokens.map((t) => t.text)).toEqual(['吾', '輩', 'は']); + expectBox(tokens[0]!.bbox, { + x0: 2 * 143, + y0: 2 * (300 - 250), + x1: 2 * 157, + y1: 2 * (300 - 236), + }); + expect(tokens[1]!.bbox.y0).toBeCloseTo(2 * (300 - 236), 6); + expect(tokens[2]!.bbox.y1).toBeCloseTo(2 * (300 - 208), 6); + expect(new Set(tokens.map((t) => t.line)).size).toBe(1); + }); + + it('lays a run rotated by 90 degrees along y', () => { + // Text that reads upward: the x axis of the font points up the page. + const tokens = textItemsToTokens( + [item({ str: 'ab', transform: [0, 12, -12, 0, 50, 100], width: 24, height: 12 })], + 0, + toPixel, + ); + expect(tokens[0]!.bbox.y0).toBeGreaterThan(tokens[1]!.bbox.y0); + expect(tokens[0]!.bbox.x0).toBeCloseTo(tokens[1]!.bbox.x0, 6); + }); + + it('gives two baselines two lines, and one baseline one line', () => { + const tokens = textItemsToTokens( + [ + item({ str: 'The ', width: 40 }), + item({ str: 'sun', width: 30, transform: [12, 0, 0, 12, 50, 101] }), + item({ str: 'rose', width: 40, transform: [12, 0, 0, 12, 10, 80] }), + ], + 1, + toPixel, + ); + expect(tokens.map((t) => t.text).join('')).toBe('Thesunrose'); + const lines = tokens.map((t) => t.line); + expect(lines.slice(0, 6).every((l) => l === lines[0])).toBe(true); + expect(lines[6]).not.toBe(lines[0]); + expect(lines[0]).toBe(100000); + expect(bookText(tokens)).toBe('The sun\nrose\n'); + }); + + it('starts a new line after hasEOL, even on the same baseline', () => { + const tokens = textItemsToTokens( + [item({ str: 'ab', hasEOL: true }), item({ str: 'cd' })], + 0, + toPixel, + ); + expect(tokens[0]!.line).not.toBe(tokens[2]!.line); + }); + + it('drops spaces and bumps the word id at each', () => { + const tokens = textItemsToTokens([item({ str: 'a b' })], 0, toPixel); + expect(tokens.map((t) => t.text)).toEqual(['a', 'b']); + expect(tokens[0]!.word).not.toBe(tokens[1]!.word); + }); +}); + +describe('textChars', () => { + it('counts the non-space characters', () => { + expect(textChars([item({ str: 'a b ' }), item({ str: ' ' }), item({ str: 'cd' })])).toBe(4); + }); +});