commit 7d4e78f0806db099eb4760b71bea6af74a7364c6 truex <truex@equwal.com> 2026-09-26 19:53:53 -0700 Give tesseract.js a PPM image of the page, not the canvas tesseract.js makes a PNG file of a canvas with canvas.toBlob. On the test phone (Android 11, WebView 153), this PNG encoding took about 13 s for each page of 1600 x 2263 pixels. A scan of 300 pages needed about 80 minutes of OCR. The OCR now reads the pixels of the page canvas with getImageData and gives tesseract.js a binary PPM image (P6): a short text header and the RGB bytes. The conversion takes about 14 ms in Node for a page of that size. Leptonica reads PPM, and the pixels do not change, so the tokens do not change. A new e2e test reads sample-eng-2p-1.png and sample-jpn.png as PNG and as PPM, and gets the same symbols and boxes.
README.md | 17 ++++++++-- src/ocr.ts | 27 +++++++++++++-- src/ppm.ts | 37 ++++++++++++++++++++ test/e2e/ocr-ppm.test.ts | 87 ++++++++++++++++++++++++++++++++++++++++++++++++ test/ocr.test.ts | 31 +++++++++++++++-- test/ppm.test.ts | 71 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 261 insertions(+), 9 deletions(-)
diff --git a/README.md b/README.md index 5a6e0ba..99eae94 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,13 @@ It is a setting of the book: another book opens with its own setting. The status strip says which one was used for each page. A new setting stops the OCR of the reading before at once. +The app renders a scanned page to a canvas 1600 pixels wide. tesseract.js +gets the pixels of the canvas as a PPM image: a short text header and the +RGB bytes. The app does not give it the canvas itself. With a canvas, +tesseract.js first makes a PNG file, and on a test phone (Android +WebView) that took about 13 s for each page. The PPM image takes +milliseconds and has the same pixels, so OCR gives the same tokens. + Both give the same tokens: one box per character with a line id and a word id. The alignment, the mark, the page turn and the lookup work the same way on a scan and on a text PDF. @@ -270,8 +277,9 @@ npm run format:check network. `npm run test:e2e` runs real OCR on the fixture images in Node and checks that every cue is marked on the correct line and page, and reads the text-layer PDFs with pdf.js in Node and checks that the text -layer gives every character in order. It needs network on the first run -to fetch language data into `.tessdata/`. +layer gives every character in order. It also checks that OCR reads the +same symbols from a PPM image of a page as from its PNG file. It needs +network on the first run to fetch language data into `.tessdata/`. `npm run fixtures` rebuilds the fixtures from `fixtures/sample-text.ts`. The Japanese text-layer PDF needs `C:\Windows\Fonts\yumin.ttf`; without @@ -406,7 +414,8 @@ Development: fixtures (render text to a page image, wrap it in an image-only PDF, write the same text as a text-layer PDF, write a silent WAV). `@pdf-lib/fontkit` lets pdf-lib embed a TrueType font for the - Japanese text-layer page. + Japanese text-layer page. `sharp` also gives the e2e test the RGBA + pixels of a page image, for the PPM image. The SRT/VTT parser is written by hand (`src/subtitles.ts`). The `subtitle` package imports the Node `stream` module at load time, so it does not run @@ -427,6 +436,8 @@ in a browser bundle. and the audio element (web). - `src/text-layer.ts`: pure conversion of pdf.js text items to tokens. - `src/ocr-tokens.ts`: pure conversion of a tesseract result to tokens. +- `src/ppm.ts`: pure PPM image of the pixels of a page canvas, for + tesseract.js. - `src/align.ts`: pure alignment of subtitle cues to the page text. - `src/subtitles.ts`: pure SRT/VTT parser and active-cue lookup. - `src/hit-test.ts`: pure tap on the page. Hit test, scan string, the diff --git a/src/ocr.ts b/src/ocr.ts index 4a6ff1b..a5110ba 100644 --- a/src/ocr.ts +++ b/src/ocr.ts @@ -5,7 +5,7 @@ // comes from jsDelivr at the first OCR run of each language. tesseract.js // keeps it in IndexedDB after that. -import { createWorker, PSM, type Worker as TesseractWorker } from 'tesseract.js'; +import { createWorker, PSM, type ImageLike, type Worker as TesseractWorker } from 'tesseract.js'; // Vite copies the two files into the build and gives their URLs. import workerPath from 'tesseract.js/dist/worker.min.js?url'; // One core file that holds the WebAssembly, so the worker makes one request. @@ -15,6 +15,7 @@ import corePath from 'tesseract.js-core/tesseract-core-simd-lstm.wasm.js?url'; import type { OcrToken } from './align'; import { OcrStartError } from './ocr-job'; import { pageToTokens } from './ocr-tokens'; +import { imageToPpm } from './ppm'; export { OcrStartError }; @@ -54,9 +55,9 @@ export async function createOcr(lang: string, onProgress: (p: number) => void): // A stop while no page runs is no error. stopped.catch(() => undefined); return { - recognize(image, pageIndex) { + async recognize(canvas, pageIndex) { const page = worker - .recognize(image, {}, { blocks: true, text: false }) + .recognize(ppmOf(canvas), {}, { blocks: true, text: false }) .then(({ data }) => pageToTokens(data, pageIndex)); return Promise.race([page, stopped]); }, @@ -67,6 +68,26 @@ export async function createOcr(lang: string, onProgress: (p: number) => void): }; } +/** + * The pixels of a page canvas as a PPM file. tesseract.js makes a PNG file of + * a canvas with canvas.toBlob, and in the Android WebView of the test phone + * that took about 13 s for each page. The PPM file has the same pixels, so the + * OCR gives the same tokens (test/e2e/ocr-ppm.test.ts). The ImageData is not + * kept after the conversion. + */ +function ppmOf(canvas: HTMLCanvasElement): ImageLike { + // pdf.js made the context with willReadFrequently (its default without + // enableHWA), so getImageData reads memory and not the GPU. + const context = canvas.getContext('2d', { willReadFrequently: true }); + if (!context) throw new Error('The page canvas has no 2D context.'); + const ppm = imageToPpm(context.getImageData(0, 0, canvas.width, canvas.height)); + // tesseract.js uses the bytes of an image file as they are: its loadImage + // returns `new Uint8Array(image)` for an input that is not a string, an + // element, a canvas or a Blob (src/worker/browser/loadImage.js). Its type + // ImageLike does not list Uint8Array. + return ppm as unknown as ImageLike; +} + /** * Starts the worker of tesseract.js. When the language data does not load, * tesseract.js gives the error to errorHandler, and the promise of diff --git a/src/ppm.ts b/src/ppm.ts new file mode 100644 index 0000000..118b33f --- /dev/null +++ b/src/ppm.ts @@ -0,0 +1,37 @@ +// Encode RGBA pixels as a binary PPM image. Pure: no DOM. +// +// tesseract.js reads image files. When it gets a canvas, it makes a PNG file +// of it with canvas.toBlob, and in the Android WebView of the test phone that +// took about 13 s for each page. A binary PPM file is a short text header and +// the RGB bytes, so it takes milliseconds to make, and it keeps each pixel. +// Leptonica, the image library of tesseract.js, reads it. + +/** RGBA pixels, 4 bytes for each pixel, row by row from the top: the layout of ImageData. */ +export interface RgbaImage { + width: number; + height: number; + data: Uint8ClampedArray; +} + +/** + * A binary PPM file ("P6", maximum value 255) of `image`. The alpha channel + * is dropped: the canvas of a pdf.js page is opaque. + */ +export function imageToPpm(image: RgbaImage): Uint8Array { + const { width, height, data } = image; + const pixels = width * height; + if (data.length !== pixels * 4) { + throw new RangeError( + `An RGBA image of ${width}x${height} has ${pixels * 4} bytes, not ${data.length}.`, + ); + } + const header = new TextEncoder().encode(`P6\n${width} ${height}\n255\n`); + const ppm = new Uint8Array(header.length + pixels * 3); + ppm.set(header); + for (let from = 0, to = header.length; from < data.length; from += 4, to += 3) { + ppm[to] = data[from]!; + ppm[to + 1] = data[from + 1]!; + ppm[to + 2] = data[from + 2]!; + } + return ppm; +} diff --git a/test/e2e/ocr-ppm.test.ts b/test/e2e/ocr-ppm.test.ts new file mode 100644 index 0000000..d2db369 --- /dev/null +++ b/test/e2e/ocr-ppm.test.ts @@ -0,0 +1,87 @@ +// End-to-end check without a browser: tesseract.js reads the same symbols +// from the PNG file of a page and from a PPM file of the same pixels. The app +// gives tesseract.js a PPM file (src/ppm.ts), because the PNG encoding of the +// page canvas took about 13 s for each page in the Android WebView. +// Needs network on the first run to fetch language data into .tessdata/. +// Run: npm run test:e2e + +import { mkdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; +import sharp from 'sharp'; +import { createWorker } from 'tesseract.js'; +import { describe, expect, it } from 'vitest'; +import type { OcrToken } from '../../src/align'; +import { pageToTokens } from '../../src/ocr-tokens'; +import { imageToPpm } from '../../src/ppm'; + +const TIMEOUT = 300_000; +const CACHE = '.tessdata'; + +/** A PPM file of the pixels of a PNG file: RGBA pixels, as getImageData gives them in the app. */ +async function ppmOf(png: Buffer): Promise<Buffer> { + const { data, info } = await sharp(png).ensureAlpha().raw().toBuffer({ resolveWithObject: true }); + expect(info.channels).toBe(4); + const ppm = imageToPpm({ + width: info.width, + height: info.height, + data: new Uint8ClampedArray(data.buffer, data.byteOffset, data.byteLength), + }); + return Buffer.from(ppm.buffer, ppm.byteOffset, ppm.byteLength); +} + +/** How far apart two readings of one page are. */ +function difference(a: OcrToken[], b: OcrToken[]): string { + const text = (tokens: OcrToken[]) => tokens.map((t) => t.text).join(''); + let box = 0; + let unequal = 0; + for (let i = 0; i < Math.min(a.length, b.length); i++) { + const p = a[i]!; + const q = b[i]!; + const d = Math.max( + Math.abs(p.bbox.x0 - q.bbox.x0), + Math.abs(p.bbox.y0 - q.bbox.y0), + Math.abs(p.bbox.x1 - q.bbox.x1), + Math.abs(p.bbox.y1 - q.bbox.y1), + ); + box = Math.max(box, d); + if (d > 0 || p.text !== q.text || p.line !== q.line || p.word !== q.word) unequal++; + } + return ( + `${a.length} and ${b.length} symbols, text ${text(a) === text(b) ? 'equal' : 'not equal'}, ` + + `${unequal} unequal symbols, largest box difference ${box} px` + ); +} + +describe.each([ + { file: 'sample-eng-2p-1.png', lang: 'eng' }, + { file: 'sample-jpn.png', lang: 'jpn' }, +])('OCR of $file', ({ file, lang }) => { + it( + 'reads the same symbols from the PNG file and from a PPM file of its pixels', + async () => { + const png = await readFile(path.resolve('fixtures', file)); + const ppm = await ppmOf(png); + // tesseract.js writes the cache file only when the directory exists. + await mkdir(CACHE, { recursive: true }); + const worker = await createWorker(lang, 1, { cachePath: CACHE }); + try { + const read = async (image: Buffer): Promise<OcrToken[]> => { + const start = performance.now(); + const { data } = await worker.recognize(image, {}, { blocks: true, text: false }); + console.log( + `[${file}] ${image.length} bytes read in ${Math.round(performance.now() - start)} ms`, + ); + return pageToTokens(data, 0); + }; + const fromPng = await read(png); + const fromPpm = await read(ppm); + console.log(`[${file}] PNG and PPM: ${difference(fromPng, fromPpm)}`); + expect(fromPng.length).toBeGreaterThan(0); + expect(fromPpm).toEqual(fromPng); + } finally { + await worker.terminate(); + } + }, + TIMEOUT, + ); +}); diff --git a/test/ocr.test.ts b/test/ocr.test.ts index c9bc2fc..4d25586 100644 --- a/test/ocr.test.ts +++ b/test/ocr.test.ts @@ -1,3 +1,4 @@ +import type Tesseract from 'tesseract.js'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { createOcr, OcrStartError } from '../src/ocr'; @@ -20,11 +21,20 @@ function options(): Options { function fakeWorker() { return { setParameters: vi.fn(() => Promise.resolve({})), - recognize: vi.fn(() => Promise.reject(new Error('The image is empty.'))), + recognize: vi.fn((_image: unknown): Promise<{ data: Pick<Tesseract.Page, 'blocks'> }> => + Promise.reject(new Error('The image is empty.')), + ), terminate: vi.fn(() => Promise.resolve({})), }; } +/** A page canvas of 2 x 1 pixels: red, then blue with half alpha. */ +function fakeCanvas(): HTMLCanvasElement { + const data = new Uint8ClampedArray([255, 0, 0, 255, 0, 0, 255, 128]); + const context = { getImageData: () => ({ width: 2, height: 1, data }) }; + return { width: 2, height: 1, getContext: () => context } as unknown as HTMLCanvasElement; +} + /** A path on the origin of the app, for example "/assets/worker.min-abc.js". */ const LOCAL = /^\/(?!\/)/; @@ -97,7 +107,7 @@ describe('createOcr', () => { worker.recognize.mockImplementation(() => new Promise(() => undefined)); createWorker.mockResolvedValue(worker); const ocr = await createOcr('eng', () => undefined); - const page = ocr.recognize({} as HTMLCanvasElement, 0); + const page = ocr.recognize(fakeCanvas(), 0); await ocr.terminate(); await expect(page).rejects.toThrow('OCR stopped.'); expect(worker.terminate).toHaveBeenCalledTimes(1); @@ -106,8 +116,23 @@ describe('createOcr', () => { it('gives the error of a page as it is, not as OcrStartError', async () => { createWorker.mockResolvedValue(fakeWorker()); const ocr = await createOcr('eng', () => undefined); - const error = await ocr.recognize({} as HTMLCanvasElement, 0).catch((e: unknown) => e); + const error = await ocr.recognize(fakeCanvas(), 0).catch((e: unknown) => e); expect(error).toEqual(new Error('The image is empty.')); expect(error).not.toBeInstanceOf(OcrStartError); }); + + it('gives tesseract.js a PPM file of the canvas pixels, not the canvas', async () => { + // tesseract.js makes a PNG file of a canvas with canvas.toBlob. The phone + // finding: that took about 13 s for each page in the Android WebView. + const worker = fakeWorker(); + worker.recognize.mockResolvedValue({ data: { blocks: [] } }); + createWorker.mockResolvedValue(worker); + const ocr = await createOcr('eng', () => undefined); + await expect(ocr.recognize(fakeCanvas(), 0)).resolves.toEqual([]); + const image = worker.recognize.mock.calls[0]?.[0]; + expect(image).toBeInstanceOf(Uint8Array); + // The header, then red and blue. The alpha is dropped. + const header = [...Buffer.from('P6\n2 1\n255\n', 'latin1')]; + expect(Array.from(image as Uint8Array)).toEqual([...header, 255, 0, 0, 0, 0, 255]); + }); }); diff --git a/test/ppm.test.ts b/test/ppm.test.ts new file mode 100644 index 0000000..68b1e9d --- /dev/null +++ b/test/ppm.test.ts @@ -0,0 +1,71 @@ +import fc from 'fast-check'; +import { describe, expect, it } from 'vitest'; +import { imageToPpm, type RgbaImage } from '../src/ppm'; + +/** Reads a binary PPM file: the header fields and the RGB bytes after the header. */ +function decodePpm(ppm: Uint8Array): { width: number; height: number; max: number; rgb: number[] } { + // The header is ASCII: "P6", then width, height and maximum value, each after white space, + // then one white space character. + const head = new TextDecoder('latin1').decode(ppm.subarray(0, 64)); + const match = /^P6\s+(\d+)\s+(\d+)\s+(\d+)\s/.exec(head); + if (!match) throw new Error(`No PPM header: ${JSON.stringify(head)}`); + return { + width: Number(match[1]), + height: Number(match[2]), + max: Number(match[3]), + rgb: Array.from(ppm.subarray(match[0].length)), + }; +} + +/** The RGB bytes of RGBA pixels. */ +function rgbOf(data: Uint8ClampedArray): number[] { + return Array.from(data).filter((_, i) => i % 4 !== 3); +} + +describe('imageToPpm', () => { + it('writes the header, then the RGB of each pixel row by row, without the alpha', () => { + // 3 x 2 pixels. Each byte holds its own index, so the order shows. + const data = new Uint8ClampedArray(Array.from({ length: 24 }, (_, i) => i)); + const ppm = imageToPpm({ width: 3, height: 2, data }); + const header = 'P6\n3 2\n255\n'; + expect(new TextDecoder('latin1').decode(ppm.subarray(0, header.length))).toBe(header); + expect(Array.from(ppm.subarray(header.length))).toEqual([ + 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, + ]); + }); + + it('has 3 bytes for each pixel of a page canvas of the app, after the header', () => { + const width = 1600; + const height = 2263; + const ppm = imageToPpm({ width, height, data: new Uint8ClampedArray(width * height * 4) }); + const header = 'P6\n1600 2263\n255\n'; + expect(new TextDecoder('latin1').decode(ppm.subarray(0, header.length))).toBe(header); + expect(ppm.length).toBe(header.length + width * height * 3); + }); + + it('refuses pixel data of the wrong length', () => { + const data = new Uint8ClampedArray(4 * 5); + expect(() => imageToPpm({ width: 2, height: 2, data })).toThrow(RangeError); + expect(() => imageToPpm({ width: 3, height: 2, data })).toThrow(RangeError); + }); + + it('gives back the size and the RGB of each pixel when the file is read', () => { + const image: fc.Arbitrary<RgbaImage> = fc + .record({ width: fc.integer({ min: 1, max: 12 }), height: fc.integer({ min: 1, max: 12 }) }) + .chain(({ width, height }) => + fc + .uint8Array({ minLength: width * height * 4, maxLength: width * height * 4 }) + .map((bytes) => ({ width, height, data: new Uint8ClampedArray(bytes) })), + ); + fc.assert( + fc.property(image, (input) => { + expect(decodePpm(imageToPpm(input))).toEqual({ + width: input.width, + height: input.height, + max: 255, + rgb: rgbOf(input.data), + }); + }), + ); + }); +});