commit e281873fac8cea7c64862bda2a3638eb5237bb61 truex <truex@equwal.com> 2026-09-26 14:19:56 -0700 Keep the last book, its page and its subtitles in IndexedDB A reload or an app restart forgot the book, the page and the subtitles. On Android the system stops the reader while the user is in the dictionary or in the player app, so the user came back to an empty reader. Version 3 of the database adds two stores: - "books" keeps a copy of the last book under the key "last". putLastBook replaces it and gives false when IndexedDB fails, for example when the quota is full. getLastBook gives the file. - "meta" keeps, for each book key, the page, the subtitles and the "Force OCR" setting. putMeta merges a patch in one transaction, and a field that is undefined in the patch is removed. The upgrade moves each text of the "srt" store to meta.subtitles, with the source "subread" and the name of the book with ".srt", and deletes "srt". The upgrade still deletes the stores of the old dictionary. clearPages(key) removes only the pages of that book. getSrt and putSrt stay as thin wrappers over the meta store, because main.ts still uses them. The next change of main.ts removes them.
src/token-cache.ts | 176 ++++++++++++++++++++++++++++++++++++++++------ test/token-cache.test.ts | 177 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 318 insertions(+), 35 deletions(-)
diff --git a/src/token-cache.ts b/src/token-cache.ts index 5a92872..73c528d 100644 --- a/src/token-cache.ts +++ b/src/token-cache.ts @@ -1,14 +1,26 @@ -// The tokens of each page, and the subtitles of each book, in IndexedDB. -// Thin browser layer. +// What the reader keeps in IndexedDB. Thin browser layer. // -// OCR is slow, so a page is read once. The key names the file, the page, -// the OCR language and the source (text layer or OCR). The subtitles that -// SubRead made are kept under the key of the PDF. +// - "pages": the tokens of each page. OCR is slow, so a page is read once. +// The key names the file, the page, the OCR language and the source (text +// layer or OCR). +// - "books": a copy of the last book, under the key "last". Android can stop +// the reader while the user is in the dictionary or in the player, and a +// reload forgets the file that the user opened. +// - "meta": for each book, the page that was open, the subtitles and the +// "Force OCR" setting. // -// The reader does not need the cache. When IndexedDB fails, a read finds -// nothing, a write does nothing, and one warning goes to the console. - -import { openDB, unwrap, type DBSchema, type IDBPDatabase } from 'idb'; +// The reader does not need IndexedDB. When IndexedDB fails, a read finds +// nothing, a write does nothing, and one warning goes to the console. No +// function throws. + +import { + openDB, + unwrap, + type DBSchema, + type IDBPDatabase, + type IDBPTransaction, + type StoreNames, +} from 'idb'; import type { OcrToken } from './align'; export type TextSource = 'text' | 'ocr'; @@ -21,28 +33,65 @@ export interface PageEntry { source: TextSource; } +/** The subtitles of a book. */ +export interface BookSubtitles { + /** The file name, for example "book.srt". */ + name: string; + /** The text of the subtitle file. */ + text: string; + /** A file that the user loaded, or the result of SubRead. */ + source: 'file' | 'subread'; +} + +/** What the reader keeps for a book, other than its pages. */ +export interface BookMeta { + /** The page that was open. 0 is the first page. */ + page?: number; + subtitles?: BookSubtitles; + /** The "Force OCR" setting for the book. */ + forceOcr?: boolean; +} + +/** The record of the last book. */ +interface LastBook { + /** See bookKey. */ + key: string; + name: string; + /** The MIME type of the file. */ + type: string; + blob: Blob; +} + interface Schema extends DBSchema { pages: { key: string; value: PageEntry }; + books: { key: string; value: LastBook }; + meta: { key: string; value: BookMeta }; + /** Only in version 2. The upgrade to version 3 moves the subtitles to "meta". */ srt: { key: string; value: string }; } const DB_NAME = 'scan-subread'; /** - * Earlier builds of the app made version 1 of this database with the - * stores of their dictionary. Version 2 makes sure that the stores of - * this schema are there, and deletes the old stores. + * Version 1: earlier builds kept the stores of their dictionary. + * Version 2: "pages" and "srt". The upgrade deletes the stores of the + * dictionary. + * Version 3: "pages", "books" and "meta". The upgrade moves each text of + * "srt" to "meta", and deletes "srt". */ -const DB_VERSION = 2; -const STORES = ['pages', 'srt'] as const; +const DB_VERSION = 3; +const STORES = ['pages', 'books', 'meta'] as const; const OLD_STORES = ['dictionaries', 'terms']; +/** The key of the one record in "books". */ +const LAST = 'last'; + let dbPromise: Promise<IDBPDatabase<Schema>> | undefined; let warned = false; function db(): Promise<IDBPDatabase<Schema>> { dbPromise ??= openDB<Schema>(DB_NAME, DB_VERSION, { - upgrade(database) { + async upgrade(database, _oldVersion, _newVersion, transaction) { // The typed database knows only the stores of the schema. const raw = unwrap(database); for (const name of OLD_STORES) { @@ -51,11 +100,34 @@ function db(): Promise<IDBPDatabase<Schema>> { for (const name of STORES) { if (!raw.objectStoreNames.contains(name)) raw.createObjectStore(name); } + if (raw.objectStoreNames.contains('srt')) { + await moveSrtToMeta(transaction); + raw.deleteObjectStore('srt'); + } }, }); return dbPromise; } +/** Copies each text of "srt" to "meta", as subtitles from SubRead. */ +async function moveSrtToMeta( + transaction: IDBPTransaction<Schema, StoreNames<Schema>[], 'versionchange'>, +): Promise<void> { + const meta = transaction.objectStore('meta'); + let cursor = await transaction.objectStore('srt').openCursor(); + while (cursor) { + await meta.put({ subtitles: subreadSubtitles(cursor.key, cursor.value) }, cursor.key); + cursor = await cursor.continue(); + } +} + +/** Subtitles from SubRead for the book `key`. "book.pdf|123" gives the name "book.srt". */ +function subreadSubtitles(key: string, text: string): BookSubtitles { + const bar = key.lastIndexOf('|'); + const file = bar < 0 ? key : key.slice(0, bar); + return { name: `${file.replace(/\.pdf$/i, '')}.srt`, text, source: 'subread' }; +} + /** Runs `op` on the database. Gives undefined when IndexedDB fails, and warns once. */ async function tryDb<T>(op: (d: IDBPDatabase<Schema>) => Promise<T>): Promise<T | undefined> { try { @@ -63,7 +135,7 @@ async function tryDb<T>(op: (d: IDBPDatabase<Schema>) => Promise<T>): Promise<T } catch (err) { if (!warned) { warned = true; - console.warn('The page cache is off: IndexedDB failed.', err); + console.warn('IndexedDB failed. The reader continues without it.', err); } return undefined; } @@ -88,14 +160,76 @@ export async function putPage(key: string, entry: PageEntry): Promise<void> { await tryDb((d) => d.put('pages', entry, key)); } -export async function clearPages(): Promise<void> { - await tryDb((d) => d.clear('pages')); +/** Removes the pages of the book `key` (see bookKey). Without a key, removes all pages. */ +export async function clearPages(key?: string): Promise<void> { + await tryDb((d) => + key === undefined + ? d.clear('pages') + : // "}" follows "|" in the order of the code units. So this range holds + // each key that starts with `${key}|`, and no other key. + d.delete('pages', IDBKeyRange.bound(`${key}|`, `${key}}`, false, true)), + ); +} + +/** + * Keeps a copy of `file` as the last book, in place of the book before it. + * Gives false when IndexedDB fails, for example when the file is larger than + * the free quota. + */ +export async function putLastBook(file: File): Promise<boolean> { + const record: LastBook = { key: bookKey(file), name: file.name, type: file.type, blob: file }; + const done = await tryDb(async (d) => { + await d.put('books', record, LAST); + return true; + }); + return done === true; +} + +/** The last book, or undefined when there is no last book or IndexedDB fails. */ +export function getLastBook(): Promise<File | undefined> { + return tryDb(async (d) => { + const book = await d.get('books', LAST); + return book && new File([book.blob], book.name, { type: book.type }); + }); } -export function getSrt(key: string): Promise<string | undefined> { - return tryDb((d) => d.get('srt', key)); +/** What the reader keeps for the book `key`. An empty object when there is nothing, or IndexedDB fails. */ +export async function getMeta(key: string): Promise<BookMeta> { + return (await tryDb((d) => d.get('meta', key))) ?? {}; } +/** + * Merges `patch` into what the reader keeps for the book `key`. A field that + * is undefined in `patch` is removed. + */ +export async function putMeta(key: string, patch: Partial<BookMeta>): Promise<void> { + await tryDb((d) => { + // One transaction reads and writes, so two merges at the same time keep + // the fields of both. + const tx = d.transaction('meta', 'readwrite'); + const merge = async (): Promise<void> => { + const meta: BookMeta = { ...(await tx.store.get(key)), ...patch }; + for (const field of Object.keys(patch) as (keyof BookMeta)[]) { + if (patch[field] === undefined) delete meta[field]; + } + await tx.store.put(meta, key); + }; + return Promise.all([merge(), tx.done]); + }); +} + +/** + * The text of the subtitles of the book `key`. + * @deprecated Use getMeta. The next change of main.ts removes this function. + */ +export async function getSrt(key: string): Promise<string | undefined> { + return (await getMeta(key)).subtitles?.text; +} + +/** + * Keeps the subtitles that SubRead made for the book `key`. + * @deprecated Use putMeta. The next change of main.ts removes this function. + */ export async function putSrt(key: string, srt: string): Promise<void> { - await tryDb((d) => d.put('srt', srt, key)); + await putMeta(key, { subtitles: subreadSubtitles(key, srt) }); } diff --git a/test/token-cache.test.ts b/test/token-cache.test.ts index 1bb572b..98f900d 100644 --- a/test/token-cache.test.ts +++ b/test/token-cache.test.ts @@ -1,8 +1,9 @@ import 'fake-indexeddb/auto'; import { IDBFactory } from 'fake-indexeddb'; +import fc from 'fast-check'; import { openDB } from 'idb'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { PageEntry } from '../src/token-cache'; +import type { BookSubtitles, PageEntry } from '../src/token-cache'; const entry: PageEntry = { tokens: [{ text: '猫', page: 0, line: 0, word: 0, bbox: { x0: 1, y0: 2, x1: 3, y1: 4 } }], @@ -19,6 +20,14 @@ async function freshCache() { return import('../src/token-cache'); } +/** The version and the store names of the database. Open the cache first. */ +async function schema(): Promise<{ version: number; stores: string[] }> { + const d = await openDB('scan-subread'); + const result = { version: d.version, stores: [...d.objectStoreNames].sort() }; + d.close(); + return result; +} + describe('token cache', () => { beforeEach(() => { // Each test gets an empty IndexedDB. @@ -26,7 +35,7 @@ describe('token cache', () => { }); afterEach(() => vi.restoreAllMocks()); - it('upgrades the database of an earlier build that has other stores', async () => { + it('upgrades version 1, which has the stores of the dictionary', async () => { // Earlier builds made version 1 with the stores of the dictionary. const old = await openDB('scan-subread', 1, { upgrade(d) { @@ -40,27 +49,148 @@ describe('token cache', () => { const cache = await freshCache(); await cache.putPage('book.pdf|10|0|jpn|ocr', entry); expect(await cache.getPage('book.pdf|10|0|jpn|ocr')).toEqual(entry); - await cache.putSrt('book.pdf|10', srt); - expect(await cache.getSrt('book.pdf|10')).toBe(srt); + expect(await schema()).toEqual({ version: 3, stores: ['books', 'meta', 'pages'] }); + }); + + it('upgrades version 2: the pages stay, and the subtitles move to the meta data', async () => { + const old = await openDB('scan-subread', 2, { + upgrade(d) { + d.createObjectStore('pages'); + d.createObjectStore('srt'); + }, + }); + await old.put('pages', entry, 'book.pdf|10|0|jpn|ocr'); + await old.put('srt', srt, 'book.pdf|10'); + await old.put('srt', 'x', 'Neko.PDF|20'); + await old.put('srt', 'y', 'a|b|30'); + old.close(); - const after = await openDB('scan-subread'); - expect(after.version).toBe(2); - expect([...after.objectStoreNames].sort()).toEqual(['pages', 'srt']); - after.close(); + const cache = await freshCache(); + expect(await cache.getPage('book.pdf|10|0|jpn|ocr')).toEqual(entry); + expect(await cache.getMeta('book.pdf|10')).toEqual({ + subtitles: { name: 'book.srt', text: srt, source: 'subread' }, + }); + expect(await cache.getMeta('Neko.PDF|20')).toEqual({ + subtitles: { name: 'Neko.srt', text: 'x', source: 'subread' }, + }); + expect(await cache.getMeta('a|b|30')).toEqual({ + subtitles: { name: 'a|b.srt', text: 'y', source: 'subread' }, + }); + expect(await schema()).toEqual({ version: 3, stores: ['books', 'meta', 'pages'] }); }); it('makes the stores in a new database', async () => { const cache = await freshCache(); expect(await cache.getPage('missing')).toBeUndefined(); - await cache.putPage('p', entry); - expect(await cache.getPage('p')).toEqual(entry); + expect(await cache.getMeta('missing')).toEqual({}); + expect(await cache.getLastBook()).toBeUndefined(); + expect(await schema()).toEqual({ version: 3, stores: ['books', 'meta', 'pages'] }); + }); + + it('keeps the last book, and a new book replaces it', async () => { + const cache = await freshCache(); + const neko = new File(['%PDF-1.7 猫'], 'neko.pdf', { type: 'application/pdf' }); + expect(await cache.putLastBook(neko)).toBe(true); + const back = await cache.getLastBook(); + expect(back).toBeInstanceOf(File); + expect(back?.name).toBe('neko.pdf'); + expect(back?.type).toBe('application/pdf'); + expect(await back?.text()).toBe('%PDF-1.7 猫'); + expect(back && cache.bookKey(back)).toBe(cache.bookKey(neko)); + + const inu = new File(['%PDF-1.7 犬と猫'], 'inu.pdf', { type: 'application/pdf' }); + expect(await cache.putLastBook(inu)).toBe(true); + expect((await cache.getLastBook())?.name).toBe('inu.pdf'); + + const d = await openDB('scan-subread'); + expect(await d.getAllKeys('books')).toEqual(['last']); + const record = await d.get('books', 'last'); + d.close(); + expect(record).toMatchObject({ + key: cache.bookKey(inu), + name: 'inu.pdf', + type: 'application/pdf', + }); + expect(record.blob).toBeInstanceOf(Blob); + }); + + it('merges the meta data, and removes a field that is undefined in the patch', async () => { + const cache = await freshCache(); + const subtitles: BookSubtitles = { name: 'neko.srt', text: srt, source: 'file' }; + await cache.putMeta('neko.pdf|10', { page: 3 }); + await cache.putMeta('neko.pdf|10', { subtitles, forceOcr: true }); + expect(await cache.getMeta('neko.pdf|10')).toEqual({ page: 3, subtitles, forceOcr: true }); + + await cache.putMeta('neko.pdf|10', { page: undefined, forceOcr: false }); + const meta = await cache.getMeta('neko.pdf|10'); + expect(meta).toEqual({ subtitles, forceOcr: false }); + expect('page' in meta).toBe(false); + expect(await cache.getMeta('inu.pdf|20')).toEqual({}); + }); + + it('keeps the fields of two merges at the same time', async () => { + const cache = await freshCache(); + await Promise.all([ + cache.putMeta('neko.pdf|10', { page: 1 }), + cache.putMeta('neko.pdf|10', { forceOcr: true }), + ]); + expect(await cache.getMeta('neko.pdf|10')).toEqual({ page: 1, forceOcr: true }); + }); + + it('keeps the subtitles of putSrt in the meta data', async () => { + const cache = await freshCache(); + await cache.putMeta('neko.pdf|10', { page: 2 }); + await cache.putSrt('neko.pdf|10', srt); + expect(await cache.getSrt('neko.pdf|10')).toBe(srt); + expect(await cache.getMeta('neko.pdf|10')).toEqual({ + page: 2, + subtitles: { name: 'neko.srt', text: srt, source: 'subread' }, + }); + expect(await cache.getSrt('inu.pdf|20')).toBeUndefined(); + }); + + it('clears the pages of one book, or the pages of all books', async () => { + const cache = await freshCache(); + const a = { name: 'a.pdf', size: 1 }; + // The key of this book, "a.pdf|10", starts with the key of `a`, "a.pdf|1". + const a10 = { name: 'a.pdf', size: 10 }; + const keys = [ + cache.pageKey(a, 0, 'jpn', 'ocr'), + cache.pageKey(a, 1, 'jpn', 'text'), + cache.pageKey(a10, 0, 'jpn', 'ocr'), + ] as const; + for (const key of keys) await cache.putPage(key, entry); + await cache.putMeta(cache.bookKey(a), { page: 1 }); + + await cache.clearPages(cache.bookKey(a)); + expect(await cache.getPage(keys[0])).toBeUndefined(); + expect(await cache.getPage(keys[1])).toBeUndefined(); + expect(await cache.getPage(keys[2])).toEqual(entry); + expect(await cache.getMeta(cache.bookKey(a))).toEqual({ page: 1 }); + await cache.clearPages(); - expect(await cache.getPage('p')).toBeUndefined(); - await cache.putSrt('b', srt); - expect(await cache.getSrt('b')).toBe(srt); + expect(await cache.getPage(keys[2])).toBeUndefined(); + }); + + it('clears each page key that starts with the book key and "|", and no other key', async () => { + const cache = await freshCache(); + const separator = fc.constantFrom('|', '||', '}', '{', '', ''); + const tails = fc.array(fc.tuple(separator, fc.string({ unit: 'binary' })), { maxLength: 8 }); + await fc.assert( + fc.asyncProperty(fc.string({ unit: 'binary' }), tails, async (book, parts) => { + await cache.clearPages(); + const keys = parts.map(([sep, tail]) => `${book}${sep}${tail}`); + for (const key of keys) await cache.putPage(key, entry); + await cache.clearPages(book); + for (const key of keys) { + const cleared = key.startsWith(`${book}|`); + expect(await cache.getPage(key), key).toEqual(cleared ? undefined : entry); + } + }), + ); }); - it('works without the cache when IndexedDB fails, and warns once', async () => { + it('works without IndexedDB, and warns once', async () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); vi.spyOn(indexedDB, 'open').mockImplementation(() => { throw new DOMException('IndexedDB is off.', 'SecurityError'); @@ -69,8 +199,27 @@ describe('token cache', () => { await expect(cache.getPage('p')).resolves.toBeUndefined(); await expect(cache.putPage('p', entry)).resolves.toBeUndefined(); await expect(cache.clearPages()).resolves.toBeUndefined(); + await expect(cache.clearPages('b')).resolves.toBeUndefined(); + await expect(cache.getMeta('b')).resolves.toEqual({}); + await expect(cache.putMeta('b', { page: 1 })).resolves.toBeUndefined(); + await expect(cache.putLastBook(new File(['x'], 'b.pdf'))).resolves.toBe(false); + await expect(cache.getLastBook()).resolves.toBeUndefined(); await expect(cache.getSrt('b')).resolves.toBeUndefined(); await expect(cache.putSrt('b', srt)).resolves.toBeUndefined(); expect(warn).toHaveBeenCalledTimes(1); }); + + it('gives false when the book does not fit, and keeps the other data', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined); + const cache = await freshCache(); + await cache.putPage('p', entry); + const put = vi.spyOn(IDBObjectStore.prototype, 'put').mockImplementation(() => { + throw new DOMException('The quota is full.', 'QuotaExceededError'); + }); + await expect(cache.putLastBook(new File(['x'], 'b.pdf'))).resolves.toBe(false); + put.mockRestore(); + expect(await cache.getPage('p')).toEqual(entry); + expect(await cache.getLastBook()).toBeUndefined(); + expect(warn).toHaveBeenCalledTimes(1); + }); });