commit e89967b2337be28e40e08f9466b251d3940663e9 truex <truex@equwal.com> 2026-09-26 13:41:31 -0700 Find the sentence around a tapped token sentenceAround gives the sentence that holds a token, for a card in SubRead Anki. The sentence starts after the end of the sentence before it and stops at the next sentence end, across the lines of the page. The words are joined with joinWords, the same as in the book text. A sentence that is too long is cut to the characters around the tapped token. The tests have Japanese and English examples and a property test: the result holds the tapped token and has at most max characters.
src/sentence.ts | 106 +++++++++++++++++++++++++++++++++++++ test/sentence.test.ts | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 247 insertions(+)
diff --git a/src/sentence.ts b/src/sentence.ts new file mode 100644 index 0000000..1655f40 --- /dev/null +++ b/src/sentence.ts @@ -0,0 +1,106 @@ +// The sentence around a tapped token, for a SubRead Anki card. Pure: no DOM. + +import type { OcrToken } from './align'; +import { joinWords } from './book-text'; + +/** Longest sentence, in characters. */ +export const SENTENCE_LENGTH = 200; + +/** + * A sentence end: one or more of 。．！？.!? and the closing marks after + * them, as in 。」 or ." so that the closing mark stays with its sentence. + */ +const END = /[。．！？.!?]+[」』）)"'”’]*/gu; + +/** An end of only ".", "!" and "?", with no closing mark. */ +const LATIN_END = /^[.!?]+$/; + +/** After such an end, a Latin letter or a digit means no sentence end: 3.14, e.g. */ +const LATIN_NEXT = /[A-Za-z0-9]/; + +/** True when two tokens are in one word: the same page, line id and word id. */ +function sameWord(a: OcrToken, b: OcrToken): boolean { + return a.page === b.page && a.line === b.line && a.word === b.word; +} + +/** + * The text of the page that holds token `index`, and the offset of that + * token in it. The words of a line and the lines of the page are joined + * with `joinWords`, the same as in the book text. The tokens of one page + * are next to each other in `tokens`. + */ +function pageText(tokens: readonly OcrToken[], index: number): { text: string; at: number } { + const page = tokens[index]!.page; + let i = index; + while (i > 0 && tokens[i - 1]!.page === page) i--; + let text = ''; + let at = 0; + while (i < tokens.length && tokens[i]!.page === page) { + const first = tokens[i]!; + let word = ''; + let tapped = -1; + do { + if (i === index) tapped = word.length; + word += tokens[i]!.text; + i++; + } while (i < tokens.length && sameWord(first, tokens[i]!)); + const trimmed = word.trim(); + const joined = joinWords([text, trimmed]); + if (tapped >= 0) { + const lead = word.length - word.trimStart().length; + at = joined.length - trimmed.length + Math.max(0, tapped - lead); + } + text = joined; + } + return { text, at }; +} + +/** The start and the end of the sentence that holds offset `at` of `text`. */ +function sentenceSpan(text: string, at: number): [number, number] { + let start = 0; + for (const m of text.matchAll(END)) { + const end = m.index + m[0].length; + if (LATIN_END.test(m[0]) && LATIN_NEXT.test(text.charAt(end))) continue; + if (end > at) return [start, end]; + start = end; + } + return [start, text.length]; +} + +/** + * The sentence that holds token `index`: from the end of the sentence + * before it to the end of its own sentence, across the lines of its page. + * + * A sentence ends at 。．！？ and at .!?, with the closing marks that follow + * them (」』） and closing quotes). A "." before a Latin letter or a digit is + * not an end (3.14, e.g.). A 」 with no end mark before it is not an end, so + * 「はい」と言った。 is one sentence. On a page with no sentence end, the + * sentence runs to the edge of the page. + * + * The words are joined with `joinWords`, and the result is trimmed. When the + * sentence is longer than `max` characters, the result is the `max` + * characters around the tapped token. + */ +export function sentenceAround( + tokens: readonly OcrToken[], + index: number, + max = SENTENCE_LENGTH, +): string { + const tapped = tokens[index]; + if (!tapped) return ''; + const { text, at } = pageText(tokens, index); + const [start, end] = sentenceSpan(text, at); + const sentence = text.slice(start, end); + const trimmed = sentence.trim(); + const chars = [...trimmed]; + if (chars.length <= max) return trimmed; + // A window of `max` characters, with the tapped token in the middle. + const lead = sentence.length - sentence.trimStart().length; + const tap = [...trimmed.slice(0, Math.max(0, at - start - lead))].length; + const size = [...tapped.text].length; + const from = Math.max(0, Math.min(tap - Math.floor((max - size) / 2), chars.length - max)); + return chars + .slice(from, from + max) + .join('') + .trim(); +} diff --git a/test/sentence.test.ts b/test/sentence.test.ts new file mode 100644 index 0000000..b654e6d --- /dev/null +++ b/test/sentence.test.ts @@ -0,0 +1,141 @@ +import fc from 'fast-check'; +import { describe, expect, it } from 'vitest'; +import type { OcrToken } from '../src/align'; +import { SENTENCE_LENGTH, sentenceAround } from '../src/sentence'; + +const box = { x0: 0, y0: 0, x1: 1, y1: 1 }; + +/** Tokens for one page, one per character: `lines` is a list of lines, each a list of words. */ +function pageTokens(page: number, lines: readonly (readonly string[])[]): OcrToken[] { + const out: OcrToken[] = []; + lines.forEach((words, li) => { + words.forEach((word, wi) => { + for (const ch of word) { + out.push({ + text: ch, + page, + line: page * 1000 + li, + word: page * 1000 + li * 100 + wi, + bbox: box, + }); + } + }); + }); + return out; +} + +/** The index of token number `n` (from 0) with the text `ch`. */ +function find(tokens: readonly OcrToken[], ch: string, n = 0): number { + let seen = 0; + for (let i = 0; i < tokens.length; i++) { + if (tokens[i]!.text !== ch) continue; + if (seen === n) return i; + seen++; + } + throw new Error(`No token ${ch} number ${n}.`); +} + +describe('sentenceAround', () => { + // 吾輩は猫である。名前は / まだ無い。どこで生れたか + const jpn = pageTokens(0, [['吾輩は猫である。名前は'], ['まだ無い。どこで生れたか']]); + + // It was a dark / night. The rain fell. + const eng = pageTokens(0, [ + ['It', 'was', 'a', 'dark'], + ['night.', 'The', 'rain', 'fell.'], + ]); + + it('reads a Japanese sentence over two lines, with no space at the line break', () => { + expect(sentenceAround(jpn, find(jpn, '前'))).toBe('名前はまだ無い。'); + }); + + it('reads an English sentence over two lines, with a space at the line break', () => { + expect(sentenceAround(eng, find(eng, 'k'))).toBe('It was a dark night.'); + expect(sentenceAround(eng, find(eng, 'f'))).toBe('The rain fell.'); + }); + + it('gives the same sentence for a tap on its first and on its last character', () => { + expect(sentenceAround(jpn, find(jpn, '名'))).toBe('名前はまだ無い。'); + expect(sentenceAround(jpn, find(jpn, '。', 1))).toBe('名前はまだ無い。'); + expect(sentenceAround(jpn, find(jpn, '吾'))).toBe('吾輩は猫である。'); + expect(sentenceAround(jpn, find(jpn, '。', 0))).toBe('吾輩は猫である。'); + expect(sentenceAround(eng, find(eng, 'I'))).toBe('It was a dark night.'); + expect(sentenceAround(eng, find(eng, '.', 0))).toBe('It was a dark night.'); + expect(sentenceAround(eng, find(eng, 'T'))).toBe('The rain fell.'); + expect(sentenceAround(eng, find(eng, '.', 1))).toBe('The rain fell.'); + }); + + it('runs to the edge of the page when no sentence end follows', () => { + expect(sentenceAround(jpn, find(jpn, 'ど'))).toBe('どこで生れたか'); + const open = pageTokens(0, [['吾輩は猫で'], ['ある']]); + expect(sentenceAround(open, find(open, '猫'))).toBe('吾輩は猫である'); + const words = pageTokens(0, [ + ['no', 'end', 'on'], + ['this', 'page'], + ]); + expect(sentenceAround(words, find(words, 'h'))).toBe('no end on this page'); + }); + + it('stays on the page of the tapped token', () => { + const tokens = [...pageTokens(0, [['前の頁の文']]), ...pageTokens(1, [['次の頁の文。']])]; + expect(sentenceAround(tokens, find(tokens, '前'))).toBe('前の頁の文'); + expect(sentenceAround(tokens, find(tokens, '次'))).toBe('次の頁の文。'); + }); + + it('keeps a closing bracket with the sentence that it ends', () => { + const quotes = pageTokens(0, [['「おはよう。」「はい。」']]); + expect(sentenceAround(quotes, find(quotes, 'よ'))).toBe('「おはよう。」'); + expect(sentenceAround(quotes, find(quotes, 'い'))).toBe('「はい。」'); + const english = pageTokens(0, [['"Go', 'away."', 'He', 'left.']]); + expect(sentenceAround(english, find(english, 'G'))).toBe('"Go away."'); + expect(sentenceAround(english, find(english, 'H'))).toBe('He left.'); + }); + + it('does not end a sentence at a closing bracket with no end mark before it', () => { + const said = pageTokens(0, [['「はい」と言った。次']]); + expect(sentenceAround(said, find(said, 'と'))).toBe('「はい」と言った。'); + }); + + it('does not end a sentence at a decimal point', () => { + const price = pageTokens(0, [['It', 'costs', '3.14', 'now.', 'Then', 'more.']]); + expect(sentenceAround(price, find(price, 'c'))).toBe('It costs 3.14 now.'); + }); + + it('cuts a long sentence around the tapped token', () => { + const long = pageTokens(0, [['あ'.repeat(150) + '猫' + 'い'.repeat(150) + '。']]); + expect(sentenceAround(long, find(long, '猫'), 21)).toBe( + 'あ'.repeat(10) + '猫' + 'い'.repeat(10), + ); + expect(sentenceAround(long, 0)).toBe('あ'.repeat(150) + '猫' + 'い'.repeat(49)); + expect([...sentenceAround(long, 301)].length).toBe(SENTENCE_LENGTH); + }); + + it('returns an empty string for a token that does not exist', () => { + expect(sentenceAround(jpn, jpn.length)).toBe(''); + expect(sentenceAround([], 0)).toBe(''); + }); + + it('always holds the tapped token and is at most max characters', () => { + const ch = fc.constantFrom('吾', 'は', 'a', 'Z', '7', '。', '！', '.', '?', '」', '"'); + const word = fc.array(ch, { minLength: 1, maxLength: 5 }).map((cs) => cs.join('')); + const line = fc.array(word, { minLength: 1, maxLength: 4 }); + const page = fc.array(line, { minLength: 1, maxLength: 4 }); + const book = fc.array(page, { minLength: 1, maxLength: 3 }); + fc.assert( + fc.property(book, fc.nat(), fc.integer({ min: 1, max: 40 }), (pages, pick, max) => { + const tokens = pages.flatMap((lines, p) => pageTokens(p, lines)); + const index = pick % tokens.length; + const tapped = tokens[index]!; + const sentence = sentenceAround(tokens, index, max); + expect(sentence).toContain(tapped.text); + expect([...sentence].length).toBeLessThanOrEqual(max); + // Without the spaces between words, the sentence is a piece of its page. + const pageChars = tokens + .filter((t) => t.page === tapped.page) + .map((t) => t.text) + .join(''); + expect(pageChars).toContain(sentence.replace(/ /g, '')); + }), + ); + }); +});