commit 54fd74a089fc7706fffc4c9471546f3ccc47b02d truex <truex@equwal.com> 2026-09-26 15:02:27 -0700 Grow the box over CJK punctuation in its own OCR word On the sample-jpn scan the box of a line stopped before the 。 at the end of the cue. Tesseract gave that 。 a word of its own, and the box grew only over tokens with the same word id. CJK has no spaces, so its OCR words do not show where a word ends. After a CJK character the box now grows over the punctuation next to the span on the same line, whatever its word id: an opening mark before the span, another mark after it. After a letter that is not CJK, the rule of the same word stays.
src/line-boxes.ts | 29 ++++++++++++++--------------- test/line-boxes.test.ts | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 15 deletions(-)
diff --git a/src/line-boxes.ts b/src/line-boxes.ts index 34640fa..9849e8c 100644 --- a/src/line-boxes.ts +++ b/src/line-boxes.ts @@ -13,25 +13,24 @@ const OPENING = /^[\p{Ps}\p{Pi}]+$/u; /** * True when the box of a line can grow from the edge token of the span - * over `next`, the token before it (`back`) or after it. Both must have the - * same page, line and word id. + * over `next`, the token before it (`back`) or after it, on the same line. * - * In a word that is not CJK, all its tokens join: the spaces show where - * the word ends. Japanese OCR can give one word for a whole line, so in a - * CJK word only punctuation joins: an opening mark before the span, and - * another mark after it. A 。 before the span ends the cue before. + * After a letter that is not CJK, the tokens of the same word join: a + * space, which is no token, gives the next word another word id. CJK has + * no spaces, and its OCR words do not show where a word ends: Japanese OCR + * can give one word for a whole line, or one for a 。 alone. So after a + * CJK character only punctuation joins: an opening mark before the span, + * and another mark after it. A 。 before the span ends the cue before. */ function joins(edge: OcrToken, next: OcrToken | undefined, back: boolean): next is OcrToken { - if ( - next === undefined || - edge.word === undefined || - next.page !== edge.page || - next.line !== edge.line || - next.word !== edge.word - ) { - return false; + if (next === undefined || next.page !== edge.page || next.line !== edge.line) return false; + if (!isCjk(edge.text)) { + return ( + edge.word !== undefined && + next.word === edge.word && + (!isCjk(next.text) || PUNCTUATION.test(next.text)) + ); } - if (!isCjk(edge.text)) return !isCjk(next.text) || PUNCTUATION.test(next.text); if (back) return OPENING.test(next.text); return PUNCTUATION.test(next.text) && !OPENING.test(next.text); } diff --git a/test/line-boxes.test.ts b/test/line-boxes.test.ts index 03ef14e..b88b76d 100644 --- a/test/line-boxes.test.ts +++ b/test/line-boxes.test.ts @@ -96,6 +96,22 @@ describe('lineBoxes', () => { expect(lineBoxes(quote, spanOf(3, 5), 0)).toEqual([{ x0: 20, y0: 0, x1: 60, y1: 20 }]); }); + it('grows over CJK punctuation that OCR gave a word of its own', () => { + // Tesseract jpn on sample-jpn: the 。 at the end of the line is a word. + const words = ['吾輩は', '猫である', '。', '名前はまだ無い', '。']; + const line: OcrToken[] = []; + words.forEach((w, word) => { + for (const ch of w) { + const x0 = line.length * 10; + line.push({ text: ch, page: 0, line: 0, word, bbox: { x0, y0: 0, x1: x0 + 10, y1: 20 } }); + } + }); + // The span is 名前はまだ無い: the last 。 joins. + expect(lineBoxes(line, spanOf(8, 15), 0)).toEqual([{ x0: 80, y0: 0, x1: 160, y1: 20 }]); + // The span is 猫である: the 。 after it joins, は before it does not. + expect(lineBoxes(line, spanOf(3, 7), 0)).toEqual([{ x0: 30, y0: 0, x1: 80, y1: 20 }]); + }); + it('pads each box on each side', () => { const line = textTokens(['quiet hills,']); expect(lineBoxes(line, spanOf(0, 9), 0, 3)).toEqual([{ x0: -3, y0: -3, x1: 123, y1: 23 }]);