commit 53030aa7458877f682aae813a74dd03f4891a33f truex <truex@equwal.com> 2026-09-26 13:29:41 -0700 Put no space around ー and the other shared kana marks The CJK test used the Script property. The Script of ー (U+30FC) is Common, so joinWords put a space around it when OCR gave it a word of its own. The lookup text and the book text for SubRead had "ニャ ー 泣いていた". The test now uses Script_Extensions, which also include ・, ゛, ゜, 々 and 〆. The CJK punctuation ranges stay. joinWords also read the last UTF-16 unit of the previous word. For a character outside the BMP, that unit is half of a surrogate pair and failed the CJK test. It now reads the last code point. book-text.ts exports the test as isCjk, for the lookup.
src/book-text.ts | 17 ++++++++++++++--- test/book-text.test.ts | 22 ++++++++++++++++++++++ test/hit-test.test.ts | 5 +++++ 3 files changed, 41 insertions(+), 3 deletions(-)
diff --git a/src/book-text.ts b/src/book-text.ts index 170b668..efc050a 100644 --- a/src/book-text.ts +++ b/src/book-text.ts @@ -5,12 +5,23 @@ import type { OcrToken } from './align'; -/** Han, kana and CJK punctuation: scripts that are written without spaces. */ -const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana} -〿-]/u; +/** + * Han, kana and CJK punctuation: scripts that are written without spaces. + * Script_Extensions (scx) also include the characters that these scripts + * share, for example ー, ・, ゛ and ゜. Their Script is Common. + */ +const CJK = /^[\p{scx=Han}\p{scx=Hiragana}\p{scx=Katakana} -〿-]/u; + +/** True when the first character of `text` is Han, kana or CJK punctuation. */ +export function isCjk(text: string): boolean { + return CJK.test(text); +} /** True when a space belongs between two neighboring words. Not between two CJK words. */ function needsSpace(prev: string, next: string): boolean { - return !(CJK.test(prev.slice(-1)) && CJK.test(next.slice(0, 1))); + // Split into code points: a character outside the BMP is two UTF-16 units. + const last = [...prev].at(-1) ?? ''; + return !(isCjk(last) && isCjk(next)); } /** One line from its words: a space between words, except between CJK words. */ diff --git a/test/book-text.test.ts b/test/book-text.test.ts index 9a5b632..3c8bec5 100644 --- a/test/book-text.test.ts +++ b/test/book-text.test.ts @@ -51,6 +51,28 @@ describe('bookText', () => { expect(joinWords(['名前', '。', 'まだ'])).toBe('名前。まだ'); }); + it('puts no space around the prolonged sound mark and the other kana marks', () => { + // OCR put ー in a word of its own. Its Script is Common, not Katakana. + const words = ['ニャ', 'ー', '泣いていた事だけは記憶している。']; + expect(joinWords(words)).toBe('ニャー泣いていた事だけは記憶している。'); + expect(bookText(pageTokens(0, [words]))).toBe('ニャー泣いていた事だけは記憶している。\n'); + for (const mark of ['ー', '・', '゛', '゜', '々', '〆']) { + expect(joinWords(['ア', mark, 'イ'])).toBe(`ア${mark}イ`); + } + }); + + it('puts no space between words of Han, kana, ー and ・, outside the BMP too', () => { + // Each list of words is one split of the string that the words make. + const word = fc.stringMatching( + /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}ー・]{1,6}$/u, + ); + fc.assert( + fc.property(fc.array(word, { minLength: 1, maxLength: 8 }), (words) => { + expect(joinWords(words)).toBe(words.join('')); + }), + ); + }); + it('returns an empty string for no tokens', () => { expect(bookText([])).toBe(''); }); diff --git a/test/hit-test.test.ts b/test/hit-test.test.ts index 222af98..41155ab 100644 --- a/test/hit-test.test.ts +++ b/test/hit-test.test.ts @@ -99,6 +99,11 @@ describe('lookupText', () => { expect(lookupText(tokens, 3)).toBe('猫である名前はまだ無い'); }); + it('puts no space around ー when OCR gives it a word of its own', () => { + const tokens = wordTokens(['ニャ', 'ー', '泣いていた事だけは記憶している。']); + expect(lookupText(tokens, 0)).toBe('ニャー泣いていた事だけは記憶している。'); + }); + it('joins a short line and the next line with a space', () => { const tokens = [...wordTokens(['the', 'end'], 0, 0), ...wordTokens(['of', 'it'], 0, 1)]; expect(lookupText(tokens, 3)).toBe('end of it');