scan-subread/src/book-text.ts

src/book-text.ts (2583 bytes)
1 // The page text as a plain-text book. Pure: no DOM.
2 //
3 // SubRead reads the book as text lines when it makes the subtitles. This
4 // module turns the page tokens (OCR or text layer) into that text.
5 
6 import type { OcrToken } from './align';
7 
8 /**
9  * Han, kana and CJK punctuation: scripts that are written without spaces.
10  * Script_Extensions (scx) also include the characters that these scripts
11  * share, for example ー, ・, ゛ and ゜. Their Script is Common.
12  */
13 const CJK = /^[\p{scx=Han}\p{scx=Hiragana}\p{scx=Katakana}　-〿＀-￯]/u;
14 
15 /** True when the first character of `text` is Han, kana or CJK punctuation. */
16 export function isCjk(text: string): boolean {
17   return CJK.test(text);
18 }
19 
20 /** True when a space belongs between two neighboring words. Not between two CJK words. */
21 function needsSpace(prev: string, next: string): boolean {
22   // Split into code points: a character outside the BMP is two UTF-16 units.
23   const last = [...prev].at(-1) ?? '';
24   return !(isCjk(last) && isCjk(next));
25 }
26 
27 /** One line from its words: a space between words, except between CJK words. */
28 export function joinWords(words: readonly string[]): string {
29   let out = '';
30   for (const w of words) {
31     if (out.length > 0 && w.length > 0 && needsSpace(out, w)) out += ' ';
32     out += w;
33   }
34   return out;
35 }
36 
37 /**
38  * One text line per OCR line, pages in order, one blank line between
39  * pages. Words on a line are joined with `joinWords`. Blank lines inside
40  * a page are dropped. The result ends with a newline.
41  */
42 export function bookText(tokens: readonly OcrToken[]): string {
43   const pages: string[][] = [];
44   let lines: string[] = [];
45   let words: string[] = [];
46   let word = '';
47   let page = -1;
48   let lineId = -1;
49   let wordId: number | undefined;
50 
51   const endWord = (): void => {
52     const text = word.trim();
53     if (text.length > 0) words.push(text);
54     word = '';
55   };
56   const endLine = (): void => {
57     endWord();
58     const text = joinWords(words);
59     if (text.length > 0) lines.push(text);
60     words = [];
61   };
62   const endPage = (): void => {
63     endLine();
64     if (lines.length > 0) pages.push(lines);
65     lines = [];
66   };
67 
68   for (const t of tokens) {
69     if (t.page !== page) {
70       endPage();
71       page = t.page;
72       lineId = t.line;
73       wordId = t.word;
74     } else if (t.line !== lineId) {
75       endLine();
76       lineId = t.line;
77       wordId = t.word;
78     } else if (t.word !== wordId) {
79       endWord();
80       wordId = t.word;
81     }
82     word += t.text;
83   }
84   endPage();
85   return pages.map((p) => p.join('\n')).join('\n\n') + (pages.length > 0 ? '\n' : '');
86 }