Recently Written · git

commit b986fd30bbbad4b4c54d11a79545967fcd3b51d2
truex <truex@equwal.com>
2026-09-26 13:49:11 -0700

Make normalizeText stable when case folding moves marks

NFKC, the case map and the filter can leave combining marks out of canonical order. A second pass then changed the text, and the idempotence property failed in about 3.5% of the runs. normalizeText now repeats the pass until the text stays the same.

 src/align.ts       | 24 +++++++++++++++++++++---
 test/align.test.ts |  9 +++++++++
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/src/align.ts b/src/align.ts
index e632e94..3809f66 100644
--- a/src/align.ts
+++ b/src/align.ts
@@ -77,16 +77,34 @@ const MAX_LOCAL_EDIT = 2;
 
 const KEEP = /[\p{L}\p{N}\p{M}]/u;
 
+/** A limit on the passes of normalizeText. Real text needs one or two. */
+const MAX_NORMALIZE_PASSES = 4;
+
+/** One pass: NFKC, lowercase, then only letters, numbers and marks. */
+function normalizePass(text: string): string {
+  let out = '';
+  for (const ch of text.normalize('NFKC').toLowerCase()) {
+    if (KEEP.test(ch)) out += ch;
+  }
+  return out;
+}
+
 /**
  * Normalize text for comparison. NFKC folds full-width forms. Lowercase
  * folds case. Only letters, numbers and marks stay. Spaces and punctuation
  * go, so word boundaries do not matter. This works for CJK, where OCR and
  * subtitles rarely agree on spacing.
+ *
+ * The case map and the filter can leave combining marks out of canonical
+ * order, and then one more pass changes the text. The passes repeat until
+ * the text stays the same, so the result is stable.
  */
 export function normalizeText(text: string): string {
-  let out = '';
-  for (const ch of text.normalize('NFKC').toLowerCase()) {
-    if (KEEP.test(ch)) out += ch;
+  let out = normalizePass(text);
+  for (let pass = 1; pass < MAX_NORMALIZE_PASSES; pass++) {
+    const next = normalizePass(out);
+    if (next === out) break;
+    out = next;
   }
   return out;
 }
diff --git a/test/align.test.ts b/test/align.test.ts
index 5e97872..cc3c16a 100644
--- a/test/align.test.ts
+++ b/test/align.test.ts
@@ -33,6 +33,15 @@ describe('normalizeText', () => {
   it('keeps kana, kanji and the long vowel mark', () => {
     expect(normalizeText('コーヒーを飲む。')).toBe('コーヒーを飲む');
   });
+
+  it('gives a stable result when case folding moves combining marks', () => {
+    // fast-check found these inputs. The case map and the filter left the
+    // combining marks out of canonical order, so a second pass changed the text.
+    for (const s of ['\u{1D157}\u{1D165}\u{FC5E}', '\u{130}\u{1D15E}']) {
+      const once = normalizeText(s);
+      expect(normalizeText(once)).toBe(once);
+    }
+  });
 });
 
 describe('alignCuesToTokens on one page', () => {