tools/make_fullbook.py (1405 bytes)
1 """Whole-book input for the scale test: every cached chapter transcript of one 2 audiobook, end to end, plus the full text. No reference output - the reference 3 cannot align a whole book at once, which is rather the point. 4 5 Local only (golden-local/ is gitignored).""" 6 import argparse, ast, json, re, sys 7 from pathlib import Path 8 sys.path.insert(0, str(Path(__file__).parent)) 9 from make_golden import aozora_paragraphs, chapter_files 10 11 ap = argparse.ArgumentParser() 12 ap.add_argument("--cache", type=Path, required=True) 13 ap.add_argument("--book", required=True) 14 ap.add_argument("--aozora", type=Path, required=True) 15 ap.add_argument("--out", type=Path, required=True) 16 a = ap.parse_args() 17 18 segments, offset, language = [], 0.0, None 19 for p in chapter_files(a.cache, a.book): 20 d = ast.literal_eval(p.read_text(encoding="utf-8")) 21 language = language or d["language"] 22 for s in d["segments"]: 23 segments.append({"text": s["text"], "start": s["start"] + offset, "end": s["end"] + offset}) 24 if d["segments"]: 25 offset = segments[-1]["end"] 26 paragraphs = aozora_paragraphs(a.aozora) 27 a.out.write_text(json.dumps({"language": language, "transcript": segments, "paragraphs": paragraphs}, 28 ensure_ascii=False), encoding="utf-8") 29 print(len(segments), "segments,", round(offset / 3600, 1), "hours;", len(paragraphs), "paragraphs,", 30 sum(map(len, paragraphs)), "chars")