backend/gen_languages.py (2007 bytes)
1 """Generate languages.json from the actually-installed pysbd + stanza. 2 3 Run once (or after upgrading either package): 4 python -m backend.gen_languages 5 6 The registry is what makes "works for all languages" concrete: every language 7 subplz can segment, tagged with the splitter backend it needs. 8 """ 9 10 import json 11 from pathlib import Path 12 13 OUT = Path(__file__).with_name("languages.json") 14 15 # Languages where subplz's default splitter (pysbd) works. Fast, no model download. 16 # Everything else must run with --nlp so subplz uses stanza instead. 17 def pysbd_languages() -> set[str]: 18 from pysbd.languages import LANGUAGE_CODES 19 20 return set(LANGUAGE_CODES.keys()) 21 22 23 def stanza_languages() -> set[str]: 24 from stanza.resources.common import list_available_languages 25 26 return set(list_available_languages()) 27 28 29 def english_name(code: str) -> str: 30 import pycountry 31 32 for attr in ("alpha_2", "alpha_3"): 33 try: 34 hit = pycountry.languages.get(**{attr: code}) 35 except (KeyError, LookupError): 36 hit = None 37 if hit is not None: 38 return getattr(hit, "name", code) 39 return code 40 41 42 def build() -> dict: 43 pysbd = pysbd_languages() 44 stanza = stanza_languages() 45 46 entries = [] 47 for code in sorted(pysbd | stanza): 48 entries.append( 49 { 50 "code": code, 51 "name": english_name(code), 52 # pysbd wins when available: no model download, much faster start. 53 "splitter": "pysbd" if code in pysbd else "stanza", 54 } 55 ) 56 return { 57 "generated_from": {"pysbd": len(pysbd), "stanza": len(stanza)}, 58 "languages": entries, 59 } 60 61 62 if __name__ == "__main__": 63 data = build() 64 OUT.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") 65 n = len(data["languages"]) 66 n_pysbd = sum(1 for e in data["languages"] if e["splitter"] == "pysbd") 67 print(f"wrote {OUT} - {n} languages ({n_pysbd} pysbd, {n - n_pysbd} stanza)")