docs/build-docs.py (14623 bytes)
1 #!/usr/bin/env python3 2 """Build docs/index.html: cheat sheets on top, the implementation spec below. 3 4 Hand-written parts: docs/forth-cheatsheet.txt, docs/forthlisp-cheatsheet.txt. 5 Everything else comes from the sources, so the page follows the code: 6 NOTES.md (spec prose), prims.lisp (primitive table), lisp.fs (special forms, 7 word list with stack comments), host.lisp (commands, demo checks), 8 build-firmware.sh / demo.sh / stm32-lisp (build and run), transcript.txt. 9 A source file that is missing or renamed is skipped, not fatal. 10 """ 11 import html, os, re, subprocess, sys 12 13 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 14 esc = html.escape 15 16 17 def read(name): 18 p = name if os.path.isabs(name) else os.path.join(ROOT, name) 19 try: 20 with open(p, encoding="utf-8", errors="replace") as f: 21 return f.read() 22 except OSError: 23 return None 24 25 26 def slug(s): 27 return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-") 28 29 30 def inline(s): 31 out = [] 32 for i, part in enumerate(re.split(r"(`[^`]*`)", s)): 33 if i % 2: 34 out.append("<code>%s</code>" % esc(part[1:-1])) 35 else: 36 t = esc(part) 37 t = re.sub(r"\*\*(.+?)\*\*", r"<b>\1</b>", t) 38 t = re.sub(r"\[([^\]]+)\]\(([^)]+)\)", r'<a href="\2">\1</a>', t) 39 out.append(t) 40 return "".join(out) 41 42 43 def md(text, toc, prefix, base=2): 44 """Small Markdown subset: headings, paragraphs, lists, tables, code.""" 45 out, lines, i = [], text.splitlines(), 0 46 while i < len(lines): 47 l = lines[i] 48 m = re.match(r"(#+)\s+(.*)", l) 49 if m: 50 lvl = min(len(m.group(1)) - 1 + base, 6) 51 if lvl < base: 52 i += 1 53 continue 54 a = prefix + slug(m.group(2)) 55 if lvl <= base + 1: 56 toc.append((lvl, a, m.group(2))) 57 out.append('<h%d id="%s">%s</h%d>' % (lvl, a, inline(m.group(2)), lvl)) 58 i += 1 59 elif l.startswith("```"): 60 j = i + 1 61 while j < len(lines) and not lines[j].startswith("```"): 62 j += 1 63 out.append("<pre>%s</pre>" % esc("\n".join(lines[i + 1:j]))) 64 i = j + 1 65 elif l.startswith(" "): 66 j = i 67 while j < len(lines) and (lines[j].startswith(" ") or not lines[j].strip()): 68 j += 1 69 out.append("<pre>%s</pre>" % esc("\n".join(x[4:] for x in lines[i:j]).rstrip())) 70 i = j 71 elif l.startswith("|"): 72 rows = [] 73 while i < len(lines) and lines[i].startswith("|"): 74 cells = [c.strip() for c in lines[i].strip().strip("|").split("|")] 75 if not all(re.fullmatch(r":?-+:?", c) for c in cells): 76 rows.append(cells) 77 i += 1 78 h = "".join("<th>%s</th>" % inline(c) for c in rows[0]) 79 b = "".join("<tr>%s</tr>" % "".join("<td>%s</td>" % inline(c) for c in r) for r in rows[1:]) 80 out.append('<div class="tw"><table><tr>%s</tr>%s</table></div>' % (h, b)) 81 elif re.match(r"- ", l): 82 items = [] 83 while i < len(lines) and (lines[i].startswith("- ") or lines[i].startswith(" ")): 84 if lines[i].startswith("- "): 85 items.append(lines[i][2:]) 86 else: 87 items[-1] += " " + lines[i].strip() 88 i += 1 89 out.append("<ul>%s</ul>" % "".join("<li>%s</li>" % inline(x) for x in items)) 90 elif not l.strip(): 91 i += 1 92 else: 93 j = i 94 while j < len(lines) and lines[j].strip() and not re.match(r"(#|\||- |```| )", lines[j]): 95 j += 1 96 out.append("<p>%s</p>" % inline(" ".join(lines[i:j]))) 97 i = j 98 return "\n".join(out) 99 100 101 def table(head, rows): 102 h = "".join("<th>%s</th>" % esc(x) for x in head) 103 b = "".join("<tr>%s</tr>" % "".join("<td>%s</td>" % c for c in r) for r in rows) 104 return '<div class="tw"><table><tr>%s</tr>%s</table></div>' % (h, b) 105 106 107 def code(s): 108 return "<code>%s</code>" % esc(s) 109 110 111 def prims_section(): 112 src = read("prims.lisp") 113 if not src: 114 return "" 115 defs = {m.group(1): (m.group(2), m.group(3)) for m in re.finditer( 116 r"^\(defun\s+(\S+)\s+\(([^)]*)\)\s+(.*)\)\s*$", src, re.M)} 117 reg = re.search(r"\(defprims(.*?)\)\s*$", src, re.S) 118 pairs = re.findall(r"\((\S+)\s+(\S+)\)", reg.group(1)) if reg else [] 119 rows = [] 120 for name, word in pairs: 121 params, body = defs.get(word, ("", "")) 122 nargs = len(set(re.findall(r"\(car(?: \(cdr)* a", body))) 123 rows.append([code(name), str(nargs), code(word), code(body)]) 124 fs = read("lisp.fs") or "" 125 for m in re.finditer(r"' (\S+) defprim (\S+)", fs): 126 rows.append([code(m.group(2)), "0", code(m.group(1)), 127 "defined in Forth in lisp.fs"]) 128 return ("<p>Generated from <code>prims.lisp</code> (the <code>defprims</code> table and " 129 "each <code>defun</code> body) and the <code>defprim</code> lines of " 130 "<code>lisp.fs</code>. Arity = distinct list positions the body reads. " 131 "Primitives check argument types but not arity.</p>" + table(["Lisp name", "Arity", "Forth word", "Semantics (host-dialect body)"], rows)) 132 133 134 def forms_section(): 135 fs = read("lisp.fs") 136 if not fs: 137 return "" 138 rows = [[code(n), code(w)] for w, n in re.findall(r"' (\S+) defform (\S+)", fs)] 139 return ("<p>Extracted from the <code>defform</code> lines of <code>lisp.fs</code>. A special " 140 "form name is bound to a marker value, so a local variable of the same name shadows it.</p>" 141 + table(["Special form", "Forth handler"], rows)) 142 143 144 def prelude_section(): 145 s = read("prelude.scm") 146 if not s: 147 return "" 148 names = re.findall(r"^\(define \((\S+)", s, re.M) 149 return ("<p>Procedures written in the chip Scheme itself (<code>prelude.scm</code>), loaded " 150 "through the REPL at boot:</p><p>%s</p>" % " ".join(code(n) for n in names)) 151 152 153 def conformance_section(): 154 t = read("r7rs-tests.scm") 155 if not t: 156 return "" 157 rows = [] 158 for l in t.splitlines(): 159 if not l.strip() or l.startswith(";"): 160 continue 161 e, _, w = l.partition("==>") 162 rows.append([code(e.strip()), code(w.strip())]) 163 res = read("conformance.txt") or "" 164 m = re.search(r"conformance: .*", res) 165 head = ("<p><b>Last run on the emulated chip: %s</b> (<code>conformance.txt</code>).</p>" 166 % esc(m.group(0))) if m else "" 167 return head + "<details><summary>All %d cases (<code>r7rs-tests.scm</code>)</summary>%s</details>" % ( 168 len(rows), table(["Expression", "Expected print"], rows)) 169 170 171 def words_section(): 172 fs = read("lisp.fs") 173 if not fs: 174 return "" 175 rows, group = [], "" 176 for l in fs.splitlines(): 177 g = re.match(r"\\ --- (.*) ---", l) 178 if g: 179 group = g.group(1) 180 m = re.match(r": (\S+)\s*(\([^)]*\))?\s*(.*)", l) 181 if m: 182 note = re.search(r"\\\s*(.*)$", l) 183 rows.append([esc(group), code(m.group(1)), code(m.group(2) or ""), 184 esc(note.group(1) if note else "")]) 185 return ("<p>Every colon definition in <code>lisp.fs</code>, in load order, with its stack " 186 "comment.</p>" + table(["Section", "Word", "Stack effect", "Comment"], rows)) 187 188 189 def header_comment(name, mark): 190 s = read(name) 191 if not s: 192 return "" 193 lines = [] 194 for l in s.splitlines(): 195 if l.startswith("#!"): 196 continue 197 if l.startswith(mark): 198 lines.append(l[len(mark):].strip()) 199 elif lines: 200 break 201 return " ".join(lines) 202 203 204 def build_section(): 205 rows = [] 206 for f in ("build-firmware.sh", "demo.sh", "stm32-lisp", "docs/deploy.sh"): 207 if read(f) is not None: 208 rows.append([code(f), esc(header_comment(f, "#"))]) 209 host = read("host.lisp") or "" 210 cmds = re.findall(r";;;\s{3}(\S+(?: \S+)?)\s{2,}(.*)", host) 211 out = table(["File", "Purpose (from its header comment)"], rows) 212 if cmds: 213 out += "<h3 id=\"spec-host-commands\">host.lisp commands</h3>" + table( 214 ["sbcl --script host.lisp ...", "What"], [[code(a), esc(b)] for a, b in cmds]) 215 for f in ("build-firmware.sh", "demo.sh", "stm32-lisp"): 216 s = read(f) 217 if s: 218 out += "<details><summary><code>%s</code></summary><pre>%s</pre></details>" % (f, esc(s)) 219 return out 220 221 222 def demo_section(): 223 host = read("host.lisp") or "" 224 checks = re.findall(r'\("((?:[^"\\]|\\.)*)" "((?:[^"\\]|\\.)*)"\)', host) 225 out = "" 226 if checks: 227 out += ("<p>The expressions <code>demo.sh</code> sends to the chip and the answers it " 228 "requires (from <code>host.lisp</code>). Any mismatch fails the demo.</p>" 229 + table(["Expression", "Expected"], [[code(e), code(w)] for e, w in checks])) 230 t = read("transcript.txt") 231 if t: 232 out += "<details><summary>Last recorded transcript.txt</summary><pre>%s</pre></details>" % esc(t) 233 return out 234 235 236 def results_section(): 237 rows = [] 238 for layer, f, pat in (("1-2 assembler vs GNU as", "asm-tests.txt", r"asm-test: .*"), 239 ("3 Forth kernel words", "kernel-tests.txt", r"kernel tests: .*"), 240 ("4 Scheme: chibi-scheme R7RS suite (pass / fail / skip / total)", 241 "tests/r7rs/results.txt", r"r7rs suite: .*"), 242 ("4 Scheme: our own regression suite", "conformance.txt", r"conformance: .*"), 243 ("CI: all three public repos in a clean ubuntu:24.04 container", "ci-summary.txt", r"CI: .*")): 244 m = re.search(pat, read(f) or "") 245 if m: 246 rows.append([esc(layer), code(f), esc(m.group(0))]) 247 return ("<p>Measured on the current stack (own kernel, no third-party Forth). Each file is " 248 "written by its test run and committed with the code.</p>" 249 + table(["Layer", "File", "Last result"], rows)) if rows else "" 250 251 252 CSS = """ 253 :root{--bg:#fdfdfb;--fg:#1d1d1b;--mut:#666;--line:#ddd;--code:#f2f1ec;--acc:#1f5fbf} 254 @media (prefers-color-scheme:dark){:root{--bg:#16171a;--fg:#e6e6e3;--mut:#9a9a9a;--line:#33353a;--code:#202227;--acc:#7fb0ff}} 255 *{box-sizing:border-box}html{-webkit-text-size-adjust:100%} 256 body{margin:0 auto;max-width:72rem;padding:1rem 16px 4rem;background:var(--bg);color:var(--fg); 257 font:16px/1.55 system-ui,-apple-system,"Segoe UI",sans-serif} 258 a{color:var(--acc)}h1{font-size:1.7rem;margin:.2rem 0}h2{border-bottom:1px solid var(--line);padding-bottom:.2rem;margin-top:2.5rem} 259 code,pre{font-family:ui-monospace,"SFMono-Regular",Menlo,Consolas,monospace;font-size:.88em} 260 code{background:var(--code);padding:.05em .3em;border-radius:3px} 261 pre{background:var(--code);padding:.8rem;overflow-x:auto;border-radius:6px;line-height:1.4} 262 pre code{background:none;padding:0} 263 .tw{overflow-x:auto}table{border-collapse:collapse;margin:.6rem 0;font-size:.92em} 264 th,td{border:1px solid var(--line);padding:.3rem .5rem;text-align:left;vertical-align:top} 265 th{background:var(--code)}.cs{display:grid;gap:1rem;grid-template-columns:repeat(auto-fit,minmax(min(100%,34rem),1fr))} 266 .cs pre{margin:0;font-size:.82em}nav ol{padding-left:1.2rem}nav li{margin:.1rem 0}.mut{color:var(--mut);font-size:.9em} 267 details{margin:.5rem 0}summary{cursor:pointer} 268 """ 269 270 271 def main(): 272 toc, body = [], [] 273 274 def sec(lvl, title, content): 275 if not content: 276 return 277 a = slug(title) 278 toc.append((lvl, a, title)) 279 body.append('<h%d id="%s">%s</h%d>\n%s' % (lvl, a, esc(title), lvl, content)) 280 281 # TOP: cheat sheets 282 cs = [] 283 for f, t in (("docs/forth-cheatsheet.txt", "Mecrisp Forth"), ("docs/forthlisp-cheatsheet.txt", "Chip Lisp")): 284 s = read(f) 285 if s: 286 cs.append('<div><h3 id="cheat-%s">%s</h3><pre>%s</pre></div>' % (slug(t), t, esc(s))) 287 sec(2, "Cheat sheets", '<div class="cs">%s</div>' % "".join(cs)) 288 289 # BELOW: spec 290 toc.append((2, "spec", "Implementation spec")) 291 body.append('<h2 id="spec">Implementation spec</h2>') 292 notes = read("NOTES.md") 293 if notes: 294 body.append(md(notes, toc, "spec-", base=3)) 295 bg = read("docs/background.md") 296 if bg: 297 bg = "\n".join(l for l in bg.split("\n") if "TODO (jose)" not in l and not l.startswith("<!--")) 298 sec(3, "Background (outline, with sources)", md(bg.split("\n", 1)[1], [], "bg-", base=4)) 299 sec(3, "Special forms (from lisp.fs)", forms_section()) 300 sec(3, "Primitives (from prims.lisp)", prims_section()) 301 sec(3, "Library procedures (from prelude.scm)", prelude_section()) 302 sec(3, "R7RS conformance subset", conformance_section()) 303 sec(3, "Forth words of the interpreter (from lisp.fs)", words_section()) 304 sec(3, "Build, flash, run", build_section()) 305 sec(3, "Test results by layer", results_section()) 306 sec(3, "Verified examples", demo_section()) 307 z = read("z80/NOTES.md") 308 if z: 309 sec(2, "Z80 version (z80/NOTES.md)", md(z, [], "z80-", base=3)) 310 311 try: 312 rev = subprocess.run(["git", "-C", ROOT, "log", "-1", "--format=%h %cd", "--date=short"], 313 capture_output=True, text=True).stdout.strip() 314 except OSError: 315 rev = "" 316 nav = "<nav><b>Contents</b><ol>%s</ol></nav>" % "".join( 317 '<li style="margin-left:%drem"><a href="#%s">%s</a></li>' % ((lvl - 2) * 1.2, a, inline(t)) 318 for lvl, a, t in toc) 319 page = """<!doctype html><html lang="en"><head><meta charset="utf-8"> 320 <meta name="viewport" content="width=device-width,initial-scale=1"> 321 <meta name="color-scheme" content="light dark"> 322 <title>Forth + Lisp on STM32</title><style>%s</style></head><body> 323 <h1>Assembly → SBCL → Forth → Lisp on STM32</h1> 324 <p>Source: <a href="https://github.com/equwal/forthlisp">equwal/forthlisp</a> (core: host Lisp, Scheme, R7RS harness) · <a href="https://github.com/equwal/forthlisp-stm32">equwal/forthlisp-stm32</a> (Thumb-2 kernel, QEMU) · <a href="https://github.com/equwal/forthlisp-z80">equwal/forthlisp-z80</a> (Z80 kernel, cpmsim). MIT licence.</p> 325 <p class="mut">Generated at %s by <code>docs/build-docs.py</code>. Do not edit the HTML; edit the sources.</p> 326 %s 327 %s 328 </body></html> 329 """ % (CSS, esc(rev or "working tree"), nav, "\n".join(body)) 330 out = sys.argv[1] if len(sys.argv) > 1 else os.path.join(ROOT, "docs", "index.html") 331 with open(out, "w", encoding="utf-8") as f: 332 f.write(page) 333 print(out) 334 335 336 if __name__ == "__main__": 337 main()