commit 2a0eed7223819bf9a00d6595d15f1e9f45086247 equwal <13551856+equwal@users.noreply.github.com> 2026-10-04 03:42:33 -0700 plan9port cluster: shared 9P storage and CPU offload for agent swarms p9c-server runs plan9port venti and fossil on one node, 9P on 127.0.0.1 only, with SSH as the access control. p9c is the 9p client aimed at it. tests/roundtrip.sh writes, reads, runs a job through /share, and checks the server survives rm. patches/fossil-remove-uaf.diff fixes the stock fossil crash on remove (fileRemove wrote to the freed source).
.gitignore | 7 ++++ bin/p9c | 7 ++++ bin/p9c-server | 74 +++++++++++++++++++++++++++++++++++++++ patches/fossil-remove-uaf.diff | 12 +++++++ skills/plan9port-cluster/SKILL.md | 71 +++++++++++++++++++++++++++++++++++++ tests/roundtrip.sh | 31 ++++++++++++++++ 6 files changed, 202 insertions(+)
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..89f4b4f --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.env +*.key +*.disk +arenas* +isect* +bloom +venti.conf diff --git a/bin/p9c b/bin/p9c new file mode 100755 index 0000000..bfef6c4 --- /dev/null +++ b/bin/p9c @@ -0,0 +1,7 @@ +#!/bin/sh +# p9c: plan9port's 9p client, aimed at the cluster file server. +# +# p9c ls -l /share +# p9c create /share/out.txt; cmd | p9c write /share/out.txt +# p9c read /share/out.txt +exec "${PLAN9:-/etc/plan9}/bin/9p" -a "tcp!127.0.0.1!${P9C_PORT:-5640}" "$@" diff --git a/bin/p9c-server b/bin/p9c-server new file mode 100755 index 0000000..d70f4a5 --- /dev/null +++ b/bin/p9c-server @@ -0,0 +1,74 @@ +#!/bin/sh +# p9c-server: the storage node. Runs plan9port venti (archival block store) +# and fossil (live 9P file server on top of it) from files in $P9C_DIR. +# The first run formats the files. 9P listens on 127.0.0.1 only, with +# authentication off: other nodes reach it through SSH tunnels (p9c-tunnel), +# so SSH does the authentication and encryption. +# +# p9c-server # foreground; run it under a supervisor (runit) +# +# Environment: PLAN9 (default /etc/plan9), P9C_DIR (default ~/plan9), +# P9C_ARENAS (arena file size, default 4G), P9C_PORT (default 5640), +# P9C_FOSSIL (fossil binary; use one built with patches/fossil-remove-uaf.diff: +# stock plan9port fossil crashes when a client removes a file). +set -eu +PLAN9=${PLAN9:-/etc/plan9} +PATH=$PATH:$PLAN9/bin +export PLAN9 PATH +d=${P9C_DIR:-$HOME/plan9} +port=${P9C_PORT:-5640} +vaddr=tcp!127.0.0.1!17034 +mkdir -p "$d" +cd "$d" + +if [ ! -f venti.conf ]; then + # ponytail: one arena file and one index section; add more arena + # files (venti/fmtarenas) when this one fills up. + truncate -s "${P9C_ARENAS:-4G}" arenas0 + truncate -s 512M isect0 + truncate -s 64M bloom + cat >venti.conf <<EOF +index main +arenas $d/arenas0 +isect $d/isect0 +bloom $d/bloom +mem 16M +bcmem 32M +icmem 32M +addr $vaddr +httpaddr tcp!127.0.0.1!17035 +EOF + "$PLAN9/bin/venti/fmtisect" isect0. isect0 + "$PLAN9/bin/venti/fmtarenas" arenas0. arenas0 + "$PLAN9/bin/venti/fmtbloom" bloom + "$PLAN9/bin/venti/fmtindex" venti.conf +fi + +# A fossil left over from an earlier run holds the disk and the port. +pkill -f "fsys main config $d/fossil.disk" || true +"$PLAN9/bin/venti/venti" -s -c venti.conf & +venti=$! +# wait until venti answers before fossil needs it +i=0 +until nc -z 127.0.0.1 17034 2>/dev/null || [ $i -ge 30 ]; do sleep 1; i=$((i+1)); done + +if [ ! -f fossil.disk ]; then + truncate -s 2G fossil.disk + "$PLAN9/bin/fossil/flfmt" -y fossil.disk +fi + +# fossil puts itself in the background. open -A turns authentication off. +# snaptime archives to venti daily at 04:00 and keeps a temporary snapshot +# every hour. /share is the writable tree for all nodes (unames differ per +# host, so world-writable); on later starts create fails harmlessly +# because it exists. +"${P9C_FOSSIL:-$PLAN9/bin/fossil/fossil}" -c "fsys main config $d/fossil.disk" \ + -c "fsys main venti $vaddr" \ + -c 'fsys main open -A -c 3000' \ + -c 'fsys main' \ + -c 'snaptime -a 0400 -s 60' \ + -c 'create /active/share adm adm d777' \ + -c "listen tcp!127.0.0.1!$port" +# The supervisor watches venti: when it dies this script exits, and the +# restart replaces the fossil too (pkill above). +wait $venti diff --git a/patches/fossil-remove-uaf.diff b/patches/fossil-remove-uaf.diff new file mode 100644 index 0000000..439f5d5 --- /dev/null +++ b/patches/fossil-remove-uaf.diff @@ -0,0 +1,12 @@ +--- a/src/cmd/fossil/file.c ++++ b/src/cmd/fossil/file.c +@@ -1239,8 +1239,8 @@ + for(ff=f->down; ff; ff=ff->next) + assert(ff->removed); + +- sourceRemove(f->source); + f->source->file = nil; /* erase back pointer */ ++ sourceRemove(f->source); + f->source = nil; + if(f->msource){ + sourceRemove(f->msource); diff --git a/skills/plan9port-cluster/SKILL.md b/skills/plan9port-cluster/SKILL.md new file mode 100644 index 0000000..ae913d5 --- /dev/null +++ b/skills/plan9port-cluster/SKILL.md @@ -0,0 +1,71 @@ +--- +name: plan9port-cluster +description: Shared network storage and CPU offload for an AI agent swarm, built on plan9port (fossil + venti over 9P). Use when a job is too heavy for this machine, when agents on different machines must share files, or when the user says "plan9 cluster", "p9c", or "shared storage". +--- + +# plan9port cluster (p9c) + +One storage node serves a 9P file tree. Agents put inputs in `/share`, run +the heavy command on the node, and write the result back to `/share`. Other +agents read it from there. + +| Part | Where | +|---|---| +| Storage node | `$P9C_HOST` (SSH login, see "This setup" at the end if present) | +| File server | plan9port fossil on `tcp!127.0.0.1!5640` on the node, backed by venti; data in `~/plan9` there | +| Writable tree | `/share` (world-writable; every agent uses it) | +| Client | `p9c` in `~/.local/bin` on the node: plan9port's `9p` aimed at the server | +| CPU | the storage node; check `uptime` before a heavy job | + +The server listens on 127.0.0.1 only and has authentication off. SSH is the +access control: never open port 5640 to a network. + +## Use it + +On the node: + +```sh +p9c ls -l /share +p9c create /share/in.txt # create an empty file +p9c write /share/in.txt <in.txt +p9c read /share/in.txt >copy.txt +p9c rm /share/in.txt +``` + +From another machine with SSH to the node (the server is only on the node's +loopback). A non-interactive SSH shell may not have `~/.local/bin` in +`PATH`, so each line adds it: + +```sh +ssh "$P9C_HOST" 'PATH=$PATH:~/.local/bin; p9c create /share/in.bin; p9c write /share/in.bin' <in.bin +ssh "$P9C_HOST" 'PATH=$PATH:~/.local/bin; p9c read /share/in.bin | xz -9 -T0 | (p9c create /share/out.xz; p9c write /share/out.xz)' +ssh "$P9C_HOST" 'PATH=$PATH:~/.local/bin; p9c read /share/out.xz' >out.xz +``` + +The second line is the CPU-offload pattern: the input comes from `/share`, +the work runs on the node, and the result goes back to `/share`. + +On Windows, PowerShell has no `<` redirection and adds CRLF to piped text: +run these lines through `cmd /c "..."`. + +## Set up a node + +1. Install plan9port. Build fossil with `patches/fossil-remove-uaf.diff` + (stock fossil crashes when a client removes a file). +2. Copy `bin/p9c-server` and `bin/p9c` to `~/.local/bin`. +3. Run `P9C_FOSSIL=/path/to/patched/fossil p9c-server` under a supervisor. + The first start formats `~/plan9` (arena size `P9C_ARENAS`, default 4G). + +## Check that it works + +On the node, in a clone of github.com/equwal/plan9port-cluster: + +```sh +P9C=~/.local/bin/p9c sh tests/roundtrip.sh # prints PASS +``` + +## Gotchas + +- `p9c create` fails if the file exists; `p9c write` then overwrites it. +- If `p9c` says "Connection refused", the server is down: restart it. +- fossil archives to venti daily at 04:00 and keeps an hourly snapshot. diff --git a/tests/roundtrip.sh b/tests/roundtrip.sh new file mode 100644 index 0000000..7f00a8a --- /dev/null +++ b/tests/roundtrip.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Round-trip test against a running p9c-server. Run it on the storage node. +# Writes 1 MiB of random bytes to /share, reads them back, and compares +# checksums; then runs a CPU job whose input and output live in /share. +# Prints PASS or FAIL and exits 0 or 1. +set -u +p9c=${P9C:-$(dirname "$0")/../bin/p9c} +t=/share/p9c-test.$$ +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +head -c 1048576 /dev/urandom >"$tmp/in" +$p9c create $t.in && $p9c write $t.in <"$tmp/in" || { echo "FAIL: write"; exit 1; } +$p9c read $t.in >"$tmp/back" || { echo "FAIL: read"; exit 1; } +a=$(sha256sum <"$tmp/in" | cut -d' ' -f1) +b=$(sha256sum <"$tmp/back" | cut -d' ' -f1) +[ "$a" = "$b" ] || { echo "FAIL: read back $b, wrote $a"; exit 1; } +echo "storage round trip: $a" + +# CPU job: input from /share, result to /share, result checked by a reader. +$p9c read $t.in | gzip -9 | wc -c >"$tmp/job" +$p9c create $t.out && $p9c write $t.out <"$tmp/job" || { echo "FAIL: job output"; exit 1; } +[ "$($p9c read $t.out)" = "$(cat "$tmp/job")" ] || { echo "FAIL: job output differs"; exit 1; } +echo "job output: $(cat "$tmp/job") bytes gzip" + +# Removing files must not take the server down (stock plan9port fossil +# freed the source and then wrote to it in fileRemove: segfault). +$p9c rm $t.in $t.out || { echo "FAIL: rm"; exit 1; } +sleep 1 +$p9c stat /share >/dev/null || { echo "FAIL: server gone after rm"; exit 1; } +echo PASS