commit f36b6f28526cf674433a5a45f5a5b8e57a7856b4 equwal <13551856+equwal@users.noreply.github.com> 2026-09-25 01:17:36 -0700 Serve the links as plain text and Atom, not JSON Remove /links/links.json. Add /links/links.txt: the links as sbm lines, for cut, awk, dmenu and bm --merge. Add /links/atom.xml: an Atom feed for sfeed and other feed readers. Add /links/list.html: an HTML fragment that a page includes with nginx SSI, so that the page needs no script. A feed needs the time of each link. So the admin page writes a date line above the links that it adds. bm ignores the line. Add deploy/deploy.sh. It installs the program and the nginx snippet.
.gitattributes | 1 + README.md | 88 ++++++++++++++---- admin.html | 2 +- deploy/deploy.sh | 43 +++++++++ deploy/nginx-location.conf | 14 ++- deploy/sbm-webpublish.service | 10 +- links.go | 43 ++++++++- links_test.go | 51 +++++++++++ main.go | 92 ++++++++----------- public.go | 186 +++++++++++++++++++++++++++++++++++++ server_test.go | 208 +++++++++++++++++++++++++++++++++++++----- 11 files changed, 632 insertions(+), 106 deletions(-)
diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfdb8b7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/README.md b/README.md index 5811fd0..1d71cda 100644 --- a/README.md +++ b/README.md @@ -4,29 +4,64 @@ sbm-webpublish publishes an [sbm](https://github.com/equwal/sbm) bookmark file as a list of links on a web site. The links page of [recentlywritten.com](https://recentlywritten.com/links.html) uses it. -It is one small Go program with no dependencies outside the standard -library. It keeps the links in one plain sbm file: one line for each link, +It is one small Go program. At run time it needs only the Go standard +library. The tests use rapid for property tests. The program keeps the +links in one plain sbm file, one line for each link: URL<tab>description<tab>tag tag tag -so `bm` can read and edit the same file. +so `bm`, `cut`, `awk` and `grep` can read the same file. -## Pages +## Get the links -| Path | Who | What | -|---|---|---| -| `GET /links/links.json` | everyone | The links, newest first, as JSON: `url`, `title`, `host`, `tags`. Only http and https links are in the list. | -| `GET /links/admin` | owner | Add a link, delete a link, import a file. | +The program makes each output from the file at each request. The outputs +contain only http and https links. + +| Path | Format | +|---|---| +| `/links/links.txt` | The links as sbm lines, oldest first. | +| `/links/atom.xml` | An Atom feed, newest first. Each tag of a link is a category. | +| `/links/list.html` | The links as an HTML fragment, under each of their tags. A page of the site includes it with nginx SSI. | + +Each output sends `Last-Modified` and answers `If-Modified-Since` with +`304 Not Modified`. + +To follow the links in [sfeed](https://codemadness.org/sfeed.html), add +a line to `feeds()` in your sfeedrc: + + feed 'recentlywritten links' 'https://recentlywritten.com/links/atom.xml' + +The plain text works with the usual tools: + + curl -s https://recentlywritten.com/links/links.txt | cut -f1 + curl -s https://recentlywritten.com/links/links.txt | bm --merge + curl -s https://recentlywritten.com/links/links.txt | dmenu -l 20 | cut -f1 | xargs xdg-open + +## Dates + +A feed needs the time of each link, and an sbm line has no time. So the +admin page writes a date line above the links that it adds: -The admin pages use HTTP basic auth. Any user name works. The password is -the value of `ADMIN_PASSWORD`. The server refuses a form POST that comes -from another site. + # 2026-09-25T08:00:00Z + https://suckless.org/ software that sucks less code unix + +`bm` ignores each line that starts with `#`, so the file stays an sbm file. +In the feed, a link gets the time of the last date line above it. A link +below no date line gets the time of the file. If you add links to the file +with an editor, write a date line above them. + +## The admin page + +`/links/admin` lets the owner add a link, delete a link and import a file. +It uses HTTP basic auth. Any user name works. The password is the value of +`ADMIN_PASSWORD`. The server refuses a form POST that comes from another +site. The import takes an sbm file or the Netscape HTML that browsers export. -The folders of a browser bookmark become its tags. A URL that is a link -already is skipped. Two URLs are the same link when they differ only in -scheme, a leading `www.`, trailing slashes or the case of the host, as in -`bm`. +The folders of a browser bookmark become its tags. The import skips a URL +that is a link already. Two URLs are the same link when they differ only +in scheme, a leading `www.`, trailing slashes or the case of the host, as +in `bm`. ## Run @@ -36,15 +71,30 @@ Then open http://127.0.0.1:8082/links/admin. | Variable | Default | Meaning | |---|---|---| -| `ADMIN_PASSWORD` | none, required | The password of the admin pages. | +| `ADMIN_PASSWORD` | none, required | The password of the admin page. | | `LINKS_FILE` | `links.sbm` | The sbm file of the links. | | `ADDR` | `127.0.0.1:8082` | The address to listen on. | +The feed has no settings. It gets its address and name from the request. + ## Put it on a site -1. Install the service with `deploy/sbm-webpublish.service`. -2. Add `deploy/nginx-location.conf` to the nginx server block of the site. -3. On a page of the site, get `links/links.json` and show the links. See +1. Install the program on a server with systemd and nginx: + + sh deploy/deploy.sh root@example.com + + The script builds the program, installs `deploy/sbm-webpublish.service`, + makes an admin password when there is none, and installs + `deploy/nginx-location.conf` as `/etc/nginx/snippets/sbm-webpublish.conf`. +2. Add `include snippets/sbm-webpublish.conf;` to the server block of the + site, then run `nginx -t && systemctl reload nginx`. The snippet sends + `/links/` to the program and turns on SSI for `/links.html`. +3. Put this line in the page of the links: + + <!--# include virtual="/links/list.html" --> + + Put `<link rel="alternate" type="application/atom+xml" href="links/atom.xml">` + in its head, so that feed readers and `sfeed_web` find the feed. See `pages/links.md` in [recentlywritten](https://github.com/equwal/recentlywritten) for an example. diff --git a/admin.html b/admin.html index bab0bca..0a5f265 100644 --- a/admin.html +++ b/admin.html @@ -16,7 +16,7 @@ li form { display: inline; } </head> <body> <h1>Links</h1> -<p><a href="links.json">links.json</a> · <a href="/links.html">public page</a></p> +<p><a href="links.txt">links.txt</a> · <a href="atom.xml">atom.xml</a> · <a href="/links.html">public page</a></p> {{with .Message}}<p class="message">{{.}}</p>{{end}} <h2>Add a link</h2> diff --git a/deploy/deploy.sh b/deploy/deploy.sh new file mode 100644 index 0000000..93af47b --- /dev/null +++ b/deploy/deploy.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# deploy.sh: build sbm-webpublish for Linux and install it on a server with +# systemd and nginx. +# +# sh deploy/deploy.sh root@example.com +# +# When the server has no admin password, the script makes one. Read it on +# the server with: cat /etc/sbm-webpublish.env +set -e +host=${1:?usage: sh deploy/deploy.sh user@host} +cd "$(dirname "$0")/.." +GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -o sbm-webpublish . +scp sbm-webpublish deploy/sbm-webpublish.service deploy/nginx-location.conf "$host:/tmp/" +rm sbm-webpublish +ssh "$host" sh -s << 'EOF' +set -e +install -m 755 /tmp/sbm-webpublish /usr/local/bin/sbm-webpublish +install -m 644 /tmp/sbm-webpublish.service /etc/systemd/system/sbm-webpublish.service +install -m 644 /tmp/nginx-location.conf /etc/nginx/snippets/sbm-webpublish.conf +rm /tmp/sbm-webpublish /tmp/sbm-webpublish.service /tmp/nginx-location.conf +if [ ! -s /etc/sbm-webpublish.env ]; then + (umask 077 && printf 'ADMIN_PASSWORD=%s\n' "$(head -c 18 /dev/urandom | base64)" > /etc/sbm-webpublish.env) + echo "made an admin password: cat /etc/sbm-webpublish.env" +fi +systemctl daemon-reload +systemctl enable sbm-webpublish +systemctl restart sbm-webpublish +sleep 1 +if ! systemctl is-active --quiet sbm-webpublish; then + journalctl -u sbm-webpublish -n 20 --no-pager + exit 1 +fi +echo "sbm-webpublish is running" +if grep -qs 'snippets/sbm-webpublish.conf' /etc/nginx/sites-enabled/*; then + nginx -t + systemctl reload nginx + echo "nginx reloaded" +else + echo "Add this line to the server block of the site:" + echo " include snippets/sbm-webpublish.conf;" + echo "Then run: nginx -t && systemctl reload nginx" +fi +EOF diff --git a/deploy/nginx-location.conf b/deploy/nginx-location.conf index a13b214..c863f4a 100644 --- a/deploy/nginx-location.conf +++ b/deploy/nginx-location.conf @@ -1,10 +1,20 @@ -# Example for nginx. Add this block to the server block of the site. +# nginx snippet for sbm-webpublish. deploy.sh installs it as +# /etc/nginx/snippets/sbm-webpublish.conf. Include it in the server block of +# the site: +# +# include snippets/sbm-webpublish.conf; +# # nginx continues to serve the static site. It sends only /links/ to # sbm-webpublish. Serve the site over HTTPS: the admin page uses basic auth. location /links/ { proxy_pass http://127.0.0.1:8082; proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; client_max_body_size 10m; } + +# The page of the links includes /links/list.html with SSI. Change the path +# to the path of your page. This location gets the root of the server block. +location = /links.html { + ssi on; +} diff --git a/deploy/sbm-webpublish.service b/deploy/sbm-webpublish.service index 9c746c5..4c4d2d1 100644 --- a/deploy/sbm-webpublish.service +++ b/deploy/sbm-webpublish.service @@ -1,10 +1,6 @@ -# systemd unit for sbm-webpublish. -# Install: -# GOOS=linux GOARCH=amd64 go build -o sbm-webpublish . -# copy the binary to /usr/local/bin/sbm-webpublish -# put ADMIN_PASSWORD=... in /etc/sbm-webpublish.env (mode 600) -# copy this file to /etc/systemd/system/sbm-webpublish.service -# systemctl daemon-reload && systemctl enable --now sbm-webpublish +# systemd unit for sbm-webpublish. deploy.sh installs it as +# /etc/systemd/system/sbm-webpublish.service. /etc/sbm-webpublish.env +# (mode 600) holds ADMIN_PASSWORD=... [Unit] Description=sbm-webpublish: links from an sbm bookmark file After=network-online.target diff --git a/links.go b/links.go index 18d2699..9c960df 100644 --- a/links.go +++ b/links.go @@ -6,12 +6,20 @@ package main // // The parse, import and duplicate rules are the rules of bm and sbm-sync, // so a file from bm works here, and the file here works in bm. +// +// A feed needs the time of each link, and a bm line has no time. So the +// admin page writes a date line above the links that it adds: +// +// # 2026-09-25T08:00:00Z +// +// bm ignores each line that starts with "#", so the file stays an sbm file. import ( "html" "regexp" "slices" "strings" + "time" ) var ( @@ -20,10 +28,12 @@ var ( imported = regexp.MustCompile(`^(https?|ftp|file)://`) ) -// Link is one link of the file. +// Link is one link of the file. Added is the time of the last date line +// above the link, or zero when there is no date line above it. type Link struct { URL, Desc string Tags []string + Added time.Time } // Title gives the text of the link. @@ -119,11 +129,30 @@ func parseLine(line string) (Link, bool) { return l, l.URL != "" } +// dateLine gives the date line of t. +func dateLine(t time.Time) string { + return "# " + t.UTC().Format(time.RFC3339) +} + +// dateOf gives the time of a date line, and false for all other lines. +func dateOf(line string) (time.Time, bool) { + s, ok := strings.CutPrefix(line, "#") + if !ok { + return time.Time{}, false + } + t, err := time.Parse(time.RFC3339, strings.TrimSpace(s)) + return t, err == nil +} + // parse gives the links of a file, in the order of the file. func parse(text string) []Link { var out []Link + var added time.Time for _, s := range lines(text) { - if l, ok := parseLine(s); ok { + if t, ok := dateOf(s); ok { + added = t + } else if l, ok := parseLine(s); ok { + l.Added = added out = append(out, l) } } @@ -244,6 +273,16 @@ func merge(text string, rows []string) (string, int, int) { return join(ls), added, skipped } +// addDated adds rows to text as merge does, below a date line of t. It +// writes the date line only when it adds a row. +func addDated(text string, rows []string, t time.Time) (string, int, int) { + out, added, skipped := merge(join(append(lines(text), dateLine(t))), rows) + if added == 0 { + return text, 0, skipped + } + return out, added, skipped +} + // remove gives text without the lines whose URL has the normal form k. func remove(text, k string) string { var out []string diff --git a/links_test.go b/links_test.go index 4354436..a41970f 100644 --- a/links_test.go +++ b/links_test.go @@ -4,6 +4,7 @@ import ( "slices" "strings" "testing" + "time" "pgregory.net/rapid" ) @@ -81,6 +82,56 @@ func TestRemove(t *testing.T) { }) } +func TestDateLineParsesBack(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + // From the year 1 to the year 9999: RFC 3339 has four digits for + // the year. + u := time.Unix(rapid.Int64Range(-62135596800, 253402300799).Draw(t, "unix"), rapid.Int64Range(0, 999999999).Draw(t, "ns")) + got, ok := dateOf(dateLine(u)) + if !ok || !got.Equal(u.Truncate(time.Second)) { + t.Fatalf("%q gave %v %v", dateLine(u), got, ok) + } + }) +} + +func TestParseDates(t *testing.T) { + d := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + e := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + ls := parse("a.com\tA\t\n# 2026-01-02T03:04:05Z\nb.com\tB\t\n# a comment\nc.com\tC\t\n#2026-01-01T00:00:00Z\nd.com\tD\t\n") + want := []time.Time{{}, d, d, e} + if len(ls) != len(want) { + t.Fatalf("got %d links", len(ls)) + } + for i, l := range ls { + if !l.Added.Equal(want[i]) { + t.Errorf("%s: added %v, want %v", l.URL, l.Added, want[i]) + } + } +} + +func TestAddDatedDatesOnlyTheNewLinks(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + base, _, _ := merge("", rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "base")) + rows := rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "rows") + when := time.Unix(rapid.Int64Range(0, 4102444800).Draw(t, "when"), 0) + out, added, _ := addDated(base, rows, when) + old, ls := parse(base), parse(out) + switch { + case added == 0 && out != base: + t.Fatalf("no link added, but the text changed to %q", out) + case added > 0 && !strings.HasPrefix(out, base+dateLine(when)+"\n"): + t.Fatalf("no date line after the old links in %q", out) + case len(ls) != len(old)+added: + t.Fatalf("%d links, want %d + %d", len(ls), len(old), added) + } + for i, l := range ls { + if i < len(old) && !l.Added.Equal(old[i].Added) || i >= len(old) && !l.Added.Equal(when) { + t.Fatalf("link %d %q has the time %v", i, l.URL, l.Added) + } + } + }) +} + func TestHrefIsOnlyHTTP(t *testing.T) { for u, want := range map[string]string{ "https://a.com": "https://a.com", diff --git a/main.go b/main.go index 48878bb..69a98df 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,12 @@ // sbm-webpublish publishes an sbm bookmark file as a list of links on a -// web site. The site gets the links as JSON from /links/links.json. The -// owner adds, deletes and imports links on /links/admin. +// web site. Everyone can get the links as sbm lines, as an Atom feed and as +// HTML (see public.go). The owner adds, deletes and imports links on +// /links/admin. package main import ( "crypto/subtle" _ "embed" - "encoding/json" "errors" "html/template" "io" @@ -18,6 +18,7 @@ import ( "slices" "strconv" "sync" + "time" ) //go:embed admin.html @@ -34,30 +35,47 @@ type store struct { path string } -func (s *store) read() (string, error) { - b, err := os.ReadFile(s.path) +// read gives the text of the file and the time of its last change. A +// missing file is an empty file. +func (s *store) read() (string, time.Time, error) { + f, err := os.Open(s.path) if errors.Is(err, fs.ErrNotExist) { - return "", nil + return "", time.Time{}, nil } - return string(b), err + if err != nil { + return "", time.Time{}, err + } + defer f.Close() + // The time comes from the open file and not from its name, so that the + // time and the text are of the same version of the file. + st, err := f.Stat() + if err != nil { + return "", time.Time{}, err + } + b, err := io.ReadAll(f) + return string(b), st.ModTime(), err } -// change reads the file, gives its text to f, and writes the result. It -// writes a temporary file and renames it, so that a reader never sees half -// a file. +// change reads the file, gives its text to f, and writes the result when it +// is different. It writes a temporary file and renames it, so that a reader +// never sees half a file. func (s *store) change(f func(string) string) error { s.mu.Lock() defer s.mu.Unlock() - text, err := s.read() + text, _, err := s.read() if err != nil { return err } + out := f(text) + if out == text { + return nil + } tmp, err := os.CreateTemp(filepath.Dir(s.path), ".links-*") if err != nil { return err } defer os.Remove(tmp.Name()) - if _, err := tmp.WriteString(f(text)); err != nil { + if _, err := tmp.WriteString(out); err != nil { tmp.Close() return err } @@ -70,42 +88,8 @@ func (s *store) change(f func(string) string) error { type server struct { links *store password string -} - -// jsonLink is one link in links.json. -type jsonLink struct { - URL string `json:"url"` - Title string `json:"title"` - Host string `json:"host"` - Tags []string `json:"tags"` -} - -// publicLinks gives the links that the page can show: only the links with -// an http or https address. The newest link is first. -func publicLinks(text string) []jsonLink { - out := []jsonLink{} - for _, l := range parse(text) { - if h := l.Href(); h != "" { - tags := l.Tags - if tags == nil { - tags = []string{} - } - out = append([]jsonLink{{h, l.Title(), l.Host(), tags}}, out...) - } - } - return out -} - -func (s *server) linksJSON(w http.ResponseWriter, r *http.Request) { - text, err := s.links.read() - if err != nil { - http.Error(w, "cannot read the links", http.StatusInternalServerError) - log.Print(err) - return - } - w.Header().Set("Content-Type", "application/json; charset=utf-8") - w.Header().Set("Cache-Control", "max-age=60") - json.NewEncoder(w).Encode(publicLinks(text)) + // now gives the time for the date lines. Tests set a fixed time. + now func() time.Time } // admin lets only the owner through: HTTP basic auth with any user name @@ -128,7 +112,7 @@ type adminData struct { } func (s *server) show(w http.ResponseWriter, message string) { - text, err := s.links.read() + text, _, err := s.links.read() if err != nil { http.Error(w, "cannot read the links", http.StatusInternalServerError) log.Print(err) @@ -159,7 +143,7 @@ func (s *server) add(w http.ResponseWriter, r *http.Request) { } var added int err := s.links.change(func(text string) string { - text, added, _ = merge(text, []string{row}) + text, added, _ = addDated(text, []string{row}, s.now()) return text }) switch { @@ -198,7 +182,7 @@ func (s *server) importFile(w http.ResponseWriter, r *http.Request) { } var added, skipped int err = s.links.change(func(text string) string { - text, added, skipped = merge(text, importRows(string(b))) + text, added, skipped = addDated(text, importRows(string(b)), s.now()) return text }) if err != nil { @@ -213,7 +197,9 @@ func (s *server) importFile(w http.ResponseWriter, r *http.Request) { // so that nginx can send /links/ here and serve the rest of the site. func (s *server) routes() http.Handler { mux := http.NewServeMux() - mux.HandleFunc("GET /links/links.json", s.linksJSON) + mux.HandleFunc("GET /links/links.txt", s.plain) + mux.HandleFunc("GET /links/atom.xml", s.feed) + mux.HandleFunc("GET /links/list.html", s.list) mux.HandleFunc("GET /links/admin", s.admin(s.adminPage)) mux.HandleFunc("POST /links/admin/add", s.admin(s.add)) mux.HandleFunc("POST /links/admin/delete", s.admin(s.delete)) @@ -235,7 +221,7 @@ func main() { if pw == "" { log.Fatal("set ADMIN_PASSWORD") } - s := &server{links: &store{path: env("LINKS_FILE", "links.sbm")}, password: pw} + s := &server{links: &store{path: env("LINKS_FILE", "links.sbm")}, password: pw, now: time.Now} addr := env("ADDR", "127.0.0.1:8082") log.Printf("listening on %s, links file %s", addr, s.links.path) log.Fatal(http.ListenAndServe(addr, s.routes())) diff --git a/public.go b/public.go new file mode 100644 index 0000000..c9a32f3 --- /dev/null +++ b/public.go @@ -0,0 +1,186 @@ +package main + +// The public outputs. The server makes each one from the links file at each +// request, and puts in only the links with an http or https address: +// +// /links/links.txt the links as sbm lines, for cut, awk, dmenu and bm +// /links/atom.xml an Atom feed, for sfeed and other feed readers +// /links/list.html the links as HTML, for a page of the site to include + +import ( + "bytes" + "encoding/xml" + "html/template" + "log" + "maps" + "net/http" + "slices" + "strings" + "time" +) + +// published gives the links of text that the outputs show, in the order of +// the file. The URL of each link is the address that Href gives. +func published(text string) []Link { + var out []Link + for _, l := range parse(text) { + if l.URL = l.Href(); l.URL != "" { + out = append(out, l) + } + } + return out +} + +// serve sends the output that render makes from the links. The time of the +// file is the Last-Modified time, so that a client can send +// If-Modified-Since and get 304 Not Modified. +func (s *server) serve(w http.ResponseWriter, r *http.Request, contentType string, render func([]Link, time.Time) []byte) { + text, mod, err := s.links.read() + if err != nil { + log.Print(err) + http.Error(w, "cannot read the links", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", contentType) + http.ServeContent(w, r, "", mod, bytes.NewReader(render(published(text), mod))) +} + +// plain sends the links as sbm lines, oldest first: URL, description and +// tags, with a tab between them. +func (s *server) plain(w http.ResponseWriter, r *http.Request) { + s.serve(w, r, "text/plain; charset=utf-8", func(ls []Link, _ time.Time) []byte { + var b strings.Builder + for _, l := range ls { + b.WriteString(line(l.URL, l.Desc, strings.Join(l.Tags, " ")) + "\n") + } + return []byte(b.String()) + }) +} + +// The Atom feed (RFC 4287). +type atomFeed struct { + XMLName xml.Name `xml:"http://www.w3.org/2005/Atom feed"` + Title string `xml:"title"` + ID string `xml:"id"` + Link atomLink `xml:"link"` + Updated string `xml:"updated"` + Author string `xml:"author>name"` + Entries []atomEntry `xml:"entry"` +} + +type atomEntry struct { + Title string `xml:"title"` + Link atomLink `xml:"link"` + ID string `xml:"id"` + Updated string `xml:"updated"` + Categories []atomCategory `xml:"category"` +} + +type atomLink struct { + Rel string `xml:"rel,attr,omitempty"` + Href string `xml:"href,attr"` +} + +type atomCategory struct { + Term string `xml:"term,attr"` +} + +func atomTime(t time.Time) string { + return t.UTC().Format(time.RFC3339) +} + +// proto gives the scheme that the client used. Behind nginx, +// X-Forwarded-Proto gives it. +func proto(r *http.Request) string { + if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { + return "https" + } + return "http" +} + +// feed sends the links as an Atom feed, newest first. The time of a link is +// the time of its date line. A link without a date line gets the time of the +// file. Each tag is a category. The feed gets its address and name from the +// request, so it needs no settings. +func (s *server) feed(w http.ResponseWriter, r *http.Request) { + s.serve(w, r, "application/atom+xml; charset=utf-8", func(ls []Link, mod time.Time) []byte { + self := proto(r) + "://" + r.Host + r.URL.Path + f := atomFeed{ + Title: r.Host + " links", + ID: self, + Link: atomLink{Rel: "self", Href: self}, + Updated: atomTime(mod), + Author: r.Host, + } + for _, l := range slices.Backward(ls) { + t := l.Added + if t.IsZero() { + t = mod + } + e := atomEntry{Title: l.Title(), Link: atomLink{Href: l.URL}, ID: l.URL, Updated: atomTime(t)} + for _, tag := range l.Tags { + e.Categories = append(e.Categories, atomCategory{tag}) + } + f.Entries = append(f.Entries, e) + } + // Marshal fails only for types that XML cannot hold, and the feed + // has only strings. It writes U+FFFD for each character that XML + // cannot hold. + b, _ := xml.MarshalIndent(f, "", "\t") + return append([]byte(xml.Header), append(b, '\n')...) + }) +} + +// group is the links of one tag, for list.html. +type group struct { + Tag string + Links []Link +} + +// groups puts each link under each of its tags, newest link first. The tags +// are in alphabetical order. The links without tags come last, under +// "untagged". +func groups(ls []Link) []group { + byTag := map[string][]Link{} + for _, l := range slices.Backward(ls) { + tags := l.Tags + if len(tags) == 0 { + tags = []string{""} + } + for _, t := range slices.Compact(slices.Sorted(slices.Values(tags))) { + byTag[t] = append(byTag[t], l) + } + } + var out []group + for _, t := range slices.Sorted(maps.Keys(byTag)) { + if t != "" { + out = append(out, group{t, byTag[t]}) + } + } + if untagged := byTag[""]; untagged != nil { + out = append(out, group{"untagged", untagged}) + } + return out +} + +var listTmpl = template.Must(template.New("list").Parse(`{{with .}}<p>{{range .}}<a href="#{{.Tag}}">{{.Tag}}</a> {{end}}</p> +{{range .}}<h2 id="{{.Tag}}">{{.Tag}}</h2> +<ul> +{{range .Links}}<li><a href="{{.URL}}">{{.Title}}</a> <small>{{.Host}}</small></li> +{{end}}</ul> +{{end}}{{else}}<p>No links yet.</p> +{{end}}`)) + +// list sends the links as an HTML fragment, under each of their tags. A page +// of the site includes it with nginx SSI, so the page needs no script: +// +// <!--# include virtual="/links/list.html" --> +func (s *server) list(w http.ResponseWriter, r *http.Request) { + s.serve(w, r, "text/html; charset=utf-8", func(ls []Link, _ time.Time) []byte { + var b bytes.Buffer + if err := listTmpl.Execute(&b, groups(ls)); err != nil { + log.Print(err) + } + return b.Bytes() + }) +} diff --git a/server_test.go b/server_test.go index a02a218..04bf1c8 100644 --- a/server_test.go +++ b/server_test.go @@ -2,19 +2,33 @@ package main import ( "bytes" - "encoding/json" + "encoding/xml" "mime/multipart" "net/http" "net/http/httptest" "net/url" "os" "path/filepath" + "reflect" + "slices" "strings" "testing" + "time" + + "pgregory.net/rapid" ) +// day is the time of the clock of the test server. +var day = time.Date(2026, 9, 25, 8, 0, 0, 0, time.UTC) + +const dayLine = "# 2026-09-25T08:00:00Z\n" + func newServer(t *testing.T) (*server, http.Handler) { - s := &server{links: &store{path: filepath.Join(t.TempDir(), "links.sbm")}, password: "pw"} + s := &server{ + links: &store{path: filepath.Join(t.TempDir(), "links.sbm")}, + password: "pw", + now: func() time.Time { return day }, + } return s, s.routes() } @@ -27,19 +41,33 @@ func do(h http.Handler, r *http.Request, auth bool) *httptest.ResponseRecorder { return w } +func get(h http.Handler, path string) *httptest.ResponseRecorder { + return do(h, httptest.NewRequest("GET", path, nil), false) +} + func form(path string, v url.Values) *http.Request { r := httptest.NewRequest("POST", path, strings.NewReader(v.Encode())) r.Header.Set("Content-Type", "application/x-www-form-urlencoded") return r } -func publicJSON(t *testing.T, h http.Handler) []jsonLink { - w := do(h, httptest.NewRequest("GET", "/links/links.json", nil), false) - var ls []jsonLink - if err := json.Unmarshal(w.Body.Bytes(), &ls); err != nil { - t.Fatal(err, w.Body.String()) +// fataler is *testing.T or *rapid.T. +type fataler interface { + Fatalf(format string, args ...any) +} + +func write(t fataler, s *server, text string) { + if err := os.WriteFile(s.links.path, []byte(text), 0o644); err != nil { + t.Fatalf("%v", err) + } +} + +func feedOf(t fataler, w *httptest.ResponseRecorder) atomFeed { + var f atomFeed + if err := xml.Unmarshal(w.Body.Bytes(), &f); err != nil { + t.Fatalf("%v in %s", err, w.Body) } - return ls + return f } func TestAdminNeedsPassword(t *testing.T) { @@ -59,13 +87,13 @@ func TestAdminNeedsPassword(t *testing.T) { } } -func TestAddDeleteAndJSON(t *testing.T) { +func TestAddAndDelete(t *testing.T) { s, h := newServer(t) - if ls := publicJSON(t, h); len(ls) != 0 { - t.Fatalf("empty store gave %v", ls) + if body := get(h, "/links/links.txt").Body.String(); body != "" { + t.Fatalf("empty store gave %q", body) } do(h, form("/links/admin/add", url.Values{"url": {"https://a.com"}, "desc": {"A"}, "tags": {"x y"}}), true) - w := do(h, form("/links/admin/add", url.Values{"url": {"https://b.com"}}), true) + w := do(h, form("/links/admin/add", url.Values{"url": {"b.com"}}), true) if w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/links/admin?m=Added." { t.Fatalf("add: %d %q", w.Code, w.Header().Get("Location")) } @@ -73,28 +101,164 @@ func TestAddDeleteAndJSON(t *testing.T) { if !strings.Contains(w.Header().Get("Location"), "already") { t.Fatalf("duplicate add: %q", w.Header().Get("Location")) } - ls := publicJSON(t, h) - if len(ls) != 2 || ls[0].URL != "https://b.com" || ls[1].Title != "A" || len(ls[1].Tags) != 2 { - t.Fatalf("json: %+v", ls) + if body := get(h, "/links/links.txt").Body.String(); body != "https://a.com\tA\tx y\nhttps://b.com\t\t\n" { + t.Fatalf("links.txt: %q", body) } do(h, form("/links/admin/delete", url.Values{"url": {"https://b.com"}}), true) + // The date line of the deleted link stays. bm ignores it. b, _ := os.ReadFile(s.links.path) - if string(b) != "https://a.com\tA\tx y\n" { + if string(b) != dayLine+"https://a.com\tA\tx y\n"+dayLine { t.Fatalf("file: %q", b) } } -func TestJSONHidesUnsafeLinks(t *testing.T) { +func TestOutputsHideUnsafeLinks(t *testing.T) { + s, h := newServer(t) + write(t, s, "javascript:alert(1)\tx\t\ndata:text/html,x\ty\t\nhttps://a.com\tA\t\n") + for _, path := range []string{"/links/links.txt", "/links/atom.xml", "/links/list.html"} { + body := get(h, path).Body.String() + if strings.Contains(body, "javascript") || strings.Contains(body, "data:") || !strings.Contains(body, "https://a.com") { + t.Errorf("%s: %s", path, body) + } + } +} + +func TestAtom(t *testing.T) { s, h := newServer(t) - os.WriteFile(s.links.path, []byte("javascript:alert(1)\tx\t\nhttps://a.com\tA\t\n"), 0o644) - if ls := publicJSON(t, h); len(ls) != 1 || ls[0].URL != "https://a.com" { - t.Fatalf("json: %+v", ls) + write(t, s, "https://old.com\tOld\t\n# 2026-01-02T03:04:05Z\nhttps://a.com\tA & B\tx y\n# a comment\nb.com\t\t\n") + r := httptest.NewRequest("GET", "/links/atom.xml", nil) + r.Host = "example.org" + r.Header.Set("X-Forwarded-Proto", "https") + w := do(h, r, false) + if ct := w.Header().Get("Content-Type"); ct != "application/atom+xml; charset=utf-8" { + t.Errorf("Content-Type %q", ct) + } + f := feedOf(t, w) + self := "https://example.org/links/atom.xml" + if f.ID != self || f.Link != (atomLink{"self", self}) || f.Title != "example.org links" || f.Author != "example.org" { + t.Errorf("feed: %+v", f) + } + st, err := os.Stat(s.links.path) + if err != nil { + t.Fatal(err) + } + mod := atomTime(st.ModTime()) + if f.Updated != mod { + t.Errorf("feed updated %q, want the time of the file %q", f.Updated, mod) + } + date := "2026-01-02T03:04:05Z" + want := []atomEntry{ + {Title: "https://b.com", Link: atomLink{Href: "https://b.com"}, ID: "https://b.com", Updated: date}, + {Title: "A & B", Link: atomLink{Href: "https://a.com"}, ID: "https://a.com", Updated: date, + Categories: []atomCategory{{"x"}, {"y"}}}, + {Title: "Old", Link: atomLink{Href: "https://old.com"}, ID: "https://old.com", Updated: mod}, + } + if !reflect.DeepEqual(f.Entries, want) { + t.Errorf("entries:\n%+v\nwant:\n%+v", f.Entries, want) + } +} + +// genPublicRow makes a file line with the characters that XML and HTML must +// escape, and with links that the outputs must leave out. +func genPublicRow(t *rapid.T) string { + u := rapid.SampledFrom([]string{"https://a.com/", "http://b.org/?x=1&y=<2>", "c.net/", "javascript:alert(1)//", "ftp://d.com/"}).Draw(t, "url") + path := rapid.StringMatching(`[a-z]{0,3}`).Draw(t, "path") + desc := rapid.StringMatching(`[a-z<>&"' \]\t]{0,10}`).Draw(t, "desc") + tags := rapid.StringMatching(`[a-z<>&" ]{0,8}`).Draw(t, "tags") + return line(u+path, desc, tags) +} + +// TestOutputsRoundTrip: links.txt parses back to the published links, and the +// feed gives each published link, newest first, with its time and tags. +func TestOutputsRoundTrip(t *testing.T) { + s, h := newServer(t) + rapid.Check(t, func(t *rapid.T) { + text := "" + for range rapid.IntRange(0, 3).Draw(t, "batches") { + rows := rapid.SliceOf(rapid.Custom(genPublicRow)).Draw(t, "rows") + when := time.Unix(rapid.Int64Range(0, 4102444800).Draw(t, "when"), 0) + text, _, _ = addDated(text, rows, when) + } + write(t, s, text) + want := published(text) + + got := parse(get(h, "/links/links.txt").Body.String()) + if len(got) != len(want) { + t.Fatalf("links.txt has %d links, want %d", len(got), len(want)) + } + for i := range want { + if got[i].URL != want[i].URL || got[i].Desc != want[i].Desc || !slices.Equal(got[i].Tags, want[i].Tags) { + t.Fatalf("links.txt link %d is %+v, want %+v", i, got[i], want[i]) + } + } + + f := feedOf(t, get(h, "/links/atom.xml")) + if len(f.Entries) != len(want) { + t.Fatalf("feed has %d entries, want %d", len(f.Entries), len(want)) + } + for i, e := range f.Entries { + l := want[len(want)-1-i] + var tags []string + for _, c := range e.Categories { + tags = append(tags, c.Term) + } + if e.Title != l.Title() || e.Link.Href != l.URL || e.ID != l.URL || e.Updated != atomTime(l.Added) || !slices.Equal(tags, l.Tags) { + t.Fatalf("entry %d is %+v, want %+v", i, e, l) + } + } + }) +} + +func TestAtomReplacesCharactersThatXMLCannotHold(t *testing.T) { + s, h := newServer(t) + write(t, s, "https://a.com\tbell \x07 and ]]>\t\n") + if f := feedOf(t, get(h, "/links/atom.xml")); f.Entries[0].Title != "bell � and ]]>" { + t.Fatalf("title %q", f.Entries[0].Title) + } +} + +func TestNotModified(t *testing.T) { + s, h := newServer(t) + write(t, s, "https://a.com\tA\t\n") + for _, path := range []string{"/links/links.txt", "/links/atom.xml", "/links/list.html"} { + lm := get(h, path).Header().Get("Last-Modified") + r := httptest.NewRequest("GET", path, nil) + r.Header.Set("If-Modified-Since", lm) + if w := do(h, r, false); lm == "" || w.Code != http.StatusNotModified { + t.Errorf("%s: Last-Modified %q, code %d", path, lm, w.Code) + } + } +} + +func TestList(t *testing.T) { + s, h := newServer(t) + if body := get(h, "/links/list.html").Body.String(); body != "<p>No links yet.</p>\n" { + t.Fatalf("empty list: %q", body) + } + write(t, s, "https://a.com\t<A>\tx y\nhttps://b.com\tB\t\nhttps://c.com\tC\ty y\n") + want := `<p><a href="#x">x</a> <a href="#y">y</a> <a href="#untagged">untagged</a> </p> +<h2 id="x">x</h2> +<ul> +<li><a href="https://a.com"><A></a> <small>a.com</small></li> +</ul> +<h2 id="y">y</h2> +<ul> +<li><a href="https://c.com">C</a> <small>c.com</small></li> +<li><a href="https://a.com"><A></a> <small>a.com</small></li> +</ul> +<h2 id="untagged">untagged</h2> +<ul> +<li><a href="https://b.com">B</a> <small>b.com</small></li> +</ul> +` + if body := get(h, "/links/list.html").Body.String(); body != want { + t.Fatalf("list:\n%s\nwant:\n%s", body, want) } } func TestImport(t *testing.T) { s, h := newServer(t) - os.WriteFile(s.links.path, []byte("https://a.com\tA\t\n"), 0o644) + write(t, s, "https://a.com\tA\t\n") var body bytes.Buffer mw := multipart.NewWriter(&body) f, _ := mw.CreateFormFile("file", "bookmarks.sbm") @@ -107,7 +271,7 @@ func TestImport(t *testing.T) { t.Fatalf("import: %d %q", w.Code, loc) } b, _ := os.ReadFile(s.links.path) - if string(b) != "https://a.com\tA\t\nhttps://b.com\tB\tt\n" { + if string(b) != "https://a.com\tA\t\n"+dayLine+"https://b.com\tB\tt\n" { t.Fatalf("file: %q", b) } }