commit 216c3de79a6b214bdfcd2b2e2311b22787e5e4d0 equwal <13551856+equwal@users.noreply.github.com> 2026-09-25 02:41:29 -0700 Add a feed to each link A link can have the address of the RSS or Atom feed of its site, as a fourth field of its line. A line without a feed keeps the three fields of bm. bm reads only the first three fields. The admin page takes a feed with a new link, and shows the feed of each link in a form that changes it. The import, and adding a URL that is a link already, give the feed to a link that has none. links.txt has the feed as the fourth field. The HTML list puts an rss link after each link with a feed, and shows the address without the scheme in place of the host, because many descriptions do not name the page. Add the command 'sbm-webpublish merge', which adds the links of its input as the import does. Write the file with mode 0644, so that the service can read a file that root wrote with merge. Fix the import, which read the file two times after the change to mergeFrom.
README.md | 52 +++++++++++++------ admin.html | 10 ++-- links.go | 155 +++++++++++++++++++++++++++++++++++++++++---------------- links_test.go | 129 ++++++++++++++++++++++++++++++++++++++++------- main.go | 104 +++++++++++++++++++++++++++++++------- public.go | 19 ++++--- server_test.go | 85 +++++++++++++++++++++++++------ 7 files changed, 435 insertions(+), 119 deletions(-)
diff --git a/README.md b/README.md index 1d71cda..98a3cd9 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,12 @@ It is one small Go program. At run time it needs only the Go standard library. The tests use rapid for property tests. The program keeps the links in one plain sbm file, one line for each link: - URL<tab>description<tab>tag tag tag + URL<tab>description<tab>tag tag tag<tab>feed -so `bm`, `cut`, `awk` and `grep` can read the same file. +so `bm`, `cut`, `awk` and `grep` can read the same file. The feed is the +address of the RSS or Atom feed of the site. It is optional: a line without +a feed has the three fields of `bm`. `bm` reads only the first three +fields. `bm --merge` does not copy the feed. ## Get the links @@ -19,9 +22,9 @@ contain only http and https links. | Path | Format | |---|---| -| `/links/links.txt` | The links as sbm lines, oldest first. | +| `/links/links.txt` | The links as sbm lines, oldest first. The fourth field is the feed of the site, if there is one. | | `/links/atom.xml` | An Atom feed, newest first. Each tag of a link is a category. | -| `/links/list.html` | The links as an HTML fragment, under each of their tags. A page of the site includes it with nginx SSI. | +| `/links/list.html` | The links as an HTML fragment, under each of their tags, with an "rss" link after each link that has a feed. A page of the site includes it with nginx SSI. | Each output sends `Last-Modified` and answers `If-Modified-Since` with `304 Not Modified`. @@ -37,6 +40,13 @@ The plain text works with the usual tools: curl -s https://recentlywritten.com/links/links.txt | bm --merge curl -s https://recentlywritten.com/links/links.txt | dmenu -l 20 | cut -f1 | xargs xdg-open +To follow the sites of the links, make a `feed` line for sfeedrc from each +link that has a feed. The name of each feed comes from its URL: + + curl -s https://recentlywritten.com/links/links.txt | awk -F'\t' '$4 { + n = $1; sub(/^[a-z]+:\/\//, "", n); gsub(/[^A-Za-z0-9._-]/, "-", n) + printf "\tfeed '\''%s'\'' '\''%s'\''\n", n, $4 }' + ## Dates A feed needs the time of each link, and an sbm line has no time. So the @@ -52,16 +62,26 @@ with an editor, write a date line above them. ## The admin page -`/links/admin` lets the owner add a link, delete a link and import a file. -It uses HTTP basic auth. Any user name works. The password is the value of -`ADMIN_PASSWORD`. The server refuses a form POST that comes from another -site. +`/links/admin` lets the owner add a link with its feed, set the feed of a +link, delete a link and import a file. It uses HTTP basic auth. Any user +name works. The password is the value of `ADMIN_PASSWORD`. The server +refuses a form POST that comes from another site. The import takes an sbm file or the Netscape HTML that browsers export. The folders of a browser bookmark become its tags. The import skips a URL -that is a link already. Two URLs are the same link when they differ only -in scheme, a leading `www.`, trailing slashes or the case of the host, as -in `bm`. +that is a link already, but a line with a feed gives its feed to a link +that has none. Two URLs are the same link when they differ only in scheme, +a leading `www.`, trailing slashes or the case of the host, as in `bm`. + +## Add links in a shell + +On the server, `sbm-webpublish merge` adds the links of its input, as the +import does: + + bm --list | ssh root@example.com 'LINKS_FILE=/var/lib/sbm-webpublish/links.sbm sbm-webpublish merge' + +It does not lock the file against the service, so do not use the admin +page at the same time. ## Run @@ -79,15 +99,17 @@ The feed has no settings. It gets its address and name from the request. ## Put it on a site -1. Install the program on a server with systemd and nginx: +1. Install the program on a server with systemd and nginx. The second + argument is the name of the site file in `/etc/nginx/sites-enabled`: - sh deploy/deploy.sh root@example.com + sh deploy/deploy.sh root@example.com example The script builds the program, installs `deploy/sbm-webpublish.service`, makes an admin password when there is none, and installs `deploy/nginx-location.conf` as `/etc/nginx/snippets/sbm-webpublish.conf`. -2. Add `include snippets/sbm-webpublish.conf;` to the server block of the - site, then run `nginx -t && systemctl reload nginx`. The snippet sends +2. The script adds `include snippets/sbm-webpublish.conf;` to each HTTPS + server block of the site file, runs `nginx -t` and reloads nginx. If + `nginx -t` fails, it puts the old site file back. The snippet sends `/links/` to the program and turns on SSI for `/links.html`. 3. Put this line in the page of the links: diff --git a/admin.html b/admin.html index 0a5f265..633eb29 100644 --- a/admin.html +++ b/admin.html @@ -12,6 +12,8 @@ li { margin: 0.4rem 0; } .host, .tags { color: #666; font-size: 0.9em; } .message { background: #ffd; padding: 0.5rem; } li form { display: inline; } +li form.feed { display: block; margin: 0.2rem 0; } +form.feed input[type=url] { width: 70%; } </style> </head> <body> @@ -24,11 +26,12 @@ li form { display: inline; } <input type="url" name="url" placeholder="https://example.com" required> <input type="text" name="desc" placeholder="Description"> <input type="text" name="tags" placeholder="Tags, with spaces between them"> +<input type="url" name="feed" placeholder="Feed (RSS or Atom) of the site, if it has one"> <button>Add</button> </form> <h2>Import a file</h2> -<p>An sbm bookmark file, or the HTML that a browser exports. URLs that are links already are skipped.</p> +<p>An sbm bookmark file, or the HTML that a browser exports. URLs that are links already are skipped. A line with a feed (the fourth field) gives its feed to a link that has none.</p> <form method="post" action="admin/import" enctype="multipart/form-data"> <input type="file" name="file" required> <button>Import</button> @@ -37,9 +40,10 @@ li form { display: inline; } <h2>{{len .Links}} links, newest first</h2> <ul> {{range .Links}}<li>{{if .Href}}<a href="{{.Href}}">{{.Title}}</a>{{else}}{{.Title}}{{end}} -<span class="host">{{.Host}}</span> +<span class="host">{{.Address}}</span> {{with .Tags}}<span class="tags">{{range .}}{{.}} {{end}}</span>{{end}} -<form method="post" action="admin/delete"><input type="hidden" name="url" value="{{.URL}}"><button>Delete</button></form></li> +<form method="post" action="admin/delete"><input type="hidden" name="url" value="{{.URL}}"><button>Delete</button></form> +<form class="feed" method="post" action="admin/feed"><input type="hidden" name="url" value="{{.URL}}"><input type="url" name="feed" value="{{.Feed}}" placeholder="Feed (RSS or Atom)"> <button>Save feed</button></form></li> {{end}}</ul> </body> </html> diff --git a/links.go b/links.go index 9c960df..8cb2452 100644 --- a/links.go +++ b/links.go @@ -2,7 +2,12 @@ package main // The links file is an sbm bookmark file. Each line is one link: // -// URL<tab>description<tab>tag tag tag +// URL<tab>description<tab>tag tag tag<tab>feed +// +// The feed is the address of the RSS or Atom feed of the site. It is +// optional: a line without a feed has only the three fields of bm. bm reads +// only the first three fields, and it keeps the full line of each other link +// when it deletes a link. bm --merge does not copy the feed. // // The parse, import and duplicate rules are the rules of bm and sbm-sync, // so a file from bm works here, and the file here works in bm. @@ -28,12 +33,13 @@ var ( imported = regexp.MustCompile(`^(https?|ftp|file)://`) ) -// Link is one link of the file. Added is the time of the last date line -// above the link, or zero when there is no date line above it. +// Link is one link of the file. Feed is the address of the feed of the site, +// or "". Added is the time of the last date line above the link, or zero +// when there is no date line above it. type Link struct { - URL, Desc string - Tags []string - Added time.Time + URL, Desc, Feed string + Tags []string + Added time.Time } // Title gives the text of the link. @@ -53,16 +59,33 @@ func (l Link) Host() string { return u } -// Href gives the address for the link on the page. Only http and https -// addresses come out, so that a line of the file cannot run script in the -// page. An address without a scheme gets https. Other schemes give "". +// Address gives the URL without its scheme and without a slash at the end. +// Pages show it after the link, because many descriptions do not name the +// page, and many links have the same host. +func (l Link) Address() string { + return strings.TrimSuffix(scheme.ReplaceAllString(l.URL, ""), "/") +} + +// Href gives the address for the link on the page (see href). func (l Link) Href() string { - s := strings.ToLower(l.URL) + return href(l.URL) +} + +// FeedHref gives the address for the feed of the link on the page (see href). +func (l Link) FeedHref() string { + return href(l.Feed) +} + +// href gives the address u for a page. Only http and https addresses come +// out, so that a line of the file cannot run script in the page. An address +// without a scheme gets https. Other schemes, and "", give "". +func href(u string) string { + s := strings.ToLower(u) switch { case strings.HasPrefix(s, "https://") || strings.HasPrefix(s, "http://"): - return l.URL - case !strings.Contains(l.URL, ":"): - return "https://" + l.URL + return u + case u != "" && !strings.Contains(u, ":"): + return "https://" + u } return "" } @@ -126,6 +149,9 @@ func parseLine(line string) (Link, bool) { if len(fields) > 2 { l.Tags = strings.Fields(fields[2]) } + if len(fields) > 3 { + l.Feed = strings.TrimSpace(fields[3]) + } return l, l.URL != "" } @@ -161,15 +187,34 @@ func parse(text string) []Link { // line gives the file line of a link, as bm writes it: the URL without // white space, the description without tabs and line breaks, and the tags -// with one space between them. It gives "" when the URL is empty or starts -// with "#". -func line(url, desc, tags string) string { - u := strings.Join(strings.Fields(url), "") +// with one space between them. A feed, without white space, is a fourth +// field. It gives "" when the URL is empty or starts with "#". +func line(url, desc, tags, feed string) string { + u := squeeze(url) if u == "" || strings.HasPrefix(u, "#") { return "" } d := strings.TrimSpace(strings.NewReplacer("\t", " ", "\r", " ", "\n", " ").Replace(desc)) - return u + "\t" + d + "\t" + strings.Join(strings.Fields(tags), " ") + s := u + "\t" + d + "\t" + strings.Join(strings.Fields(tags), " ") + if f := squeeze(feed); f != "" { + s += "\t" + f + } + return s +} + +// squeeze gives s without white space. +func squeeze(s string) string { + return strings.Join(strings.Fields(s), "") +} + +// fields gives the URL, description, tags and feed of a line. A line without +// a tab has the old format "URL description". +func fields(row string) []string { + if !strings.Contains(row, "\t") { + words := strings.Fields(row) + return []string{words[0], strings.Join(words[1:], " "), "", ""} + } + return append(strings.SplitN(row, "\t", 5), "", "", "")[:4] } // importRows gives the link lines of an imported file: the Netscape HTML @@ -244,43 +289,67 @@ func folderTag(name string) string { } // merge adds to text the rows whose URL is not in text and not in a row -// before them, as bm --merge does. It gives the new text, the number of rows -// that it added, and the number that it skipped as duplicates. -func merge(text string, rows []string) (string, int, int) { +// before them, as bm --merge does. A row whose URL is a link already gives +// its feed to that link, when the link has no feed. merge gives the new text, +// the number of rows that it added, the number that it skipped as +// duplicates, and the number of links that got a feed. +func merge(text string, rows []string) (string, int, int, int) { ls := lines(text) - seen := map[string]bool{} - for _, s := range ls { - seen[key(s)] = true + at := map[string]int{} + for i, s := range ls { + if k := key(s); k != "" { + if _, ok := at[k]; !ok { + at[k] = i + } + } } - added, skipped := 0, 0 + added, skipped, feeds := 0, 0, 0 for _, row := range rows { k := key(row) - switch { - case k == "": - case seen[k]: - skipped++ - default: - seen[k] = true - f := append(strings.SplitN(row, "\t", 4), "", "") - if !strings.Contains(row, "\t") { - words := strings.Fields(row) - f = []string{words[0], strings.Join(words[1:], " "), ""} - } - ls = append(ls, line(f[0], f[1], f[2])) + if k == "" { + continue + } + f := fields(row) + i, ok := at[k] + if !ok { + at[k] = len(ls) + ls = append(ls, line(f[0], f[1], f[2], f[3])) added++ + continue + } + skipped++ + if l, _ := parseLine(ls[i]); l.Feed == "" && squeeze(f[3]) != "" { + ls[i] = line(l.URL, l.Desc, strings.Join(l.Tags, " "), f[3]) + feeds++ } } - return join(ls), added, skipped + return join(ls), added, skipped, feeds } // addDated adds rows to text as merge does, below a date line of t. It // writes the date line only when it adds a row. -func addDated(text string, rows []string, t time.Time) (string, int, int) { - out, added, skipped := merge(join(append(lines(text), dateLine(t))), rows) - if added == 0 { - return text, 0, skipped +func addDated(text string, rows []string, t time.Time) (string, int, int, int) { + out, added, skipped, feeds := merge(join(append(lines(text), dateLine(t))), rows) + switch { + case added > 0: + return out, added, skipped, feeds + case feeds > 0: + out, _, _, _ = merge(text, rows) + return out, 0, skipped, feeds + } + return text, 0, skipped, 0 +} + +// setFeed gives text with feed as the feed of the links whose URL has the +// normal form k. An empty feed removes the feed. +func setFeed(text, k, feed string) string { + ls := lines(text) + for i, s := range ls { + if l, ok := parseLine(s); ok && k != "" && key(s) == k { + ls[i] = line(l.URL, l.Desc, strings.Join(l.Tags, " "), feed) + } } - return out, added, skipped + return join(ls) } // remove gives text without the lines whose URL has the normal form k. diff --git a/links_test.go b/links_test.go index a41970f..f126229 100644 --- a/links_test.go +++ b/links_test.go @@ -9,15 +9,27 @@ import ( "pgregory.net/rapid" ) -// genRow makes a file line: a URL, a description and tags, with tabs, -// spaces and line breaks in awkward places. +// genRow makes a file line: a URL, a description, tags and sometimes a feed, +// with tabs, spaces and line breaks in awkward places. func genRow(t *rapid.T) string { host := rapid.SampledFrom([]string{"a.com", "www.a.com", "b.org", "c.net"}).Draw(t, "host") path := rapid.SampledFrom([]string{"", "/", "/x", "/x/", "/y"}).Draw(t, "path") sch := rapid.SampledFrom([]string{"https://", "http://", ""}).Draw(t, "scheme") desc := rapid.StringMatching(`[a-z \t\n]{0,8}`).Draw(t, "desc") tags := rapid.StringMatching(`[a-z ]{0,8}`).Draw(t, "tags") - return line(sch+host+path, desc, tags) + feed := rapid.SampledFrom([]string{"", "", "https://a.com/feed", "b.org/rss", " c.net/atom.xml "}).Draw(t, "feed") + return line(sch+host+path, desc, tags, feed) +} + +// countDates gives the number of date lines of text. +func countDates(text string) int { + n := 0 + for _, s := range lines(text) { + if _, ok := dateOf(s); ok { + n++ + } + } + return n } func TestLineParsesBack(t *testing.T) { @@ -25,15 +37,21 @@ func TestLineParsesBack(t *testing.T) { u := rapid.StringMatching(`[a-z:/. ]{1,12}`).Draw(t, "url") desc := rapid.String().Draw(t, "desc") tags := rapid.StringMatching(`[a-z \t]{0,12}`).Draw(t, "tags") - row := line(u, desc, tags) + feed := rapid.StringMatching(`[a-z:/. \t]{0,12}`).Draw(t, "feed") + row := line(u, desc, tags, feed) if row == "" { return } - if strings.ContainsAny(row, "\n\r") || strings.Count(row, "\t") != 2 { + // A line without a feed has the three fields of bm. + tabs := 2 + if squeeze(feed) != "" { + tabs = 3 + } + if strings.ContainsAny(row, "\n\r") || strings.Count(row, "\t") != tabs { t.Fatalf("bad line %q", row) } l, ok := parseLine(row) - if !ok || l.URL != strings.Join(strings.Fields(u), "") || !slices.Equal(l.Tags, strings.Fields(tags)) { + if !ok || l.URL != squeeze(u) || !slices.Equal(l.Tags, strings.Fields(tags)) || l.Feed != squeeze(feed) { t.Fatalf("line %q parsed as %+v", row, l) } }) @@ -41,9 +59,9 @@ func TestLineParsesBack(t *testing.T) { func TestMergeKeepsOneLineForEachURL(t *testing.T) { rapid.Check(t, func(t *rapid.T) { - base, _, _ := merge("", rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "base")) + base, _, _, _ := merge("", rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "base")) rows := rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "rows") - text, added, skipped := merge(base, rows) + text, added, skipped, _ := merge(base, rows) if added+skipped != len(rows) { t.Fatalf("added %d + skipped %d != %d rows", added, skipped, len(rows)) } @@ -59,15 +77,70 @@ func TestMergeKeepsOneLineForEachURL(t *testing.T) { t.Fatalf("row %q is missing", r) } } - if again, n, _ := merge(text, rows); n != 0 || again != text { + if again, n, _, f := merge(text, rows); n != 0 || f != 0 || again != text { t.Fatalf("second merge changed the file") } }) } +// TestMergeFeeds: after merge, each link has the first feed of its URL: the +// feed of its line in the text, else the feed of the first row with that +// URL that has a feed. +func TestMergeFeeds(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + base, _, _, _ := merge("", rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "base")) + rows := rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "rows") + out, _, _, feeds := merge(base, rows) + want, seen, filled := map[string]string{}, map[string]bool{}, 0 + for _, s := range append(lines(base), rows...) { + k := key(s) + l, _ := parseLine(s) + switch { + case k == "": + case !seen[k]: + seen[k], want[k] = true, l.Feed + case want[k] == "" && l.Feed != "": + want[k] = l.Feed + filled++ + } + } + if feeds != filled { + t.Fatalf("merge gave %d feeds, want %d", feeds, filled) + } + for _, s := range lines(out) { + if l, _ := parseLine(s); l.Feed != want[key(s)] { + t.Fatalf("%q has the feed %q, want %q", s, l.Feed, want[key(s)]) + } + } + }) +} + +func TestSetFeed(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + text, _, _, _ := merge("", rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "rows")) + k := key(genRow(t)) + feed := rapid.SampledFrom([]string{"", "https://x.org/feed", " y.org/rss "}).Draw(t, "feed") + old, ls := lines(text), lines(setFeed(text, k, feed)) + if len(ls) != len(old) { + t.Fatalf("%d lines, want %d", len(ls), len(old)) + } + for i := range old { + a, _ := parseLine(old[i]) + b, _ := parseLine(ls[i]) + want := a.Feed + if key(old[i]) == k { + want = squeeze(feed) + } + if b.URL != a.URL || b.Desc != a.Desc || !slices.Equal(b.Tags, a.Tags) || b.Feed != want { + t.Fatalf("line %q became %q, want the feed %q", old[i], ls[i], want) + } + } + }) +} + func TestRemove(t *testing.T) { rapid.Check(t, func(t *rapid.T) { - text, _, _ := merge("", rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "rows")) + text, _, _, _ := merge("", rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "rows")) k := key(genRow(t)) out := remove(text, k) var want []string @@ -111,21 +184,25 @@ func TestParseDates(t *testing.T) { func TestAddDatedDatesOnlyTheNewLinks(t *testing.T) { rapid.Check(t, func(t *rapid.T) { - base, _, _ := merge("", rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "base")) + base, _, _, _ := merge("", rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "base")) rows := rapid.SliceOf(rapid.Custom(genRow)).Draw(t, "rows") when := time.Unix(rapid.Int64Range(0, 4102444800).Draw(t, "when"), 0) - out, added, _ := addDated(base, rows, when) + out, added, _, feeds := addDated(base, rows, when) old, ls := parse(base), parse(out) + dates := countDates(base) + if added > 0 { + dates++ + } switch { - case added == 0 && out != base: - t.Fatalf("no link added, but the text changed to %q", out) - case added > 0 && !strings.HasPrefix(out, base+dateLine(when)+"\n"): - t.Fatalf("no date line after the old links in %q", out) + case added == 0 && feeds == 0 && out != base: + t.Fatalf("no link and no feed added, but the text changed to %q", out) + case countDates(out) != dates: + t.Fatalf("%d date lines in %q, want %d", countDates(out), out, dates) case len(ls) != len(old)+added: t.Fatalf("%d links, want %d + %d", len(ls), len(old), added) } for i, l := range ls { - if i < len(old) && !l.Added.Equal(old[i].Added) || i >= len(old) && !l.Added.Equal(when) { + if i < len(old) && (l.URL != old[i].URL || !l.Added.Equal(old[i].Added)) || i >= len(old) && !l.Added.Equal(when) { t.Fatalf("link %d %q has the time %v", i, l.URL, l.Added) } } @@ -137,6 +214,7 @@ func TestHrefIsOnlyHTTP(t *testing.T) { "https://a.com": "https://a.com", "HTTP://a.com": "HTTP://a.com", "a.com/x": "https://a.com/x", + "": "", "javascript:alert(1)": "", "data:text/html,x": "", "ftp://a.com": "", @@ -145,6 +223,21 @@ func TestHrefIsOnlyHTTP(t *testing.T) { if got := (Link{URL: u}).Href(); got != want { t.Errorf("Href(%q) = %q, want %q", u, got, want) } + if got := (Link{Feed: u}).FeedHref(); got != want { + t.Errorf("FeedHref(%q) = %q, want %q", u, got, want) + } + } +} + +func TestAddress(t *testing.T) { + for u, want := range map[string]string{ + "https://github.com/equwal/sbm/": "github.com/equwal/sbm", + "http://a.com": "a.com", + "a.com/x?y=1": "a.com/x?y=1", + } { + if got := (Link{URL: u}).Address(); got != want { + t.Errorf("Address(%q) = %q, want %q", u, got, want) + } } } @@ -165,7 +258,7 @@ func TestFromHTML(t *testing.T) { } func TestMergeOldFormat(t *testing.T) { - text, added, _ := merge("", []string{"https://a.com some site"}) + text, added, _, _ := merge("", []string{"https://a.com some site"}) if added != 1 || text != "https://a.com\tsome site\t\n" { t.Fatalf("got %q", text) } diff --git a/main.go b/main.go index 69a98df..65565b2 100644 --- a/main.go +++ b/main.go @@ -1,13 +1,16 @@ // sbm-webpublish publishes an sbm bookmark file as a list of links on a // web site. Everyone can get the links as sbm lines, as an Atom feed and as // HTML (see public.go). The owner adds, deletes and imports links on -// /links/admin. +// /links/admin, or adds links in a shell on the server: +// +// sbm-webpublish merge < bookmarks package main import ( "crypto/subtle" _ "embed" "errors" + "fmt" "html/template" "io" "io/fs" @@ -79,12 +82,42 @@ func (s *store) change(f func(string) string) error { tmp.Close() return err } + // The links are public. With mode 0644 the service can read the file + // after root runs "sbm-webpublish merge" on the server. + if err := tmp.Chmod(0o644); err != nil { + tmp.Close() + return err + } if err := tmp.Close(); err != nil { return err } return os.Rename(tmp.Name(), s.path) } +// importMessage tells what an import did. +func importMessage(added, skipped, feeds int) string { + m := "Imported " + strconv.Itoa(added) + " links. Skipped " + strconv.Itoa(skipped) + " duplicates." + if feeds > 0 { + m += " Added " + strconv.Itoa(feeds) + " feeds to links that had none." + } + return m +} + +// mergeFrom adds the links of in, as the import page does, and tells what it +// did. in holds sbm lines or the HTML that browsers export. +func mergeFrom(links *store, in io.Reader, now time.Time) (string, error) { + b, err := io.ReadAll(in) + if err != nil { + return "", err + } + var added, skipped, feeds int + err = links.change(func(text string) string { + text, added, skipped, feeds = addDated(text, importRows(string(b)), now) + return text + }) + return importMessage(added, skipped, feeds), err +} + type server struct { links *store password string @@ -135,21 +168,34 @@ func done(w http.ResponseWriter, r *http.Request, message string) { http.Redirect(w, r, "../admin?m="+template.URLQueryEscaper(message), http.StatusSeeOther) } +// badFeed is true for a feed that the outputs cannot show: an address that +// is not http or https. +func badFeed(feed string) bool { + f := squeeze(feed) + return f != "" && href(f) == "" +} + func (s *server) add(w http.ResponseWriter, r *http.Request) { - row := line(r.FormValue("url"), r.FormValue("desc"), r.FormValue("tags")) - if row == "" { + row := line(r.FormValue("url"), r.FormValue("desc"), r.FormValue("tags"), r.FormValue("feed")) + switch { + case row == "": done(w, r, "Give a URL.") return + case badFeed(r.FormValue("feed")): + done(w, r, "Give an http or https address for the feed.") + return } - var added int + var added, feeds int err := s.links.change(func(text string) string { - text, added, _ = addDated(text, []string{row}, s.now()) + text, added, _, feeds = addDated(text, []string{row}, s.now()) return text }) switch { case err != nil: log.Print(err) http.Error(w, "cannot write the links", http.StatusInternalServerError) + case feeds > 0: + done(w, r, "That URL is a link already. It got the feed.") case added == 0: done(w, r, "That URL is a link already.") default: @@ -157,6 +203,22 @@ func (s *server) add(w http.ResponseWriter, r *http.Request) { } } +// editFeed sets the feed of one link. An empty feed removes the feed. +func (s *server) editFeed(w http.ResponseWriter, r *http.Request) { + feed := r.FormValue("feed") + if badFeed(feed) { + done(w, r, "Give an http or https address for the feed.") + return + } + k := norm(r.FormValue("url")) + if err := s.links.change(func(text string) string { return setFeed(text, k, feed) }); err != nil { + log.Print(err) + http.Error(w, "cannot write the links", http.StatusInternalServerError) + return + } + done(w, r, "Saved the feed.") +} + func (s *server) delete(w http.ResponseWriter, r *http.Request) { k := norm(r.FormValue("url")) if err := s.links.change(func(text string) string { return remove(text, k) }); err != nil { @@ -175,22 +237,13 @@ func (s *server) importFile(w http.ResponseWriter, r *http.Request) { return } defer f.Close() - b, err := io.ReadAll(f) - if err != nil { - done(w, r, "Cannot read the file.") - return - } - var added, skipped int - err = s.links.change(func(text string) string { - text, added, skipped = addDated(text, importRows(string(b)), s.now()) - return text - }) + message, err := mergeFrom(s.links, f, s.now()) if err != nil { log.Print(err) http.Error(w, "cannot write the links", http.StatusInternalServerError) return } - done(w, r, "Imported "+strconv.Itoa(added)+" links. Skipped "+strconv.Itoa(skipped)+" duplicates.") + done(w, r, message) } // routes gives the handler of the server. The paths start with /links/, @@ -203,6 +256,7 @@ func (s *server) routes() http.Handler { mux.HandleFunc("GET /links/admin", s.admin(s.adminPage)) mux.HandleFunc("POST /links/admin/add", s.admin(s.add)) mux.HandleFunc("POST /links/admin/delete", s.admin(s.delete)) + mux.HandleFunc("POST /links/admin/feed", s.admin(s.editFeed)) mux.HandleFunc("POST /links/admin/import", s.admin(s.importFile)) // The admin forms use basic auth, which the browser sends with each // request. CrossOriginProtection refuses a POST from another site. @@ -217,11 +271,27 @@ func env(name, fallback string) string { } func main() { + links := &store{path: env("LINKS_FILE", "links.sbm")} + if len(os.Args) > 1 { + // "merge" adds the links of stdin, as bm --merge does. It does not + // lock the file against the service: do not use the admin page at + // the same time. + if os.Args[1] != "merge" || len(os.Args) > 2 { + fmt.Fprintln(os.Stderr, "usage: sbm-webpublish [merge < bookmarks]") + os.Exit(2) + } + message, err := mergeFrom(links, os.Stdin, time.Now()) + if err != nil { + log.Fatal(err) + } + fmt.Println(message) + return + } pw := os.Getenv("ADMIN_PASSWORD") if pw == "" { log.Fatal("set ADMIN_PASSWORD") } - s := &server{links: &store{path: env("LINKS_FILE", "links.sbm")}, password: pw, now: time.Now} + s := &server{links: links, password: pw, now: time.Now} addr := env("ADDR", "127.0.0.1:8082") log.Printf("listening on %s, links file %s", addr, s.links.path) log.Fatal(http.ListenAndServe(addr, s.routes())) diff --git a/public.go b/public.go index c9a32f3..8949a23 100644 --- a/public.go +++ b/public.go @@ -20,11 +20,12 @@ import ( ) // published gives the links of text that the outputs show, in the order of -// the file. The URL of each link is the address that Href gives. +// the file. The URL of each link is the address that Href gives, and the +// feed is the address that FeedHref gives. func published(text string) []Link { var out []Link for _, l := range parse(text) { - if l.URL = l.Href(); l.URL != "" { + if l.URL, l.Feed = l.Href(), l.FeedHref(); l.URL != "" { out = append(out, l) } } @@ -45,13 +46,14 @@ func (s *server) serve(w http.ResponseWriter, r *http.Request, contentType strin http.ServeContent(w, r, "", mod, bytes.NewReader(render(published(text), mod))) } -// plain sends the links as sbm lines, oldest first: URL, description and -// tags, with a tab between them. +// plain sends the links as sbm lines, oldest first: URL, description, tags +// and feed, with a tab between them. A link without a feed has no fourth +// field. func (s *server) plain(w http.ResponseWriter, r *http.Request) { s.serve(w, r, "text/plain; charset=utf-8", func(ls []Link, _ time.Time) []byte { var b strings.Builder for _, l := range ls { - b.WriteString(line(l.URL, l.Desc, strings.Join(l.Tags, " ")) + "\n") + b.WriteString(line(l.URL, l.Desc, strings.Join(l.Tags, " "), l.Feed) + "\n") } return []byte(b.String()) }) @@ -166,13 +168,14 @@ func groups(ls []Link) []group { var listTmpl = template.Must(template.New("list").Parse(`{{with .}}<p>{{range .}}<a href="#{{.Tag}}">{{.Tag}}</a> {{end}}</p> {{range .}}<h2 id="{{.Tag}}">{{.Tag}}</h2> <ul> -{{range .Links}}<li><a href="{{.URL}}">{{.Title}}</a> <small>{{.Host}}</small></li> +{{range .Links}}<li><a href="{{.URL}}">{{.Title}}</a> <small>{{.Address}}</small>{{with .Feed}} <a class="feed" href="{{.}}">rss</a>{{end}}</li> {{end}}</ul> {{end}}{{else}}<p>No links yet.</p> {{end}}`)) -// list sends the links as an HTML fragment, under each of their tags. A page -// of the site includes it with nginx SSI, so the page needs no script: +// list sends the links as an HTML fragment, under each of their tags. A link +// with a feed has an "rss" link to the feed after it. A page of the site +// includes the fragment with nginx SSI, so the page needs no script: // // <!--# include virtual="/links/list.html" --> func (s *server) list(w http.ResponseWriter, r *http.Request) { diff --git a/server_test.go b/server_test.go index 04bf1c8..045df11 100644 --- a/server_test.go +++ b/server_test.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "slices" "strings" "testing" @@ -75,6 +76,7 @@ func TestAdminNeedsPassword(t *testing.T) { for _, r := range []*http.Request{ httptest.NewRequest("GET", "/links/admin", nil), form("/links/admin/add", url.Values{"url": {"https://a.com"}}), + form("/links/admin/feed", url.Values{"url": {"https://a.com"}, "feed": {"https://a.com/rss"}}), } { if w := do(h, r, false); w.Code != http.StatusUnauthorized { t.Errorf("%s %s: code %d", r.Method, r.URL, w.Code) @@ -92,7 +94,7 @@ func TestAddAndDelete(t *testing.T) { if body := get(h, "/links/links.txt").Body.String(); body != "" { t.Fatalf("empty store gave %q", body) } - do(h, form("/links/admin/add", url.Values{"url": {"https://a.com"}, "desc": {"A"}, "tags": {"x y"}}), true) + do(h, form("/links/admin/add", url.Values{"url": {"https://a.com"}, "desc": {"A"}, "tags": {"x y"}, "feed": {"https://a.com/rss"}}), true) w := do(h, form("/links/admin/add", url.Values{"url": {"b.com"}}), true) if w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/links/admin?m=Added." { t.Fatalf("add: %d %q", w.Code, w.Header().Get("Location")) @@ -101,23 +103,57 @@ func TestAddAndDelete(t *testing.T) { if !strings.Contains(w.Header().Get("Location"), "already") { t.Fatalf("duplicate add: %q", w.Header().Get("Location")) } - if body := get(h, "/links/links.txt").Body.String(); body != "https://a.com\tA\tx y\nhttps://b.com\t\t\n" { + if body := get(h, "/links/links.txt").Body.String(); body != "https://a.com\tA\tx y\thttps://a.com/rss\nhttps://b.com\t\t\n" { t.Fatalf("links.txt: %q", body) } do(h, form("/links/admin/delete", url.Values{"url": {"https://b.com"}}), true) // The date line of the deleted link stays. bm ignores it. b, _ := os.ReadFile(s.links.path) - if string(b) != dayLine+"https://a.com\tA\tx y\n"+dayLine { + if string(b) != dayLine+"https://a.com\tA\tx y\thttps://a.com/rss\n"+dayLine { t.Fatalf("file: %q", b) } } +func location(w *httptest.ResponseRecorder) string { + m, _ := url.ParseQuery(strings.TrimPrefix(w.Header().Get("Location"), "/links/admin?")) + return m.Get("m") +} + +func TestAddGivesTheFeedToALinkWithout(t *testing.T) { + s, h := newServer(t) + do(h, form("/links/admin/add", url.Values{"url": {"https://a.com"}, "desc": {"A"}}), true) + w := do(h, form("/links/admin/add", url.Values{"url": {"a.com/"}, "desc": {"other"}, "feed": {"https://a.com/rss"}}), true) + if m := location(w); m != "That URL is a link already. It got the feed." { + t.Fatalf("message %q", m) + } + // One date line only: no link was added the second time. + if b, _ := os.ReadFile(s.links.path); string(b) != dayLine+"https://a.com\tA\t\thttps://a.com/rss\n" { + t.Fatalf("file: %q", b) + } +} + +func TestEditFeed(t *testing.T) { + s, h := newServer(t) + write(t, s, dayLine+"https://a.com\tA\tx\nhttps://b.com\tB\t\n") + for _, c := range []struct{ feed, message, file string }{ + {"https://a.com/rss", "Saved the feed.", dayLine + "https://a.com\tA\tx\thttps://a.com/rss\nhttps://b.com\tB\t\n"}, + {"javascript:alert(1)", "Give an http or https address for the feed.", dayLine + "https://a.com\tA\tx\thttps://a.com/rss\nhttps://b.com\tB\t\n"}, + {"", "Saved the feed.", dayLine + "https://a.com\tA\tx\nhttps://b.com\tB\t\n"}, + } { + w := do(h, form("/links/admin/feed", url.Values{"url": {"https://a.com"}, "feed": {c.feed}}), true) + b, _ := os.ReadFile(s.links.path) + if m := location(w); m != c.message || string(b) != c.file { + t.Errorf("feed %q: message %q, file %q", c.feed, m, b) + } + } +} + func TestOutputsHideUnsafeLinks(t *testing.T) { s, h := newServer(t) - write(t, s, "javascript:alert(1)\tx\t\ndata:text/html,x\ty\t\nhttps://a.com\tA\t\n") + write(t, s, "javascript:alert(1)\tx\t\ndata:text/html,x\ty\t\nhttps://a.com\tA\t\nhttps://b.com\tB\t\tjavascript:alert(2)\n") for _, path := range []string{"/links/links.txt", "/links/atom.xml", "/links/list.html"} { body := get(h, path).Body.String() - if strings.Contains(body, "javascript") || strings.Contains(body, "data:") || !strings.Contains(body, "https://a.com") { + if strings.Contains(body, "javascript") || strings.Contains(body, "data:") || !strings.Contains(body, "https://a.com") || !strings.Contains(body, "https://b.com") { t.Errorf("%s: %s", path, body) } } @@ -165,7 +201,8 @@ func genPublicRow(t *rapid.T) string { path := rapid.StringMatching(`[a-z]{0,3}`).Draw(t, "path") desc := rapid.StringMatching(`[a-z<>&"' \]\t]{0,10}`).Draw(t, "desc") tags := rapid.StringMatching(`[a-z<>&" ]{0,8}`).Draw(t, "tags") - return line(u+path, desc, tags) + feed := rapid.SampledFrom([]string{"", "https://a.com/feed", "http://b.org/rss?x=<1>&y", "c.net/atom.xml", "javascript:alert(2)"}).Draw(t, "feed") + return line(u+path, desc, tags, feed) } // TestOutputsRoundTrip: links.txt parses back to the published links, and the @@ -177,7 +214,7 @@ func TestOutputsRoundTrip(t *testing.T) { for range rapid.IntRange(0, 3).Draw(t, "batches") { rows := rapid.SliceOf(rapid.Custom(genPublicRow)).Draw(t, "rows") when := time.Unix(rapid.Int64Range(0, 4102444800).Draw(t, "when"), 0) - text, _, _ = addDated(text, rows, when) + text, _, _, _ = addDated(text, rows, when) } write(t, s, text) want := published(text) @@ -187,7 +224,7 @@ func TestOutputsRoundTrip(t *testing.T) { t.Fatalf("links.txt has %d links, want %d", len(got), len(want)) } for i := range want { - if got[i].URL != want[i].URL || got[i].Desc != want[i].Desc || !slices.Equal(got[i].Tags, want[i].Tags) { + if got[i].URL != want[i].URL || got[i].Desc != want[i].Desc || !slices.Equal(got[i].Tags, want[i].Tags) || got[i].Feed != want[i].Feed { t.Fatalf("links.txt link %d is %+v, want %+v", i, got[i], want[i]) } } @@ -235,16 +272,16 @@ func TestList(t *testing.T) { if body := get(h, "/links/list.html").Body.String(); body != "<p>No links yet.</p>\n" { t.Fatalf("empty list: %q", body) } - write(t, s, "https://a.com\t<A>\tx y\nhttps://b.com\tB\t\nhttps://c.com\tC\ty y\n") + write(t, s, "https://a.com/x/\t<A>\tx y\thttps://a.com/rss\nhttps://b.com\tB\t\nhttps://c.com\tC\ty y\n") want := `<p><a href="#x">x</a> <a href="#y">y</a> <a href="#untagged">untagged</a> </p> <h2 id="x">x</h2> <ul> -<li><a href="https://a.com"><A></a> <small>a.com</small></li> +<li><a href="https://a.com/x/"><A></a> <small>a.com/x</small> <a class="feed" href="https://a.com/rss">rss</a></li> </ul> <h2 id="y">y</h2> <ul> <li><a href="https://c.com">C</a> <small>c.com</small></li> -<li><a href="https://a.com"><A></a> <small>a.com</small></li> +<li><a href="https://a.com/x/"><A></a> <small>a.com/x</small> <a class="feed" href="https://a.com/rss">rss</a></li> </ul> <h2 id="untagged">untagged</h2> <ul> @@ -262,20 +299,38 @@ func TestImport(t *testing.T) { var body bytes.Buffer mw := multipart.NewWriter(&body) f, _ := mw.CreateFormFile("file", "bookmarks.sbm") - f.Write([]byte("https://a.com/\tagain\t\nhttps://b.com\tB\tt\n")) + f.Write([]byte("https://a.com/\tagain\t\thttps://a.com/rss\nhttps://b.com\tB\tt\n")) mw.Close() r := httptest.NewRequest("POST", "/links/admin/import", &body) r.Header.Set("Content-Type", mw.FormDataContentType()) w := do(h, r, true) - if loc := w.Header().Get("Location"); !strings.Contains(loc, "Imported+1+links.+Skipped+1") { - t.Fatalf("import: %d %q", w.Code, loc) + if m := location(w); m != "Imported 1 links. Skipped 1 duplicates. Added 1 feeds to links that had none." { + t.Fatalf("import: %d %q", w.Code, m) } + // The link a.com keeps its description, and gets the feed. b, _ := os.ReadFile(s.links.path) - if string(b) != "https://a.com\tA\t\n"+dayLine+"https://b.com\tB\tt\n" { + if string(b) != "https://a.com\tA\t\thttps://a.com/rss\n"+dayLine+"https://b.com\tB\tt\n" { t.Fatalf("file: %q", b) } } +func TestMergeCommand(t *testing.T) { + s, _ := newServer(t) + write(t, s, "https://a.com\tA\t\n") + m, err := mergeFrom(s.links, strings.NewReader("https://b.com\tB\tt\thttps://b.com/rss\nhttps://a.com\tagain\t\n"), day) + if err != nil || m != "Imported 1 links. Skipped 1 duplicates." { + t.Fatalf("%q %v", m, err) + } + b, _ := os.ReadFile(s.links.path) + if string(b) != "https://a.com\tA\t\n"+dayLine+"https://b.com\tB\tt\thttps://b.com/rss\n" { + t.Fatalf("file: %q", b) + } + // The service must be able to read a file that root wrote. + if st, err := os.Stat(s.links.path); runtime.GOOS != "windows" && (err != nil || st.Mode().Perm() != 0o644) { + t.Fatalf("mode %v %v", st.Mode(), err) + } +} + func TestCrossOriginPostRefused(t *testing.T) { s, h := newServer(t) r := form("/links/admin/add", url.Values{"url": {"https://evil.com"}})