commit 71ab31e7a64250413289c0c472dd5e81f1a4ba49 equwal <13551856+equwal@users.noreply.github.com> 2026-09-21 12:30:08 -0700 ideamine 0.5.0: search by meaning, groups, and a dashboard /ideas-find searches by meaning and /ideas-groups groups the ideas by meaning. The hook answers both with no model call. idea_list gets the parameters semantic and groups, so Claude can do the same. Embeddings come from nomic-embed-text through an OpenAI-compatible /v1/embeddings server (Ollama, or the llama.cpp server). The design follows memstate: the search_document and search_query prefixes, one cached vector for each idea (a content hash tells when to embed again), cosine similarity with a threshold, and word search when the server does not answer. A text that is too long is halved until the server takes it. Groups use average-linkage clustering on the same vectors, so no graph database is necessary. The thresholds (0.5 for search, 0.65 for groups) come from the 17 real ideas: an unrelated query stays below 0.46. ideamine publish uploads index.html and data.json with HTTP PUT. The page shows a board of tickets, a timeline, the groups, and search by meaning. After the first publish, the hook starts a background publish when the archive changes, like the watcher. updateIdea records started when an idea goes to doing, for the timeline. Settings are in ~/.ideamine/config.json (ideamine config), and environment variables override them. fast-check is a new development dependency for the property tests of the vector codec and the clustering. The plugin itself still installs nothing.
.claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- README.md | 101 ++++- bin/ideamine.js | 56 ++- dashboard/index.html | 899 ++++++++++++++++++++++++++++++++++++++++ package-lock.json | 62 +++ package.json | 8 +- skills/ideas-find/SKILL.md | 11 + skills/ideas-groups/SKILL.md | 11 + skills/ideas/SKILL.md | 2 +- src/config.js | 89 ++++ src/embed.js | 318 ++++++++++++++ src/hook.js | 57 ++- src/mcp.js | 24 +- src/publish.js | 234 +++++++++++ src/render.js | 23 + src/store.js | 39 +- tests/embed.test.js | 163 ++++++++ tests/fixtures/fake-server.js | 51 +++ tests/hook.test.js | 41 +- tests/mcp.test.js | 2 +- tests/publish.test.js | 126 ++++++ tests/store.test.js | 12 + 23 files changed, 2285 insertions(+), 48 deletions(-)
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c72f57c..51b5526 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ { "name": "ideamine", "description": "An idea inbox for Claude Code. /idea saves an idea without a model call. Claude triages the ideas and picks the cheapest model that can build each one.", - "version": "0.4.0", + "version": "0.5.0", "author": { "name": "equwal" }, diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 5e4f1f7..b449e53 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ideamine", - "version": "0.4.0", + "version": "0.5.0", "description": "An idea inbox for Claude Code. /idea saves an idea without a model call, so it works at any time, even at your usage limit. Claude triages the ideas and picks the cheapest model that can build each one.", "author": { "name": "equwal", diff --git a/README.md b/README.md index 39ad54a..dba01b9 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,8 @@ Marketplaces you add yourself do not auto-update. To upgrade, run `claude plugin | `/ideas-rm 12 14` | Delete ideas for good | **No** | | `/ideas-done 12 shipped it` | Mark an idea done, with a note | **No** | | `/ideas-reopen 12` | Put an idea back in the queue, for example one that the triage skipped | **No** | +| `/ideas-find sync subtitles` | Search every lane by meaning, not only by the words. See [Search by meaning](#search-by-meaning-and-groups). | **No** | +| `/ideas-groups` · `done` · `-a` | Show the ideas grouped by meaning | **No** | | `/ideas-go [12]` | Build the idea that fits this chat, else the first in the queue, or #12, on its recommended model. New ideas are triaged first. | Yes, this is the build | | `/ideas-all` | Claude reads every idea, takes the ones that fit this chat out of the queue, and does them. The others stay in the queue. | Yes, this is the build | | `/ideas-sort` | Triage the inbox now and show the queue. You do not have to: `/ideas-go` triages when it must. | Yes, briefly | @@ -67,6 +69,68 @@ Run `/ideas-watch` once. After that, a few seconds after you save an idea, Haiku The watcher is not a process that stays alive, because a process like that can stop: a crash, a reboot, a full context, or a usage limit. Instead, the ideamine hook starts a short background pass when you send a prompt and there is work. Thus the watcher keeps going while you use Claude Code, and it costs nothing while no new ideas come in. A failed call gets 2 more tries in the same pass. After a failed pass, the watcher waits 10 minutes before it tries again. `/ideas-watch` shows the last passes, and `/ideas-watch off` turns the watcher off. In a terminal, `ideamine watch` does the same. The log is `~/.ideamine/watch.log`. +## Search by meaning and groups + +``` +> /ideas-find make money + ideas like "make money", by meaning + 53% done #4 sonnet S ▲3 Research monetization models for device-hacking utility apps +``` + +`/ideas-find` finds ideas that mean the same thing as your words, also when they use other words. `/ideas-groups` puts ideas that are about the same thing in one group, for example all the ideas for one app. Both come from embeddings: an embedding model turns each idea into a vector, and ideas with similar meaning get similar vectors. The default model is [nomic-embed-text](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5). The design follows [memstate](https://github.com/map588/memstate): the `search_document:` and `search_query:` task prefixes of nomic, one cached vector for each idea that ideamine computes again when the idea changes, cosine similarity with a threshold, and word search when the embedding server does not answer. The groups and the related ideas come from the same vectors, so ideamine needs no graph database. + +ideamine sends the text of each new or changed idea to an embedding server that has an OpenAI-compatible `/v1/embeddings` endpoint. Two servers work: + +- [Ollama](https://ollama.com) on your machine: `ollama pull nomic-embed-text`. This is the default (`http://127.0.0.1:11434/v1`). +- The [llama.cpp](https://github.com/ggml-org/llama.cpp) server on another machine, for example a small VPS. The model file is [`nomic-embed-text-v1.5.Q8_0.gguf`](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF) (146 MB). With `-c 512` the server uses about 20 MB of memory plus the model file, which the kernel can page out: + + ```bash + llama-server -m nomic-embed-text-v1.5.Q8_0.gguf --embeddings --pooling mean -c 512 -b 512 -ub 512 --alias nomic-embed-text --host 127.0.0.1 --port 8081 + ``` + +Then tell ideamine where the server is: + +```bash +ideamine config embed_url http://10.66.0.1/v1 +``` + +```bash +ideamine embed +``` + +`ideamine embed` embeds the ideas that have no vector yet. It also shows the similarity of each idea to its nearest neighbour, like `memstated embed status`. Use these numbers to set the thresholds for a new model. With nomic-embed-text, a search result must have a similarity of 0.5 or more (`search_threshold`), and the ideas in a group must have a mean similarity of 0.65 or more (`group_threshold`). A group gets a label from the words that its ideas share and other ideas do not use. If the server does not answer, `/ideas-find` searches by words and tells you why, and `/ideas-groups` tells you that it needs the server. The vectors are in `~/.ideamine/vectors.json`. When you delete an idea, its vector goes too. + +Claude can search by meaning with `idea_list` and `semantic: true`, and group with `groups: true`. + +## Dashboard + +`ideamine publish` makes a web page of your ideas: a board with a ticket for each idea, a timeline (a Gantt chart) of how long each idea waited and how long the work took, the groups, and search by meaning. A ticket opens a preview with the brief, the notes, and the related ideas. The page is one static file, `index.html`, and it reads a snapshot, `data.json`. It loads nothing from the internet. + +```bash +ideamine publish http://10.66.0.1/ +``` + +This uploads both files with HTTP PUT. It also saves the address, so the ideamine hook publishes again in the background after each change, at no model cost. `ideamine publish off` stops that. `ideamine publish --dir <folder>` writes the two files to a folder instead. The timeline uses the time when an idea went to `doing`. ideamine records that time from 0.5.0 on, so for older ideas the timeline shows an estimated bar. + +The snapshot contains the full text of your ideas. Serve it only where only you can open it. For example, this nginx site answers only on the address of a WireGuard interface, and it accepts uploads only from one peer: + +```nginx +server { + listen 10.66.0.1:80; # the WireGuard address of the server + allow 10.66.0.0/24; # WireGuard peers only + deny all; + root /var/www/ideamine; + location ~ ^/(index\.html|data\.json)$ { + limit_except GET HEAD { allow 10.66.0.2; deny all; } # only your PC uploads + dav_methods PUT; + client_max_body_size 16m; + } + location /v1/ { proxy_pass http://127.0.0.1:8081; } # the llama.cpp server, for search on the page +} +``` + +`data.json` has `version` (1), `generated`, `embed` (model, query prefix, thresholds, and whether vectors are present), `counts`, `ideas`, and `groups`. Each idea has its ticket key (`IDEA-12`), lane, rank in the queue, triage, times (`created`, `triaged`, `started`, `closed`), timeline `phases`, notes, `group`, `related` ideas with their similarity, and `vec`, the vector as base64 of little-endian float32. + ## Model routing The triage gives each idea a verdict (`do`, `maybe`, `skip`), an impact from 1 to 5, a size from `xs` to `xl`, a one-line reason, and a short brief that an agent can act on without the original chat. It also picks the cheapest model that is likely to finish the idea in one pass. If a weaker model fails and has to retry, that costs more than using the right model once. @@ -106,6 +170,11 @@ ideamine next # what to build next ideamine go 12 # opens Claude Code on the right model, in the idea's project ideamine sort # headless triage (see above) ideamine watch [off] # the watcher (see above) +ideamine find sync subtitles # search by meaning +ideamine groups [-a] # ideas grouped by meaning +ideamine embed # embed new ideas, show the similarity numbers +ideamine publish [url|off] [--dir folder] # the dashboard (see above) +ideamine config [key [value]] # show or change a setting ideamine export IDEAS.md # Markdown copy of everything ``` @@ -125,14 +194,21 @@ Tools: `idea_add`, `idea_list`, `idea_triage`, `idea_update`, `idea_next`, `idea ## Where your ideas live -`~/.ideamine/ideas.json` is plain, readable JSON. Set `IDEAMINE_HOME` to move it, for example into a synced folder. Each write takes a lock and then replaces the file in one step, so many sessions can write at the same time without losing an idea. The previous version is kept as `ideas.json.bak`. If the file becomes damaged, ideamine stops and does not overwrite it. Nothing leaves your machine, except when you run triage or a build, which go through Claude as usual. So that it can pair ideas with projects, the triage also sends the paths of your project folders and the first line of each README. +`~/.ideamine/ideas.json` is plain, readable JSON. Set `IDEAMINE_HOME` to move it, for example into a synced folder. Each write takes a lock and then replaces the file in one step, so many sessions can write at the same time without losing an idea. The previous version is kept as `ideas.json.bak`. If the file becomes damaged, ideamine stops and does not overwrite it. Nothing leaves your machine, except in these cases: triage and builds go through Claude as usual; search by meaning and groups send the text of your ideas to the embedding server that you set; `ideamine publish` uploads a snapshot to the dashboard server that you set. So that it can pair ideas with projects, the triage also sends the paths of your project folders and the first line of each README. -| Variable | Default | Purpose | -|---|---|---| -| `IDEAMINE_HOME` | `~/.ideamine` | archive location | -| `IDEAMINE_TRIAGE_MODEL` | `sonnet` | model for the headless triage | -| `IDEAMINE_CLAUDE_BIN` | `claude` | Claude Code executable | -| `IDEAMINE_SETTING_SOURCES` | *(empty)* | set to `user` if your login needs `settings.json` (e.g. `apiKeyHelper`) | +`ideamine config` shows each setting and where its value comes from. `ideamine config <key> <value>` saves a setting in `~/.ideamine/config.json`, and an empty value restores the default. An environment variable wins over the file. + +| Variable | Setting | Default | Purpose | +|---|---|---|---| +| `IDEAMINE_HOME` | | `~/.ideamine` | archive location | +| `IDEAMINE_TRIAGE_MODEL` | | `sonnet` | model for the headless triage | +| `IDEAMINE_CLAUDE_BIN` | | `claude` | Claude Code executable | +| `IDEAMINE_SETTING_SOURCES` | | *(empty)* | set to `user` if your login needs `settings.json` (e.g. `apiKeyHelper`) | +| `IDEAMINE_EMBED_URL` | `embed_url` | `http://127.0.0.1:11434/v1` | OpenAI-compatible embeddings API (Ollama, llama.cpp server) | +| `IDEAMINE_EMBED_MODEL` | `embed_model` | `nomic-embed-text` | embedding model; nomic models get their task prefixes | +| `IDEAMINE_SEARCH_THRESHOLD` | `search_threshold` | `0.5` | lowest similarity of a search result | +| `IDEAMINE_GROUP_THRESHOLD` | `group_threshold` | `0.65` | lowest mean similarity in a group, and of a related idea | +| `IDEAMINE_PUBLISH_URL` | `publish_url` | *(off)* | dashboard server for `ideamine publish` | ## How it works @@ -146,17 +222,24 @@ Tools: `idea_add`, `idea_list`, `idea_triage`, `idea_update`, `idea_next`, `idea /ideas-all ──► Claude ──► MCP idea_list (full) ──► idea_remove for the ideas that fit this chat ──► builds them watcher on: any prompt ──► hook ──► background pass ──► claude -p (Haiku) ──► verdicts + a project for each idea + +/ideas-find, /ideas-groups ──► hook ──► embedding server (new ideas only) ──► cosine similarity ──► answer +publish on: any prompt after a change ──► hook ──► background publish ──► PUT index.html + data.json ``` -The plugin contains a Node MCP server with no dependencies, eleven skills (the slash commands), and one hook. The hook answers `/idea`, `/ideas`, and the local `/ideas-*` commands before any API call, and it lets every other prompt through. The hook runs directly, not through a shell, and takes about 130 ms per prompt on Windows. The skills are user-only, so their descriptions add no tokens to your sessions. If the archive cannot be read, the hook lets the prompt through, so the `/idea` skill can still save it with the MCP tool. Your text is never dropped. +The plugin contains a Node MCP server with no dependencies, thirteen skills (the slash commands), and one hook. The hook answers `/idea`, `/ideas`, and the local `/ideas-*` commands before any API call, and it lets every other prompt through. The hook runs directly, not through a shell, and takes about 130 ms per prompt on Windows. The skills are user-only, so their descriptions add no tokens to your sessions. If the archive cannot be read, the hook lets the prompt through, so the `/idea` skill can still save it with the MCP tool. Your text is never dropped. ## Development +```bash +npm install +``` + ```bash npm test ``` -The tests use `node:test` only. They cover the store, including concurrent writers from several processes, the hook, the MCP protocol, headless triage with pairing, and the watcher. Triage runs against a stand-in `claude`, so the tests spend no tokens. +The tests use `node:test`, and [fast-check](https://fast-check.dev) for the property tests. fast-check is a development dependency only: the plugin itself installs nothing. The tests cover the store, including concurrent writers from several processes, the hook, the MCP protocol, headless triage with pairing, the watcher, embeddings and groups, and the dashboard upload. Triage runs against a stand-in `claude`, and embeddings and uploads run against a stand-in server, so the tests spend no tokens and need no network. ## License diff --git a/bin/ideamine.js b/bin/ideamine.js index 2cef36f..60050e2 100644 --- a/bin/ideamine.js +++ b/bin/ideamine.js @@ -18,6 +18,13 @@ const HELP = `ideamine: an idea inbox for Claude Code triage the inbox with one headless \`claude -p\` call ideamine watch [off] the watcher: Haiku triages new ideas and pairs them with projects ideamine watch-pass one pass of the watcher, now, in the foreground + ideamine find <words...> search by meaning (nomic-embed-text), in every lane + ideamine groups [lane|-a] ideas grouped by meaning + ideamine embed embed new ideas, and show how the vectors and thresholds fit + ideamine publish [url|off] [--dir <folder>] + upload the dashboard (index.html, data.json) now; a url also turns + on the upload after each change; --dir writes the files to a folder + ideamine config [key [value]] show or change a setting (an empty value restores the default) ideamine export [file.md] Markdown export of the whole archive ideamine path where the archive lives (override with IDEAMINE_HOME) ideamine mcp run the MCP server on stdio @@ -31,7 +38,7 @@ function parseArgs(argv) { if (a.startsWith('--')) { const [key, inline] = a.slice(2).split('=', 2); if (inline !== undefined) flags[key] = inline; - else if (argv[i + 1] !== undefined && !argv[i + 1].startsWith('--') && ['model', 'limit', 'budget', 'query'].includes(key)) flags[key] = argv[++i]; + else if (argv[i + 1] !== undefined && !argv[i + 1].startsWith('--') && ['model', 'limit', 'budget', 'query', 'dir'].includes(key)) flags[key] = argv[++i]; else flags[key] = true; } else words.push(a); } @@ -171,6 +178,53 @@ async function main() { if ((await watch.pass()) === 'error') process.exitCode = 1; break; } + case 'find': { + const embed = await import('../src/embed.js'); + const query = words.join(' ') || fail('usage: ideamine find <words...>'); + console.log(render.renderFound(await embed.find(store.load(), query), query, { cwd })); + break; + } + case 'groups': { + const embed = await import('../src/embed.js'); + const lower = words.map((w) => (w === '-a' ? 'all' : w.toLowerCase())); + const filter = lower.find((w) => store.FILTERS.includes(w)) || 'open'; + const ideas = store.listIdeas(store.load(), { filter }); + console.log(render.renderGroups(await embed.groupIdeas(ideas), ideas, { cwd, scope: filter })); + break; + } + case 'embed': { + const embed = await import('../src/embed.js'); + console.log(await embed.status(store.load())); + break; + } + case 'publish': { + const publish = await import('../src/publish.js'); + if (flags.background) { + process.exitCode = (await publish.backgroundPublish()) === 'error' ? 1 : 0; + break; + } + const config = await import('../src/config.js'); + if (words[0] === 'off') { + config.set('publish_url', ''); + console.log(publish.status()); + break; + } + if (words[0]) config.set('publish_url', words[0]); + const out = await publish.publish({ dir: typeof flags.dir === 'string' ? flags.dir : null }); + console.log(`Published ${out.ideas} ideas and ${out.groups} groups to ${out.where}`); + if (out.note) console.log(out.note); + break; + } + case 'config': { + const config = await import('../src/config.js'); + if (words.length >= 1 && rest.length >= 2) config.set(words[0], words.slice(1).join(' ')); + else if (words.length === 1 && rest.length === 1) { + console.log(config.get(words[0])); + break; + } + console.log(config.describe()); + break; + } case 'export': { const md = render.renderMarkdown(store.load()); if (words[0]) { diff --git a/dashboard/index.html b/dashboard/index.html new file mode 100644 index 0000000..54331a5 --- /dev/null +++ b/dashboard/index.html @@ -0,0 +1,899 @@ +<!doctype html> +<html lang="en"> +<head> +<meta charset="utf-8"> +<meta name="viewport" content="width=device-width, initial-scale=1"> +<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self' data:"> +<meta name="color-scheme" content="light dark"> +<title>ideamine</title> +<style> +/* Theme tokens. Dark mode overrides the same names. */ +:root { + color-scheme: light dark; + --bg: #f9f9f7; + --surface: #fcfcfb; + --surface-2: #f1f1ee; + --text: #0b0b0b; + --text-dim: #52514e; + --text-muted: #898781; + --border: #e1e0d9; + --border-strong: #c3c2b7; + --accent: #2a78d6; + --focus-ring: #2a78d6; + --shadow: 0 8px 24px rgba(11, 11, 11, 0.16); + --lane-doing: #2a78d6; + --lane-do: #1baf7a; + --lane-maybe: #eda100; + --lane-inbox: #898781; + --lane-skip: #4a3aa7; + --lane-done: #0ca30c; + --lane-dropped: #d03b3b; + --phase-inbox: #86b6ef; + --phase-queued: #3987e5; + --phase-doing: #184f95; + --now-line: #0b0b0b; +} +@media (prefers-color-scheme: dark) { + :root { + --bg: #0d0d0d; + --surface: #1a1a19; + --surface-2: #232322; + --text: #ffffff; + --text-dim: #c3c2b7; + --text-muted: #898781; + --border: #2c2c2a; + --border-strong: #383835; + --accent: #3987e5; + --focus-ring: #86b6ef; + --shadow: 0 8px 24px rgba(0, 0, 0, 0.6); + --lane-doing: #3987e5; + --lane-do: #199e70; + --lane-maybe: #c98500; + --lane-inbox: #898781; + --lane-skip: #9085e9; + --lane-done: #0ca30c; + --lane-dropped: #e66767; + --phase-inbox: #184f95; + --phase-queued: #3987e5; + --phase-doing: #86b6ef; + --now-line: #ffffff; + } +} + +* { box-sizing: border-box; } +html, body { margin: 0; padding: 0; } +body { + background: var(--bg); + color: var(--text); + font-family: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; + font-size: 14px; + line-height: 1.45; +} +h1, h2, h3, p, dl, dd { margin: 0; } +button { font: inherit; color: inherit; } +code, .mono { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; } + +/* header */ +.app-header { position: sticky; top: 0; z-index: 5; background: var(--surface); border-bottom: 1px solid var(--border); } +.header-row { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; padding: 6px 12px; } +.header-top { justify-content: space-between; padding-top: 10px; } +.brand { font-size: 18px; font-weight: 700; } +.updated { color: var(--text-muted); font-size: 12px; } +.counts { padding-top: 0; } +.count-chip { display: inline-flex; align-items: center; gap: 5px; font-size: 12px; color: var(--text-dim); } +.count-chip-dot { width: 8px; height: 8px; border-radius: 50%; background: var(--chip-color); flex: none; } +.header-controls { padding-top: 0; padding-bottom: 10px; } +.tabs { display: flex; gap: 4px; } +.tabs button { padding: 6px 12px; border: 1px solid var(--border); background: var(--surface-2); border-radius: 6px; cursor: pointer; } +.tabs button.active { background: var(--accent); border-color: var(--accent); color: #fff; } +.parked-toggle { display: flex; align-items: center; gap: 6px; font-size: 12px; color: var(--text-dim); white-space: nowrap; cursor: pointer; } +.search-input { flex: 1 1 160px; min-width: 120px; padding: 6px 10px; border: 1px solid var(--border); border-radius: 6px; background: var(--surface); color: var(--text); } + +/* layout */ +main { padding: 12px; } +.empty-note { color: var(--text-muted); padding: 24px 4px; } +.load-error { padding: 24px; color: var(--lane-dropped); } + +/* board */ +.board { display: flex; align-items: flex-start; gap: 12px; overflow-x: auto; padding-bottom: 8px; } +.column { flex: 0 0 260px; background: var(--surface); border: 1px solid var(--border); border-radius: 8px; } +.column-header { display: flex; align-items: center; gap: 6px; padding: 8px 10px; border-bottom: 1px solid var(--border); font-weight: 600; } +.column-dot { width: 8px; height: 8px; border-radius: 50%; background: var(--chip-color); flex: none; } +.column-title { flex: 1; } +.column-count { color: var(--text-muted); font-weight: 400; } +.column-body { display: flex; flex-direction: column; gap: 8px; padding: 8px; min-height: 12px; } +.column-empty { color: var(--text-muted); font-size: 12px; padding: 4px; } + +/* ticket cards */ +.card { display: block; width: 100%; text-align: left; background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 8px 10px; cursor: pointer; } +.card:hover { border-color: var(--border-strong); } +.card-compact { padding: 6px 8px; } +.card-top { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; margin-bottom: 4px; } +.card-key { font-size: 11px; color: var(--text-muted); } +.card-chips { display: flex; gap: 4px; flex-wrap: wrap; } +.card-title { font-size: 13px; } +.card-meta { display: flex; gap: 6px; flex-wrap: wrap; margin-top: 6px; } +.card-project { font-size: 11px; color: var(--text-muted); } +.card-score { margin-left: auto; font-size: 11px; font-weight: 600; color: var(--accent); } + +/* chips (lane / model / size / impact / tag) */ +.chip { display: inline-flex; align-items: center; border: 1px solid var(--border-strong); border-radius: 999px; padding: 1px 8px; font-size: 11px; color: var(--text-dim); background: var(--surface-2); white-space: nowrap; } +.chip-lane { border-color: var(--chip-color); color: var(--chip-color); font-weight: 600; background: transparent; } +.chip-model { text-transform: capitalize; } +.chip-tag::before { content: "#"; opacity: .6; } + +/* groups view */ +.group-section { margin-bottom: 20px; } +.group-title { font-size: 15px; font-weight: 700; margin-bottom: 8px; } +.group-count { color: var(--text-muted); font-weight: 400; } +.group-cards { display: flex; flex-wrap: wrap; gap: 8px; } +.group-cards .card { width: 220px; } + +/* search results */ +.search-mode { color: var(--text-muted); font-size: 12px; margin-bottom: 8px; } +.search-list { display: flex; flex-direction: column; gap: 8px; max-width: 520px; } + +/* timeline (gantt) */ +.tl-legend { display: flex; flex-wrap: wrap; gap: 12px; align-items: center; padding: 6px 2px 10px; font-size: 12px; color: var(--text-dim); } +.tl-legend-item { display: inline-flex; align-items: center; gap: 5px; } +.tl-legend-swatch { display: inline-block; width: 14px; height: 10px; border-radius: 2px; background: var(--chip-color); } +.tl-legend-hatch { background-image: repeating-linear-gradient(45deg, var(--chip-color) 0 2px, transparent 2px 4px); } +.tl-legend-mark { font-weight: 700; } +.tl-scroll { overflow-x: auto; border: 1px solid var(--border); border-radius: 8px; } +.tl-inner { display: flex; align-items: flex-start; width: max-content; } +.tl-labels { position: sticky; left: 0; z-index: 2; background: var(--surface); border-right: 1px solid var(--border); flex: 0 0 auto; } +.tl-label-header { border-bottom: 1px solid var(--border); } +.tl-label-row { display: flex; align-items: center; gap: 6px; width: 220px; padding: 0 8px; border: 0; border-bottom: 1px solid var(--border); background: none; cursor: pointer; text-align: left; } +.tl-label-row:hover, .tl-label-row:focus-visible { background: var(--surface-2); } +.tl-label-key { font-size: 11px; color: var(--text-muted); flex: none; } +.tl-label-title { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.tl-chart { flex: 0 0 auto; } +.tl-grid { stroke: var(--border); stroke-width: 1; } +.tl-tick-label { fill: var(--text-muted); font-size: 10px; } +.tl-now { stroke: var(--now-line); stroke-width: 1.5; stroke-dasharray: 4 3; } +.tl-now-label { fill: var(--now-line); font-size: 10px; } +.tl-seg-inbox { fill: var(--phase-inbox); } +.tl-seg-queued { fill: var(--phase-queued); } +.tl-seg-doing { fill: var(--phase-doing); } +.tl-seg-estimated.tl-seg-inbox { fill: url(#hatch-inbox); } +.tl-seg-estimated.tl-seg-queued { fill: url(#hatch-queued); } +.tl-seg-estimated.tl-seg-doing { fill: url(#hatch-doing); } +.tl-seg { cursor: pointer; } +.tl-seg:hover, .tl-seg:focus-visible { stroke: var(--text); stroke-width: 1.5; outline: none; } +.tl-end-mark { font-size: 13px; font-weight: 700; dominant-baseline: middle; } +.tl-end-done { fill: var(--lane-done); } +.tl-end-dropped { fill: var(--lane-dropped); } + +/* detail drawer */ +.drawer-backdrop { position: fixed; inset: 0; background: rgba(0, 0, 0, .4); z-index: 20; } +.drawer-backdrop[hidden] { display: none; } +.drawer { position: fixed; top: 0; right: 0; bottom: 0; width: 420px; max-width: 100%; background: var(--surface); box-shadow: var(--shadow); z-index: 21; overflow-y: auto; padding: 16px; } +.drawer[hidden] { display: none; } +@media (max-width: 640px) { .drawer { width: 100%; } } +.drawer-close { position: sticky; top: 0; float: right; background: none; border: none; font-size: 22px; line-height: 1; cursor: pointer; color: var(--text-dim); } +.drawer-title { font-size: 16px; font-weight: 700; margin: 4px 0 10px; clear: both; } +.drawer-key { color: var(--text-muted); font-size: 13px; margin-right: 6px; } +.drawer-chips { display: flex; gap: 6px; flex-wrap: wrap; margin-bottom: 10px; } +.drawer-meta { color: var(--text-dim); font-size: 12px; margin-bottom: 10px; } +.date-grid { display: grid; grid-template-columns: auto 1fr; gap: 4px 10px; font-size: 12px; margin-bottom: 14px; } +.date-grid dt { color: var(--text-muted); } +.drawer-section { margin-bottom: 16px; } +.drawer-section h3 { font-size: 12px; text-transform: uppercase; letter-spacing: .04em; color: var(--text-muted); margin-bottom: 6px; } +.drawer-text { white-space: pre-wrap; font-size: 13px; } +.notes { display: flex; flex-direction: column; gap: 8px; } +.note { border-left: 2px solid var(--border-strong); padding-left: 8px; } +.note-at { font-size: 11px; color: var(--text-muted); } +.note-text { white-space: pre-wrap; font-size: 13px; } +.link-button { background: none; border: none; color: var(--accent); cursor: pointer; padding: 0; text-align: left; font: inherit; } +.related-list { display: flex; flex-direction: column; gap: 6px; } +.related-item { display: flex; align-items: center; gap: 8px; width: 100%; } +.related-title { flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--text); } +.related-score { color: var(--text-muted); font-size: 11px; } +.text-muted { color: var(--text-muted); } + +/* hover / tap preview for timeline bars */ +.tooltip { position: fixed; z-index: 30; max-width: 240px; background: var(--surface-2); border: 1px solid var(--border-strong); border-radius: 6px; padding: 6px 8px; font-size: 12px; box-shadow: var(--shadow); pointer-events: none; } +.tooltip[hidden] { display: none; } +.tooltip-key { font-weight: 700; } +.tooltip-title { color: var(--text-dim); margin: 2px 0; } +.tooltip-phase { color: var(--text-muted); } + +/* focus */ +button:focus-visible, input:focus-visible, [tabindex]:focus-visible { + outline: 2px solid var(--focus-ring); + outline-offset: 2px; +} +</style> +</head> +<body> +<header class="app-header"> + <div class="header-row header-top"> + <h1 class="brand">ideamine</h1> + <span class="updated" id="updated"></span> + </div> + <div class="header-row counts" id="counts" aria-label="Idea counts by lane"></div> + <div class="header-row header-controls"> + <nav class="tabs" id="tabs" role="tablist" aria-label="Views"> + <button type="button" role="tab" data-view="board" aria-controls="main">Board</button> + <button type="button" role="tab" data-view="timeline" aria-controls="main">Timeline</button> + <button type="button" role="tab" data-view="groups" aria-controls="main">Groups</button> + </nav> + <label class="parked-toggle"> + <input type="checkbox" id="parked-toggle"> Show parked + </label> + <input type="search" id="search-input" class="search-input" placeholder="Search ideas…" aria-label="Search ideas"> + </div> +</header> + +<main id="main" role="tabpanel"></main> + +<div class="drawer-backdrop" id="drawer-backdrop" hidden></div> +<aside class="drawer" id="drawer" hidden role="dialog" aria-modal="true" aria-labelledby="drawer-title"> + <button class="drawer-close" id="drawer-close" type="button" aria-label="Close">×</button> + <div id="drawer-body"></div> +</aside> + +<div class="tooltip" id="tooltip" hidden role="status"></div> + +<script> +'use strict'; + +/* ideamine dashboard. One file, no build step, no external calls + besides data.json and v1/embeddings on the same origin. */ + +const SVG_NS = 'http://www.w3.org/2000/svg'; +const MIN = 60000, HOUR = 3600000, DAY = 86400000; +const TL_STEPS = [MIN, 5 * MIN, 15 * MIN, 30 * MIN, HOUR, 3 * HOUR, 6 * HOUR, 12 * HOUR, DAY, 2 * DAY, 7 * DAY, 30 * DAY, 90 * DAY]; +const TL_ROW_H = 34, TL_HEADER_H = 28, TL_BAR_H = 16; +const BOARD_LANES = ['doing', 'do', 'maybe', 'inbox', 'done']; +const PARKED_LANES = ['skip', 'dropped']; +const ALL_LANES = ['doing', 'do', 'maybe', 'inbox', 'skip', 'done', 'dropped']; +const LANE_LABELS = { doing: 'Doing', do: 'Do', maybe: 'Maybe', inbox: 'Inbox', skip: 'Skip', done: 'Done', dropped: 'Dropped' }; +const PHASE_LABELS = { inbox: 'Inbox', queued: 'Queued', doing: 'Doing' }; +const SEARCH_DEBOUNCE_MS = 300; +const EMBED_TIMEOUT_MS = 4000; + +const state = { + data: null, + byId: new Map(), + byKey: new Map(), + view: 'board', + showParked: false, + openKey: null, + query: '', + searchResult: null, +}; + +let searchTimer = null; +let searchToken = 0; +let lastFocused = null; +const vecCache = new Map(); + +const $main = document.getElementById('main'); +const $counts = document.getElementById('counts'); +const $updated = document.getElementById('updated'); +const $tabs = document.getElementById('tabs'); +const $parkedToggle = document.getElementById('parked-toggle'); +const $searchInput = document.getElementById('search-input'); +const $drawer = document.getElementById('drawer'); +const $drawerBackdrop = document.getElementById('drawer-backdrop'); +const $drawerClose = document.getElementById('drawer-close'); +const $drawerBody = document.getElementById('drawer-body'); +const $tooltip = document.getElementById('tooltip'); + +/* ---- small DOM builders. Text children become text nodes, never HTML. ---- */ + +function el(tag, attrs, ...kids) { + const node = document.createElement(tag); + if (attrs) { + for (const [k, v] of Object.entries(attrs)) { + if (v == null || v === false) continue; + if (k === 'class') node.className = v; + else if (k.startsWith('on') && typeof v === 'function') node.addEventListener(k.slice(2), v); + else node.setAttribute(k, v === true ? '' : v); + } + } + for (const kid of kids.flat(Infinity)) { + if (kid == null || kid === false) continue; + node.append(kid.nodeType ? kid : String(kid)); + } + return node; +} + +function svgEl(tag, attrs, ...kids) { + const node = document.createElementNS(SVG_NS, tag); + if (attrs) { + for (const [k, v] of Object.entries(attrs)) { + if (v == null || v === false) continue; + node.setAttribute(k, v === true ? '' : v); + } + } + for (const kid of kids.flat(Infinity)) { + if (kid == null || kid === false) continue; + node.append(kid.nodeType ? kid : String(kid)); + } + return node; +} + +/* ---- formatting ---- */ + +// The page text is English, so dates and "5 minutes ago" are English too, whatever the browser locale. +const LOCALE = 'en'; + +function fmtDate(iso) { + if (!iso) return null; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return iso; + return d.toLocaleString(LOCALE, { year: 'numeric', month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' }); +} + +function fmtRelative(iso) { + const then = new Date(iso).getTime(); + if (Number.isNaN(then)) return 'at an unknown time'; + const diffSec = Math.round((then - Date.now()) / 1000); + const rtf = new Intl.RelativeTimeFormat(LOCALE, { numeric: 'auto' }); + const units = [['year', 31536000], ['month', 2592000], ['day', 86400], ['hour', 3600], ['minute', 60], ['second', 1]]; + for (const [unit, secs] of units) { + if (Math.abs(diffSec) >= secs || unit === 'second') return rtf.format(Math.round(diffSec / secs), unit); + } +} + +function formatTick(t, step) { + const d = new Date(t); + if (step < HOUR) return d.toLocaleTimeString(LOCALE, { hour: '2-digit', minute: '2-digit' }); + if (step < DAY) return d.toLocaleString(LOCALE, { month: 'short', day: 'numeric', hour: '2-digit' }); + if (step < 30 * DAY) return d.toLocaleDateString(LOCALE, { month: 'short', day: 'numeric' }); + return d.toLocaleDateString(LOCALE, { month: 'short', year: 'numeric' }); +} + +/* ---- vectors: decode base64 little-endian float32, dot product ---- */ + +function decodeVec(b64) { + if (!b64) return null; + const bin = atob(b64); + const bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); + const view = new DataView(bytes.buffer); + const n = Math.floor(bin.length / 4); + const out = new Float32Array(n); + for (let i = 0; i < n; i++) out[i] = view.getFloat32(i * 4, true); + return out; +} + +function getIdeaVec(idea) { + if (!idea.vec) return null; + if (vecCache.has(idea.id)) return vecCache.get(idea.id); + const v = decodeVec(idea.vec); + vecCache.set(idea.id, v); + return v; +} + +function normalize(arr) { + let sum = 0; + for (let i = 0; i < arr.length; i++) sum += arr[i] * arr[i]; + const norm = Math.sqrt(sum) || 1; + const out = new Float64Array(arr.length); + for (let i = 0; i < arr.length; i++) out[i] = arr[i] / norm; + return out; +} + +function dot(a, b) { + const n = Math.min(a.length, b.length); + let s = 0; + for (let i = 0; i < n; i++) s += a[i] * b[i]; + return s; +} + +function findByKey(key) { return state.byKey.get(key.toUpperCase()) || null; } + +/* ---- ticket card, shared by board, groups, and search results ---- */ + +function ideaCard(idea, opts) { + opts = opts || {}; + const chips = [el('span', { class: 'chip chip-lane', style: '--chip-color:var(--lane-' + idea.lane + ')' }, LANE_LABELS[idea.lane] || idea.lane)]; + if (idea.model) chips.push(el('span', { class: 'chip chip-model' }, idea.model)); + if (idea.size) chips.push(el('span', { class: 'chip' }, idea.size.toUpperCase())); + if (idea.impact != null) chips.push(el('span', { class: 'chip chip-impact' }, '▲' + idea.impact)); + + const top = el('div', { class: 'card-top' }, el('code', { class: 'card-key' }, idea.key), el('span', { class: 'card-chips' }, chips)); + if (opts.scoreLabel) top.append(el('span', { class: 'card-score' }, opts.scoreLabel)); + + const meta = []; + if (idea.project) meta.push(el('span', { class: 'card-project' }, idea.project)); + for (const t of idea.tags || []) meta.push(el('span', { class: 'chip chip-tag' }, t)); + + return el('button', { class: 'card' + (opts.compact ? ' card-compact' : ''), type: 'button', onclick: () => openTicket(idea.key) }, + top, + el('div', { class: 'card-title' }, idea.title), + meta.length ? el('div', { class: 'card-meta' }, meta) : null); +} + +/* ---- header: counts + relative update time ---- */ + +function renderHeader() { + $counts.textContent = ''; + const counts = state.data.counts || {}; + for (const lane of ALL_LANES) { + $counts.append(el('span', { class: 'count-chip' }, + el('span', { class: 'count-chip-dot', style: '--chip-color:var(--lane-' + lane + ')' }), + LANE_LABELS[lane] + ' ' + (counts[lane] ?? 0))); + } + $updated.textContent = 'updated ' + fmtRelative(state.data.generated); + $updated.title = fmtDate(state.data.generated) || ''; +} + +function renderTabsActive() { + for (const btn of $tabs.querySelectorAll('button')) { + const active = btn.dataset.view === state.view; + btn.classList.toggle('active', active); + btn.setAttribute('aria-selected', String(active)); + } +} + +/* ---- board view ---- */ + +function renderColumn(lane) { + const ideas = state.data.ideas.filter((i) => i.lane === lane).sort((a, b) => a.rank - b.rank); + return el('div', { class: 'column' }, + el('div', { class: 'column-header' }, + el('span', { class: 'column-dot', style: '--chip-color:var(--lane-' + lane + ')' }), + el('span', { class: 'column-title' }, LANE_LABELS[lane]), + el('span', { class: 'column-count' }, String(ideas.length))), + el('div', { class: 'column-body' }, ideas.length ? ideas.map((i) => ideaCard(i)) : el('p', { class: 'column-empty' }, 'No tickets'))); +} + +function renderBoard() { + $main.textContent = ''; + const lanes = state.showParked ? BOARD_LANES.concat(PARKED_LANES) : BOARD_LANES; + $main.append(el('div', { class: 'board' }, lanes.map(renderColumn))); +} + +/* ---- groups view ---- */ + +function renderGroupSection(label, ideas) { + return el('section', { class: 'group-section' }, + el('h2', { class: 'group-title' }, label + ' ', el('span', { class: 'group-count' }, '(' + ideas.length + ')')), + el('div', { class: 'group-cards' }, ideas.map((i) => ideaCard(i, { compact: true })))); +} + +function renderGroups() { + $main.textContent = ''; + const wrap = el('div', { class: 'groups' }); + const grouped = new Set(); + for (const g of state.data.groups || []) { + const ideas = g.ids.map((id) => state.byId.get(id)).filter(Boolean); + for (const id of g.ids) grouped.add(id); + wrap.append(renderGroupSection(g.label, ideas)); + } + const ungrouped = state.data.ideas.filter((i) => !grouped.has(i.id)); + if (ungrouped.length) wrap.append(renderGroupSection('Ungrouped', ungrouped)); + $main.append(wrap); +} + +/* ---- timeline (gantt) view ---- */ + +function chooseStep(spanMs, targetTicks) { + for (const step of TL_STEPS) { if (spanMs / step <= targetTicks) return step; } + return TL_STEPS[TL_STEPS.length - 1]; +} + +function buildTimeScale(domainStart, domainEnd) { + const span = Math.max(domainEnd - domainStart, 1); + const step = chooseStep(span, 8); + const ticksCount = Math.ceil(span / step) + 1; + const width = Math.max(640, ticksCount * 90); + const pxPerMs = width / span; + const ticks = []; + const first = Math.ceil(domainStart / step) * step; + for (let t = first; t <= domainEnd; t += step) ticks.push(t); + return { width, step, ticks, x: (t) => (t - domainStart) * pxPerMs }; +} + +function makeHatchPattern(id, colorVar) { + return svgEl('pattern', { id, patternUnits: 'userSpaceOnUse', width: 6, height: 6, patternTransform: 'rotate(45)' }, + svgEl('rect', { width: 6, height: 6, fill: colorVar, 'fill-opacity': 0.35 }), + svgEl('line', { x1: 0, y1: 0, x2: 0, y2: 6, stroke: colorVar, 'stroke-width': 3 })); +} + +function tooltipContentForSegment(idea, seg) { + const to = seg.to ? fmtDate(seg.to) : 'now'; + return el('div', {}, + el('div', { class: 'tooltip-key' }, idea.key + (seg.estimated ? ' (estimated)' : '')), + el('div', { class: 'tooltip-title' }, idea.title), + el('div', { class: 'tooltip-phase' }, (PHASE_LABELS[seg.phase] || seg.phase) + ': ' + fmtDate(seg.from) + ' → ' + to)); +} + +function positionTooltip(evt, node) { + const rect = node.getBoundingClientRect(); + const x = evt.clientX ?? (rect.left + rect.width / 2); + const y = evt.clientY ?? rect.top; + const pad = 12, tw = 240, th = 90; + let left = x + pad, top = y + pad; + if (left + tw > window.innerWidth) left = x - tw - pad; + if (top + th > window.innerHeight) top = y - th - pad; + $tooltip.style.left = Math.max(4, left) + 'px'; + $tooltip.style.top = Math.max(4, top) + 'px'; +} + +function attachTooltip(node, contentFn) { + const show = (evt) => { + $tooltip.textContent = ''; + $tooltip.append(contentFn()); + $tooltip.hidden = false; + positionTooltip(evt, node); + }; + const hide = () => { $tooltip.hidden = true; }; + node.addEventListener('mouseenter', show); + node.addEventListener('mousemove', (e) => { if (!$tooltip.hidden) positionTooltip(e, node); }); + node.addEventListener('mouseleave', hide); + node.addEventListener('focus', show); + node.addEventListener('blur', hide); + node.addEventListener('click', (e) => { e.stopPropagation(); show(e); }); +} + +function renderTimelineRow(idea, y, scale) { + const g = svgEl('g', { class: 'tl-row' }); + const barY = y + (TL_ROW_H - TL_BAR_H) / 2; + const phases = idea.phases || []; + let endX = null; + phases.forEach((seg, i) => { + const from = Date.parse(seg.from); + if (Number.isNaN(from)) return; + const to = seg.to ? Date.parse(seg.to) : Date.now(); + let x1 = scale.x(from); + let x2 = scale.x(Math.max(to, from)); + if (i > 0) x1 += 1; + if (i < phases.length - 1) x2 -= 1; + const w = Math.max(x2 - x1, 1); + const phaseName = seg.phase || 'inbox'; + const rect = svgEl('rect', { + x: x1, y: barY, width: w, height: TL_BAR_H, rx: 3, + class: 'tl-seg tl-seg-' + phaseName + (seg.estimated ? ' tl-seg-estimated' : ''), + tabindex: '0', role: 'img', + 'aria-label': idea.key + ' ' + (PHASE_LABELS[phaseName] || phaseName) + ' from ' + fmtDate(seg.from) + ' to ' + (seg.to ? fmtDate(seg.to) : 'now'), + }); + attachTooltip(rect, () => tooltipContentForSegment(idea, seg)); + g.append(rect); + endX = x1 + w; + }); + if (endX != null && (idea.lane === 'done' || idea.lane === 'dropped')) { + const mark = idea.lane === 'done' ? '✓' : '✗'; + g.append(svgEl('text', { x: endX + 6, y: y + TL_ROW_H / 2 + 4, class: 'tl-end-mark tl-end-' + idea.lane }, mark)); + } + return g; +} + +function renderTimelineLegend() { + function item(colorVar, label, hatch) { + return el('span', { class: 'tl-legend-item' }, el('span', { class: 'tl-legend-swatch' + (hatch ? ' tl-legend-hatch' : ''), style: '--chip-color:' + colorVar }), label); + } + return el('div', { class: 'tl-legend' }, + item('var(--phase-inbox)', 'Inbox'), + item('var(--phase-queued)', 'Queued'), + item('var(--phase-doing)', 'Doing'), + item('var(--phase-doing)', 'Estimated', true), + el('span', { class: 'tl-legend-item' }, el('span', { class: 'tl-legend-mark', style: 'color:var(--lane-done)' }, '✓'), 'Done'), + el('span', { class: 'tl-legend-item' }, el('span', { class: 'tl-legend-mark', style: 'color:var(--lane-dropped)' }, '✗'), 'Dropped')); +} + +function renderTimeline() { + $main.textContent = ''; + const ideas = state.data.ideas + .filter((i) => state.showParked || (i.lane !== 'skip' && i.lane !== 'dropped')) + .slice() + .sort((a, b) => Date.parse(a.created) - Date.parse(b.created)); + + if (!ideas.length) { + $main.append(el('p', { class: 'empty-note' }, 'No ideas to show.')); + return; + } + + const now = Date.now(); + let domainStart = Infinity, domainEnd = -Infinity; + for (const idea of ideas) { + for (const seg of idea.phases || []) { + const from = Date.parse(seg.from); + const to = seg.to ? Date.parse(seg.to) : now; + if (!Number.isNaN(from)) domainStart = Math.min(domainStart, from); + if (!Number.isNaN(to)) domainEnd = Math.max(domainEnd, to); + } + } + if (!Number.isFinite(domainStart)) domainStart = now - DAY; + domainEnd = Math.max(domainEnd, now); + const pad = (domainEnd - domainStart) * 0.03 || HOUR; + domainStart -= pad; + domainEnd += pad; + + const scale = buildTimeScale(domainStart, domainEnd); + const chartHeight = TL_HEADER_H + ideas.length * TL_ROW_H; + + const defs = svgEl('defs', {}, makeHatchPattern('hatch-inbox', 'var(--phase-inbox)'), makeHatchPattern('hatch-queued', 'var(--phase-queued)'), makeHatchPattern('hatch-doing', 'var(--phase-doing)')); + + const axis = svgEl('g', { class: 'tl-axis' }); + for (const t of scale.ticks) { + const x = scale.x(t); + axis.append(svgEl('line', { x1: x, x2: x, y1: TL_HEADER_H, y2: chartHeight, class: 'tl-grid' }), svgEl('text', { x: x + 4, y: TL_HEADER_H - 8, class: 'tl-tick-label' }, formatTick(t, scale.step))); + } + + const rows = svgEl('g', { class: 'tl-rows' }); + ideas.forEach((idea, i) => rows.append(renderTimelineRow(idea, TL_HEADER_H + i * TL_ROW_H, scale))); + + const svgRoot = svgEl('svg', { width: scale.width, height: chartHeight, viewBox: '0 0 ' + scale.width + ' ' + chartHeight, role: 'img', 'aria-label': 'Idea timeline' }, defs, axis, rows); + + if (now >= domainStart && now <= domainEnd) { + const nx = scale.x(now); + svgRoot.append(svgEl('line', { x1: nx, x2: nx, y1: 0, y2: chartHeight, class: 'tl-now' }), svgEl('text', { x: nx - 4, y: 11, 'text-anchor': 'end', class: 'tl-now-label' }, 'now')); + } + + const labels = el('div', { class: 'tl-labels' }, + el('div', { class: 'tl-label-header', style: 'height:' + TL_HEADER_H + 'px' }), + ideas.map((idea) => el('button', { class: 'tl-label-row', type: 'button', style: 'height:' + TL_ROW_H + 'px', onclick: () => openTicket(idea.key) }, + el('code', { class: 'tl-label-key' }, idea.key), + el('span', { class: 'tl-label-title' }, idea.title)))); + + const scroll = el('div', { class: 'tl-scroll' }, el('div', { class: 'tl-inner' }, labels, el('div', { class: 'tl-chart' }, svgRoot))); + $main.append(renderTimelineLegend(), scroll); +} + +/* ---- search: semantic first, word fallback ---- */ + +function wordSearch(q) { + const words = q.toLowerCase().split(/\s+/).filter(Boolean); + const items = []; + for (const idea of state.data.ideas) { + const hay = [idea.title, idea.text, (idea.tags || []).join(' '), idea.brief].filter(Boolean).join(' \n ').toLowerCase(); + let score = 0; + for (const w of words) { if (hay.includes(w)) score++; } + if (score >= 1) items.push({ idea, score }); + } + items.sort((a, b) => b.score - a.score); + return { mode: 'words', items }; +} + +async function trySemanticSearch(q, embed) { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), EMBED_TIMEOUT_MS); + try { + const res = await fetch('v1/embeddings', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: embed.model, input: [embed.query_prefix + q] }), + signal: ctrl.signal, + }); + if (!res.ok) return null; + const json = await res.json(); + const raw = json && json.data && json.data[0] && json.data[0].embedding; + if (!Array.isArray(raw) || raw.length === 0) return null; + const qvec = normalize(raw); + const threshold = embed.search_threshold ?? 0.5; + const items = []; + for (const idea of state.data.ideas) { + const vec = getIdeaVec(idea); + if (!vec) continue; + const score = dot(qvec, vec); + if (score >= threshold) items.push({ idea, score }); + } + items.sort((a, b) => b.score - a.score); + return { mode: 'meaning', items }; + } catch (err) { + return null; + } finally { + clearTimeout(timer); + } +} + +async function runSearch(query) { + const q = query.trim(); + if (!q) { state.searchResult = null; render(); return; } + const token = ++searchToken; + state.searchResult = null; + render(); + const embed = state.data.embed || {}; + const hasVecs = state.data.ideas.some((i) => i.vec); + let result = null; + if (embed.available && hasVecs) result = await trySemanticSearch(q, embed); + if (token !== searchToken) return; + if (!result) result = wordSearch(q); + state.searchResult = result; + render(); +} + +function searchResultCard(item) { + const label = state.searchResult.mode === 'meaning' ? Math.round(item.score * 100) + '%' : item.score + (item.score === 1 ? ' word' : ' words'); + return ideaCard(item.idea, { compact: true, scoreLabel: label }); +} + +function renderSearch() { + $main.textContent = ''; + if (!state.searchResult) { $main.append(el('p', { class: 'empty-note' }, 'Searching…')); return; } + const { mode, items } = state.searchResult; + const modeLabel = mode === 'meaning' ? 'Showing results by meaning' : 'Showing results by words, meaning search is not available'; + const wrap = el('div', { class: 'search-results' }, el('p', { class: 'search-mode' }, modeLabel)); + if (!items.length) wrap.append(el('p', { class: 'empty-note' }, 'No matching ideas.')); + else wrap.append(el('div', { class: 'search-list' }, items.map(searchResultCard))); + $main.append(wrap); +} + +/* ---- detail drawer ---- */ + +function renderDateGrid(idea) { + const grid = el('dl', { class: 'date-grid' }); + for (const [label, value] of [['Created', idea.created], ['Triaged', idea.triaged], ['Started', idea.started], ['Closed', idea.closed]]) { + grid.append(el('dt', {}, label), el('dd', {}, value ? fmtDate(value) : '—')); + } + return grid; +} + +function renderNotes(notes) { + if (!notes || !notes.length) return null; + const list = el('div', { class: 'notes' }); + for (const note of notes) list.append(el('div', { class: 'note' }, el('div', { class: 'note-at' }, fmtDate(note.at)), el('div', { class: 'note-text' }, note.text))); + return el('section', { class: 'drawer-section' }, el('h3', {}, 'Notes'), list); +} + +function renderDupOf(dupId) { + const target = state.byId.get(dupId); + const key = target ? target.key : 'IDEA-' + dupId; + const content = target + ? el('button', { class: 'link-button', type: 'button', onclick: () => openTicket(target.key) }, key + ' — ' + target.title) + : el('span', { class: 'text-muted' }, key + ' (not found)'); + return el('section', { class: 'drawer-section' }, el('h3', {}, 'Duplicate of'), content); +} + +function renderRelated(related) { + const list = el('div', { class: 'related-list' }); + for (const r of related) { + const target = state.byId.get(r.id); + if (!target) continue; + const pct = Math.round((r.score ?? 0) * 100) + '%'; + list.append(el('button', { class: 'link-button related-item', type: 'button', onclick: () => openTicket(target.key) }, + el('code', { class: 'card-key' }, target.key), el('span', { class: 'related-title' }, target.title), el('span', { class: 'related-score' }, pct))); + } + if (!list.childNodes.length) return null; + return el('section', { class: 'drawer-section' }, el('h3', {}, 'Related'), list); +} + +function buildDrawerContent(idea) { + const frag = document.createDocumentFragment(); + const chips = el('div', { class: 'drawer-chips' }, el('span', { class: 'chip chip-lane', style: '--chip-color:var(--lane-' + idea.lane + ')' }, LANE_LABELS[idea.lane] || idea.lane)); + if (idea.model) chips.append(el('span', { class: 'chip chip-model' }, idea.model)); + if (idea.size) chips.append(el('span', { class: 'chip' }, idea.size.toUpperCase())); + if (idea.impact != null) chips.append(el('span', { class: 'chip chip-impact' }, '▲' + idea.impact)); + if (idea.verdict) chips.append(el('span', { class: 'chip' }, 'verdict: ' + idea.verdict)); + + frag.append(el('h2', { id: 'drawer-title', class: 'drawer-title' }, el('code', { class: 'drawer-key' }, idea.key), idea.title), chips); + + const metaLines = []; + if (idea.project) metaLines.push('Project: ' + idea.project); + if (idea.tags && idea.tags.length) metaLines.push('Tags: ' + idea.tags.join(', ')); + if (metaLines.length) frag.append(el('p', { class: 'drawer-meta' }, metaLines.join(' · '))); + + frag.append(renderDateGrid(idea)); + + if (idea.text) frag.append(el('section', { class: 'drawer-section' }, el('h3', {}, 'Text'), el('p', { class: 'drawer-text' }, idea.text))); + if (idea.why) frag.append(el('section', { class: 'drawer-section' }, el('h3', {}, 'Why'), el('p', { class: 'drawer-text' }, idea.why))); + if (idea.brief) frag.append(el('section', { class: 'drawer-section' }, el('h3', {}, 'Build brief'), el('p', { class: 'drawer-text' }, idea.brief))); + + const notes = renderNotes(idea.notes); + if (notes) frag.append(notes); + if (idea.dup_of != null) frag.append(renderDupOf(idea.dup_of)); + if (idea.related && idea.related.length) { + const related = renderRelated(idea.related); + if (related) frag.append(related); + } + return frag; +} + +function renderDrawer() { + const idea = state.openKey ? findByKey(state.openKey) : null; + const wasHidden = $drawer.hidden; + if (!idea) { + if (!wasHidden) { + $drawer.hidden = true; + $drawerBackdrop.hidden = true; + if (lastFocused && document.contains(lastFocused)) lastFocused.focus(); + lastFocused = null; + } + return; + } + if (wasHidden) lastFocused = document.activeElement; + $drawer.hidden = false; + $drawerBackdrop.hidden = false; + $drawerBody.textContent = ''; + $drawerBody.append(buildDrawerContent(idea)); + if (wasHidden) requestAnimationFrame(() => $drawerClose.focus()); +} + +function openTicket(key) { location.hash = key; } +function closeDrawer() { location.hash = state.view; } + +/* ---- load error ---- */ + +class HttpError extends Error { + constructor(status) { super('HTTP ' + status); this.status = status; } +} + +function renderLoadError(err) { + $main.textContent = ''; + $counts.textContent = ''; + $updated.textContent = ''; + const msg = err instanceof HttpError ? 'Could not load data.json (HTTP ' + err.status + ').' : 'Could not load data.json (' + (err && err.message ? err.message : 'network error') + ').'; + $main.append(el('div', { class: 'load-error' }, el('p', {}, msg))); +} + +/* ---- top-level render and hash routing ---- */ + +function render() { + renderHeader(); + renderTabsActive(); + if (state.query.trim() !== '') renderSearch(); + else if (state.view === 'timeline') renderTimeline(); + else if (state.view === 'groups') renderGroups(); + else renderBoard(); + renderDrawer(); +} + +function applyHash() { + const raw = decodeURIComponent(location.hash.slice(1)); + if (raw === 'board' || raw === 'timeline' || raw === 'groups') { + state.view = raw; + state.openKey = null; + } else if (/^IDEA-\d+$/i.test(raw)) { + state.openKey = raw.toUpperCase(); + } else { + state.openKey = null; + if (raw) state.view = 'board'; + } +} + +function onHashChange() { applyHash(); render(); } + +function onSearchInput() { + const q = $searchInput.value; + state.query = q; + clearTimeout(searchTimer); + if (q.trim() === '') { state.searchResult = null; render(); return; } + searchTimer = setTimeout(() => runSearch(q), SEARCH_DEBOUNCE_MS); +} + +function bindStaticEvents() { + for (const btn of $tabs.querySelectorAll('button')) { + btn.addEventListener('click', () => { + $searchInput.value = ''; + state.query = ''; + state.searchResult = null; + location.hash = btn.dataset.view; + }); + } + $parkedToggle.addEventListener('change', () => { state.showParked = $parkedToggle.checked; render(); }); + $searchInput.addEventListener('input', onSearchInput); + $searchInput.addEventListener('keydown', (e) => { if (e.key === 'Enter') { clearTimeout(searchTimer); runSearch($searchInput.value); } }); + $drawerClose.addEventListener('click', closeDrawer); + $drawerBackdrop.addEventListener('click', closeDrawer); + document.addEventListener('keydown', (e) => { if (e.key === 'Escape' && !$drawer.hidden) closeDrawer(); }); + document.addEventListener('click', () => { $tooltip.hidden = true; }); +} + +async function main() { + bindStaticEvents(); + try { + const res = await fetch('data.json', { cache: 'no-store' }); + if (!res.ok) throw new HttpError(res.status); + const data = await res.json(); + state.data = data; + for (const idea of data.ideas) { + state.byId.set(idea.id, idea); + state.byKey.set(idea.key.toUpperCase(), idea); + } + window.addEventListener('hashchange', onHashChange); + applyHash(); + render(); + } catch (err) { + renderLoadError(err); + } +} + +main(); +</script> +</body> +</html> diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..3c4bbe0 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,62 @@ +{ + "name": "ideamine", + "version": "0.4.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "ideamine", + "version": "0.4.0", + "license": "MIT", + "bin": { + "ideamine": "bin/ideamine.js" + }, + "devDependencies": { + "fast-check": "^4.10.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/fast-check": { + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.10.2.tgz", + "integrity": "sha512-iK2f+YrcmoeGqk6fA0ea2bptcu/itMIm4NfEozq6N25+aG6h7s5HZbB/k1aV7b5w5sFLMCbbtRUsTVR+BgC3xw==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^8.0.0" + }, + "engines": { + "node": ">=12.17.0" + } + }, + "node_modules/pure-rand": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.2.tgz", + "integrity": "sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + } + } +} diff --git a/package.json b/package.json index a5b0a54..e7da825 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ideamine", - "version": "0.4.0", + "version": "0.5.0", "description": "An idea inbox for Claude Code. /idea saves an idea without calling the model; Claude later triages each idea and picks the cheapest model that can build it.", "type": "module", "bin": { @@ -9,6 +9,7 @@ "files": [ "bin", "src", + "dashboard", "skills", "hooks", ".claude-plugin", @@ -39,5 +40,8 @@ "ideas", "backlog", "vibecoding" - ] + ], + "devDependencies": { + "fast-check": "^4.10.2" + } } diff --git a/skills/ideas-find/SKILL.md b/skills/ideas-find/SKILL.md new file mode 100644 index 0000000..e81e87d --- /dev/null +++ b/skills/ideas-find/SKILL.md @@ -0,0 +1,11 @@ +--- +name: ideas-find +description: Search the ideamine archive by meaning (nomic-embed-text), in every lane. The ideamine hook answers this with no model call. +argument-hint: "<words>" +disable-model-invocation: true +allowed-tools: mcp__plugin_ideamine_ideamine__idea_list +--- + +Call `idea_list` with `semantic: true`, `filter: "all"`, and `query` set to: $ARGUMENTS + +Show the result exactly as returned. diff --git a/skills/ideas-groups/SKILL.md b/skills/ideas-groups/SKILL.md new file mode 100644 index 0000000..8626e2b --- /dev/null +++ b/skills/ideas-groups/SKILL.md @@ -0,0 +1,11 @@ +--- +name: ideas-groups +description: Show the ideamine ideas grouped by meaning (nomic-embed-text). A lane name or -a picks the ideas. The ideamine hook answers this with no model call. +argument-hint: "[inbox|do|maybe|skip|doing|done|dropped|-a]" +disable-model-invocation: true +allowed-tools: mcp__plugin_ideamine_ideamine__idea_list +--- + +Call `idea_list` with `groups: true` for this request: $ARGUMENTS + +A lane name is the `filter`, and `-a` is `filter: "all"`. Show the result exactly as returned. diff --git a/skills/ideas/SKILL.md b/skills/ideas/SKILL.md index a4b54df..47d8532 100644 --- a/skills/ideas/SKILL.md +++ b/skills/ideas/SKILL.md @@ -8,4 +8,4 @@ allowed-tools: mcp__plugin_ideamine_ideamine__idea_list, mcp__plugin_ideamine_id Request: $ARGUMENTS -Answer from `idea_list`. Pass `filter`, `id`, `query`, `here`, or `full` when the request asks for them. If the request is empty, show the board exactly as returned. To change a status, use `idea_update`. To delete ideas, use `idea_remove`. Keep the answer short. Do not start to build an idea: /ideas-go and /ideas-all do that. +Answer from `idea_list`. Pass `filter`, `id`, `query`, `here`, or `full` when the request asks for them. For a question about a topic, pass the topic as `query` with `semantic: true`, which finds ideas by meaning. For a question about themes or clusters, pass `groups: true`. If the request is empty, show the board exactly as returned. To change a status, use `idea_update`. To delete ideas, use `idea_remove`. Keep the answer short. Do not start to build an idea: /ideas-go and /ideas-all do that. diff --git a/src/config.js b/src/config.js new file mode 100644 index 0000000..63af809 --- /dev/null +++ b/src/config.js @@ -0,0 +1,89 @@ +// Settings. `ideamine config` writes them to ~/.ideamine/config.json. An environment variable +// overrides the file, so a test or a single command can use other values. + +import fs from 'node:fs'; +import path from 'node:path'; +import { home, withLock, writeAtomic } from './store.js'; + +export const SETTINGS = { + embed_url: { + env: 'IDEAMINE_EMBED_URL', + value: 'http://127.0.0.1:11434/v1', + about: 'OpenAI-compatible embeddings API (Ollama, llama.cpp server)', + }, + embed_model: { env: 'IDEAMINE_EMBED_MODEL', value: 'nomic-embed-text', about: 'embedding model' }, + search_threshold: { + env: 'IDEAMINE_SEARCH_THRESHOLD', + value: 0.5, + about: 'lowest similarity (0 to 1) of a search result to the query', + }, + group_threshold: { + env: 'IDEAMINE_GROUP_THRESHOLD', + value: 0.65, + about: 'lowest mean similarity (0 to 1) of the ideas in a group; also the floor for related ideas', + }, + publish_url: { + env: 'IDEAMINE_PUBLISH_URL', + value: '', + about: 'dashboard server that gets index.html and data.json by HTTP PUT (empty: off)', + }, +}; + +const configPath = () => path.join(home(), 'config.json'); + +function readFile() { + try { + return JSON.parse(fs.readFileSync(configPath(), 'utf8')); + } catch { + return {}; + } +} + +function setting(key) { + const s = SETTINGS[key]; + if (!s) throw new Error(`unknown setting "${key}" (use: ${Object.keys(SETTINGS).join(', ')})`); + return s; +} + +function parse(key, raw) { + if (typeof setting(key).value !== 'number') return String(raw).trim(); + const n = Number(raw); + if (String(raw).trim() === '' || !Number.isFinite(n) || n < 0 || n > 1) throw new Error(`${key} must be a number from 0 to 1`); + return n; +} + +/** The value and where it comes from: env, file, or default. */ +function lookup(key) { + const s = setting(key); + const env = process.env[s.env]; + if (env !== undefined && env !== '') return { value: parse(key, env), from: s.env }; + const saved = readFile()[key]; + if (saved !== undefined) return { value: parse(key, saved), from: 'config.json' }; + return { value: s.value, from: 'default' }; +} + +export function get(key) { + return lookup(key).value; +} + +/** Save a setting. An empty value removes it from the file, so the default applies again. */ +export function set(key, value) { + const clean = value == null || String(value).trim() === '' ? undefined : parse(key, value); + withLock(() => { + const all = readFile(); + if (clean === undefined) delete all[key]; + else all[key] = clean; + writeAtomic(configPath(), JSON.stringify(all, null, 2) + '\n'); + }); + return get(key); +} + +/** Every setting, one line each, with its source. */ +export function describe() { + return Object.keys(SETTINGS) + .map((key) => { + const { value, from } = lookup(key); + return `${key.padEnd(17)} ${String(value === '' ? '(off)' : value).padEnd(28)} ${from} · ${SETTINGS[key].about}`; + }) + .join('\n'); +} diff --git a/src/embed.js b/src/embed.js new file mode 100644 index 0000000..13b21d5 --- /dev/null +++ b/src/embed.js @@ -0,0 +1,318 @@ +// Search by meaning, and groups of ideas that are close in meaning. This follows memstate +// (github.com/map588/memstate): nomic-embed-text with its task prefixes, one vector for each idea +// that a content hash keeps current, cosine similarity with a threshold, and word search when the +// embedding server does not answer. The groups and the "related" links come from the vectors, so +// ideamine needs no graph database. + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import * as config from './config.js'; +import { home, listIdeas, load, withLock, writeAtomic } from './store.js'; +import { clip, wordSet } from './text.js'; + +const MAX_CHARS = 1500; // the first cut; embed() halves a text again when the server says it is too long +const MIN_CHARS = 100; +const BATCH = 32; + +/** The embedding server cannot answer. Callers then use word search. */ +export class EmbedError extends Error {} + +const vectorsPath = () => path.join(home(), 'vectors.json'); + +/** The text that stands for an idea: its title (when the text does not start with it), text, and tags. */ +export function ideaText(idea) { + const text = String(idea.text || ''); + const title = String(idea.title || '').replace(/…$/, ''); + const head = text.startsWith(title) ? text : `${idea.title}\n${text}`; + return idea.tags?.length ? `${head}\ntags: ${idea.tags.join(', ')}` : head; +} + +// nomic-embed models expect a task prefix on each side of a search. Other models get the raw text. +const isNomic = (model) => /^nomic-embed/i.test(model); +export const documentText = (model, text) => (isNomic(model) ? `search_document: ${text}` : text); +export const queryText = (model, text) => (isNomic(model) ? `search_query: ${text}` : text); + +/** Base64 of the vector as little-endian float32, the form that data.json and vectors.json use. */ +export function encodeVec(vec) { + const buf = Buffer.alloc(vec.length * 4); + for (let i = 0; i < vec.length; i++) buf.writeFloatLE(vec[i], i * 4); + return buf.toString('base64'); +} + +export function decodeVec(b64) { + const buf = Buffer.from(b64, 'base64'); + const vec = new Float32Array(buf.length >> 2); + for (let i = 0; i < vec.length; i++) vec[i] = buf.readFloatLE(i * 4); + return vec; +} + +/** The vector scaled to length 1. */ +export function unit(values) { + const vec = Float32Array.from(values); + let sum = 0; + for (const x of vec) sum += x * x; + const norm = Math.sqrt(sum); + if (!norm || !Number.isFinite(norm)) throw new EmbedError('the embedding server returned an empty vector'); + for (let i = 0; i < vec.length; i++) vec[i] /= norm; + return vec; +} + +/** Cosine similarity of two unit vectors. */ +export function cosine(a, b) { + let sum = 0; + for (let i = 0; i < a.length; i++) sum += a[i] * b[i]; + return sum; +} + +/** Cut at a code point, never inside a surrogate pair. */ +const cut = (text, max) => (text.length <= max ? text : [...text].slice(0, max).join('')); + +async function post(texts, { url, model, timeoutMs }) { + const endpoint = `${url.replace(/\/+$/, '')}/embeddings`; + let res; + try { + res = await fetch(endpoint, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ model, input: texts }), + signal: AbortSignal.timeout(timeoutMs), + }); + } catch (e) { + const why = e.name === 'TimeoutError' ? `no answer in ${timeoutMs / 1000} s` : e.cause?.code || e.cause?.message || e.message; + throw new EmbedError(`cannot reach ${endpoint} (${why})`); + } + const body = await res.text(); + if (!res.ok) { + const err = new EmbedError(`${endpoint} answered ${res.status}: ${clip(body, 160)}`); + err.tooLong = /too large|too long|context/i.test(body); + throw err; + } + let data; + try { + data = JSON.parse(body).data; + } catch { + data = null; + } + if (!Array.isArray(data) || data.length !== texts.length) { + throw new EmbedError(`${endpoint} did not return ${texts.length} embeddings: ${clip(body, 160)}`); + } + return [...data].sort((a, b) => a.index - b.index).map((d) => unit(d.embedding)); +} + +/** + * Unit vectors for the texts, with the document or the query prefix. When the server says that a + * text is too long, it embeds the texts one at a time and halves each text that is still too long. + */ +export async function embed(texts, { query = false, timeoutMs = 30000 } = {}) { + const url = config.get('embed_url'); + const model = config.get('embed_model'); + const wrap = (t) => (query ? queryText(model, t) : documentText(model, t)); + const first = texts.map((t) => cut(String(t), MAX_CHARS)); + try { + return await post(first.map(wrap), { url, model, timeoutMs }); + } catch (e) { + if (!e.tooLong) throw e; + } + const out = []; + for (let text of first) { + for (;;) { + try { + out.push((await post([wrap(text)], { url, model, timeoutMs }))[0]); + break; + } catch (e) { + if (!e.tooLong || text.length <= MIN_CHARS) throw e; + text = cut(text, Math.floor([...text].length / 2)); + } + } + } + return out; +} + +const hashOf = (model, text) => crypto.createHash('sha256').update(`${model}\n${text}`).digest('hex').slice(0, 16); + +function readVectors(model) { + try { + const file = JSON.parse(fs.readFileSync(vectorsPath(), 'utf8')); + return file.model === model && file.items ? file.items : {}; + } catch { + return {}; + } +} + +/** Add vectors to the cache. Vectors of deleted ideas go, so a deleted idea leaves nothing behind. */ +function saveVectors(model, entries) { + withLock(() => { + const live = new Set(load().ideas.map((i) => String(i.id))); + const items = Object.fromEntries(Object.entries(readVectors(model)).filter(([id]) => live.has(id))); + for (const [id, item] of entries) if (live.has(String(id))) items[id] = item; + writeAtomic(vectorsPath(), JSON.stringify({ model, items }) + '\n'); + }); +} + +/** A vector for each idea. Only new and changed ideas go to the server; the others come from the cache. */ +export async function vectorsFor(ideas, { timeoutMs } = {}) { + const model = config.get('embed_model'); + const cached = readVectors(model); + const vectors = new Map(); + const todo = []; + for (const idea of ideas) { + const text = ideaText(idea); + const hash = hashOf(model, text); + if (cached[idea.id]?.hash === hash) vectors.set(idea.id, decodeVec(cached[idea.id].vec)); + else todo.push({ id: idea.id, text, hash }); + } + for (let i = 0; i < todo.length; i += BATCH) { + const part = todo.slice(i, i + BATCH); + const vecs = await embed(part.map((t) => t.text), { timeoutMs }); + part.forEach((t, k) => vectors.set(t.id, vecs[k])); + saveVectors(model, part.map((t, k) => [t.id, { hash: t.hash, vec: encodeVec(vecs[k]) }])); + } + return vectors; +} + +/** + * Groups of ideas that are close in meaning, by average-linkage clustering: two groups join while + * the mean similarity of the pairs across them is at least `threshold`. The ids are sorted first, + * so the order of the input does not change the result. Only groups of 2 or more ideas are returned. + */ +export function groupIds(ids, vectors, threshold) { + const sorted = [...new Set(ids)].sort((a, b) => a - b); + const n = sorted.length; + // sum[i][j]: sum of the similarities of all pairs across clusters i and j. + const sum = sorted.map((a) => sorted.map((b) => cosine(vectors.get(a), vectors.get(b)))); + const clusters = sorted.map((id) => [id]); + const alive = new Array(n).fill(true); + for (;;) { + let best = -Infinity; + let pair = null; + for (let i = 0; i < n; i++) { + if (!alive[i]) continue; + for (let j = i + 1; j < n; j++) { + if (!alive[j]) continue; + const avg = sum[i][j] / (clusters[i].length * clusters[j].length); + if (avg > best) { + best = avg; + pair = [i, j]; + } + } + } + if (!pair || best < threshold) break; + const [i, j] = pair; + clusters[i] = [...clusters[i], ...clusters[j]].sort((a, b) => a - b); + alive[j] = false; + for (let k = 0; k < n; k++) { + sum[i][k] += sum[j][k]; + sum[k][i] = sum[i][k]; + } + } + return clusters.filter((c, i) => alive[i] && c.length > 1); +} + +// Words that many ideas use and that say little about a group ("Add full support for ..."). +const LABEL_SKIP = new Set( + 'add added also app apps better build built fix full idea ideas made need new nicer now should support want work'.split(' '), +); + +/** A few words that the ideas of a group share and other ideas do not, most distinctive first. */ +export function groupLabel(members, all) { + const words = (idea) => [...wordSet(`${idea.title} ${idea.text} ${(idea.tags || []).join(' ')}`)].filter((w) => !LABEL_SKIP.has(w)); + const count = (ideas) => { + const df = new Map(); + for (const idea of ideas) for (const w of words(idea)) df.set(w, (df.get(w) || 0) + 1); + return df; + }; + const inside = count(members); + const overall = count(all); + const best = [...inside] + .filter(([, n]) => n >= 2) + .map(([w, n]) => [w, (n / members.length) * Math.log((all.length + 1) / (overall.get(w) || 1))]) + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, 3) + .map(([w]) => w); + return best.length ? best.join(' · ') : clip(members[0].title, 40); +} + +/** Groups for these ideas: [{ ids, label }], the biggest first. Throws EmbedError when the server does not answer. */ +export async function groupIdeas(ideas, { threshold = config.get('group_threshold'), vectors = null, timeoutMs } = {}) { + vectors ||= await vectorsFor(ideas, { timeoutMs }); + const byId = new Map(ideas.map((i) => [i.id, i])); + return groupIds(ideas.map((i) => i.id), vectors, threshold) + .map((ids) => ({ ids, label: groupLabel(ids.map((id) => byId.get(id)), ideas) })) + .sort((a, b) => b.ids.length - a.ids.length || a.ids[0] - b.ids[0]); +} + +/** The `k` ideas closest to idea `id`, with a similarity of at least `threshold`. */ +export function nearest(id, ids, vectors, { k = 3, threshold = 0 } = {}) { + const own = vectors.get(id); + if (!own) return []; + return ids + .filter((other) => other !== id && vectors.has(other)) + .map((other) => ({ id: other, score: Math.round(cosine(own, vectors.get(other)) * 1000) / 1000 })) + .filter((r) => r.score >= threshold) + .sort((a, b) => b.score - a.score || a.id - b.id) + .slice(0, k); +} + +/** Word search for when meaning search is not available: more query words found ranks higher. */ +function byWords(ideas, query) { + const words = [...wordSet(query)]; + if (!words.length) words.push(...String(query).toLowerCase().split(/\s+/).filter(Boolean)); + return ideas + .map((idea) => { + const hay = `${ideaText(idea)}\n${idea.triage?.brief || ''}`.toLowerCase(); + return { idea, score: words.filter((w) => hay.includes(w)).length / words.length }; + }) + .filter((r) => r.score > 0) + .sort((a, b) => b.score - a.score || b.idea.id - a.idea.id); +} + +/** + * Ideas like the query, best first: { mode: 'meaning' | 'words', results: [{ idea, score }], note }. + * When the embedding server does not answer, it falls back to word search and says why in `note`. + */ +export async function find(db, query, { filter = 'all', project = null, limit = 10, threshold, timeoutMs = 8000 } = {}) { + const ideas = listIdeas(db, { filter, project }); + threshold ??= config.get('search_threshold'); + try { + const vectors = await vectorsFor(ideas, { timeoutMs }); + const [q] = await embed([query], { query: true, timeoutMs }); + const results = ideas + .map((idea) => ({ idea, score: cosine(q, vectors.get(idea.id)) })) + .filter((r) => r.score >= threshold) + .sort((a, b) => b.score - a.score || a.idea.id - b.idea.id); + return { mode: 'meaning', results: limit ? results.slice(0, limit) : results, note: '' }; + } catch (e) { + if (!(e instanceof EmbedError)) throw e; + const results = byWords(ideas, query); + return { mode: 'words', results: limit ? results.slice(0, limit) : results, note: e.message }; + } +} + +/** Percentile `p` (0 to 1) of sorted numbers. */ +const percentile = (sorted, p) => sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))]; + +/** + * Embed every idea that has no current vector, then describe the set, like memstate's `embed + * status`: coverage, the nearest-neighbour similarity of each idea, and how many ideas the + * thresholds keep. Use it to set group_threshold and search_threshold for a new model. + */ +export async function status(db, { timeoutMs } = {}) { + const url = config.get('embed_url'); + const model = config.get('embed_model'); + const groupThreshold = config.get('group_threshold'); + const lines = [`model ${model} at ${url}`]; + const vectors = await vectorsFor(db.ideas, { timeoutMs }); + const ids = db.ideas.map((i) => i.id); + const dim = vectors.size ? vectors.values().next().value.length : 0; + lines.push(`vectors ${vectors.size}/${ids.length}${dim ? `, ${dim} dimensions` : ''} · ${vectorsPath()}`); + if (ids.length > 1) { + const nn = ids.map((id) => nearest(id, ids, vectors, { k: 1 })[0]?.score ?? 0).sort((a, b) => a - b); + const p = (x) => percentile(nn, x).toFixed(2); + lines.push(`nearest neighbour similarity: p10 ${p(0.1)} · p50 ${p(0.5)} · p90 ${p(0.9)} · max ${nn[nn.length - 1].toFixed(2)}`); + const groups = groupIds(ids, vectors, groupThreshold); + const grouped = groups.reduce((n, g) => n + g.length, 0); + lines.push(`group_threshold ${groupThreshold}: ${groups.length} groups hold ${grouped} of ${ids.length} ideas`); + } + return lines.join('\n'); +} diff --git a/src/hook.js b/src/hook.js index ce9a813..0af0987 100644 --- a/src/hook.js +++ b/src/hook.js @@ -1,11 +1,13 @@ // UserPromptSubmit hook: answers /idea, /ideas, and the local /ideas-* commands (ls, cat, rm, done, -// reopen, watch) and blocks the prompt, so the model is never called. That makes capture free, -// instant, and possible even when the session is out of usage. Every other prompt passes through -// untouched, including /ideas-go, /ideas-all, /ideas-sort, and questions, which their skills answer. -// After each prompt, the hook lets the watcher start a background pass when there is work. +// reopen, find, groups, watch) and blocks the prompt, so the model is never called. That makes +// capture free, instant, and possible even when the session is out of usage. Every other prompt +// passes through untouched, including /ideas-go, /ideas-all, /ideas-sort, and questions, which their +// skills answer. After each prompt, the hook lets the watcher and the dashboard catch up. -import { renderAdded, renderBoard, renderIdea } from './render.js'; -import { addIdeas, FILTERS, findIdea, lane, load, removeIdeas, updateIdea } from './store.js'; +import * as embed from './embed.js'; +import * as publish from './publish.js'; +import { renderAdded, renderBoard, renderFound, renderGroups, renderIdea } from './render.js'; +import { addIdeas, FILTERS, findIdea, lane, listIdeas, load, removeIdeas, updateIdea } from './store.js'; import { clip, splitIdeas } from './text.js'; import * as watch from './watch.js'; @@ -21,6 +23,8 @@ const USAGE = `Usage: /idea <text> add an idea (a bulleted list a /ideas-cat N... show ideas in full /ideas-rm N... delete ideas for good /ideas-done N [note] · /ideas-reopen N + /ideas-find <words> search by meaning (all lanes) + /ideas-groups [lane|-a] ideas grouped by meaning These call the model: /ideas-go [N] build the next idea, or #N, on its model. New ideas are triaged first. /ideas-all do every idea that fits this chat. The others stay in the queue. @@ -40,8 +44,11 @@ function readStdin() { }); } -/** Handle one prompt. Returns the text to show the user, or null to let the prompt through. */ -export function handlePrompt(prompt, { cwd = process.cwd(), session = null } = {}) { +// A search by meaning must not hold the prompt for long when the embedding server is away. +const SEARCH_TIMEOUT_MS = 4000; + +/** Handle one prompt. Resolves to the text to show the user, or null to let the prompt through. */ +export async function handlePrompt(prompt, { cwd = process.cwd(), session = null } = {}) { const m = COMMAND.exec(prompt || ''); if (!m) return null; const command = m[1].toLowerCase(); @@ -66,6 +73,26 @@ export function handlePrompt(prompt, { cwd = process.cwd(), session = null } = { if (command === 'ideas' && !words.length) return renderBoard(db, { cwd, hints: true }); + if (command === 'ideas-find') { + if (!arg) return USAGE; + return renderFound(await embed.find(db, arg, { timeoutMs: SEARCH_TIMEOUT_MS }), arg, { cwd }); + } + + if (command === 'ideas-groups') { + const lower = words.map((w) => (w === '-a' ? 'all' : w.toLowerCase())); + if (lower.length <= 1 && lower.every((w) => FILTERS.includes(w))) { + const filter = lower[0] || 'open'; + const ideas = listIdeas(db, { filter }); + try { + const groups = await embed.groupIdeas(ideas, { timeoutMs: SEARCH_TIMEOUT_MS }); + return renderGroups(groups, ideas, { cwd, scope: filter }); + } catch (e) { + if (!(e instanceof embed.EmbedError)) throw e; + return `Groups need the embedding server, which does not answer: ${e.message}`; + } + } + } + if (command === 'ideas-ls') { const lower = words.map((w) => (w === '-a' ? 'all' : w.toLowerCase())); const filters = lower.filter((w) => FILTERS.includes(w)); @@ -103,7 +130,7 @@ export async function runHook() { } let message = null; try { - message = handlePrompt(typeof input?.prompt === 'string' ? input.prompt : '', { + message = await handlePrompt(typeof input?.prompt === 'string' ? input.prompt : '', { cwd: input.cwd || process.cwd(), session: input.session_id || null, }); @@ -120,10 +147,12 @@ export async function runHook() { hookSpecificOutput: { hookEventName: 'UserPromptSubmit', suppressOriginalPrompt: true }, })); } - // Each prompt lets the watcher catch up, so it keeps going while it is on. - try { - watch.kick(); - } catch { - // The watcher must never stop a prompt. + // Each prompt lets the watcher and the dashboard catch up, so they keep going while they are on. + for (const kick of [watch.kick, publish.kick]) { + try { + kick(); + } catch { + // The watcher and the dashboard must never stop a prompt. + } } } diff --git a/src/mcp.js b/src/mcp.js index da75325..0827e9f 100644 --- a/src/mcp.js +++ b/src/mcp.js @@ -6,8 +6,9 @@ import path from 'node:path'; import readline from 'node:readline'; import { fileURLToPath } from 'node:url'; import { headlessTriage, triageFirst } from './claude.js'; +import * as embed from './embed.js'; import { knownProjects } from './projects.js'; -import { renderAdded, renderBoard, renderIdea } from './render.js'; +import { renderAdded, renderBoard, renderFound, renderGroups, renderIdea } from './render.js'; import { BATCH_SCHEMA, MODELS, pendingIdeas, triagePrompt } from './rubric.js'; import { addIdeas, @@ -58,13 +59,17 @@ const TOOLS = [ }, { name: 'idea_list', - description: 'Show the idea board, or one idea in full when id is given. full=true shows every listed idea in full.', + description: + 'Show the idea board, or one idea in full when id is given. full=true shows every listed idea in full. ' + + 'semantic=true ranks ideas by meaning for the query. groups=true groups the listed ideas by meaning.', inputSchema: { type: 'object', properties: { id: { type: 'integer', description: 'Show this idea in full.' }, filter: { type: 'string', enum: FILTERS, description: 'Default: open (doing, do, maybe, inbox).' }, - query: { type: 'string', description: 'Only ideas containing all of these words.' }, + query: { type: 'string', description: 'Only ideas containing all of these words (with semantic: ideas like it).' }, + semantic: { type: 'boolean', description: 'Rank by meaning (nomic-embed-text), best first. Falls back to words.' }, + groups: { type: 'boolean', description: 'Group the listed ideas by meaning.' }, here: { type: 'boolean', description: 'Only ideas from the current project.' }, limit: { type: 'integer' }, full: { type: 'boolean', description: 'Every listed idea in full (brief, text, notes), not one line each.' }, @@ -156,6 +161,8 @@ const PROMPTS = [ { name: 'ideas-ls', description: 'List the queue, one lane, or every lane (-a)', arguments: [{ name: 'lane', required: false }] }, { name: 'ideas-cat', description: 'Show ideas in full', arguments: [{ name: 'ids', required: true }] }, { name: 'ideas-rm', description: 'Delete ideas for good', arguments: [{ name: 'ids', required: true }] }, + { name: 'ideas-find', description: 'Search ideas by meaning', arguments: [{ name: 'words', required: true }] }, + { name: 'ideas-groups', description: 'Show ideas grouped by meaning', arguments: [{ name: 'lane', required: false }] }, { name: 'ideas-done', description: 'Mark an idea done', arguments: [{ name: 'id', required: true }, { name: 'note', required: false }] }, { name: 'ideas-reopen', description: 'Put an idea back in the queue', arguments: [{ name: 'id', required: true }] }, { name: 'ideas-go', description: 'Build the next idea on its recommended model', arguments: [{ name: 'id', required: false }] }, @@ -202,7 +209,7 @@ const handlers = { return renderAdded(results, load()); }, - idea_list({ id, filter = 'open', query = '', here = false, limit = 0, full = false }) { + async idea_list({ id, filter = 'open', query = '', here = false, limit = 0, full = false, semantic = false, groups = false }) { const db = load(); if (id != null) { const idea = findIdea(db, id); @@ -210,6 +217,15 @@ const handlers = { return renderIdea(idea); } const project = here ? currentProject() : null; + if (semantic && query) { + const found = await embed.find(db, query, { filter, project, limit: limit || 10 }); + if (!full) return renderFound(found, query, { cwd: currentProject() }); + return found.results.length ? found.results.map((r) => renderIdea(r.idea)).join('\n\n') : 'No ideas match.'; + } + if (groups) { + const ideas = listIdeas(db, { filter, query, limit, project }); + return renderGroups(await embed.groupIdeas(ideas), ideas, { cwd: currentProject(), scope: filter }); + } if (full) { const ideas = listIdeas(db, { filter, query, limit, project }); return ideas.length ? ideas.map(renderIdea).join('\n\n') : 'No ideas match.'; diff --git a/src/publish.js b/src/publish.js new file mode 100644 index 0000000..17a0a68 --- /dev/null +++ b/src/publish.js @@ -0,0 +1,234 @@ +// The dashboard: a static page (dashboard/index.html) and a snapshot of the archive (data.json). +// `ideamine publish` uploads both with HTTP PUT, for example to nginx on a WireGuard address, or +// writes them to a folder. Like the watcher, no process stays alive: the hook calls kick() for +// each prompt, and kick() starts a background publish when the archive changed since the last one. + +import { spawn } from 'node:child_process'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as config from './config.js'; +import * as embed from './embed.js'; +import { counts, dbPath, home, lane, listIdeas, load } from './store.js'; + +const PAGE = new URL('../dashboard/index.html', import.meta.url); +const WAIT_AFTER_ERROR_MS = 10 * 60 * 1000; // after a failed publish, before kick() tries again +const LOCK_STALE_MS = 10 * 60 * 1000; + +const statePath = () => path.join(home(), 'publish.json'); +const lockPath = () => path.join(home(), '.publish'); + +/** The earliest of the given times; null when none is set. */ +const earliest = (...times) => times.filter(Boolean).sort()[0] || null; + +/** + * Bars for the timeline: the time an idea waited in the inbox, waited in the queue, and was in work. + * `to: null` means "until now". Ideas that started before ideamine recorded start times get an + * `estimated` work bar from the triage (or the creation) to the end. + */ +export function phases(idea) { + const triaged = idea.triage?.at || null; + const started = idea.started || null; + const closed = idea.closed || null; + const out = []; + const add = (phase, from, to, estimated = false) => { + if (from && (!to || to >= from)) out.push(estimated ? { phase, from, to, estimated } : { phase, from, to }); + }; + if (!started && (idea.status === 'doing' || idea.status === 'done')) { + if (triaged) add('inbox', idea.created, triaged); + add('doing', triaged || idea.created, closed, true); + return out; + } + add('inbox', idea.created, earliest(triaged, started, closed)); + if (triaged && (!started || triaged < started)) add('queued', triaged, started || closed); + if (started) add('doing', started, closed); + return out; +} + +/** data.json (version 1). The dashboard page reads it; its format is in the README. */ +export function snapshot(db, { vectors = null, groups = [], generated = new Date().toISOString() } = {}) { + const model = config.get('embed_model'); + const groupThreshold = config.get('group_threshold'); + const rank = new Map(listIdeas(db, { filter: 'all' }).map((idea, n) => [idea.id, n])); + const groupOf = new Map(); + groups.forEach((g, n) => g.ids.forEach((id) => groupOf.set(id, n))); + const ids = db.ideas.map((i) => i.id); + return { + version: 1, + generated, + embed: { + model, + dim: vectors?.size ? vectors.values().next().value.length : 0, + query_prefix: embed.queryText(model, ''), + search_threshold: config.get('search_threshold'), + group_threshold: groupThreshold, + available: !!vectors, + }, + counts: counts(db), + ideas: db.ideas.map((i) => { + const t = i.triage; + return { + id: i.id, + key: `IDEA-${i.id}`, + title: i.title, + text: i.text, + lane: lane(i), + status: i.status, + rank: rank.get(i.id), + tags: i.tags || [], + project: i.project ? path.basename(i.project) : null, + created: i.created, + updated: i.updated, + triaged: t?.at || null, + started: i.started || null, + closed: i.closed || null, + verdict: t?.verdict || null, + impact: t?.impact ?? null, + size: t?.size || null, + model: t?.model || null, + why: t?.why || '', + brief: t?.brief || '', + notes: i.notes || [], + dup_of: i.dup_of || null, + group: groupOf.has(i.id) ? groupOf.get(i.id) : null, + related: vectors ? embed.nearest(i.id, ids, vectors, { k: 3, threshold: groupThreshold }) : [], + phases: phases(i), + vec: vectors?.has(i.id) ? embed.encodeVec(vectors.get(i.id)) : null, + }; + }), + groups: groups.map((g, n) => ({ id: n, label: g.label, ids: g.ids })), + }; +} + +/** The snapshot, with groups when the embedding server answers. `note` says why there are none. */ +export async function build(db, { timeoutMs = 60000 } = {}) { + let vectors = null; + let groups = []; + let note = ''; + try { + vectors = await embed.vectorsFor(db.ideas, { timeoutMs }); + groups = await embed.groupIdeas(db.ideas, { vectors }); + } catch (e) { + if (!(e instanceof embed.EmbedError)) throw e; + vectors = null; + note = `published without search by meaning or groups: ${e.message}`; + } + return { data: snapshot(db, { vectors, groups }), note }; +} + +async function put(url, body, type) { + let res; + try { + res = await fetch(url, { method: 'PUT', body, headers: { 'content-type': type }, signal: AbortSignal.timeout(30000) }); + } catch (e) { + throw new Error(`cannot reach ${url} (${e.name === 'TimeoutError' ? 'no answer in 30 s' : e.cause?.code || e.cause?.message || e.message})`); + } + if (!res.ok) throw new Error(`${url} answered ${res.status} ${res.statusText}`); +} + +const pageHash = () => crypto.createHash('sha256').update(fs.readFileSync(PAGE)).digest('hex').slice(0, 16); + +function archiveTime() { + try { + return fs.statSync(dbPath()).mtimeMs; + } catch { + return 0; + } +} + +export function readState() { + try { + return JSON.parse(fs.readFileSync(statePath(), 'utf8')); + } catch { + return {}; + } +} + +function writeState(patch) { + fs.mkdirSync(home(), { recursive: true }); + fs.writeFileSync(statePath(), JSON.stringify({ ...readState(), ...patch }, null, 2) + '\n'); +} + +/** + * Upload index.html and data.json to `url` (a folder URL on the dashboard server), or write them + * to the folder `dir`. Returns what it did, for the caller to show. + */ +export async function publish({ url = config.get('publish_url'), dir = null } = {}) { + if (!url && !dir) throw new Error('no dashboard server is set. Run `ideamine publish <url>` once, or pass --dir <folder>'); + const mtime = archiveTime(); + const { data, note } = await build(load()); + const files = [ + ['data.json', JSON.stringify(data), 'application/json'], + ['index.html', fs.readFileSync(PAGE), 'text/html; charset=utf-8'], + ]; + if (dir) { + fs.mkdirSync(dir, { recursive: true }); + for (const [name, body] of files) fs.writeFileSync(path.join(dir, name), body); + } else { + const base = url.endsWith('/') ? url : `${url}/`; + for (const [name, body, type] of files) await put(new URL(name, base), body, type); + writeState({ at: new Date().toISOString(), mtime, page: pageHash(), url, error: null, errorAt: null }); + } + return { where: dir || url, ideas: data.ideas.length, groups: data.groups.length, note }; +} + +function isRunning(now) { + try { + return now - fs.statSync(lockPath()).mtimeMs < LOCK_STALE_MS; + } catch { + return false; + } +} + +/** One background publish, from kick(). Only one runs at a time, and a failure waits 10 minutes. */ +export async function backgroundPublish() { + fs.mkdirSync(home(), { recursive: true }); + try { + fs.mkdirSync(lockPath()); + } catch (e) { + if (e.code !== 'EEXIST' || isRunning(Date.now())) return 'busy'; + fs.utimesSync(lockPath(), new Date(), new Date()); // take over the lock of a crashed publish + } + try { + await publish(); + return 'done'; + } catch (e) { + writeState({ error: e.message, errorAt: new Date().toISOString() }); + return 'error'; + } finally { + fs.rmSync(lockPath(), { recursive: true, force: true }); + } +} + +function startProcess() { + const bin = fileURLToPath(new URL('../bin/ideamine.js', import.meta.url)); + spawn(process.execPath, [bin, 'publish', '--background'], { cwd: home(), detached: true, stdio: 'ignore', windowsHide: true }).unref(); +} + +/** + * Start a background publish when a dashboard server is set, the archive or the page changed since + * the last publish, no publish runs, and no publish failed a short time ago. + */ +export function kick({ start = startProcess, now = Date.now() } = {}) { + const url = config.get('publish_url'); + if (!url) return false; + const state = readState(); + if (state.errorAt && now - Date.parse(state.errorAt) < WAIT_AFTER_ERROR_MS) return false; + if (isRunning(now)) return false; + const current = state.url === url && archiveTime() <= (state.mtime || 0) && state.page === pageHash(); + if (current) return false; + start(); + return true; +} + +/** Where the dashboard goes and how the last publish went. */ +export function status() { + const url = config.get('publish_url'); + if (!url) return 'ideamine publish: off. `ideamine publish <url>` sets the dashboard server.'; + const s = readState(); + const lines = [`ideamine publish: ${url}`]; + lines.push(s.at ? `last publish ${s.at}` : 'not published yet'); + if (s.error) lines.push(`the last background publish failed at ${s.errorAt}: ${s.error}`); + return lines.join('\n'); +} diff --git a/src/render.js b/src/render.js index be694ed..805e05e 100644 --- a/src/render.js +++ b/src/render.js @@ -71,6 +71,29 @@ export function renderBoard(db, { filter = 'open', project = null, query = '', c return out.join('\n'); } +/** Search results, best first, with the similarity (by meaning) or the share of words found. */ +export function renderFound({ mode, results, note }, query, { cwd = null } = {}) { + const how = mode === 'meaning' ? 'by meaning' : `by words, because search by meaning is not available (${note})`; + const out = [`ideas like "${clip(query, 60)}", ${how}`]; + if (!results.length) out.push(' (no match)'); + for (const { idea, score } of results) { + out.push(`${String(Math.round(score * 100)).padStart(4)}% ${lane(idea).padEnd(7)}${ideaLine(idea, { cwd }).slice(1)}`); + } + return out.join('\n'); +} + +/** Groups of ideas that are close in meaning, then the ideas in no group. */ +export function renderGroups(groups, ideas, { cwd = null, scope = 'open' } = {}) { + const out = [`ideamine groups: ${groups.length} group${groups.length === 1 ? '' : 's'} of ${scope} ideas, by meaning`]; + const byId = new Map(ideas.map((i) => [i.id, i])); + const line = (idea) => ` ${lane(idea).padEnd(7)}${ideaLine(idea, { cwd }).slice(1)}`; + for (const g of groups) out.push('', g.label.toUpperCase(), ...g.ids.map((id) => line(byId.get(id)))); + const grouped = new Set(groups.flatMap((g) => g.ids)); + const rest = ideas.filter((i) => !grouped.has(i.id)); + if (rest.length) out.push('', 'IN NO GROUP', ...rest.map(line)); + return out.join('\n'); +} + /** One idea in full. */ export function renderIdea(idea) { const t = idea.triage; diff --git a/src/store.js b/src/store.js index c0efa83..f624ec1 100644 --- a/src/store.js +++ b/src/store.js @@ -102,29 +102,42 @@ function renameWithRetry(from, to) { } } -function save(db) { - const file = dbPath(); +/** Replace `file` in one step, so a reader never sees half a file. Call it while you hold the lock. */ +export function writeAtomic(file, text, { backup = false } = {}) { const tmp = `${file}.${process.pid}.${Date.now()}.tmp`; - fs.writeFileSync(tmp, JSON.stringify(db, null, 2) + '\n'); - try { - fs.copyFileSync(file, `${file}.bak`); - } catch (e) { - if (e.code !== 'ENOENT') throw e; + fs.writeFileSync(tmp, text); + if (backup) { + try { + fs.copyFileSync(file, `${file}.bak`); + } catch (e) { + if (e.code !== 'ENOENT') throw e; + } } renameWithRetry(tmp, file); } -/** Lock, read, apply `fn(db)`, write. Nothing is written if `fn` throws. */ -export function mutate(fn) { +function save(db) { + writeAtomic(dbPath(), JSON.stringify(db, null, 2) + '\n', { backup: true }); +} + +/** Run `fn` while this process holds the archive lock. Other files in home() can share the lock. */ +export function withLock(fn) { const release = acquireLock(); try { + return fn(); + } finally { + release(); + } +} + +/** Lock, read, apply `fn(db)`, write. Nothing is written if `fn` throws. */ +export function mutate(fn) { + return withLock(() => { const db = load(); const result = fn(db); save(db); return result; - } finally { - release(); - } + }); } // --------------------------------------------------------------------------------------------- @@ -332,6 +345,8 @@ export function updateIdea(id, patch = {}) { status = STATUS_ALIASES[status] || status; if (status === 'reopen' || status === 'open') status = idea.triage ? 'triaged' : 'inbox'; if (!STATUSES.includes(status)) throw new Error(`status must be one of ${STATUSES.join(', ')}, or reopen`); + // The dashboard timeline shows work from the latest start. + if (status === 'doing' && idea.status !== 'doing') idea.started = now(); idea.status = status; if (status === 'done' || status === 'dropped') idea.closed = now(); else delete idea.closed; diff --git a/tests/embed.test.js b/tests/embed.test.js new file mode 100644 index 0000000..616ba05 --- /dev/null +++ b/tests/embed.test.js @@ -0,0 +1,163 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, test } from 'node:test'; +import fc from 'fast-check'; +import * as embed from '../src/embed.js'; +import * as store from '../src/store.js'; +import { startFakeServer } from './fixtures/fake-server.js'; + +let server; + +beforeEach(async () => { + process.env.IDEAMINE_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'ideamine-embed-')); + server = await startFakeServer(); + process.env.IDEAMINE_EMBED_URL = `${server.url}/v1`; + delete process.env.IDEAMINE_EMBED_MODEL; +}); + +afterEach(() => server.close()); + +const unitVector = (dim) => + fc + .array(fc.float({ min: -1, max: 1, noNaN: true }), { minLength: dim, maxLength: dim }) + .filter((v) => v.some((x) => Math.abs(x) > 1e-3)) + .map((v) => embed.unit(v)); + +test('property: a vector survives encode and decode unchanged', () => { + fc.assert( + fc.property(fc.float32Array(), (vec) => { + const back = embed.decodeVec(embed.encodeVec(vec)); + assert.equal(back.length, vec.length); + for (let i = 0; i < vec.length; i++) assert.ok(Object.is(back[i], vec[i]), `index ${i}: ${back[i]} != ${vec[i]}`); + }), + ); +}); + +test('property: cosine of unit vectors is symmetric, at most 1, and 1 for the same vector', () => { + fc.assert( + fc.property(unitVector(8), unitVector(8), (a, b) => { + assert.ok(Math.abs(embed.cosine(a, b) - embed.cosine(b, a)) < 1e-6); + assert.ok(Math.abs(embed.cosine(a, b)) <= 1 + 1e-5); + assert.ok(Math.abs(embed.cosine(a, a) - 1) < 1e-5); + }), + ); +}); + +test('property: groups are disjoint, follow the average-linkage rule, and do not depend on the input order', () => { + const cases = fc + .uniqueArray(fc.integer({ min: 1, max: 500 }), { minLength: 0, maxLength: 12 }) + .chain((ids) => fc.tuple(fc.constant(ids), fc.array(unitVector(4), { minLength: ids.length, maxLength: ids.length }), fc.double({ min: -1, max: 1, noNaN: true }))); + fc.assert( + fc.property(cases, ([ids, vecs, threshold]) => { + const vectors = new Map(ids.map((id, i) => [id, vecs[i]])); + const groups = embed.groupIds(ids, vectors, threshold); + const seen = groups.flat(); + assert.equal(new Set(seen).size, seen.length); + assert.ok(seen.every((id) => ids.includes(id))); + assert.ok(groups.every((g) => g.length >= 2)); + assert.deepEqual(embed.groupIds([...ids].reverse(), vectors, threshold), groups); + // Average linkage: the mean similarity inside a group is at least the threshold, and the mean + // similarity across any two clusters that stayed apart (single ideas too) is below it. + const mean = (a, b) => { + let sum = 0; + for (const x of a) for (const y of b) sum += embed.cosine(vectors.get(x), vectors.get(y)); + return sum / (a.length * b.length); + }; + for (const g of groups) { + let sum = 0; + let pairs = 0; + for (let i = 0; i < g.length; i++) for (let j = i + 1; j < g.length; j++, pairs++) sum += embed.cosine(vectors.get(g[i]), vectors.get(g[j])); + assert.ok(sum / pairs >= threshold - 1e-6, `group ${g} has mean ${sum / pairs} < ${threshold}`); + } + const clusters = [...groups, ...ids.filter((id) => !seen.includes(id)).map((id) => [id])]; + for (let i = 0; i < clusters.length; i++) { + for (let j = i + 1; j < clusters.length; j++) assert.ok(mean(clusters[i], clusters[j]) < threshold + 1e-6); + } + }), + ); +}); + +test('property: a higher threshold only splits groups, it never joins ideas from two groups', () => { + const cases = fc + .uniqueArray(fc.integer({ min: 1, max: 500 }), { minLength: 2, maxLength: 10 }) + .chain((ids) => fc.tuple(fc.constant(ids), fc.array(unitVector(4), { minLength: ids.length, maxLength: ids.length }), fc.double({ min: -1, max: 1, noNaN: true }), fc.double({ min: 0, max: 1, noNaN: true }))); + fc.assert( + fc.property(cases, ([ids, vecs, low, step]) => { + const vectors = new Map(ids.map((id, i) => [id, vecs[i]])); + const coarse = embed.groupIds(ids, vectors, low); + for (const g of embed.groupIds(ids, vectors, low + step)) { + assert.ok(coarse.some((c) => g.every((id) => c.includes(id))), `${g} is not inside one group of ${JSON.stringify(coarse)}`); + } + }), + ); +}); + +test('nomic prefixes: documents and queries get their own task prefix; other models get raw text', async () => { + store.addIdeas(['read subtitles aloud']); + await embed.find(store.load(), 'subtitles'); + assert.deepEqual(server.inputs, ['search_document: read subtitles aloud', 'search_query: subtitles']); + process.env.IDEAMINE_EMBED_MODEL = 'mxbai-embed-large'; + await embed.find(store.load(), 'subtitles'); + assert.deepEqual(server.inputs.slice(2), ['read subtitles aloud', 'subtitles']); +}); + +test('the cache embeds each idea once, again after a change, and drops deleted ideas', async () => { + store.addIdeas(['sync subtitles with audiobooks', 'dark mode for the popup']); + await embed.vectorsFor(store.load().ideas); + assert.equal(server.inputs.length, 2); + await embed.vectorsFor(store.load().ideas); + assert.equal(server.inputs.length, 2); // from the cache + store.updateIdea(2, { text: 'dark mode for the settings page' }); + await embed.vectorsFor(store.load().ideas); + assert.deepEqual(server.inputs.slice(2), ['search_document: dark mode for the settings page']); + store.removeIdeas([1]); + store.addIdeas(['a third idea']); + await embed.vectorsFor(store.load().ideas); + const cached = JSON.parse(fs.readFileSync(path.join(store.home(), 'vectors.json'), 'utf8')); + assert.deepEqual(Object.keys(cached.items).sort(), ['2', '3']); +}); + +test('a text that is too long is halved until the server takes it', async () => { + server.options.maxChars = 400; + store.addIdeas([`${'subtitle '.repeat(100)}end`, 'short idea']); + const vectors = await embed.vectorsFor(store.load().ideas); + assert.equal(vectors.size, 2); + const tries = server.inputs.filter((x) => x.startsWith('search_document: subtitle')).map((x) => x.length); + assert.deepEqual(tries, [920, 920, 468, 242]); // the batch, then one at a time, halved twice +}); + +test('find ranks by meaning and keeps results above the threshold', async () => { + store.addIdeas(['subtitles for audiobooks in the reader', 'a tor exit relay on the server', 'subtitles in the video player']); + const found = await embed.find(store.load(), 'subtitles audiobooks reader', { threshold: 0.3 }); + assert.equal(found.mode, 'meaning'); + assert.deepEqual(found.results.map((r) => r.idea.id), [1, 3]); + assert.ok(found.results[0].score > found.results[1].score); +}); + +test('find falls back to word search and says why when the server does not answer', async () => { + store.addIdeas(['subtitles for audiobooks', 'dark mode']); + server.options.down = true; + const found = await embed.find(store.load(), 'audiobooks subtitles'); + assert.equal(found.mode, 'words'); + assert.match(found.note, /503/); + assert.deepEqual(found.results.map((r) => r.idea.id), [1]); + const closed = await startFakeServer(); + await closed.close(); // now nothing listens on its port + process.env.IDEAMINE_EMBED_URL = `${closed.url}/v1`; + const offline = await embed.find(store.load(), 'dark', { timeoutMs: 5000 }); + assert.equal(offline.mode, 'words'); + assert.match(offline.note, /cannot reach http:\/\/127\.0\.0\.1:\d+\/v1\/embeddings \((ECONNREFUSED|no answer)/); +}); + +test('group labels use the distinctive shared words, not "add full support"', () => { + store.addIdeas([ + 'Add full support for subread to hoshireader', + 'Add full support for subread and whispersync to chimahon', + 'Add full support for whispersync to koreader', + 'Add a back button to the key screen', + ]); + const ideas = store.load().ideas; + assert.equal(embed.groupLabel(ideas.slice(0, 3), ideas), 'subread · whispersync'); +}); diff --git a/tests/fixtures/fake-server.js b/tests/fixtures/fake-server.js new file mode 100644 index 0000000..abc0293 --- /dev/null +++ b/tests/fixtures/fake-server.js @@ -0,0 +1,51 @@ +// A stand-in for the dashboard server in the tests: an OpenAI-compatible POST /v1/embeddings, and +// PUT for the files that `ideamine publish` uploads. The vectors are a hashed bag of words, so +// texts that share words are similar, and the same text always gets the same vector. + +import crypto from 'node:crypto'; +import http from 'node:http'; + +export const DIM = 64; + +export function fakeVector(text) { + const vec = new Array(DIM).fill(0); + const words = String(text).toLowerCase().replace(/^search_(query|document): /, '').match(/[\p{L}\p{N}]{3,}/gu) || ['empty']; + for (const w of words) vec[crypto.createHash('sha1').update(w).digest().readUInt16LE(0) % DIM] += 1; + return vec; +} + +/** + * Start the server on a free port. Options: `maxChars` makes it refuse longer inputs the way + * llama.cpp does; `down` makes it answer 503. Returns { url, inputs, files, close, options }. + */ +export async function startFakeServer(options = {}) { + const inputs = []; // every text that /v1/embeddings got, in order + const files = new Map(); // path -> body of each PUT + const server = http.createServer((req, res) => { + let body = ''; + req.setEncoding('utf8'); + req.on('data', (chunk) => (body += chunk)); + req.on('end', () => { + if (req.method === 'PUT') { + files.set(req.url, body); + res.writeHead(201).end(); + return; + } + if (req.method !== 'POST' || req.url !== '/v1/embeddings') return res.writeHead(404).end(); + if (options.down) return res.writeHead(503).end('{"error":"model is loading"}'); + const { input } = JSON.parse(body); + const texts = Array.isArray(input) ? input : [input]; + inputs.push(...texts); + const long = texts.find((t) => options.maxChars && t.length > options.maxChars); + if (long) { + const error = { code: 500, message: `input (${long.length} tokens) is too large to process. increase the physical batch size`, type: 'server_error' }; + return res.writeHead(500, { 'content-type': 'application/json' }).end(JSON.stringify({ error })); + } + const data = texts.map((t, index) => ({ object: 'embedding', index, embedding: fakeVector(t) })); + res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ object: 'list', data })); + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const url = `http://127.0.0.1:${server.address().port}`; + return { url, inputs, files, options, close: () => new Promise((resolve) => server.close(resolve)) }; +} diff --git a/tests/hook.test.js b/tests/hook.test.js index afed0d5..7b66153 100644 --- a/tests/hook.test.js +++ b/tests/hook.test.js @@ -7,6 +7,7 @@ import { beforeEach, test } from 'node:test'; import { fileURLToPath } from 'node:url'; import { handlePrompt } from '../src/hook.js'; import * as store from '../src/store.js'; +import { startFakeServer } from './fixtures/fake-server.js'; const BIN = fileURLToPath(new URL('../bin/ideamine.js', import.meta.url)); @@ -113,10 +114,10 @@ test('while the watcher is on, a saved idea is triaged in the background', async assert.equal(store.findIdea(store.load(), 1).triage?.by, 'haiku (headless)'); }); -test('handlePrompt filters by project with "here"', () => { +test('handlePrompt filters by project with "here"', async () => { store.addIdeas(['mine'], { project: '/a' }); store.addIdeas(['theirs'], { project: '/b' }); - const board = handlePrompt('/ideas-ls here', { cwd: '/a' }); + const board = await handlePrompt('/ideas-ls here', { cwd: '/a' }); assert.match(board, /mine/); assert.doesNotMatch(board, /theirs/); }); @@ -133,3 +134,39 @@ test('when the archive is unreadable the prompt goes through instead of being lo assert.equal(r.stdout, ''); assert.match(r.stderr, /not valid JSON/); }); + +test('/ideas-find searches by meaning and /ideas-groups groups by meaning, with no model call', async () => { + const server = await startFakeServer(); + process.env.IDEAMINE_EMBED_URL = `${server.url}/v1`; + process.env.IDEAMINE_GROUP_THRESHOLD = '0.5'; + try { + store.addIdeas(['subtitles for audiobooks', 'subtitles for audiobooks on android', 'a tor exit relay']); + const found = await handlePrompt('/ideas-find audiobooks with subtitles'); + assert.match(found, /^ideas like "audiobooks with subtitles", by meaning\n.*#1 .*\n.*#2 /); + assert.doesNotMatch(found, /tor exit/); + const groups = await handlePrompt('/ideas-groups'); + assert.match(groups, /^ideamine groups: 1 group of open ideas/); + assert.match(groups, /IN NO GROUP\n.*#3 /); + assert.equal(await handlePrompt('/ideas-groups what is this'), null); // a question: the skill answers + } finally { + await server.close(); + delete process.env.IDEAMINE_EMBED_URL; + delete process.env.IDEAMINE_GROUP_THRESHOLD; + } +}); + +test('/ideas-find falls back to words when the embedding server is away', async () => { + const closed = await startFakeServer(); + await closed.close(); // nothing listens on its port now + process.env.IDEAMINE_EMBED_URL = `${closed.url}/v1`; + try { + store.addIdeas(['subtitles for audiobooks', 'dark mode']); + const out = runHook('/ideas-find subtitles'); + assert.equal(out.decision, 'block'); + assert.match(out.reason, /by words, because search by meaning is not available \(cannot reach/); + assert.match(out.reason, /#1 /); + assert.doesNotMatch(out.reason, /#2 /); + } finally { + delete process.env.IDEAMINE_EMBED_URL; + } +}); diff --git a/tests/mcp.test.js b/tests/mcp.test.js index b772897..90986d8 100644 --- a/tests/mcp.test.js +++ b/tests/mcp.test.js @@ -109,7 +109,7 @@ test('tool errors come back as isError results, not protocol errors', async () = test('prompts mirror the skills', async () => { const list = await request('prompts/list'); const names = list.result.prompts.map((p) => p.name); - assert.deepEqual(names, ['idea', 'ideas', 'ideas-ls', 'ideas-cat', 'ideas-rm', 'ideas-done', 'ideas-reopen', 'ideas-go', 'ideas-all', 'ideas-sort', 'ideas-watch']); + assert.deepEqual(names, ['idea', 'ideas', 'ideas-ls', 'ideas-cat', 'ideas-rm', 'ideas-find', 'ideas-groups', 'ideas-done', 'ideas-reopen', 'ideas-go', 'ideas-all', 'ideas-sort', 'ideas-watch']); // One prompt for each skill, so that other MCP clients get the same commands as the plugin. assert.deepEqual([...names].sort(), fs.readdirSync(new URL('../skills', import.meta.url)).sort()); const go = await request('prompts/get', { name: 'ideas-go', arguments: { id: '12' } }); diff --git a/tests/publish.test.js b/tests/publish.test.js new file mode 100644 index 0000000..61376e0 --- /dev/null +++ b/tests/publish.test.js @@ -0,0 +1,126 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, test } from 'node:test'; +import * as config from '../src/config.js'; +import * as embed from '../src/embed.js'; +import * as publish from '../src/publish.js'; +import * as store from '../src/store.js'; +import { startFakeServer } from './fixtures/fake-server.js'; + +let server; + +beforeEach(async () => { + process.env.IDEAMINE_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'ideamine-publish-')); + server = await startFakeServer(); + process.env.IDEAMINE_EMBED_URL = `${server.url}/v1`; + for (const name of ['IDEAMINE_PUBLISH_URL', 'IDEAMINE_EMBED_MODEL', 'IDEAMINE_GROUP_THRESHOLD', 'IDEAMINE_SEARCH_THRESHOLD']) delete process.env[name]; +}); + +afterEach(() => server.close()); + +const T = (h) => `2026-09-21T${String(h).padStart(2, '0')}:00:00.000Z`; + +test('timeline phases follow the inbox, the queue, and the work', () => { + const base = { id: 1, created: T(1), status: 'inbox' }; + assert.deepEqual(publish.phases(base), [{ phase: 'inbox', from: T(1), to: null }]); + const queued = { ...base, status: 'triaged', triage: { at: T(2) } }; + assert.deepEqual(publish.phases(queued), [ + { phase: 'inbox', from: T(1), to: T(2) }, + { phase: 'queued', from: T(2), to: null }, + ]); + const done = { ...queued, status: 'done', started: T(3), closed: T(5) }; + assert.deepEqual(publish.phases(done), [ + { phase: 'inbox', from: T(1), to: T(2) }, + { phase: 'queued', from: T(2), to: T(3) }, + { phase: 'doing', from: T(3), to: T(5) }, + ]); + // Done before start times were recorded: the work bar is a guess, and says so. + assert.deepEqual(publish.phases({ ...queued, status: 'done', closed: T(5) }), [ + { phase: 'inbox', from: T(1), to: T(2) }, + { phase: 'doing', from: T(2), to: T(5), estimated: true }, + ]); + assert.deepEqual(publish.phases({ ...base, status: 'dropped', closed: T(4) }), [{ phase: 'inbox', from: T(1), to: T(4) }]); +}); + +test('the snapshot has a ticket for each idea, groups, related ideas, and the vectors', async () => { + process.env.IDEAMINE_GROUP_THRESHOLD = '0.5'; + store.addIdeas(['subtitles for audiobooks', 'subtitles for audiobooks on android', 'a tor exit relay'], { project: '/work/app' }); + store.applyTriage([{ id: 3, verdict: 'do', impact: 4, size: 's', model: 'haiku', title: 'Tor relay', why: 'w', brief: 'b' }]); + store.updateIdea(3, { status: 'doing' }); + const { data, note } = await publish.build(store.load()); + assert.equal(note, ''); + assert.equal(data.version, 1); + assert.equal(data.embed.available, true); + assert.equal(data.embed.query_prefix, 'search_query: '); + assert.equal(data.counts.doing, 1); + const [one, two, three] = data.ideas; + assert.deepEqual([one.key, one.lane, one.project, three.lane, three.model], ['IDEA-1', 'inbox', 'app', 'doing', 'haiku']); + assert.deepEqual(data.ideas.map((i) => i.rank), [2, 1, 0]); // doing first, then the inbox, newest first + assert.deepEqual(data.groups, [{ id: 0, label: data.groups[0].label, ids: [1, 2] }]); + assert.deepEqual([one.group, two.group, three.group], [0, 0, null]); + assert.equal(one.related[0].id, 2); + assert.ok(three.phases.some((p) => p.phase === 'doing' && p.from === three.started)); + const vectors = await embed.vectorsFor(store.load().ideas); + assert.deepEqual([...embed.decodeVec(one.vec)], [...vectors.get(1)]); +}); + +test('without the embedding server, the snapshot has no groups or vectors, and says why', async () => { + store.addIdeas(['one idea']); + server.options.down = true; + const { data, note } = await publish.build(store.load()); + assert.match(note, /without search by meaning or groups: .*503/); + assert.equal(data.embed.available, false); + assert.deepEqual([data.groups, data.ideas[0].vec, data.ideas[0].related], [[], null, []]); +}); + +test('publish uploads data.json and index.html with PUT, and kick() starts again only after a change', async () => { + store.addIdeas(['one idea']); + let started = 0; + const kick = () => publish.kick({ start: () => started++ }); + assert.equal(kick(), false); // no dashboard server set + config.set('publish_url', `${server.url}/dash`); + assert.equal(kick(), true); // never published + const out = await publish.publish(); + assert.deepEqual([out.where, out.ideas], [`${server.url}/dash`, 1]); + assert.deepEqual([...server.files.keys()], ['/dash/data.json', '/dash/index.html']); + assert.equal(JSON.parse(server.files.get('/dash/data.json')).ideas[0].title, 'one idea'); + assert.match(server.files.get('/dash/index.html'), /<html/i); + assert.equal(kick(), false); // nothing changed + await new Promise((resolve) => setTimeout(resolve, 20)); // a later modification time + store.addIdeas(['two']); + assert.equal(kick(), true); + assert.equal(started, 2); +}); + +test('a failed background publish is recorded, and kick() waits before the next try', async () => { + store.addIdeas(['one idea']); + const closed = await startFakeServer(); + await closed.close(); + config.set('publish_url', `${closed.url}/`); + assert.equal(await publish.backgroundPublish(), 'error'); + assert.match(publish.readState().error, /cannot reach/); + assert.equal(publish.kick({ start: () => assert.fail('must wait') }), false); + assert.match(publish.status(), /failed/); +}); + +test('publish --dir writes both files to a folder', async () => { + store.addIdeas(['one idea']); + const dir = path.join(store.home(), 'out'); + const out = await publish.publish({ dir }); + assert.equal(out.where, dir); + assert.deepEqual(fs.readdirSync(dir).sort(), ['data.json', 'index.html']); +}); + +test('config: the environment wins over the file, and an empty value restores the default', () => { + assert.equal(config.get('search_threshold'), 0.5); + assert.equal(config.set('search_threshold', '0.42'), 0.42); + process.env.IDEAMINE_SEARCH_THRESHOLD = '0.9'; + assert.equal(config.get('search_threshold'), 0.9); + delete process.env.IDEAMINE_SEARCH_THRESHOLD; + assert.equal(config.set('search_threshold', ''), 0.5); + assert.throws(() => config.set('search_threshold', '2'), /from 0 to 1/); + assert.throws(() => config.get('nope'), /unknown setting/); + assert.match(config.describe(), /^embed_url\s+http:\/\/127\.0\.0\.1:\d+\/v1\s+IDEAMINE_EMBED_URL/); +}); diff --git a/tests/store.test.js b/tests/store.test.js index f52abd1..8f79093 100644 --- a/tests/store.test.js +++ b/tests/store.test.js @@ -97,6 +97,18 @@ test('status changes, notes, model override, and reopen', () => { assert.throws(() => store.updateIdea(42, { note: 'x' }), /no idea #42/); }); +test('the start time is recorded when work starts, for the dashboard timeline', async () => { + store.addIdeas(['thing']); + assert.equal(store.findIdea(store.load(), 1).started, undefined); + const first = store.updateIdea(1, { status: 'doing' }).started; + assert.ok(first); + assert.equal(store.updateIdea(1, { status: 'doing', note: 'still on it' }).started, first); // no new start + assert.equal(store.updateIdea(1, { status: 'done' }).started, first); + store.updateIdea(1, { status: 'reopen' }); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.ok(store.updateIdea(1, { status: 'start' }).started > first); // the latest start +}); + test('a corrupt archive is never overwritten', () => { fs.mkdirSync(store.home(), { recursive: true }); fs.writeFileSync(store.dbPath(), '{"ideas": [oops');