commit 78466858965dee37edb88c9d6acfa6d1f8c08d90 equwal <13551856+equwal@users.noreply.github.com> 2026-09-21 19:33:09 -0700 ideamine 0.7.0: one archive for every machine, prompts and memories With sync on, an ideamine server holds the archive, and every Claude on every machine reads and changes it. Each change goes into an outbox, then to the server. When the server does not answer, the change waits in the outbox, so no idea is lost. The server applies each change only once, also when a machine sends it twice. `ideamine serve` gets a server role: /api/db and /api/ops for the machines, and /api/prompts for the prompt log. With the prompt log on, the hook sends each prompt in the background. The dashboard gets a Prompts tab, a timeline of the sessions with each prompt, and a Memory tab, which reads the memories of a memstated daemon. Triage, Build, Ask, and the watcher show only where Claude Code runs. /ideas-sync [url|off] and `ideamine sync` turn sync on or off. `ideamine prompts import` sends the prompts of older chats.
.claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- README.md | 53 +++++- bin/ideamine.js | 68 +++++-- dashboard/index.html | 383 ++++++++++++++++++++++++++++++++++++++- package-lock.json | 4 +- package.json | 2 +- skills/ideas-sync/SKILL.md | 8 + src/archive.js | 62 +++++++ src/claude.js | 23 ++- src/config.js | 20 ++ src/hook.js | 89 ++++++--- src/mcp.js | 44 ++--- src/publish.js | 8 +- src/render.js | 5 +- src/serve.js | 250 ++++++++++++++++++++++--- src/store.js | 29 ++- src/sync.js | 356 ++++++++++++++++++++++++++++++++++++ tests/fixtures/fake-memstate.js | 39 ++++ tests/fixtures/server-process.js | 52 ++++++ tests/hook.test.js | 58 ++++++ tests/mcp.test.js | 2 +- tests/serve.test.js | 97 +++++++++- tests/sync.test.js | 233 ++++++++++++++++++++++++ 24 files changed, 1766 insertions(+), 123 deletions(-)
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 5055c13..fd622de 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ { "name": "ideamine", "description": "An idea inbox for Claude Code. /idea saves an idea without a model call. Claude triages the ideas and picks the cheapest model that can build each one.", - "version": "0.6.0", + "version": "0.7.0", "author": { "name": "equwal" }, diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index a1b495a..44b2ad8 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "ideamine", - "version": "0.6.0", + "version": "0.7.0", "description": "An idea inbox for Claude Code. /idea saves an idea without a model call, so it works at any time, even at your usage limit. Claude triages the ideas and picks the cheapest model that can build each one.", "author": { "name": "equwal", diff --git a/README.md b/README.md index 948fe87..722861e 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Marketplaces you add yourself do not auto-update. To upgrade, run `claude plugin | `/ideas-find sync subtitles` | Search every lane by meaning, not only by the words. See [Search by meaning](#search-by-meaning-and-groups). | **No** | | `/ideas-groups` · `done` · `-a` | Show the ideas grouped by meaning | **No** | | `/ideas-web` · `off` | Start the dashboard with a button for each command on this PC, and show its address. See [Buttons](#buttons). | **No** | +| `/ideas-sync <url>` · `off` | Share one archive between all your machines through an ideamine server. See [One archive for every machine](#one-archive-for-every-machine). | **No** | | `/ideas-go [12]` | Build the idea that fits this chat, else the first in the queue, or #12, on its recommended model. New ideas are triaged first. | Yes, this is the build | | `/ideas-all` | Claude reads every idea, takes the ones that fit this chat out of the queue, and does them. The others stay in the queue. | Yes, this is the build | | `/ideas-sort` | Triage the inbox now and show the queue. You do not have to: `/ideas-go` triages when it must. | Yes, briefly | @@ -153,7 +154,44 @@ server { The board, the timeline, the groups, and the search show `/ideas`, `/ideas-ls`, `/ideas-cat`, `/ideas-groups`, and `/ideas-find`. `/ideas-all` has no button, because it needs the chat that it works in. -The commands run on your PC, because the archive and your Claude Code login are there. Thus the server listens on `127.0.0.1` only, and it takes commands only from its own page: each command must be JSON from the same origin, and the Host header must name the server. Another web page in your browser cannot send commands to it, and no other page can show it in a frame. After a change, the server uploads the dashboard again when `publish_url` is set, so the copy on your dashboard server stays current. That copy has no buttons: it shows "Read-only copy". `/ideas-web off` stops the server. In a terminal, `ideamine serve` runs it in the foreground. To use another port, run `ideamine config serve_port 5000`. The log is `~/.ideamine/serve.log`. +Triage, Build, Ask, and the watcher need the Claude Code login, so they show only where Claude Code runs. The commands run on your PC, because your Claude Code login is there. Thus the server listens on `127.0.0.1` only, and it takes commands only from its own page: each command must be JSON from the same origin, and the Host header must name the server. Another web page in your browser cannot send commands to it, and no other page can show it in a frame. After a change, the server uploads the dashboard again when `publish_url` is set, so the copy on your dashboard server stays current. That copy has no buttons: it shows "Read-only copy". `/ideas-web off` stops the server. In a terminal, `ideamine serve` runs it in the foreground. To use another port, run `ideamine config serve_port 5000`. The log is `~/.ideamine/serve.log`. + +## One archive for every machine + +``` +> /ideas-sync http://10.66.0.1/ + ideamine sync: http://10.66.0.1/ + last sync 2026-09-21 19:24, revision 212. +``` + +Without sync, each machine has its own archive. With sync, an ideamine server holds one archive, and every Claude on every machine reads and changes it. The server is `ideamine serve` on a machine that all your machines can reach, for example a small VPS on a WireGuard network. Its page is the same dashboard for every machine. + +Each machine keeps a copy of the archive, so that `/ideas` and the other reads stay fast. A change goes into an outbox on the machine first, and then to the server. If the server does not answer, for example because the tunnel is down, the change waits in the outbox and goes with the next sync. Thus no idea is lost. `/idea` then says "Saved here", and `/ideas` says that it shows the copy on this machine. The server applies each change only once, also when a machine sends it twice. A background sync after your prompts sends what waits, and it gets the changes of the other machines at most once a minute. Prompts for the prompt log go at most every 20 seconds, so that a prompt does not start a process each time. The first `/ideas-sync` saves the archive that was on the machine as `ideas.before-sync-<date>.json`, because the archive of the server takes its place. + +On the server, sync stays off. It serves the archive, and nginx gives it the address of the WireGuard interface: + +```bash +IDEAMINE_HOME=/var/lib/ideamine IDEAMINE_SERVE_HOSTS=10.66.0.1 ideamine serve --port 4332 +``` + +```nginx +server { + listen 10.66.0.1:80; + allow 10.66.0.0/24; + deny all; + location / { proxy_pass http://127.0.0.1:4332; proxy_set_header Host $host; client_max_body_size 2m; } + location /v1/ { proxy_pass http://127.0.0.1:8081; } # the embedding server +} +``` + +`serve_hosts` names the Host that nginx sends. Without it, the server refuses the request, so a DNS rebinding page cannot reach the archive. The server does not ask who you are: the network decides who can reach it. Put it only on a private network like WireGuard. + +### Prompts and Memory + +The dashboard can also show what your Claudes did: + +- **Prompts**: each prompt that you typed, on every machine, as a timeline of sessions. `ideamine config prompt_log on` sends each prompt to the ideamine server in the background. The full text goes, without the notes that Claude Code puts into a prompt; a paste longer than 100,000 characters is cut. `ideamine prompts import` sends the prompts of your older chats from the Claude Code transcripts. +- **Memory**: the memories of a [memstate](https://github.com/map588/memstate) daemon: a timeline of the writes of each project, the latest writes, and each memory with its versions. Set `memstate_url` on the server, for example `http://127.0.0.1:8765`. The dashboard only reads memstate. ## Model routing @@ -199,6 +237,8 @@ ideamine groups [-a] # ideas grouped by meaning ideamine embed # embed new ideas, show the similarity numbers ideamine publish [url|off] [--dir folder] # the dashboard (see above) ideamine serve [--port 4332] # the dashboard with buttons, on 127.0.0.1 +ideamine sync [url|off] # one archive for every machine (see above) +ideamine prompts import # the prompts of older chats, for the Prompts tab ideamine config [key [value]] # show or change a setting ideamine export IDEAS.md # Markdown copy of everything ``` @@ -219,7 +259,7 @@ Tools: `idea_add`, `idea_list`, `idea_triage`, `idea_update`, `idea_next`, `idea ## Where your ideas live -`~/.ideamine/ideas.json` is plain, readable JSON. Set `IDEAMINE_HOME` to move it, for example into a synced folder. Each write takes a lock and then replaces the file in one step, so many sessions can write at the same time without losing an idea. The previous version is kept as `ideas.json.bak`. If the file becomes damaged, ideamine stops and does not overwrite it. Nothing leaves your machine, except in these cases: triage and builds go through Claude as usual; search by meaning and groups send the text of your ideas to the embedding server that you set; `ideamine publish` uploads a snapshot to the dashboard server that you set. So that it can pair ideas with projects, the triage also sends the paths of your project folders and the first line of each README. +`~/.ideamine/ideas.json` is plain, readable JSON. Set `IDEAMINE_HOME` to move it, for example into a synced folder. Each write takes a lock and then replaces the file in one step, so many sessions can write at the same time without losing an idea. The previous version is kept as `ideas.json.bak`. If the file becomes damaged, ideamine stops and does not overwrite it. Nothing leaves your machine, except in these cases: triage and builds go through Claude as usual; search by meaning and groups send the text of your ideas to the embedding server that you set; `ideamine publish` uploads a snapshot to the dashboard server that you set; with sync on, the archive is on the ideamine server, and with the prompt log on, your prompts go there too. So that it can pair ideas with projects, the triage also sends the paths of your project folders and the first line of each README. `ideamine config` shows each setting and where its value comes from. `ideamine config <key> <value>` saves a setting in `~/.ideamine/config.json`, and an empty value restores the default. An environment variable wins over the file. @@ -235,6 +275,10 @@ Tools: `idea_add`, `idea_list`, `idea_triage`, `idea_update`, `idea_next`, `idea | `IDEAMINE_GROUP_THRESHOLD` | `group_threshold` | `0.65` | lowest mean similarity in a group, and of a related idea | | `IDEAMINE_PUBLISH_URL` | `publish_url` | *(off)* | dashboard server for `ideamine publish` | | `IDEAMINE_SERVE_PORT` | `serve_port` | `4332` | port of `ideamine serve` and `/ideas-web` | +| `IDEAMINE_SERVE_HOSTS` | `serve_hosts` | *(none)* | more Host names that `ideamine serve` answers, for a server behind nginx | +| `IDEAMINE_SYNC_URL` | `sync_url` | *(off)* | the ideamine server that holds the archive of every machine | +| `IDEAMINE_PROMPT_LOG` | `prompt_log` | `off` | `on` sends each prompt to the ideamine server | +| `IDEAMINE_MEMSTATE_URL` | `memstate_url` | *(off)* | the memstated daemon for the Memory tab | ## How it works @@ -253,9 +297,12 @@ watcher on: any prompt ──► hook ──► background pass ──► claude publish on: any prompt after a change ──► hook ──► background publish ──► PUT index.html + data.json /ideas-web ──► hook ──► ideamine serve on 127.0.0.1 ──► the buttons on the page ──► the same archive + +sync on: each change ──► outbox ──► ideamine server ──► the archive of every machine ──► copy here +prompt log on: any prompt ──► hook ──► prompt outbox ──► background sync ──► the Prompts tab ``` -The plugin contains a Node MCP server with no dependencies, fourteen skills (the slash commands), and one hook. The hook answers `/idea`, `/ideas`, and the local `/ideas-*` commands before any API call, and it lets every other prompt through. The hook runs directly, not through a shell, and takes about 130 ms per prompt on Windows. The skills are user-only, so their descriptions add no tokens to your sessions. If the archive cannot be read, the hook lets the prompt through, so the `/idea` skill can still save it with the MCP tool. Your text is never dropped. +The plugin contains a Node MCP server with no dependencies, fifteen skills (the slash commands), and one hook. The hook answers `/idea`, `/ideas`, and the local `/ideas-*` commands before any API call, and it lets every other prompt through. The hook runs directly, not through a shell, and takes about 130 ms per prompt on Windows. The skills are user-only, so their descriptions add no tokens to your sessions. If the archive cannot be read, the hook lets the prompt through, so the `/idea` skill can still save it with the MCP tool. Your text is never dropped. ## Development diff --git a/bin/ideamine.js b/bin/ideamine.js index 6e177af..e2960ee 100644 --- a/bin/ideamine.js +++ b/bin/ideamine.js @@ -26,6 +26,8 @@ const HELP = `ideamine: an idea inbox for Claude Code upload the dashboard (index.html, data.json) now; a url also turns on the upload after each change; --dir writes the files to a folder ideamine serve [--port 4332] the dashboard with a button for each command, at 127.0.0.1 + ideamine sync [url|off] share the archive of every machine through an ideamine server + ideamine prompts import put the prompts of older Claude Code chats into the prompt log ideamine config [key [value]] show or change a setting (an empty value restores the default) ideamine export [file.md] Markdown export of the whole archive ideamine path where the archive lives (override with IDEAMINE_HOME) @@ -71,36 +73,40 @@ async function main() { if (command === 'hook') return (await import('../src/hook.js')).runHook(); const store = await import('../src/store.js'); + const archive = await import('../src/archive.js'); const render = await import('../src/render.js'); const { splitIdeas, clip } = await import('../src/text.js'); const { flags, words } = parseArgs(rest); const cwd = process.cwd(); const needId = () => words[0] || fail(`usage: ideamine ${command} <id>`); + // A change that waits for the ideamine server: say so, and go on. + const queued = (out) => out.queued && (console.log(archive.queuedText(out)), true); switch (command) { case 'add': { const text = words.length === 1 && words[0] === '-' ? await readStdin() : words.join(' '); - const results = store.addIdeas(splitIdeas(text), { source: 'cli', project: flags['no-project'] ? null : cwd }); - console.log(render.renderAdded(results, store.load())); + const results = await archive.add(splitIdeas(text), { source: 'cli', project: flags['no-project'] ? null : cwd }); + if (!queued(results)) console.log(render.renderAdded(results, store.load())); break; } case 'ls': { const lower = words.map((w) => (w === '-a' ? 'all' : w.toLowerCase())); const filter = lower.find((w) => store.FILTERS.includes(w)) || 'open'; const here = lower.includes('here') || flags.here; - console.log(render.renderBoard(store.load(), { filter, project: here ? cwd : null, query: flags.query || '', cwd })); + console.log(render.renderBoard(await archive.fresh(), { filter, project: here ? cwd : null, query: flags.query || '', cwd })); break; } case 'cat': { needId(); - const db = store.load(); + const db = await archive.fresh(); const ideas = words.map((id) => store.findIdea(db, id) || fail(`no idea #${id.replace(/^#/, '')}`)); console.log(ideas.map(render.renderIdea).join('\n\n')); break; } case 'rm': { needId(); - for (const idea of store.removeIdeas(words)) console.log(`Removed #${idea.id} ${clip(idea.title, 60)}`); + const gone = await archive.remove(words); + if (!queued(gone)) for (const idea of gone) console.log(`Removed #${idea.id} ${clip(idea.title, 60)}`); break; } case 'done': @@ -110,23 +116,23 @@ async function main() { case 'reopen': { const status = { drop: 'dropped', start: 'doing' }[command] || command; const note = words.slice(1).join(' '); - const idea = store.updateIdea(needId(), { status, note: note || undefined }); - console.log(`#${idea.id} ${clip(idea.title, 60)} → ${store.lane(idea)}`); + const idea = await archive.update(needId(), { status, note: note || undefined }); + if (!queued(idea)) console.log(`#${idea.id} ${clip(idea.title, 60)} → ${store.lane(idea)}`); break; } case 'note': { const note = words.slice(1).join(' ') || fail('usage: ideamine note <id> <text>'); - const idea = store.updateIdea(needId(), { note }); - console.log(`#${idea.id}: note added`); + const idea = await archive.update(needId(), { note }); + if (!queued(idea)) console.log(`#${idea.id}: note added`); break; } case 'model': { - const idea = store.updateIdea(needId(), { model: words[1] || fail('usage: ideamine model <id> <haiku|sonnet|opus|fable>') }); - console.log(`#${idea.id} will be built with ${idea.triage.model}`); + const idea = await archive.update(needId(), { model: words[1] || fail('usage: ideamine model <id> <haiku|sonnet|opus|fable>') }); + if (!queued(idea)) console.log(`#${idea.id} will be built with ${idea.triage.model}`); break; } case 'next': { - const idea = store.pickNext(store.load(), { project: cwd, only: !!flags.here }); + const idea = store.pickNext(await archive.fresh(), { project: cwd, only: !!flags.here }); if (!idea) { console.log('Nothing is ready to build. Triage the inbox first: ideamine sort'); break; @@ -136,7 +142,7 @@ async function main() { } case 'go': { const { goPlan, launchSession } = await import('../src/claude.js'); - const db = store.load(); + const db = await archive.fresh(); const idea = words[0] ? store.findIdea(db, words[0]) : store.pickNext(db, { project: cwd }); if (!idea) fail(words[0] ? `no idea #${words[0]}` : 'nothing is ready to build; run: ideamine sort'); const { model, dir, prompt } = goPlan(idea, cwd); @@ -144,7 +150,7 @@ async function main() { console.log(`directory: ${dir}\nmodel: ${model}\n\n${prompt}`); break; } - store.updateIdea(idea.id, { status: 'doing' }); + queued(await archive.update(idea.id, { status: 'doing' })); console.log(`Opening Claude Code (${model}) in ${dir} for #${idea.id}…`); process.exitCode = launchSession({ model, prompt, cwd: dir }); break; @@ -186,14 +192,14 @@ async function main() { case 'find': { const embed = await import('../src/embed.js'); const query = words.join(' ') || fail('usage: ideamine find <words...>'); - console.log(render.renderFound(await embed.find(store.load(), query), query, { cwd })); + console.log(render.renderFound(await embed.find(await archive.fresh(), query), query, { cwd })); break; } case 'groups': { const embed = await import('../src/embed.js'); const lower = words.map((w) => (w === '-a' ? 'all' : w.toLowerCase())); const filter = lower.find((w) => store.FILTERS.includes(w)) || 'open'; - const ideas = store.listIdeas(store.load(), { filter }); + const ideas = store.listIdeas(await archive.fresh(), { filter }); console.log(render.renderGroups(await embed.groupIdeas(ideas), ideas, { cwd, scope: filter })); break; } @@ -230,6 +236,34 @@ async function main() { console.log(serve.logLine(`the dashboard with buttons runs at ${url} (Ctrl+C stops it)`)); break; } + case 'sync': { + const sync = await import('../src/sync.js'); + if (flags.background) { + process.exitCode = (await sync.backgroundSync()) === 'error' ? 1 : 0; + break; + } + if (words[0]) { + const backup = sync.setServer(words[0] === 'off' ? '' : words[0]); + if (backup) console.log(`The archive that was here is in ${backup}.`); + } + if (sync.enabled()) { + try { + await sync.pull({ timeoutMs: 30000, prompts: true }); + } catch (e) { + if (!(e instanceof sync.SyncError)) throw e; + process.exitCode = 1; + } + } + console.log(sync.status()); + break; + } + case 'prompts': { + const sync = await import('../src/sync.js'); + if (words[0] !== 'import') fail('usage: ideamine prompts import'); + if (!sync.enabled()) fail('the prompt log goes to an ideamine server. Turn sync on first: ideamine sync <url>'); + console.log(`Put ${sync.importPrompts()} prompts from ${sync.transcriptsDir()} into the prompt outbox. The next sync sends them.`); + break; + } case 'config': { const config = await import('../src/config.js'); if (words.length >= 1 && rest.length >= 2) config.set(words[0], words.slice(1).join(' ')); @@ -241,7 +275,7 @@ async function main() { break; } case 'export': { - const md = render.renderMarkdown(store.load()); + const md = render.renderMarkdown(await archive.fresh()); if (words[0]) { fs.writeFileSync(words[0], md); console.log(`Wrote ${words[0]}`); diff --git a/dashboard/index.html b/dashboard/index.html index ea63deb..13091e0 100644 --- a/dashboard/index.html +++ b/dashboard/index.html @@ -223,6 +223,30 @@ textarea { width: 100%; resize: vertical; } .toast-busy { border-left-color: var(--accent); } .toast-error { border-left-color: var(--lane-dropped); } +/* prompts and memory tabs */ +.view-controls { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; margin-bottom: 10px; font-size: 12px; color: var(--text-dim); } +.view-note { color: var(--text-muted); font-size: 12px; } +.section-title { font-size: 13px; font-weight: 700; margin: 18px 0 8px; } +.row-list { display: flex; flex-direction: column; gap: 6px; max-width: 860px; } +.row-item { display: grid; grid-template-columns: 150px 1fr; gap: 10px; width: 100%; text-align: left; background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 6px 10px; cursor: pointer; } +.row-item:hover { border-color: var(--border-strong); } +.row-meta { font-size: 11px; color: var(--text-muted); } +.row-text { font-size: 13px; overflow: hidden; display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; white-space: pre-wrap; word-break: break-word; } +@media (max-width: 640px) { .row-item { grid-template-columns: 1fr; gap: 2px; } } +.tl-tick { stroke: var(--text-dim); stroke-width: 2; cursor: pointer; } +.tl-tick:hover, .tl-tick:focus-visible { stroke: var(--text); stroke-width: 3; outline: none; } +.tl-span { fill: var(--phase-inbox); } +.tl-dot { stroke: var(--surface); stroke-width: 1; cursor: pointer; } +.tl-dot:hover, .tl-dot:focus-visible { stroke: var(--text); stroke-width: 2; outline: none; } +.mem-area { margin-bottom: 14px; } +.mem-area h3 { font-size: 12px; text-transform: uppercase; letter-spacing: .04em; color: var(--text-muted); margin-bottom: 6px; } +.mem-cards { display: grid; grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); gap: 8px; } +.mem-card { text-align: left; background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 8px 10px; cursor: pointer; } +.mem-card:hover { border-color: var(--border-strong); } +.mem-key { font-size: 12px; font-weight: 600; word-break: break-all; } +.mem-preview { font-size: 12px; color: var(--text-dim); white-space: pre-wrap; word-break: break-word; overflow: hidden; display: -webkit-box; -webkit-line-clamp: 4; -webkit-box-orient: vertical; margin-top: 4px; } +.version { border-left: 2px solid var(--border-strong); padding-left: 8px; margin-bottom: 12px; } + /* focus */ button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus-visible, [tabindex]:focus-visible { outline: 2px solid var(--focus-ring); @@ -242,6 +266,8 @@ button:focus-visible, input:focus-visible, textarea:focus-visible, select:focus- <button type="button" role="tab" data-view="board" aria-controls="main">Board</button> <button type="button" role="tab" data-view="timeline" aria-controls="main">Timeline</button> <button type="button" role="tab" data-view="groups" aria-controls="main">Groups</button> + <button type="button" role="tab" data-view="prompts" aria-controls="main" hidden>Prompts</button> + <button type="button" role="tab" data-view="memory" aria-controls="main" hidden>Memory</button> </nav> <label class="parked-toggle"> <input type="checkbox" id="parked-toggle"> Show parked @@ -307,6 +333,10 @@ const SEARCH_DEBOUNCE_MS = 300; const EMBED_TIMEOUT_MS = 4000; const MODELS = ['haiku', 'sonnet', 'opus', 'fable']; const OPEN_LANES = ['doing', 'do', 'maybe', 'inbox']; +const VIEWS = ['board', 'timeline', 'groups', 'prompts', 'memory']; +const PROMPT_RANGES = [[1, 'Last day'], [7, 'Last 7 days'], [30, 'Last 30 days'], [0, 'All']]; +// Memory categories share the lane colors, so that the page needs no new colors. +const CATEGORY_COLORS = { status: 'doing', decision: 'do', gotcha: 'dropped', config: 'maybe', note: 'inbox', reference: 'skip', learning: 'skip' }; const state = { data: null, @@ -318,6 +348,8 @@ const state = { query: '', searchResult: null, busy: false, + prompts: { days: 1, list: null, error: null, filter: '' }, + memory: { overview: null, error: null, project: null, memories: new Map(), histories: new Map() }, }; let searchTimer = null; @@ -493,13 +525,21 @@ function renderHeader() { el('span', { class: 'count-chip-dot', style: '--chip-color:var(--lane-' + lane + ')' }), LANE_LABELS[lane] + ' ' + (counts[lane] ?? 0))); } - $updated.textContent = 'updated ' + fmtRelative(state.data.generated); - $updated.title = fmtDate(state.data.generated) || ''; + const sync = state.data.live && state.data.live.sync; + let where = ''; + if (sync) where = sync.offline ? ' · the server does not answer: copy on this PC' : ' · shared by every machine'; + if (sync && sync.waiting) where += ' · ' + sync.waiting + ' waiting'; + $updated.textContent = 'updated ' + fmtRelative(state.data.generated) + where; + $updated.title = fmtDate(state.data.generated) + (sync ? ' · ' + sync.url + (sync.offline ? ' · ' + sync.offline : '') : ''); } function renderTabsActive() { + const live = state.data.live || {}; for (const btn of $tabs.querySelectorAll('button')) { - const active = btn.dataset.view === state.view; + const view = btn.dataset.view; + if (view === 'prompts') btn.hidden = !live.prompts; + if (view === 'memory') btn.hidden = !live.memory; + const active = view === state.view; btn.classList.toggle('active', active); btn.setAttribute('aria-selected', String(active)); } @@ -789,6 +829,304 @@ function renderSearch() { $main.append(wrap); } +/* ---- shared by the prompts and memory tabs ---- */ + +async function getJson(url) { + const res = await fetch(url, { cache: 'no-store' }); + let json = null; + try { json = await res.json(); } catch (err) { json = null; } + if (!res.ok || !json || !json.ok) throw new Error((json && json.error) || 'HTTP ' + res.status); + return json; +} + +function baseName(dir) { return String(dir || '').split(/[\\/]+/).filter(Boolean).pop() || 'no folder'; } + +const isoOf = (seconds) => new Date(seconds * 1000).toISOString(); + +/** A timeline with one row per item: a span bar, and a mark for each event in the row. */ +function renderRowsChart({ rows, label, count, span, marks, onRow, ariaLabel }) { + const now = Date.now(); + let start = Infinity, end = -Infinity; + for (const row of rows) { + const [from, to] = span(row); + start = Math.min(start, from); + end = Math.max(end, to); + } + end = Math.max(end, start + 10 * MIN); + const pad = (end - start) * 0.03; + start -= pad; + end += pad; + const scale = buildTimeScale(start, end); + const height = TL_HEADER_H + rows.length * TL_ROW_H; + const axis = svgEl('g', {}); + for (const t of scale.ticks) { + const x = scale.x(t); + axis.append(svgEl('line', { x1: x, x2: x, y1: TL_HEADER_H, y2: height, class: 'tl-grid' }), svgEl('text', { x: x + 4, y: TL_HEADER_H - 8, class: 'tl-tick-label' }, formatTick(t, scale.step))); + } + const body = svgEl('g', {}); + rows.forEach((row, i) => { + const y = TL_HEADER_H + i * TL_ROW_H; + const barY = y + (TL_ROW_H - TL_BAR_H) / 2; + const [from, to] = span(row); + const x1 = scale.x(from); + body.append(svgEl('rect', { x: x1, y: barY, width: Math.max(scale.x(to) - x1, 3), height: TL_BAR_H, rx: 3, class: 'tl-span' })); + for (const node of marks(row, scale, barY)) body.append(node); + }); + const svg = svgEl('svg', { width: scale.width, height, viewBox: '0 0 ' + scale.width + ' ' + height, role: 'img', 'aria-label': ariaLabel }, axis, body); + if (now >= start && now <= end) { + const nx = scale.x(now); + svg.append(svgEl('line', { x1: nx, x2: nx, y1: 0, y2: height, class: 'tl-now' }), svgEl('text', { x: nx - 4, y: 11, 'text-anchor': 'end', class: 'tl-now-label' }, 'now')); + } + const labels = el('div', { class: 'tl-labels' }, + el('div', { class: 'tl-label-header', style: 'height:' + TL_HEADER_H + 'px' }), + rows.map((row) => el('button', { class: 'tl-label-row', type: 'button', style: 'height:' + TL_ROW_H + 'px', onclick: () => onRow(row) }, + el('code', { class: 'tl-label-key' }, count(row)), + el('span', { class: 'tl-label-title' }, label(row))))); + return el('div', { class: 'tl-scroll' }, el('div', { class: 'tl-inner' }, labels, el('div', { class: 'tl-chart' }, svg))); +} + +/* ---- prompts tab: the prompt log, one row for each Claude Code session ---- */ + +async function loadPrompts() { + const p = state.prompts; + p.loading = true; + const days = p.days; + try { + const list = (await getJson('api/prompts?days=' + days)).prompts; + if (days === p.days) p.list = list; // the range did not change during the load + p.error = null; + } catch (err) { + p.list ||= []; + p.error = err.message; + } + p.loading = false; + render(); +} + +const sessionOf = (q) => q.session || (q.host || 'unknown') + ' ' + q.at.slice(0, 10); + +/** One session for each session id, with its prompts in time order. The newest session comes first. */ +function sessionsOf(prompts) { + const byId = new Map(); + for (const q of prompts) { + const id = sessionOf(q); + if (!byId.has(id)) byId.set(id, { id, host: q.host, cwd: q.cwd, prompts: [] }); + const s = byId.get(id); + s.prompts.push(q); + if (q.cwd) s.cwd = q.cwd; + } + const sessions = [...byId.values()]; + for (const s of sessions) { + s.from = Date.parse(s.prompts[0].at); + s.to = Date.parse(s.prompts[s.prompts.length - 1].at); + } + return sessions.sort((a, b) => b.to - a.to); +} + +// A chat without a project folder runs in a scratch folder of the desktop app. Its first prompt +// says more than the folder name. +function sessionLabel(s) { + const name = baseName(s.cwd); + const first = s.prompts && s.prompts[0]; + const label = /^scratch-\d{4}-\d{2}-\d{2}-/.test(name) && first ? '“' + first.prompt.replace(/\s+/g, ' ').slice(0, 60) + '”' : name; + return label + (s.host ? ' · ' + s.host : ''); +} + +function matchesPromptFilter(q) { + const f = state.prompts.filter.trim().toLowerCase(); + return !f || q.prompt.toLowerCase().includes(f) || String(q.cwd || '').toLowerCase().includes(f); +} + +function renderPromptBody(body) { + body.textContent = ''; + const p = state.prompts; + if (p.error) body.append(el('p', { class: 'load-error' }, 'Could not load the prompts: ' + p.error)); + const list = p.list.filter(matchesPromptFilter); + if (!list.length) { + if (!p.error) body.append(el('p', { class: 'empty-note' }, 'No prompts in this time range.')); + return; + } + const sessions = sessionsOf(list); + body.append(el('p', { class: 'view-note' }, list.length + ' prompts in ' + sessions.length + ' sessions. A mark is one prompt.')); + body.append(renderRowsChart({ + rows: sessions, + ariaLabel: 'Sessions and their prompts', + label: sessionLabel, + count: (s) => String(s.prompts.length), + span: (s) => [s.from, s.to], + onRow: (s) => openTicket('SESSION:' + s.id), + marks: (s, scale, barY) => s.prompts.map((q) => { + const x = scale.x(Date.parse(q.at)); + const tick = svgEl('line', { x1: x, x2: x, y1: barY - 3, y2: barY + TL_BAR_H + 3, class: 'tl-tick', tabindex: '0', role: 'img', 'aria-label': fmtDate(q.at) + ': ' + q.prompt.slice(0, 80) }); + attachTooltip(tick, () => el('div', {}, el('div', { class: 'tooltip-key' }, fmtDate(q.at)), el('div', { class: 'tooltip-title' }, q.prompt.slice(0, 200)))); + tick.addEventListener('click', () => openTicket('SESSION:' + s.id)); + return tick; + }), + })); + body.append(el('h2', { class: 'section-title' }, 'Latest prompts')); + body.append(el('div', { class: 'row-list' }, list.slice(-30).reverse().map((q) => + el('button', { type: 'button', class: 'row-item', onclick: () => openTicket('SESSION:' + sessionOf(q)) }, + el('span', { class: 'row-meta' }, fmtDate(q.at), el('br'), baseName(q.cwd) + (q.host ? ' · ' + q.host : '')), + el('span', { class: 'row-text' }, q.prompt))))); +} + +function renderPrompts() { + $main.textContent = ''; + const p = state.prompts; + const range = el('select', { 'aria-label': 'Time range' }, PROMPT_RANGES.map(([days, label]) => el('option', { value: String(days), selected: days === p.days }, label))); + range.addEventListener('change', () => { p.days = Number(range.value); p.list = null; render(); }); + const filter = el('input', { type: 'search', class: 'search-input', placeholder: 'Filter prompts…', 'aria-label': 'Filter prompts' }); + filter.value = p.filter; + const body = el('div', {}); + // Only the list below the filter changes while you type, so the filter keeps the focus. + filter.addEventListener('input', () => { p.filter = filter.value; if (p.list) renderPromptBody(body); }); + $main.append(el('div', { class: 'view-controls' }, range, filter), body); + if (!p.list) body.append(el('p', { class: 'empty-note' }, 'Loading the prompts…')); + else renderPromptBody(body); +} + +function buildSessionDrawer(id) { + const frag = document.createDocumentFragment(); + const list = (state.prompts.list || []).filter((q) => sessionOf(q) === id); + if (!list.length) { + frag.append(el('h2', { id: 'drawer-title', class: 'drawer-title' }, 'Session'), + el('p', { class: 'text-muted' }, state.prompts.list ? 'This session has no prompts in the time range.' : 'Loading the prompts…')); + return frag; + } + const first = list[0]; + frag.append(el('h2', { id: 'drawer-title', class: 'drawer-title' }, sessionLabel({ cwd: first.cwd, host: first.host, prompts: list }))); + frag.append(el('p', { class: 'drawer-meta' }, [first.cwd, first.session && 'session ' + first.session].filter(Boolean).join(' · '))); + frag.append(el('p', { class: 'drawer-meta' }, fmtDate(first.at) + ' → ' + fmtDate(list[list.length - 1].at) + ' · ' + list.length + ' prompts')); + const notes = el('div', { class: 'notes' }); + for (const q of list) notes.append(el('div', { class: 'note' }, el('div', { class: 'note-at' }, fmtDate(q.at)), el('div', { class: 'note-text' }, q.prompt))); + frag.append(el('section', { class: 'drawer-section' }, el('h3', {}, 'Prompts'), notes)); + return frag; +} + +/* ---- memory tab: what the memstate memories say, and when Claude wrote them ---- */ + +/** The projects and memories. With `refresh`, the texts and versions that the page has load again too. */ +async function loadMemoryOverview(opts) { + const m = state.memory; + m.loading = true; + try { + m.overview = await getJson('api/memory/overview'); + m.error = null; + if (opts && opts.refresh) { + m.memories.clear(); + m.histories.clear(); + } + } catch (err) { + m.overview ||= { projects: [], memories: [] }; + m.error = err.message; + } + m.loading = false; + render(); +} + +async function loadInto(map, key, url, pick) { + if (map.has(key)) return; + map.set(key, null); // loading + try { map.set(key, pick(await getJson(url))); } catch (err) { map.set(key, { error: err.message }); } + render(); +} + +function categoryChip(category) { + const lane = CATEGORY_COLORS[category]; + return el('span', { class: 'chip chip-lane', style: '--chip-color:' + (lane ? 'var(--lane-' + lane + ')' : 'var(--text-muted)') }, category || 'memory'); +} + +const memKey = (project, keypath) => 'MEM:' + project + '|' + keypath; + +function selectProject(id) { + state.memory.project = id; + render(); + const target = document.getElementById('mem-project'); + if (target) target.scrollIntoView({ block: 'start' }); +} + +function renderProjectMemories(id) { + const wrap = el('section', { id: 'mem-project' }, el('h2', { class: 'section-title' }, 'Memories of ' + id)); + const url = 'api/memory/project?id=' + encodeURIComponent(id); + const list = state.memory.memories.get(id); + if (list === undefined) loadInto(state.memory.memories, id, url, (json) => json.memories); + if (!list) { wrap.append(el('p', { class: 'empty-note' }, 'Loading…')); return wrap; } + if (list.error) { wrap.append(el('p', { class: 'load-error' }, list.error)); return wrap; } + const areas = new Map(); + for (const x of list) { + const area = x.keypath.split('.')[0]; + if (!areas.has(area)) areas.set(area, []); + areas.get(area).push(x); + } + for (const [area, items] of areas) { + items.sort((a, b) => b.created_at - a.created_at); + wrap.append(el('div', { class: 'mem-area' }, el('h3', {}, area), el('div', { class: 'mem-cards' }, items.map((x) => + el('button', { type: 'button', class: 'mem-card', onclick: () => openTicket(memKey(id, x.keypath)) }, + el('div', { class: 'card-top' }, categoryChip(x.category), el('span', { class: 'card-key' }, 'v' + x.version + ' · ' + fmtRelative(isoOf(x.created_at)))), + el('div', { class: 'mem-key' }, x.keypath), + el('div', { class: 'mem-preview' }, x.content || '')))))); + } + return wrap; +} + +function renderMemory() { + $main.textContent = ''; + const m = state.memory; + if (!m.overview) { $main.append(el('p', { class: 'empty-note' }, 'Loading the memories…')); return; } + if (m.error) $main.append(el('p', { class: 'load-error' }, 'Could not load the memories: ' + m.error)); + const { projects, memories } = m.overview; + if (!projects.length) { if (!m.error) $main.append(el('p', { class: 'empty-note' }, 'memstate has no projects yet.')); return; } + const sorted = projects.slice().sort((a, b) => (b.last_updated_at || b.created_at) - (a.last_updated_at || a.created_at)); + const byProject = new Map(sorted.map((p) => [p.id, []])); + for (const x of memories) if (byProject.has(x.project_id)) byProject.get(x.project_id).push(x); + $main.append(el('p', { class: 'view-note' }, projects.length + ' projects · ' + memories.length + ' memories. A dot is the last write of a memory.')); + $main.append(renderRowsChart({ + rows: sorted, + ariaLabel: 'Projects and their memory writes', + label: (p) => p.id, + count: (p) => String(p.memory_count), + span: (p) => [p.created_at * 1000, (p.last_updated_at || p.created_at) * 1000], + onRow: (p) => selectProject(p.id), + marks: (p, scale, barY) => byProject.get(p.id).map((x) => { + const lane = CATEGORY_COLORS[x.category]; + const dot = svgEl('circle', { cx: scale.x(x.created_at * 1000), cy: barY + TL_BAR_H / 2, r: 5, class: 'tl-dot', fill: lane ? 'var(--lane-' + lane + ')' : 'var(--text-muted)', tabindex: '0', role: 'img', 'aria-label': x.keypath + ', ' + fmtDate(isoOf(x.created_at)) }); + attachTooltip(dot, () => el('div', {}, el('div', { class: 'tooltip-key' }, x.keypath), el('div', { class: 'tooltip-title' }, p.id), el('div', { class: 'tooltip-phase' }, (x.category || 'memory') + ' · v' + x.version + ' · ' + fmtDate(isoOf(x.created_at))))); + dot.addEventListener('click', () => openTicket(memKey(p.id, x.keypath))); + return dot; + }), + })); + $main.append(el('h2', { class: 'section-title' }, 'Latest writes')); + const latest = memories.slice().sort((a, b) => b.created_at - a.created_at).slice(0, 12); + $main.append(el('div', { class: 'row-list' }, latest.map((x) => + el('button', { type: 'button', class: 'row-item', onclick: () => openTicket(memKey(x.project_id, x.keypath)) }, + el('span', { class: 'row-meta' }, fmtDate(isoOf(x.created_at)), el('br'), x.project_id), + el('span', { class: 'row-text' }, categoryChip(x.category), ' ', x.keypath, x.version > 1 ? ' · v' + x.version : ''))))); + if (!m.project || !byProject.has(m.project)) m.project = sorted[0].id; + $main.append(renderProjectMemories(m.project)); +} + +function buildMemoryDrawer(key) { + const cut = key.indexOf('|'); + const project = key.slice(0, cut); + const keypath = key.slice(cut + 1); + const url = 'api/memory/history?project=' + encodeURIComponent(project) + '&keypath=' + encodeURIComponent(keypath); + const versions = state.memory.histories.get(key); + if (versions === undefined) loadInto(state.memory.histories, key, url, (json) => json.versions); + const frag = document.createDocumentFragment(); + frag.append(el('h2', { id: 'drawer-title', class: 'drawer-title' }, keypath), el('p', { class: 'drawer-meta' }, 'memstate project ' + project)); + if (!versions) { frag.append(el('p', { class: 'text-muted' }, 'Loading…')); return frag; } + if (versions.error) { frag.append(el('p', { class: 'load-error' }, versions.error)); return frag; } + const list = el('div', {}); + for (const v of versions.slice().reverse()) { + list.append(el('div', { class: 'version' }, + el('div', { class: 'drawer-chips' }, categoryChip(v.category), el('span', { class: 'chip' }, 'v' + v.version), el('span', { class: 'note-at' }, fmtDate(isoOf(v.created_at)))), + v.source ? el('p', { class: 'drawer-meta' }, v.source) : null, + el('p', { class: 'drawer-text' }, v.tombstone ? '(deleted)' : v.content))); + } + frag.append(el('section', { class: 'drawer-section' }, el('h3', {}, versions.length + (versions.length === 1 ? ' version' : ' versions') + ', newest first'), list)); + return frag; +} + /* ---- detail drawer ---- */ function renderDateGrid(idea) { @@ -860,10 +1198,19 @@ function buildDrawerContent(idea) { return frag; } +/** The content of the open drawer: a ticket, a session of the prompt log, or a memory. */ +function drawerContent(key) { + if (!key) return null; + if (key.startsWith('SESSION:')) return buildSessionDrawer(key.slice('SESSION:'.length)); + if (key.startsWith('MEM:')) return buildMemoryDrawer(key.slice('MEM:'.length)); + const idea = findByKey(key); + return idea ? buildDrawerContent(idea) : null; +} + function renderDrawer() { - const idea = state.openKey ? findByKey(state.openKey) : null; + const content = drawerContent(state.openKey); const wasHidden = $drawer.hidden; - if (!idea) { + if (!content) { if (!wasHidden) { $drawer.hidden = true; $drawerBackdrop.hidden = true; @@ -876,7 +1223,7 @@ function renderDrawer() { $drawer.hidden = false; $drawerBackdrop.hidden = false; $drawerBody.textContent = ''; - $drawerBody.append(buildDrawerContent(idea)); + $drawerBody.append(content); if (wasHidden) requestAnimationFrame(() => $drawerClose.focus()); } @@ -928,7 +1275,7 @@ async function runAction(action, body, busyText) { showToast(worked ? 'Done, but the page could not load the new data: ' + err.message : err.message, 'error'); } setBusy(false); - if (state.openKey && !findByKey(state.openKey)) { + if (/^IDEA-/.test(String(state.openKey)) && !findByKey(state.openKey)) { // The idea of the open ticket is gone: close the ticket without a new history entry. history.replaceState(null, '', '#' + state.view); state.openKey = null; @@ -945,9 +1292,14 @@ function renderToolbar() { $readonlyNote.hidden = live; if (!live) return; const inbox = (state.data.counts && state.data.counts.inbox) || 0; + // Triage, Build, Ask, and the watcher need the Claude Code login: a server without it hides them. + const claude = !!state.data.live.claude; $triage.textContent = 'Triage inbox (' + inbox + ')'; $triage.disabled = inbox === 0; - $buildNext.hidden = !state.data.live.window; + $triage.hidden = !claude; + $buildNext.hidden = !(claude && state.data.live.window); + $askOpen.hidden = !claude; + $watchLabel.hidden = !claude; const watch = state.data.live.watch || {}; $watchToggle.checked = !!watch.on; $watchLabel.title = watch.status || ''; @@ -966,7 +1318,7 @@ function renderDrawerActions(idea) { const update = (patch) => runAction('update', Object.assign({ id }, patch)); const buttons = []; - if (!closed && state.data.live.window) { + if (!closed && state.data.live.window && state.data.live.claude) { buttons.push(actionButton('Build with Claude', 'primary', () => runAction('go', { id }, 'Opening Claude Code for ' + idea.key + '…'), 'Open Claude Code on the recommended model, in the project of the idea (/ideas-go ' + id + ')')); } @@ -1061,17 +1413,25 @@ function render() { if (state.query.trim() !== '') renderSearch(); else if (state.view === 'timeline') renderTimeline(); else if (state.view === 'groups') renderGroups(); + else if (state.view === 'prompts') renderPrompts(); + else if (state.view === 'memory') renderMemory(); else renderBoard(); renderDrawer(); + // The prompt log and the memories load when a tab or a drawer needs them. + const wantsPrompts = state.view === 'prompts' || String(state.openKey).startsWith('SESSION:'); + if (wantsPrompts && !state.prompts.list && !state.prompts.loading) loadPrompts(); + if (state.view === 'memory' && !state.memory.overview && !state.memory.loading) loadMemoryOverview(); } function applyHash() { const raw = decodeURIComponent(location.hash.slice(1)); - if (raw === 'board' || raw === 'timeline' || raw === 'groups') { + if (VIEWS.includes(raw)) { state.view = raw; state.openKey = null; } else if (/^IDEA-\d+$/i.test(raw)) { state.openKey = raw.toUpperCase(); + } else if (/^(SESSION|MEM):/.test(raw)) { + state.openKey = raw; } else { state.openKey = null; if (raw) state.view = 'board'; @@ -1122,6 +1482,9 @@ function bindStaticEvents() { // The watcher and other sessions change the archive too. Show their changes when the page comes back. document.addEventListener('visibilitychange', () => { if (document.hidden || !isLive() || state.busy) return; + // The old data stays on the page until the new data is here, so nothing flickers. + if (state.prompts.list && !state.prompts.loading) loadPrompts(); + if (state.memory.overview && !state.memory.loading) loadMemoryOverview({ refresh: true }); loadData().then(render, () => {}); }); } diff --git a/package-lock.json b/package-lock.json index 6613423..f7af566 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ideamine", - "version": "0.6.0", + "version": "0.7.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ideamine", - "version": "0.6.0", + "version": "0.7.0", "license": "MIT", "bin": { "ideamine": "bin/ideamine.js" diff --git a/package.json b/package.json index 7b7d487..293fa32 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ideamine", - "version": "0.6.0", + "version": "0.7.0", "description": "An idea inbox for Claude Code. /idea saves an idea without calling the model; Claude later triages each idea and picks the cheapest model that can build it.", "type": "module", "bin": { diff --git a/skills/ideas-sync/SKILL.md b/skills/ideas-sync/SKILL.md new file mode 100644 index 0000000..6c0333d --- /dev/null +++ b/skills/ideas-sync/SKILL.md @@ -0,0 +1,8 @@ +--- +name: ideas-sync +description: Share one ideamine archive between all your machines through an ideamine server, or show the state of the sync. "off" turns it off. The ideamine hook answers this with no model call. +argument-hint: "[url|off]" +disable-model-invocation: true +--- + +The ideamine hook did not answer this command, and only the hook or the ideamine CLI can change the sync. Tell the user to run `ideamine sync <url>` (or `ideamine sync off`) in a terminal. Then stop. diff --git a/src/archive.js b/src/archive.js new file mode 100644 index 0000000..72da4ec --- /dev/null +++ b/src/archive.js @@ -0,0 +1,62 @@ +// Each change of the archive goes through here: to the archive on this machine, or to the ideamine +// server when sync is on. Callers await each function in both cases. When the server does not +// answer, a change resolves to { queued: true, reason }: it waits in the outbox and goes later. + +import os from 'node:os'; +import { absolute, addIdeas, applyTriage, findIdea, load, removeIdeas, updateIdea } from './store.js'; +import * as sync from './sync.js'; + +let offline = null; + +/** + * The archive, fresh from the server when sync is on. When the server does not answer, the copy on + * this machine, and offline() tells why. + */ +export async function fresh({ timeoutMs = 1500 } = {}) { + offline = null; + if (sync.enabled()) { + try { + await sync.pull({ timeoutMs }); + } catch (e) { + if (!(e instanceof sync.SyncError)) throw e; + offline = e.message; + } + } + return load(); +} + +/** Why the last fresh() gave the copy on this machine, or null. */ +export const offlineReason = () => offline; + +export async function add(texts, { source = 'mcp', project = null, session = null, tags = [] } = {}) { + const host = os.hostname(); + if (!sync.enabled()) return addIdeas(texts, { source, project, session, tags, host }); + const items = (Array.isArray(texts) ? texts : [texts]).map((t) => String(t ?? '').trim()).filter(Boolean); + if (!items.length) throw new Error('idea text is empty'); + const out = await sync.change({ op: 'add', texts: items, source, project: project ? absolute(project) : null, session, tags, host }); + if (out.queued) return out; + const db = load(); + return out.value.map((r) => ({ idea: findIdea(db, r.id), similar: r.similar })); +} + +export async function update(id, patch = {}) { + if (!sync.enabled()) return updateIdea(id, patch); + const clean = patch.project ? { ...patch, project: absolute(patch.project) } : patch; + const out = await sync.change({ op: 'update', id, patch: clean }); + return out.queued ? out : findIdea(load(), out.value.id); +} + +export async function remove(ids) { + if (!sync.enabled()) return removeIdeas(ids); + const out = await sync.change({ op: 'remove', ids }); + return out.queued ? out : out.value; +} + +export async function triage(verdicts, { by = null, projects = null } = {}) { + if (!sync.enabled()) return applyTriage(verdicts, { by, projects }); + const out = await sync.change({ op: 'triage', verdicts, by, projects }); + return out.queued ? out : out.value; +} + +/** The text for a change that waits in the outbox. */ +export const queuedText = (out) => `The ideamine server does not answer (${out.reason}). The change waits here and goes to the server later.`; diff --git a/src/claude.js b/src/claude.js index 863b17e..f5cf049 100644 --- a/src/claude.js +++ b/src/claude.js @@ -3,10 +3,11 @@ import { spawn, spawnSync } from 'node:child_process'; import fs from 'node:fs'; +import { fresh, triage } from './archive.js'; import { knownProjects } from './projects.js'; import { renderMarkdown } from './render.js'; import { BATCH_SCHEMA, pendingIdeas, triagePrompt } from './rubric.js'; -import { applyTriage, counts, findIdea, load, normalizeModel } from './store.js'; +import { counts, findIdea, normalizeModel } from './store.js'; /** Claude Code executable: explicit override, else the one running us (desktop app), else PATH. */ export function claudeBin() { @@ -41,6 +42,17 @@ function childEnv() { return env; } +let claudeFound = null; + +/** True when the Claude Code CLI starts on this machine. The answer is kept for the life of the process. */ +export function hasClaude() { + if (claudeFound === null) { + const [bin, argv] = command(['--version']); + claudeFound = spawnSync(bin, argv, { stdio: 'ignore', windowsHide: true, timeout: 20000, env: childEnv() }).status === 0; + } + return claudeFound; +} + function parseLooseJson(text) { const s = String(text || '').replace(/^\s*```(?:json)?\s*|\s*```\s*$/g, ''); const start = s.indexOf('{'); @@ -126,7 +138,7 @@ function tokensOf(out) { */ export async function headlessTriage({ model = process.env.IDEAMINE_TRIAGE_MODEL || 'sonnet', ids = null, limit = 20, dryRun = false, budget = 1 } = {}) { const alias = normalizeModel(model) || model; - const db = load(); + const db = await fresh(); const pending = pendingIdeas(db, { ids, limit }); if (!pending.length) return { message: 'Nothing to triage: the inbox is empty.' }; @@ -150,7 +162,8 @@ export async function headlessTriage({ model = process.env.IDEAMINE_TRIAGE_MODEL const out = await runHeadless(args, prompt, { timeoutMs: 5 * 60 * 1000, env }); const data = out.structured_output ?? parseLooseJson(out.result); if (!Array.isArray(data?.verdicts)) throw new Error('claude answered without verdicts'); - const results = applyTriage(data.verdicts, { by: `${alias} (headless)`, projects }); + const results = await triage(data.verdicts, { by: `${alias} (headless)`, projects }); + if (results.queued) return { message: `The verdicts wait for the ideamine server: ${results.reason}` }; return { results, model: alias, cost: out.total_cost_usd, tokens: tokensOf(out) }; } @@ -165,7 +178,7 @@ export async function askAboutIdeas(question, { model = 'sonnet', budget = 0.5 } budget, system: 'You answer questions about a developer\'s backlog of ideas. Answer in a few short lines of plain text. Name each idea by its number, like #12.', }); - const out = await runHeadless(args, `${renderMarkdown(load())}\nQuestion: ${question}`, { timeoutMs: 2 * 60 * 1000 }); + const out = await runHeadless(args, `${renderMarkdown(await fresh())}\nQuestion: ${question}`, { timeoutMs: 2 * 60 * 1000 }); return { answer: String(out.result || '').trim(), model: alias, cost: out.total_cost_usd, tokens: tokensOf(out) }; } @@ -174,7 +187,7 @@ export async function askAboutIdeas(question, { model = 'sonnet', budget = 0.5 } * whole inbox. Returns the headlessTriage result, or null when every candidate has a verdict already. */ export async function triageFirst({ id = null, model } = {}) { - const db = load(); + const db = await fresh(); if (id != null) { const idea = findIdea(db, id); return idea && !idea.triage ? headlessTriage({ model, ids: [idea.id] }) : null; diff --git a/src/config.js b/src/config.js index 508bc13..ed235be 100644 --- a/src/config.js +++ b/src/config.js @@ -32,6 +32,26 @@ export const SETTINGS = { value: '4332', about: 'port of `ideamine serve`, the dashboard with buttons on 127.0.0.1', }, + serve_hosts: { + env: 'IDEAMINE_SERVE_HOSTS', + value: '', + about: 'more Host names that `ideamine serve` answers, comma-separated (for a server behind nginx)', + }, + sync_url: { + env: 'IDEAMINE_SYNC_URL', + value: '', + about: 'ideamine server that holds the archive for every machine (empty: the archive is here)', + }, + prompt_log: { + env: 'IDEAMINE_PROMPT_LOG', + value: 'off', + about: 'on: send each prompt to the ideamine server, for the Prompts tab of the dashboard', + }, + memstate_url: { + env: 'IDEAMINE_MEMSTATE_URL', + value: '', + about: 'memstated daemon that the Memory tab of `ideamine serve` reads (empty: no Memory tab)', + }, }; const configPath = () => path.join(home(), 'config.json'); diff --git a/src/hook.js b/src/hook.js index 7eff289..cc0cfe6 100644 --- a/src/hook.js +++ b/src/hook.js @@ -1,14 +1,17 @@ // UserPromptSubmit hook: answers /idea, /ideas, and the local /ideas-* commands (ls, cat, rm, done, -// reopen, find, groups, watch, web) and blocks the prompt, so the model is never called. That makes -// capture free, instant, and possible even when the session is out of usage. Every other prompt -// passes through untouched, including /ideas-go, /ideas-all, /ideas-sort, and questions, which their -// skills answer. After each prompt, the hook lets the watcher and the dashboard catch up. +// reopen, find, groups, watch, web, sync) and blocks the prompt, so the model is never called. That +// makes capture free, instant, and possible even when the session is out of usage. Every other +// prompt passes through untouched, including /ideas-go, /ideas-all, /ideas-sort, and questions, +// which their skills answer. With the prompt log on, each prompt goes into the prompt outbox. After +// each prompt, the hook lets the watcher, the dashboard, and the sync catch up. +import * as archive from './archive.js'; import * as embed from './embed.js'; import * as publish from './publish.js'; -import { renderAdded, renderBoard, renderFound, renderGroups, renderIdea } from './render.js'; -import { addIdeas, FILTERS, findIdea, lane, listIdeas, load, removeIdeas, updateIdea } from './store.js'; -import { clip, splitIdeas } from './text.js'; +import { renderAdded, renderBoard, renderFound, renderGroups, renderIdea, stamp } from './render.js'; +import { FILTERS, findIdea, lane, listIdeas, load } from './store.js'; +import * as sync from './sync.js'; +import { clip, deriveTitle, splitIdeas } from './text.js'; import * as watch from './watch.js'; // `/idea ...`, `/ideas ...`, `/ideas-<verb> ...`, and the plugin-qualified `/ideamine:...` forms. @@ -26,6 +29,7 @@ const USAGE = `Usage: /idea <text> add an idea (a bulleted list a /ideas-find <words> search by meaning (all lanes) /ideas-groups [lane|-a] ideas grouped by meaning /ideas-web [off] the dashboard with a button for each command, on this PC + /ideas-sync [url|off] share the archive of every machine through an ideamine server These call the model: /ideas-go [N] build the next idea, or #N, on its model. New ideas are triaged first. /ideas-all do every idea that fits this chat. The others stay in the queue. @@ -34,6 +38,14 @@ These call the model: const noIdea = (ids) => `No idea ${ids.map((id) => `#${String(id).replace(/^#/, '')}`).join(', ')}.`; +// A read that the server could not answer shows the copy on this machine, and says so. +function offline(text) { + const reason = archive.offlineReason(); + if (!reason) return text; + const pulled = sync.readState().pulled; + return `(The ideamine server does not answer. This is the copy from ${pulled ? stamp(pulled) : 'no sync yet'}.)\n${text}`; +} + function readStdin() { return new Promise((resolve) => { if (process.stdin.isTTY) return resolve(''); @@ -57,8 +69,28 @@ export async function handlePrompt(prompt, { cwd = process.cwd(), session = null if (command === 'idea') { if (!arg) return USAGE; - const results = addIdeas(splitIdeas(arg), { source: 'hook', project: cwd, session }); - return renderAdded(results, load()); + const texts = splitIdeas(arg); + const results = await archive.add(texts, { source: 'hook', project: cwd, session }); + if (results.queued) { + const what = texts.length === 1 ? clip(deriveTitle(texts[0]), 70) : `${texts.length} ideas`; + return `💡 Saved here: ${what}. The ideamine server does not answer, so it gets the idea later (${sync.waiting()} waiting).`; + } + return renderAdded(results, load()); // the copy here is current: the server answered + } + + if (command === 'ideas-sync') { + if (/^off$/i.test(arg)) sync.setServer(''); + else if (arg) { + let backup; + try { + backup = sync.setServer(arg); + } catch (e) { + return `ideamine sync: ${e.message}. Usage: /ideas-sync http://10.66.0.1/ · /ideas-sync off`; + } + await archive.fresh({ timeoutMs: 5000 }); + if (backup) return `${sync.status()}\nThe archive that was here is in ${backup}.`; + } + return sync.status(); } if (command === 'ideas-watch') { @@ -78,13 +110,13 @@ export async function handlePrompt(prompt, { cwd = process.cwd(), session = null const words = arg.split(/\s+/).filter(Boolean); const ids = words.length > 0 && words.every((w) => ID.test(w)) ? words : null; - const db = load(); + const db = await archive.fresh(); - if (command === 'ideas' && !words.length) return renderBoard(db, { cwd, hints: true }); + if (command === 'ideas' && !words.length) return offline(renderBoard(db, { cwd, hints: true })); if (command === 'ideas-find') { if (!arg) return USAGE; - return renderFound(await embed.find(db, arg, { timeoutMs: SEARCH_TIMEOUT_MS }), arg, { cwd }); + return offline(renderFound(await embed.find(db, arg, { timeoutMs: SEARCH_TIMEOUT_MS }), arg, { cwd })); } if (command === 'ideas-groups') { @@ -94,7 +126,7 @@ export async function handlePrompt(prompt, { cwd = process.cwd(), session = null const ideas = listIdeas(db, { filter }); try { const groups = await embed.groupIdeas(ideas, { timeoutMs: SEARCH_TIMEOUT_MS }); - return renderGroups(groups, ideas, { cwd, scope: filter }); + return offline(renderGroups(groups, ideas, { cwd, scope: filter })); } catch (e) { if (!(e instanceof embed.EmbedError)) throw e; return `Groups need the embedding server, which does not answer: ${e.message}`; @@ -107,22 +139,25 @@ export async function handlePrompt(prompt, { cwd = process.cwd(), session = null const filters = lower.filter((w) => FILTERS.includes(w)); const here = lower.includes('here'); if (filters.length + (here ? 1 : 0) === lower.length && filters.length <= 1) { - return renderBoard(db, { filter: filters[0] || 'open', project: here ? cwd : null, cwd, hints: true }); + return offline(renderBoard(db, { filter: filters[0] || 'open', project: here ? cwd : null, cwd, hints: true })); } } if ((command === 'ideas-cat' || command === 'ideas-rm') && ids) { const missing = ids.filter((id) => !findIdea(db, id)); if (missing.length) return noIdea(missing); - if (command === 'ideas-cat') return ids.map((id) => renderIdea(findIdea(db, id))).join('\n\n'); - return removeIdeas(ids).map((i) => `✗ Removed #${i.id} · ${clip(i.title, 60)}`).join('\n'); + if (command === 'ideas-cat') return offline(ids.map((id) => renderIdea(findIdea(db, id))).join('\n\n')); + const gone = await archive.remove(ids); + if (gone.queued) return archive.queuedText(gone); + return gone.map((i) => `✗ Removed #${i.id} · ${clip(i.title, 60)}`).join('\n'); } const status = STATUS_COMMANDS[command]; if (status && words[0] && ID.test(words[0])) { if (!findIdea(db, words[0])) return noIdea([words[0]]); const note = words.slice(1).join(' '); - const idea = updateIdea(words[0], { status, note: note || undefined }); + const idea = await archive.update(words[0], { status, note: note || undefined }); + if (idea.queued) return archive.queuedText(idea); return `✓ #${idea.id} ${clip(idea.title, 60)} → ${lane(idea)}${note ? ' (note added)' : ''}`; } @@ -137,12 +172,18 @@ export async function runHook() { } catch { return; // not a hook payload; stay out of the way } + const prompt = typeof input?.prompt === 'string' ? input.prompt : ''; + const cwd = input.cwd || process.cwd(); + const session = input.session_id || null; + try { + if (sync.logsPrompts()) sync.logPrompt({ prompt, session, cwd }); + } catch (e) { + // The prompt log must never stop a prompt. + process.stderr.write(`ideamine: the prompt log failed: ${e.message}\n`); + } let message = null; try { - message = await handlePrompt(typeof input?.prompt === 'string' ? input.prompt : '', { - cwd: input.cwd || process.cwd(), - session: input.session_id || null, - }); + message = await handlePrompt(prompt, { cwd, session }); } catch (e) { // Never swallow the user's text: let the prompt through so the /idea skill can save it via MCP. process.stderr.write(`ideamine: ${e.message}\n`); @@ -156,12 +197,12 @@ export async function runHook() { hookSpecificOutput: { hookEventName: 'UserPromptSubmit', suppressOriginalPrompt: true }, })); } - // Each prompt lets the watcher and the dashboard catch up, so they keep going while they are on. - for (const kick of [watch.kick, publish.kick]) { + // Each prompt lets the watcher, the dashboard, and the sync catch up, so they keep going. + for (const kick of [watch.kick, publish.kick, sync.kick]) { try { kick(); } catch { - // The watcher and the dashboard must never stop a prompt. + // The watcher, the dashboard, and the sync must never stop a prompt. } } } diff --git a/src/mcp.js b/src/mcp.js index 37fcbf1..2941830 100644 --- a/src/mcp.js +++ b/src/mcp.js @@ -5,25 +5,13 @@ import fs from 'node:fs'; import path from 'node:path'; import readline from 'node:readline'; import { fileURLToPath } from 'node:url'; +import * as archive from './archive.js'; import { headlessTriage, triageFirst } from './claude.js'; import * as embed from './embed.js'; import { knownProjects } from './projects.js'; import { renderAdded, renderBoard, renderFound, renderGroups, renderIdea } from './render.js'; import { BATCH_SCHEMA, MODELS, pendingIdeas, triagePrompt } from './rubric.js'; -import { - addIdeas, - applyTriage, - counts, - FILTERS, - findIdea, - lane, - listIdeas, - load, - pickNext, - removeIdeas, - STATUSES, - updateIdea, -} from './store.js'; +import { counts, FILTERS, findIdea, lane, listIdeas, load, pickNext, STATUSES } from './store.js'; import { clip, splitIdeas } from './text.js'; const ROOT = new URL('..', import.meta.url); @@ -170,6 +158,7 @@ const PROMPTS = [ { name: 'ideas-sort', description: 'Triage the inbox now and show the queue', arguments: [] }, { name: 'ideas-watch', description: 'Turn the background watcher on or off', arguments: [{ name: 'off', required: false }] }, { name: 'ideas-web', description: 'Start or stop the dashboard with buttons on this PC', arguments: [{ name: 'off', required: false }] }, + { name: 'ideas-sync', description: 'Share one archive between your machines through an ideamine server', arguments: [{ name: 'url', required: false }] }, ]; function skillBody(name, args) { @@ -205,13 +194,13 @@ function withBoard(text) { // Tool handlers return plain text for the model. const handlers = { - idea_add({ text, tags, project }) { - const results = addIdeas(splitIdeas(text || ''), { source: 'mcp', project: project || currentProject(), tags }); - return renderAdded(results, load()); + async idea_add({ text, tags, project }) { + const results = await archive.add(splitIdeas(text || ''), { source: 'mcp', project: project || currentProject(), tags }); + return results.queued ? `Saved on this machine. ${archive.queuedText(results)}` : renderAdded(results, load()); }, async idea_list({ id, filter = 'open', query = '', here = false, limit = 0, full = false, semantic = false, groups = false }) { - const db = load(); + const db = await archive.fresh(); if (id != null) { const idea = findIdea(db, id); if (!idea) throw new Error(`no idea #${id}`); @@ -239,11 +228,12 @@ const handlers = { return withBoard(headlessSummary(await headlessTriage({ model, limit: limit || 20 }))); } if (Array.isArray(verdicts) && verdicts.length) { - const projects = knownProjects(load()); - return withBoard(summarizeTriage(applyTriage(verdicts, { by: by ? clip(by, 40) : 'claude', projects }))); + const projects = knownProjects(await archive.fresh()); + const results = await archive.triage(verdicts, { by: by ? clip(by, 40) : 'claude', projects }); + return results.queued ? archive.queuedText(results) : withBoard(summarizeTriage(results)); } limit ||= 30; - const db = load(); + const db = await archive.fresh(); const pending = pendingIdeas(db, { ids, limit }); if (!pending.length) return 'Nothing to triage: the inbox is empty.'; return ( @@ -253,8 +243,9 @@ const handlers = { ); }, - idea_update({ id, ...patch }) { - const idea = updateIdea(id, patch); + async idea_update({ id, ...patch }) { + const idea = await archive.update(id, patch); + if (idea.queued) return archive.queuedText(idea); const bits = [`#${idea.id} ${clip(idea.title, 60)} · ${lane(idea)}`]; if (patch.model) bits.push(`model ${idea.triage.model}`); if (patch.note) bits.push('note added'); @@ -271,7 +262,7 @@ const handlers = { out.push(`Triage failed: ${e.message}`); } } - const db = load(); + const db = await archive.fresh(); let idea; if (id != null) { idea = findIdea(db, id); @@ -301,9 +292,10 @@ const handlers = { return out.join('\n\n'); }, - idea_remove({ ids = [] }) { + async idea_remove({ ids = [] }) { if (!ids.length) throw new Error('pass the ids to delete'); - const gone = removeIdeas(ids); + const gone = await archive.remove(ids); + if (gone.queued) return archive.queuedText(gone); return [`Removed ${gone.length} idea${gone.length === 1 ? '' : 's'}.`, ...gone.map(renderIdea)].join('\n\n'); }, }; diff --git a/src/publish.js b/src/publish.js index e925d56..10c87dd 100644 --- a/src/publish.js +++ b/src/publish.js @@ -10,7 +10,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import * as config from './config.js'; import * as embed from './embed.js'; -import { counts, dbPath, home, lane, listIdeas, load } from './store.js'; +import { baseName, counts, dbPath, home, lane, listIdeas, load } from './store.js'; const PAGE = new URL('../dashboard/index.html', import.meta.url); const WAIT_AFTER_ERROR_MS = 10 * 60 * 1000; // after a failed publish, before kick() tries again @@ -77,7 +77,8 @@ export function snapshot(db, { vectors = null, groups = [], generated = new Date status: i.status, rank: rank.get(i.id), tags: i.tags || [], - project: i.project ? path.basename(i.project) : null, + project: i.project ? baseName(i.project) : null, + host: i.host || null, created: i.created, updated: i.updated, triaged: t?.at || null, @@ -215,7 +216,8 @@ function startProcess() { */ export function kick({ start = startProcess, now = Date.now() } = {}) { const url = config.get('publish_url'); - if (!url) return false; + // With sync on, the ideamine server shows the live archive itself: nothing to upload. + if (!url || config.get('sync_url')) return false; const state = readState(); if (state.errorAt && now - Date.parse(state.errorAt) < WAIT_AFTER_ERROR_MS) return false; if (isRunning(now)) return false; diff --git a/src/render.js b/src/render.js index 805e05e..4485b65 100644 --- a/src/render.js +++ b/src/render.js @@ -1,8 +1,7 @@ // Plain-text views. Kept compact on purpose: every character here may end up in a model's context. -import path from 'node:path'; import { samePath } from './projects.js'; -import { counts, lane, listIdeas } from './store.js'; +import { baseName, counts, lane, listIdeas } from './store.js'; import { clip } from './text.js'; const LANE_TITLES = { @@ -16,7 +15,7 @@ const LANE_TITLES = { }; function projectName(p) { - return p ? path.basename(p) : ''; + return p ? baseName(p) : ''; } export function stamp(iso) { diff --git a/src/serve.js b/src/serve.js index 08c7052..65bd502 100644 --- a/src/serve.js +++ b/src/serve.js @@ -1,8 +1,15 @@ -// The dashboard with buttons. `ideamine serve` runs a web server on this PC. It serves the page of -// `ideamine publish`, a live data.json, and an API for the slash commands: add, delete, done, -// reopen, start, drop, note, model, triage, build, ask, and the watcher. The archive and the Claude -// Code login are on this PC, so the commands run here. The server listens on 127.0.0.1 only, and -// it takes commands only from its own page. +// The dashboard with buttons. `ideamine serve` runs a web server that serves the page of `ideamine +// publish`, a live data.json, and an API for the slash commands: add, delete, done, reopen, start, +// drop, note, model, triage, build, ask, and the watcher. It has two roles: +// +// - On a PC, it runs the commands there, where the Claude Code login is. When sync is on, the +// archive that it changes is on the ideamine server, and the Prompts and Memory tabs come from +// there too. +// - On a server (sync off), it holds the archive for every machine: the machines sync with /api/db +// and /api/ops, and send their prompts to /api/prompts. With memstate_url, the Memory tab shows +// the memories of a memstated daemon. nginx in front of it names the server in serve_hosts. +// +// The server listens on 127.0.0.1 only, and it takes commands only from its own page. import { spawn } from 'node:child_process'; import fs from 'node:fs'; @@ -10,10 +17,12 @@ import http from 'node:http'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import * as archive from './archive.js'; import * as config from './config.js'; import * as publish from './publish.js'; import { renderAdded, stamp } from './render.js'; import * as store from './store.js'; +import * as sync from './sync.js'; import { clip, splitIdeas } from './text.js'; import * as watch from './watch.js'; @@ -22,8 +31,12 @@ const BIN = fileURLToPath(new URL('../bin/ideamine.js', import.meta.url)); const MAX_BODY = 1024 * 1024; const EMBED_TIMEOUT_MS = 5000; // data.json must not wait long for an embedding server that is away const HEADERS = { 'cache-control': 'no-store', 'x-content-type-options': 'nosniff' }; +const DAY = 86400000; +const APPLIED_KEPT = 1000; // results of recent changes, so that a change that comes twice applies once const logPath = () => path.join(store.home(), 'serve.log'); +const promptsPath = () => path.join(store.home(), 'prompts.jsonl'); +const appliedPath = () => path.join(store.home(), 'applied.json'); const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); /** The port from the setting serve_port. */ @@ -36,14 +49,17 @@ export function port() { export const address = (p = port()) => `http://127.0.0.1:${p}/`; +/** The Host names from the setting serve_hosts, for example the address that nginx answers on. */ +const extraHosts = () => config.get('serve_hosts').split(',').map((h) => h.trim().toLowerCase()).filter(Boolean); + /** * True when the server may answer a request. The Host header must name this server, which stops a * DNS rebinding page. A POST must send JSON, and its Origin, if any, must be this server. A page of * another site cannot send JSON here without a CORS preflight, and this server allows none. */ -export function allowed({ method, host, origin, type }, port) { +export function allowed({ method, host, origin, type }, port, hosts = []) { const h = String(host ?? '').toLowerCase(); - if (h !== `127.0.0.1:${port}` && h !== `localhost:${port}`) return false; + if (h !== `127.0.0.1:${port}` && h !== `localhost:${port}` && !hosts.includes(h)) return false; if (method === 'GET' || method === 'HEAD') return true; if (method !== 'POST' || !/^application\/json\s*(;|$)/i.test(String(type ?? '').trim())) return false; return origin == null || origin === `http://${h}`; @@ -119,9 +135,23 @@ async function readJson(req) { /** The snapshot of `ideamine publish`, and `live`: what only this server can tell the page. */ async function liveData() { - const { data, note } = await publish.build(store.load(), { timeoutMs: EMBED_TIMEOUT_MS }); + const db = await archive.fresh(); + const { data, note } = await publish.build(db, { timeoutMs: EMBED_TIMEOUT_MS }); + const { hasClaude } = await import('./claude.js'); const w = watch.readState(); - return { ...data, live: { note, window: process.platform === 'win32', watch: { on: !!w.on, status: watch.status() } } }; + const synced = sync.enabled(); + return { + ...data, + live: { + note, + window: process.platform === 'win32', + claude: hasClaude(), + watch: { on: !!w.on, status: watch.status() }, + sync: synced ? { url: sync.serverUrl(), offline: archive.offlineReason(), waiting: sync.waiting() } : null, + prompts: synced || fs.existsSync(promptsPath()), + memory: synced || !!config.get('memstate_url'), + }, + }; } /** Search by meaning on the page: the page sends its query to the embedding server through here. */ @@ -142,24 +172,175 @@ async function proxyEmbeddings(req, res) { res.end(Buffer.from(await upstream.arrayBuffer())); } +/** A read of the Prompts or Memory tab on a PC with sync on: the ideamine server answers it. */ +async function forward(res, route) { + const target = new URL(route, sync.serverUrl()); + let upstream; + try { + upstream = await fetch(target, { signal: AbortSignal.timeout(15000) }); + } catch { + return send(res, 502, { ok: false, error: `cannot reach ${target.origin}` }); + } + res.writeHead(upstream.status, { ...HEADERS, 'content-type': upstream.headers.get('content-type') || 'application/json' }); + res.end(Buffer.from(await upstream.arrayBuffer())); +} + +// --------------------------------------------------------------------------------------------- +// The server role: the archive for every machine, the prompt log, and the memories. + +// A change from a machine. Each result is small, because the machine reads the archive that comes +// with the answer. +const CHANGES = { + add: (op) => + store + .addIdeas(op.texts, { source: op.source, project: op.project, session: op.session, tags: op.tags, host: op.host }) + .map(({ idea, similar }) => ({ id: idea.id, similar })), + update: (op) => ({ id: store.updateIdea(op.id, op.patch || {}).id }), + remove: (op) => store.removeIdeas(op.ids || []), + triage: (op) => store.applyTriage(op.verdicts, { by: op.by, projects: op.projects }), +}; + +function readApplied() { + try { + return JSON.parse(fs.readFileSync(appliedPath(), 'utf8')); + } catch { + return {}; + } +} + +/** + * Apply changes in order. A change that came before (the same oid) is not applied again: its first + * result comes back. A machine sends a change again when an answer got lost on the way. + */ +function applyChanges(ops) { + if (!Array.isArray(ops)) throw new Error('ops must be a list'); + const applied = readApplied(); + const results = ops.map((op) => { + if (op?.oid && applied[op.oid]) return applied[op.oid]; + let result; + try { + if (!op || !Object.hasOwn(CHANGES, op.op)) throw new Error(`unknown change "${op?.op}"`); + result = { ok: true, value: CHANGES[op.op](op) }; + } catch (e) { + result = { ok: false, error: e.message }; + } + if (op?.oid) applied[op.oid] = result; + return result; + }); + const kept = Object.entries(applied).slice(-APPLIED_KEPT); + store.withLock(() => store.writeAtomic(appliedPath(), JSON.stringify(Object.fromEntries(kept)))); + return results; +} + +let promptCache = null; + +/** The prompt log: one JSON line for each prompt. The copy in memory is kept until the file changes. */ +function loadPrompts() { + const st = fs.statSync(promptsPath(), { throwIfNoEntry: false }); + if (!st) return { prompts: [], ids: new Set() }; + if (promptCache?.mtime === st.mtimeMs && promptCache.size === st.size) return promptCache; + const prompts = []; + for (const line of fs.readFileSync(promptsPath(), 'utf8').split('\n')) { + if (!line.trim()) continue; + try { + prompts.push(JSON.parse(line)); + } catch { + // A line that a crash cut in half. + } + } + promptCache = { mtime: st.mtimeMs, size: st.size, prompts, ids: new Set(prompts.map((p) => p.id)) }; + return promptCache; +} + +const field = (v, max = 300) => (v == null ? null : String(v).slice(0, max)); + +/** Add prompts that the log does not have yet. A prompt without an id, a time, or a text is skipped. */ +function addPrompts(list) { + if (!Array.isArray(list)) throw new Error('prompts must be a list'); + const { ids } = loadPrompts(); + const add = []; + let skipped = 0; + for (const p of list) { + const bad = !p || typeof p.id !== 'string' || !p.id || typeof p.prompt !== 'string' || Number.isNaN(Date.parse(p.at)); + if (bad) skipped++; + if (bad || ids.has(p.id)) continue; + ids.add(p.id); + add.push({ id: p.id.slice(0, 64), at: new Date(p.at).toISOString(), host: field(p.host, 100), session: field(p.session, 100), cwd: field(p.cwd), prompt: p.prompt }); + } + if (add.length) store.withLock(() => fs.appendFileSync(promptsPath(), add.map((p) => `${JSON.stringify(p)}\n`).join(''))); + return { added: add.length, skipped }; +} + +/** The prompts of the last `days` days (all prompts for 0), oldest first. */ +function listPrompts(days) { + const since = days > 0 ? Date.now() - days * DAY : 0; + return loadPrompts().prompts.filter((p) => Date.parse(p.at) >= since).sort((a, b) => a.at.localeCompare(b.at)); +} + +/** A read from the memstated daemon at memstate_url. */ +async function memstate(route, body) { + const base = config.get('memstate_url'); + if (!base) throw new Error('this server shows no memories. Set memstate_url, for example: ideamine config memstate_url http://127.0.0.1:8765'); + const url = `${base.replace(/\/+$/, '')}${route}`; + const init = { signal: AbortSignal.timeout(8000) }; + if (body) Object.assign(init, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) }); + let res; + try { + res = await fetch(url, init); + } catch { + throw new Error(`cannot reach memstated at ${base}`); + } + const json = await res.json().catch(() => null); + if (!res.ok) throw new Error(`memstated answered ${res.status}: ${json?.error || 'no reason'}`); + return json; +} + +// The Memory tab reads memstated. It never writes there. +const MEMORY = { + /** Every project, and every current memory without its text, for the list and the timeline. */ + async overview() { + const { projects } = await memstate('/api/v1/projects'); + const memories = []; + for (const p of projects) { + const { memories: list } = await memstate('/api/v1/keypaths', { project_id: p.id }); + for (const m of list) memories.push({ id: m.id, project_id: p.id, keypath: m.keypath, category: m.category || null, version: m.version, created_at: m.created_at }); + } + return { projects, memories }; + }, + /** The current memories of one project, with their text. */ + async project(query) { + const { memories } = await memstate('/api/v1/keypaths', { project_id: query.get('id') || '', include_content: true }); + return { memories }; + }, + /** Every version of one memory, oldest first. */ + async history(query) { + const { versions } = await memstate('/api/v1/memories/history', { project_id: query.get('project') || '', keypath: query.get('keypath') || '' }); + return { versions }; + }, +}; + +// --------------------------------------------------------------------------------------------- // The slash commands. Each one takes the JSON body and returns the text to show on the page. + const ACTIONS = { /** /idea. A bulleted list adds one idea per bullet. */ - add({ text }) { - const results = store.addIdeas(splitIdeas(String(text ?? '')), { source: 'web' }); - return renderAdded(results, store.load()); + async add({ text }) { + const results = await archive.add(splitIdeas(String(text ?? '')), { source: 'web' }); + return results.queued ? archive.queuedText(results) : renderAdded(results, store.load()); }, /** /ideas-rm. An unknown id deletes nothing. */ - rm({ ids }) { + async rm({ ids }) { if (!Array.isArray(ids) || !ids.length) throw new Error('name the ideas to delete'); - return store.removeIdeas(ids).map((i) => `Removed #${i.id} · ${clip(i.title, 60)}`).join('\n'); + const gone = await archive.remove(ids); + return gone.queued ? archive.queuedText(gone) : gone.map((i) => `Removed #${i.id} · ${clip(i.title, 60)}`).join('\n'); }, /** /ideas-done, /ideas-reopen, and the verbs start, drop, note, and model of the CLI. */ - update({ id, status, note, model }) { + async update({ id, status, note, model }) { if (!status && !note && !model) throw new Error('nothing to change'); - const idea = store.updateIdea(id, { status, note, model }); + const idea = await archive.update(id, { status, note, model }); + if (idea.queued) return archive.queuedText(idea); const bits = [`#${idea.id} ${clip(idea.title, 60)} → ${store.lane(idea)}`]; if (model) bits.push(`model ${idea.triage.model}`); if (note) bits.push('note added'); @@ -187,12 +368,12 @@ const ACTIONS = { } catch (e) { lines.push(`Triage failed: ${e.message}`); } - const db = store.load(); + const db = await archive.fresh(); const idea = id != null ? store.findIdea(db, id) : store.pickNext(db); if (!idea) throw new Error(id != null ? `no idea #${id}` : 'Nothing is ready to build. Triage the inbox first.'); const { model, dir } = goPlan(idea, os.homedir()); await open([BIN, 'go', String(idea.id)], dir); - store.updateIdea(idea.id, { status: 'doing' }); + await archive.update(idea.id, { status: 'doing' }); lines.push(`Opened Claude Code (${model}) in ${dir} for #${idea.id} · ${clip(idea.title, 60)}`); return lines.join('\n\n'); }, @@ -218,11 +399,27 @@ const ACTIONS = { // These actions do not change the archive, so the dashboard server needs no new upload. const READ_ONLY = new Set(['ask', 'watch']); -async function handle(req, res, ctx) { - if (!allowed({ method: req.method, host: req.headers.host, origin: req.headers.origin, type: req.headers['content-type'] }, ctx.port)) { - return send(res, 403, { ok: false, error: 'forbidden' }); +/** The reads and changes of the server role. Resolves to false for a route that it does not have. */ +async function serverRoute(req, res, route, query) { + if (route === 'GET /api/db') return send(res, 200, { ok: true, db: store.load() }); + if (route === 'POST /api/ops') { + const { ops } = await readJson(req); + return send(res, 200, { ok: true, results: applyChanges(ops), db: store.load() }); + } + if (route === 'POST /api/prompts') { + const { prompts } = await readJson(req); + return send(res, 200, { ok: true, ...addPrompts(prompts) }); } - const { pathname } = new URL(req.url, 'http://127.0.0.1'); + if (route === 'GET /api/prompts') return send(res, 200, { ok: true, prompts: listPrompts(Number(query.get('days')) || 0) }); + const memory = route.startsWith('GET /api/memory/') && route.slice('GET /api/memory/'.length); + if (memory && Object.hasOwn(MEMORY, memory)) return send(res, 200, { ok: true, ...(await MEMORY[memory](query)) }); + return false; +} + +async function handle(req, res, ctx) { + const request = { method: req.method, host: req.headers.host, origin: req.headers.origin, type: req.headers['content-type'] }; + if (!allowed(request, ctx.port, extraHosts())) return send(res, 403, { ok: false, error: 'forbidden' }); + const { pathname, search, searchParams } = new URL(req.url, 'http://127.0.0.1'); const route = `${req.method === 'HEAD' ? 'GET' : req.method} ${pathname}`; if (route === 'GET /' || route === 'GET /index.html') return sendPage(res); if (route === 'GET /data.json') return send(res, 200, await liveData()); @@ -236,6 +433,15 @@ async function handle(req, res, ctx) { ctx.server.close(); return; } + if (sync.enabled()) { + if (route === 'GET /api/prompts' || route.startsWith('GET /api/memory/')) return forward(res, `${pathname.slice(1)}${search}`); + } else { + try { + if ((await serverRoute(req, res, route, searchParams)) !== false) return; + } catch (e) { + return send(res, 400, { ok: false, error: e.message }); + } + } const action = route.startsWith('POST /api/') ? route.slice('POST /api/'.length) : ''; if (!Object.hasOwn(ACTIONS, action)) return send(res, 404, { ok: false, error: 'not found' }); try { diff --git a/src/store.js b/src/store.js index f624ec1..9dcc6ce 100644 --- a/src/store.js +++ b/src/store.js @@ -2,6 +2,7 @@ // Writers take a directory lock and replace the file atomically, so concurrent sessions, // the hook, and the CLI can all write at once without losing ideas. +import crypto from 'node:crypto'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -116,10 +117,31 @@ export function writeAtomic(file, text, { backup = false } = {}) { renameWithRetry(tmp, file); } +/** + * Write the archive. `uid` names this archive and `rev` counts its writes, so that a machine that + * syncs with a server can tell an old copy of the server's archive from a new one. + */ function save(db) { + db.uid ||= crypto.randomUUID(); + db.rev = (Number(db.rev) || 0) + 1; writeAtomic(dbPath(), JSON.stringify(db, null, 2) + '\n', { backup: true }); } +/** + * The absolute form of a project folder. On a POSIX server, a Windows path from a machine that + * syncs with it stays as that machine wrote it. + */ +export function absolute(dir, platform = process.platform) { + const s = String(dir); + if (platform !== 'win32' && /^(?:[A-Za-z]:[\\/]|\\\\)/.test(s)) return s; + return path.resolve(s); +} + +/** The last part of a folder path, for Windows and POSIX paths on any machine. */ +export function baseName(dir) { + return String(dir).split(/[\\/]+/).filter(Boolean).pop() || String(dir); +} + /** Run `fn` while this process holds the archive lock. Other files in home() can share the lock. */ export function withLock(fn) { const release = acquireLock(); @@ -236,7 +258,7 @@ function similarTo(db, text, excludeId) { // --------------------------------------------------------------------------------------------- // Mutations -export function addIdeas(texts, { source = 'mcp', project = null, session = null, tags = [] } = {}) { +export function addIdeas(texts, { source = 'mcp', project = null, session = null, tags = [], host = null } = {}) { const items = (Array.isArray(texts) ? texts : [texts]).map((t) => String(t ?? '').trim()).filter(Boolean); if (!items.length) throw new Error('idea text is empty'); return mutate((db) => @@ -249,7 +271,7 @@ export function addIdeas(texts, { source = 'mcp', project = null, session = null text, status: 'inbox', tags: normalizeTags([...extractTags(text), ...normalizeTags(tags)]), - project: project ? path.resolve(project) : null, + project: project ? absolute(project) : null, source, created: stamp, updated: stamp, @@ -257,6 +279,7 @@ export function addIdeas(texts, { source = 'mcp', project = null, session = null notes: [], }; if (session) idea.session = String(session); + if (host) idea.host = String(host); db.ideas.push(idea); return { idea, similar }; }), @@ -355,7 +378,7 @@ export function updateIdea(id, patch = {}) { if (patch.title) idea.title = clip(patch.title, 90); else if (patch.text) idea.title = deriveTitle(idea.text); if (Array.isArray(patch.tags)) idea.tags = normalizeTags(patch.tags); - if (patch.project !== undefined) idea.project = patch.project ? path.resolve(patch.project) : null; + if (patch.project !== undefined) idea.project = patch.project ? absolute(patch.project) : null; if (patch.model) { const model = normalizeModel(patch.model); if (!model) throw new Error(`model must be one of ${MODEL_ALIASES.join(', ')}`); diff --git a/src/sync.js b/src/sync.js new file mode 100644 index 0000000..4dc2acb --- /dev/null +++ b/src/sync.js @@ -0,0 +1,356 @@ +// Sync with an ideamine server, so that every Claude on every machine sees one archive. The server +// (`ideamine serve`, for example behind nginx on a WireGuard address) holds the archive, and +// ideas.json on this machine is a copy of it. A change goes into the outbox first and then to the +// server. When the server does not answer, the change waits in the outbox, so no idea is lost. The +// prompt log takes the same way. No process stays alive: the hook calls kick() for each prompt, and +// kick() starts a background sync when work waits or the copy is old. + +import { spawn } from 'node:child_process'; +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as config from './config.js'; +import { stamp } from './render.js'; +import { dbPath, home, withLock, writeAtomic } from './store.js'; +import { clip } from './text.js'; + +const PULL_EVERY_MS = 60 * 1000; // a copy older than this is pulled again in the background +const PROMPTS_EVERY_MS = 20 * 1000; // prompts wait at least this long, so that a prompt starts no process each time +const WAIT_AFTER_ERROR_MS = 2 * 60 * 1000; // after a failed sync, before kick() tries again +const LOCK_STALE_MS = 10 * 60 * 1000; +const OPS_PER_REQUEST = 100; +const PROMPT_BYTES_PER_REQUEST = 256 * 1024; +const MAX_PROMPT_CHARS = 100 * 1000; // a longer paste is cut, so that each prompt fits in a request + +const outboxPath = () => path.join(home(), 'outbox.jsonl'); +const promptsPath = () => path.join(home(), 'prompts-outbox.jsonl'); +const statePath = () => path.join(home(), 'sync.json'); +const lockPath = () => path.join(home(), '.sync'); + +/** The server does not answer, or it refuses the request. The change stays in the outbox. */ +export class SyncError extends Error {} + +/** The address of the server, with a slash at the end, or '' when sync is off. */ +export function serverUrl() { + const url = config.get('sync_url'); + return url ? (url.endsWith('/') ? url : `${url}/`) : ''; +} + +export const enabled = () => serverUrl() !== ''; +export const logsPrompts = () => enabled() && /^(on|true|yes|1)$/i.test(config.get('prompt_log')); + +function readLines(file) { + let raw; + try { + raw = fs.readFileSync(file, 'utf8'); + } catch { + return []; + } + const out = []; + for (const line of raw.split('\n')) { + if (!line.trim()) continue; + try { + out.push(JSON.parse(line)); + } catch { + // A line that a crash cut in half. The rest of the file is still good. + } + } + return out; +} + +const writeLines = (file, items) => writeAtomic(file, items.map((i) => `${JSON.stringify(i)}\n`).join('')); +const append = (file, items) => withLock(() => fs.appendFileSync(file, items.map((i) => `${JSON.stringify(i)}\n`).join(''))); +const size = (file) => fs.statSync(file, { throwIfNoEntry: false })?.size || 0; + +export function readState() { + try { + return JSON.parse(fs.readFileSync(statePath(), 'utf8')); + } catch { + return {}; + } +} + +// Call it while you hold the lock. +const saveState = (patch) => writeAtomic(statePath(), `${JSON.stringify({ ...readState(), ...patch }, null, 2)}\n`); +const writeState = (patch) => withLock(() => saveState(patch)); +const failed = (e) => writeState({ error: e.message, errorAt: new Date().toISOString() }); + +async function request(route, { body, timeoutMs }) { + const target = new URL(route, serverUrl()); + const init = { signal: AbortSignal.timeout(timeoutMs) }; + if (body !== undefined) Object.assign(init, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(body) }); + let res; + try { + res = await fetch(target, init); + } catch (e) { + const why = e.name === 'TimeoutError' ? `no answer in ${timeoutMs / 1000} s` : e.cause?.code || e.cause?.message || e.message; + throw new SyncError(`cannot reach ${target.origin} (${why})`); + } + const text = await res.text(); + let json = null; + try { + json = JSON.parse(text); + } catch { + // Not JSON: the error below shows the text. + } + if (!res.ok || !json?.ok) throw new SyncError(`${target.origin} answered ${res.status}: ${json?.error || clip(text, 160)}`); + return json; +} + +/** Keep the archive of the server as the copy here, unless the copy is newer: a slow answer must not undo a later one. */ +function keep(db) { + if (!db || !Array.isArray(db.ideas)) throw new SyncError('the server sent no archive'); + withLock(() => { + const state = readState(); + if (db.uid === state.uid && (Number(db.rev) || 0) < (Number(state.rev) || 0)) return; + writeAtomic(dbPath(), `${JSON.stringify(db, null, 2)}\n`, { backup: true }); + saveState({ uid: db.uid, rev: db.rev, pulled: new Date().toISOString(), error: null, errorAt: null }); + }); +} + +/** + * Send the changes of the outbox, oldest first. `extra` must go even when another process took it + * from the outbox already: the server applies a change only once, and it answers with the result. + * Resolves to the results by oid, and to the archive of the server after the last change. + */ +async function sendOps({ extra = null, timeoutMs }) { + const ops = withLock(() => readLines(outboxPath())); + if (extra && !ops.some((op) => op.oid === extra.oid)) ops.push(extra); + const results = new Map(); + let db = null; + for (let i = 0; i < ops.length; i += OPS_PER_REQUEST) { + const part = ops.slice(i, i + OPS_PER_REQUEST); + const out = await request('api/ops', { body: { ops: part }, timeoutMs }); + part.forEach((op, k) => results.set(op.oid, out.results[k])); + withLock(() => writeLines(outboxPath(), readLines(outboxPath()).filter((op) => !results.has(op.oid)))); + db = out.db; + } + return { results, db }; +} + +async function sendPrompts({ timeoutMs }) { + const prompts = withLock(() => readLines(promptsPath())); + let part = []; + let bytes = 0; + const sendPart = async () => { + if (!part.length) return; + await request('api/prompts', { body: { prompts: part }, timeoutMs }); + const sent = new Set(part.map((p) => p.id)); + withLock(() => writeLines(promptsPath(), readLines(promptsPath()).filter((p) => !sent.has(p.id)))); + part = []; + bytes = 0; + }; + for (const p of prompts) { + const n = Buffer.byteLength(JSON.stringify(p)); + if (bytes + n > PROMPT_BYTES_PER_REQUEST) await sendPart(); + part.push(p); + bytes += n; + } + await sendPart(); +} + +/** + * Bring the copy here up to date: send the changes that wait, then take the archive of the server. + * With `prompts`, send the prompts that wait too. Throws SyncError when the server does not answer. + */ +export async function pull({ timeoutMs = 10000, prompts = false } = {}) { + let { db } = await sendOps({ timeoutMs }); + if (prompts) await sendPrompts({ timeoutMs }); + db ||= (await request('api/db', { timeoutMs })).db; + keep(db); +} + +/** + * Change the archive on the server. The change goes into the outbox, then to the server. Resolves + * to { value }, the result of the change, or to { queued, reason } when the server does not answer: + * the change then waits in the outbox. A change that the server refuses throws. + */ +export async function change(op, { timeoutMs = 3000 } = {}) { + const item = { ...op, oid: crypto.randomUUID(), at: new Date().toISOString() }; + append(outboxPath(), [item]); + let sent; + try { + sent = await sendOps({ extra: item, timeoutMs }); + } catch (e) { + if (!(e instanceof SyncError)) throw e; + failed(e); + return { queued: true, reason: e.message }; + } + keep(sent.db); + const r = sent.results.get(item.oid); + if (!r?.ok) throw new Error(r?.error || 'the server sent no result'); + return { value: r.value }; +} + +/** How many changes wait in the outbox. */ +export const waiting = () => readLines(outboxPath()).length; + +/** + * The text of a prompt for the log, or null when nothing is left. The notes that Claude Code and the + * desktop app put into a prompt (<system-reminder> blocks) go, and a huge paste is cut. + */ +function promptText(raw) { + const text = String(raw ?? '').replace(/<system-reminder>[\s\S]*?<\/system-reminder>/g, '').trim(); + if (!text) return null; + return text.length > MAX_PROMPT_CHARS ? `${text.slice(0, MAX_PROMPT_CHARS)}\n… (cut: ${text.length - MAX_PROMPT_CHARS} more characters)` : text; +} + +/** Put a prompt into the prompt outbox. The next background sync sends it. */ +export function logPrompt({ prompt, session = null, cwd = null, at = new Date().toISOString() }) { + const text = promptText(prompt); + if (!text) return; + append(promptsPath(), [{ id: crypto.randomUUID(), at, host: os.hostname(), session, cwd, prompt: text }]); + if (!readState().promptsSince) writeState({ promptsSince: at }); +} + +// The text that the user typed, from one line of a Claude Code transcript. Null for everything +// else: tool results, subagent prompts, command output, and notes that Claude Code adds itself. +function typedText(entry) { + if (entry?.type !== 'user' || entry.isMeta || entry.isSidechain || entry.toolUseResult) return null; + const content = entry.message?.content; + let text = null; + if (typeof content === 'string') text = content; + else if (Array.isArray(content) && content.every((b) => b?.type === 'text' || b?.type === 'image')) { + text = content.filter((b) => b.type === 'text').map((b) => b.text).join('\n'); + } + if (!text?.trim()) return null; + const name = text.match(/<command-name>([^<]*)<\/command-name>/); + if (name) { + const args = text.match(/<command-args>([\s\S]*?)<\/command-args>/); + return `${name[1]} ${args ? args[1] : ''}`.trim(); + } + if (/^(<(local-command-stdout|local-command-stderr|command-message|bash-input|bash-stdout|bash-stderr)>|\[Request interrupted)/.test(text.trim())) return null; + return text; +} + +/** The transcripts folder of Claude Code on this machine. */ +export function transcriptsDir() { + return path.join(process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), '.claude'), 'projects'); +} + +/** + * Prompts from the Claude Code transcripts of this machine, for the prompt log. Only prompts from + * before the live prompt log started are taken, so that no prompt comes twice. An id made from the + * session, the time, and the text lets the server skip a prompt that an earlier import sent. + */ +export function importPrompts({ dir = transcriptsDir() } = {}) { + const before = readState().promptsSince || null; + const found = []; + let files = []; + try { + files = fs.readdirSync(dir, { recursive: true }).filter((f) => String(f).endsWith('.jsonl')); + } catch { + return 0; + } + for (const file of files) { + let raw; + try { + raw = fs.readFileSync(path.join(dir, String(file)), 'utf8'); + } catch { + continue; + } + for (const line of raw.split('\n')) { + if (!line.includes('"user"')) continue; + let entry; + try { + entry = JSON.parse(line); + } catch { + continue; + } + const prompt = promptText(typedText(entry)); + if (!prompt || !entry.timestamp || (before && entry.timestamp >= before)) continue; + const id = crypto.createHash('sha256').update(`${entry.sessionId}\n${entry.timestamp}\n${prompt}`).digest('hex').slice(0, 32); + found.push({ id, at: entry.timestamp, host: os.hostname(), session: entry.sessionId || null, cwd: entry.cwd || null, prompt }); + } + } + if (found.length) append(promptsPath(), found); + return found.length; +} + +function isRunning(now) { + try { + return now - fs.statSync(lockPath()).mtimeMs < LOCK_STALE_MS; + } catch { + return false; + } +} + +function startProcess() { + const bin = fileURLToPath(new URL('../bin/ideamine.js', import.meta.url)); + const child = spawn(process.execPath, [bin, 'sync', '--background'], { cwd: home(), detached: true, stdio: 'ignore', windowsHide: true }); + // A sync that cannot start must not stop the caller. + child.on('error', (e) => failed(new Error(`cannot start a sync: ${e.message}`))); + child.unref(); +} + +/** + * Start a background sync when sync is on, no sync runs, no sync failed a short time ago, and + * changes wait, prompts wait since the last sync 20 seconds ago, or the copy is older than a minute. + */ +export function kick({ start = startProcess, now = Date.now() } = {}) { + if (!enabled()) return false; + const state = readState(); + if (state.errorAt && now - Date.parse(state.errorAt) < WAIT_AFTER_ERROR_MS) return false; + if (isRunning(now)) return false; + const age = state.pulled ? now - Date.parse(state.pulled) : Infinity; + const due = age > PULL_EVERY_MS || size(outboxPath()) > 0 || (size(promptsPath()) > 0 && age > PROMPTS_EVERY_MS); + if (!due) return false; + start(); + return true; +} + +/** One background sync, from kick(). Only one runs at a time. */ +export async function backgroundSync() { + fs.mkdirSync(home(), { recursive: true }); + try { + fs.mkdirSync(lockPath()); + } catch (e) { + if (e.code !== 'EEXIST' || isRunning(Date.now())) return 'busy'; + fs.utimesSync(lockPath(), new Date(), new Date()); // take over the lock of a crashed sync + } + try { + await pull({ timeoutMs: 30000, prompts: true }); + return 'done'; + } catch (e) { + failed(e); + return 'error'; + } finally { + fs.rmSync(lockPath(), { recursive: true, force: true }); + } +} + +/** + * Turn sync on with the server at `url`, or off with ''. Before the first sync with a server, the + * archive here goes to a backup file, because the archive of the server takes its place. + */ +export function setServer(url) { + const clean = String(url || '').trim(); + if (clean && !/^https?:\/\//i.test(clean)) throw new Error(`"${clean}" is not an http:// or https:// address`); + let backup = null; + if (clean && fs.existsSync(dbPath()) && !readState().uid) { + backup = path.join(home(), `ideas.before-sync-${new Date().toISOString().slice(0, 10)}.json`); + fs.copyFileSync(dbPath(), backup); + } + config.set('sync_url', clean); + if (clean) config.set('publish_url', ''); // the server shows the live archive: no uploads + else writeState({ uid: null, rev: null, pulled: null, error: null, errorAt: null }); + return backup; +} + +/** On or off, the last sync, and what waits. */ +export function status() { + const url = serverUrl(); + if (!url) return 'ideamine sync: off. The archive is on this machine. /ideas-sync <url> shares it through an ideamine server.'; + const s = readState(); + const lines = [`ideamine sync: ${url}`]; + lines.push(s.pulled ? `last sync ${stamp(s.pulled)}${s.rev ? `, revision ${s.rev}` : ''}.` : 'Not synced yet.'); + const changes = waiting(); + if (changes) lines.push(`${changes} change${changes === 1 ? ' waits' : 's wait'} for the server.`); + const prompts = readLines(promptsPath()).length; + if (prompts) lines.push(`${prompts} prompt${prompts === 1 ? ' waits' : 's wait'} for the server.`); + lines.push(logsPrompts() ? 'The prompt log is on.' : 'The prompt log is off. `ideamine config prompt_log on` turns it on.'); + if (s.error) lines.push(`The last sync failed at ${stamp(s.errorAt)}: ${s.error}`); + return lines.join('\n'); +} diff --git a/tests/fixtures/fake-memstate.js b/tests/fixtures/fake-memstate.js new file mode 100644 index 0000000..f7fd793 --- /dev/null +++ b/tests/fixtures/fake-memstate.js @@ -0,0 +1,39 @@ +// A stand-in for the memstated daemon: the three read routes that the Memory tab uses. It records +// every request, so that a test can check that the dashboard never writes to memstate. + +import http from 'node:http'; + +export const MEMORIES = [ + { id: 7, project_id: 'ideamine', keypath: 'task.summary.2026_09_21', content: 'Built the dashboard.', category: 'status', version: 2, created_at: 1790000000 }, + { id: 3, project_id: 'ideamine', keypath: 'decisions.sync', content: 'The server holds the archive.', category: 'decision', version: 1, created_at: 1789990000 }, +]; + +export async function startFakeMemstate() { + const requests = []; + const server = http.createServer((req, res) => { + let body = ''; + req.on('data', (d) => (body += d)); + req.on('end', () => { + requests.push(`${req.method} ${req.url}`); + const json = (status, data) => res.writeHead(status, { 'content-type': 'application/json' }).end(JSON.stringify(data)); + const input = body ? JSON.parse(body) : {}; + if (req.method === 'GET' && req.url === '/api/v1/projects') { + return json(200, { projects: [{ id: 'ideamine', created_at: 1789900000, memory_count: 2, last_updated_at: 1790000000 }] }); + } + if (req.method === 'POST' && req.url === '/api/v1/keypaths') { + const list = MEMORIES.filter((m) => m.project_id === input.project_id).map((m) => ({ ...m, content: input.include_content ? m.content : '' })); + return json(200, { memories: list, total_count: list.length }); + } + if (req.method === 'POST' && req.url === '/api/v1/memories/history') { + const versions = [ + { ...MEMORIES[0], version: 1, content: 'Started the dashboard.', created_at: 1789950000 }, + MEMORIES[0], + ].filter((m) => m.keypath === input.keypath); + return json(200, { versions, total_versions: versions.length }); + } + json(404, { error: 'not found' }); + }); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return { url: `http://127.0.0.1:${server.address().port}`, requests, close: () => new Promise((resolve) => server.close(resolve)) }; +} diff --git a/tests/fixtures/server-process.js b/tests/fixtures/server-process.js new file mode 100644 index 0000000..d59d10b --- /dev/null +++ b/tests/fixtures/server-process.js @@ -0,0 +1,52 @@ +// An ideamine server in its own process, as on a real server: its own IDEAMINE_HOME, sync off. The +// tests switch IDEAMINE_HOME between machines in the test process, so the server must not share it. + +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import net from 'node:net'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const BIN = fileURLToPath(new URL('../../bin/ideamine.js', import.meta.url)); + +/** A port that nothing listens on now. */ +export async function freePort() { + const s = net.createServer(); + await new Promise((resolve) => s.listen(0, '127.0.0.1', resolve)); + const { port } = s.address(); + await new Promise((resolve) => s.close(resolve)); + return port; +} + +/** Start `ideamine serve`. Resolves to { url, home, port, stop }. `env` adds settings. */ +export async function startServerProcess(env = {}) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'ideamine-server-')); + const port = await freePort(); + const clean = { ...process.env }; + for (const key of Object.keys(clean)) if (key.startsWith('IDEAMINE_')) delete clean[key]; + const child = spawn(process.execPath, [BIN, 'serve', '--port', String(port)], { + env: { ...clean, IDEAMINE_HOME: home, IDEAMINE_EMBED_URL: 'http://127.0.0.1:9/v1', ...env }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let output = ''; + child.stdout.on('data', (d) => (output += d)); + child.stderr.on('data', (d) => (output += d)); + const url = `http://127.0.0.1:${port}/`; + for (let i = 0; ; i++) { + try { + if ((await fetch(`${url}api/ping`)).ok) break; + } catch { + // Not listening yet. + } + if (i > 100 || child.exitCode !== null) throw new Error(`the server did not start: ${output}`); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + const stop = () => + new Promise((resolve) => { + if (child.exitCode !== null) return resolve(); + child.once('exit', resolve); + child.kill(); + }); + return { url, home, port, stop, output: () => output }; +} diff --git a/tests/hook.test.js b/tests/hook.test.js index 7b66153..cfd5b4c 100644 --- a/tests/hook.test.js +++ b/tests/hook.test.js @@ -8,6 +8,7 @@ import { fileURLToPath } from 'node:url'; import { handlePrompt } from '../src/hook.js'; import * as store from '../src/store.js'; import { startFakeServer } from './fixtures/fake-server.js'; +import { freePort, startServerProcess } from './fixtures/server-process.js'; const BIN = fileURLToPath(new URL('../bin/ideamine.js', import.meta.url)); @@ -16,6 +17,7 @@ beforeEach(() => { // A pass of the watcher must never reach the real claude or the projects of this machine. process.env.IDEAMINE_CLAUDE_BIN = fileURLToPath(new URL('./fixtures/fake-claude.js', import.meta.url)); process.env.CLAUDE_CONFIG_DIR = process.env.IDEAMINE_HOME; + for (const name of ['IDEAMINE_SYNC_URL', 'IDEAMINE_PROMPT_LOG']) delete process.env[name]; }); /** Run the real hook process the way Claude Code does: JSON on stdin, decision on stdout. */ @@ -155,6 +157,62 @@ test('/ideas-find searches by meaning and /ideas-groups groups by meaning, with } }); +const serverIdeas = async (url) => (await (await fetch(`${url}api/db`)).json()).db.ideas.map((i) => i.text); + +test('with sync on, /idea saves on the server, /ideas shows every machine, and an idea waits when the server is away', async () => { + const remote = await startServerProcess(); + try { + process.env.IDEAMINE_SYNC_URL = remote.url; + assert.match(runHook('/idea shared by every machine').reason, /Saved #1 · shared by every machine/); + assert.deepEqual(await serverIdeas(remote.url), ['shared by every machine']); + // Another machine adds an idea. The next /ideas here shows it. + const laptop = { ops: [{ op: 'add', texts: ['from the laptop'], oid: 'laptop-1' }] }; + await fetch(`${remote.url}api/ops`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(laptop) }); + assert.match(runHook('/ideas').reason, /#2 {3}from the laptop[\s\S]*#1 {3}shared by every machine/); + + process.env.IDEAMINE_SYNC_URL = `http://127.0.0.1:${await freePort()}/`; // the tunnel is down + assert.match(runHook('/idea written offline').reason, /^💡 Saved here: written offline\. The ideamine server does not answer, so it gets the idea later \(1 waiting\)\.$/); + assert.match(runHook('/ideas').reason, /^\(The ideamine server does not answer\. This is the copy from \d{4}-/); + + process.env.IDEAMINE_SYNC_URL = remote.url; // the tunnel is back + assert.match(runHook('/ideas').reason, /written offline/); + assert.deepEqual((await serverIdeas(remote.url)).sort(), ['from the laptop', 'shared by every machine', 'written offline']); + } finally { + await remote.stop(); + } +}); + +test('/ideas-sync turns sync on, keeps the archive that was here, and shows the state', async () => { + const remote = await startServerProcess(); + try { + store.addIdeas(['only here']); + const on = runHook(`/ideas-sync ${remote.url}`).reason; + assert.match(on, /^ideamine sync: http:\/\/127\.0\.0\.1:\d+\/\nlast sync \d{4}-\d{2}-\d{2} \d{2}:\d{2}\.\n/); // a new archive has no revision yet + assert.match(on, /The archive that was here is in .*ideas\.before-sync-\d{4}-\d{2}-\d{2}\.json\.$/); + assert.match(runHook('/ideamine:ideas-sync').reason, /^ideamine sync: http/); + assert.match(runHook('/ideas-sync 10.66.0.1').reason, /^ideamine sync: "10\.66\.0\.1" is not an http:\/\/ or https:\/\/ address\. Usage/); + assert.match(runHook('/ideas-sync off').reason, /^ideamine sync: off/); + } finally { + await remote.stop(); + } +}); + +test('with the prompt log on, each prompt goes to the server in the background', async () => { + const remote = await startServerProcess(); + try { + process.env.IDEAMINE_SYNC_URL = remote.url; + process.env.IDEAMINE_PROMPT_LOG = 'on'; + assert.equal(runHook('please fix the flaky test'), null); // an ordinary prompt still goes to the model + let prompts = []; + for (const deadline = Date.now() + 20000; !prompts.length && Date.now() < deadline; await new Promise((r) => setTimeout(r, 200))) { + prompts = (await (await fetch(`${remote.url}api/prompts`)).json()).prompts; + } + assert.deepEqual(prompts.map((p) => [p.prompt, p.session, p.cwd]), [['please fix the flaky test', 's1', '/work/app']]); + } finally { + await remote.stop(); + } +}); + test('/ideas-find falls back to words when the embedding server is away', async () => { const closed = await startFakeServer(); await closed.close(); // nothing listens on its port now diff --git a/tests/mcp.test.js b/tests/mcp.test.js index 82fa498..216423e 100644 --- a/tests/mcp.test.js +++ b/tests/mcp.test.js @@ -109,7 +109,7 @@ test('tool errors come back as isError results, not protocol errors', async () = test('prompts mirror the skills', async () => { const list = await request('prompts/list'); const names = list.result.prompts.map((p) => p.name); - assert.deepEqual(names, ['idea', 'ideas', 'ideas-ls', 'ideas-cat', 'ideas-rm', 'ideas-find', 'ideas-groups', 'ideas-done', 'ideas-reopen', 'ideas-go', 'ideas-all', 'ideas-sort', 'ideas-watch', 'ideas-web']); + assert.deepEqual(names, ['idea', 'ideas', 'ideas-ls', 'ideas-cat', 'ideas-rm', 'ideas-find', 'ideas-groups', 'ideas-done', 'ideas-reopen', 'ideas-go', 'ideas-all', 'ideas-sort', 'ideas-watch', 'ideas-web', 'ideas-sync']); // One prompt for each skill, so that other MCP clients get the same commands as the plugin. assert.deepEqual([...names].sort(), fs.readdirSync(new URL('../skills', import.meta.url)).sort()); const go = await request('prompts/get', { name: 'ideas-go', arguments: { id: '12' } }); diff --git a/tests/serve.test.js b/tests/serve.test.js index 8ea127f..0e577dc 100644 --- a/tests/serve.test.js +++ b/tests/serve.test.js @@ -11,7 +11,9 @@ import fc from 'fast-check'; import { handlePrompt } from '../src/hook.js'; import * as serve from '../src/serve.js'; import * as store from '../src/store.js'; +import { startFakeMemstate } from './fixtures/fake-memstate.js'; import { startFakeServer } from './fixtures/fake-server.js'; +import { startServerProcess } from './fixtures/server-process.js'; const BIN = fileURLToPath(new URL('../bin/ideamine.js', import.meta.url)); @@ -26,6 +28,7 @@ beforeEach(async () => { // The triage and the questions must never reach the real claude or the projects of this machine. process.env.IDEAMINE_CLAUDE_BIN = fileURLToPath(new URL('./fixtures/fake-claude.js', import.meta.url)); process.env.CLAUDE_CONFIG_DIR = process.env.IDEAMINE_HOME; + for (const name of ['IDEAMINE_SYNC_URL', 'IDEAMINE_MEMSTATE_URL', 'IDEAMINE_SERVE_HOSTS', 'IDEAMINE_PROMPT_LOG']) delete process.env[name]; embedServer = await startFakeServer(); process.env.IDEAMINE_EMBED_URL = `${embedServer.url}/v1`; opened = []; @@ -91,6 +94,17 @@ test('only this page may use the server: a property over hosts, origins, and con assert.equal(serve.allowed(post({ host: '127.0.0.1:4332', origin: 'http://127.0.0.1:4332', type: 'application/json; charset=utf-8' }), port), true); assert.equal(serve.allowed(post({ host: '127.0.0.1:4332' }), port), true); // no Origin: a program on this PC for (const method of ['PUT', 'DELETE', 'OPTIONS', 'PATCH']) assert.equal(serve.allowed({ method, host: '127.0.0.1:4332' }, port), false); + // Behind nginx, the server also answers the names in serve_hosts, and a POST must come from there. + const hosts = ['10.66.0.1']; + assert.equal(serve.allowed({ method: 'GET', host: '10.66.0.1' }, port), false); + assert.equal(serve.allowed({ method: 'GET', host: '10.66.0.1' }, port, hosts), true); + assert.equal(serve.allowed(post({ host: '10.66.0.1', origin: 'http://10.66.0.1' }), port, hosts), true); + assert.equal(serve.allowed(post({ host: '10.66.0.1', origin: 'http://127.0.0.1:4332' }), port, hosts), false); + fc.assert( + fc.property(fc.string(), (host) => { + assert.equal(serve.allowed({ method: 'GET', host }, port, hosts), ['127.0.0.1:4332', 'localhost:4332', '10.66.0.1'].includes(host.toLowerCase())); + }), + ); }); test('requests from other pages change nothing', async () => { @@ -116,7 +130,15 @@ test('the page comes with live data, and no other page can frame it', async () = const data = await (await fetch(new URL('data.json', url))).json(); assert.equal(data.version, 1); assert.equal(data.ideas[0].key, 'IDEA-1'); - assert.deepEqual(data.live, { note: '', window: process.platform === 'win32', watch: { on: false, status: data.live.watch.status } }); + assert.deepEqual(data.live, { + note: '', + window: process.platform === 'win32', + claude: true, // the stand-in claude of the tests starts + watch: { on: false, status: data.live.watch.status }, + sync: null, + prompts: false, + memory: false, + }); assert.match(data.live.watch.status, /^ideamine watch: off/); }); @@ -202,6 +224,79 @@ test('search by meaning on the page goes through the server to the embedding ser assert.deepEqual(embedServer.inputs, ['search_query: subtitles']); }); +test('the server role keeps the prompts of every machine, each once, and gives them by days', async () => { + const recent = new Date(Date.now() - 3600000).toISOString(); + const prompts = [ + { id: 'a', at: recent, host: 'pc', session: 's1', cwd: 'C:\\work\\app', prompt: 'fix the tests' }, + { id: 'b', at: '2026-01-01T00:00:00.000Z', host: 'mac', session: 's2', cwd: '/Users/me/app', prompt: 'an old one' }, + { id: 'bad', prompt: 'no time' }, + ]; + const first = await api('prompts', { prompts }); + assert.deepEqual([first.added, first.skipped], [2, 1]); + assert.equal((await api('prompts', { prompts })).added, 0); // the same prompts again + const get = async (route) => (await (await fetch(new URL(route, url))).json()).prompts; + assert.deepEqual((await get('api/prompts')).map((p) => p.id), ['b', 'a']); // oldest first + assert.deepEqual((await get('api/prompts?days=1')).map((p) => p.prompt), ['fix the tests']); + assert.equal((await (await fetch(new URL('data.json', url))).json()).live.prompts, true); +}); + +test('the Memory tab reads memstated, and never writes there', async () => { + const memstate = await startFakeMemstate(); + process.env.IDEAMINE_MEMSTATE_URL = memstate.url; + const get = async (route) => (await fetch(new URL(route, url))).json(); + try { + const overview = await get('api/memory/overview'); + assert.deepEqual(overview.projects.map((p) => p.id), ['ideamine']); + assert.deepEqual(overview.memories.map((m) => [m.keypath, m.category, m.version]), [['task.summary.2026_09_21', 'status', 2], ['decisions.sync', 'decision', 1]]); + assert.equal(overview.memories[0].content, undefined); // the overview carries no texts + assert.equal((await get('api/memory/project?id=ideamine')).memories[0].content, 'Built the dashboard.'); + const history = await get('api/memory/history?project=ideamine&keypath=task.summary.2026_09_21'); + assert.deepEqual(history.versions.map((v) => v.content), ['Started the dashboard.', 'Built the dashboard.']); + assert.equal((await get('data.json')).live.memory, true); + const reads = /^(GET \/api\/v1\/projects|POST \/api\/v1\/keypaths|POST \/api\/v1\/memories\/history)$/; + assert.ok(memstate.requests.every((r) => reads.test(r)), memstate.requests.join(', ')); + await memstate.close(); + assert.match((await get('api/memory/overview')).error, /cannot reach memstated/); + } finally { + await memstate.close().catch(() => {}); + delete process.env.IDEAMINE_MEMSTATE_URL; + } + assert.match((await get('api/memory/overview')).error, /this server shows no memories/); +}); + +test('behind nginx, the server answers the names in serve_hosts', async () => { + const ask = async (host) => (await raw('GET', '/api/ping', { host })).statusCode; + assert.equal(await ask('10.66.0.1'), 403); + process.env.IDEAMINE_SERVE_HOSTS = '10.66.0.1, ideas.lan'; + try { + assert.deepEqual([await ask('10.66.0.1'), await ask('IDEAS.lan'), await ask('evil.example')], [200, 200, 403]); + } finally { + delete process.env.IDEAMINE_SERVE_HOSTS; + } +}); + +test('on a PC with sync on, the page reads the server, and its buttons change the archive there', async () => { + const memstate = await startFakeMemstate(); + const remote = await startServerProcess({ IDEAMINE_MEMSTATE_URL: memstate.url }); + process.env.IDEAMINE_SYNC_URL = remote.url; + try { + assert.match((await api('add', { text: 'from the page of the PC' })).message, /Saved #1 · from the page of the PC/); + const onServer = (await (await fetch(`${remote.url}api/db`)).json()).db; + assert.deepEqual(onServer.ideas.map((i) => i.text), ['from the page of the PC']); + const hello = { id: 'p1', at: new Date().toISOString(), prompt: 'hello' }; + await fetch(`${remote.url}api/prompts`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ prompts: [hello] }) }); + assert.deepEqual((await (await fetch(new URL('api/prompts', url))).json()).prompts.map((p) => p.prompt), ['hello']); + assert.deepEqual((await (await fetch(new URL('api/memory/overview', url))).json()).projects.map((p) => p.id), ['ideamine']); + const { live } = await (await fetch(new URL('data.json', url))).json(); + assert.deepEqual([live.sync.url, live.sync.offline, live.prompts, live.memory], [remote.url, null, true, true]); + assert.equal((await api('ops', { ops: [] })).status, 404); // a PC is no server for other machines + } finally { + delete process.env.IDEAMINE_SYNC_URL; + await remote.stop(); + await memstate.close(); + } +}); + test('serve_port must be a port number', () => { process.env.IDEAMINE_SERVE_PORT = 'abc'; try { diff --git a/tests/sync.test.js b/tests/sync.test.js new file mode 100644 index 0000000..5b248cd --- /dev/null +++ b/tests/sync.test.js @@ -0,0 +1,233 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, before, beforeEach, test } from 'node:test'; +import fc from 'fast-check'; +import * as archive from '../src/archive.js'; +import * as store from '../src/store.js'; +import * as sync from '../src/sync.js'; +import { freePort, startServerProcess } from './fixtures/server-process.js'; + +let server; +let deadUrl; // an address where nothing answers: the tunnel of a machine is down + +before(async () => { + server = await startServerProcess(); + deadUrl = `http://127.0.0.1:${await freePort()}/`; +}); + +after(() => server.stop()); + +/** An empty archive on the server: its archive and its record of applied changes go. */ +function resetServer() { + for (const name of ['ideas.json', 'ideas.json.bak', 'applied.json', 'prompts.jsonl']) fs.rmSync(path.join(server.home, name), { force: true }); +} + +const newHome = (name) => fs.mkdtempSync(path.join(os.tmpdir(), `ideamine-${name}-`)); + +/** Act as one machine: its own archive copy and outbox, and the server at `url`. */ +function machine(home, url = server.url) { + process.env.IDEAMINE_HOME = home; + process.env.IDEAMINE_SYNC_URL = url; +} + +beforeEach(() => { + resetServer(); + delete process.env.IDEAMINE_PROMPT_LOG; + machine(newHome('pc')); +}); + +const serverDb = async () => (await (await fetch(`${server.url}api/db`)).json()).db; + +test('a change goes to the server, and the copy here follows the server', async () => { + const [{ idea }] = await archive.add(['alpha #web'], { source: 'test', project: '/work/app' }); + assert.equal(idea.id, 1); + assert.equal(idea.host, os.hostname()); + const onServer = await serverDb(); + assert.deepEqual(onServer.ideas.map((i) => [i.id, i.title, i.tags[0]]), [[1, 'alpha', 'web']]); + assert.deepEqual(store.load().ideas, onServer.ideas); + assert.equal(sync.waiting(), 0); + const done = await archive.update(1, { status: 'done', note: 'shipped' }); + assert.deepEqual([done.status, done.notes.map((n) => n.text)], ['done', ['shipped']]); + assert.deepEqual((await archive.remove([1])).map((i) => i.id), [1]); + assert.deepEqual((await serverDb()).ideas, []); +}); + +test('when the server does not answer, a change waits in the outbox and goes with the next sync', async () => { + const home = store.home(); + machine(home, deadUrl); + const out = await archive.add(['written on a train']); + assert.equal(out.queued, true); + assert.match(out.reason, /cannot reach/); + assert.equal(sync.waiting(), 1); + assert.equal((await serverDb()).ideas.length, 0); + assert.match(sync.status(), /1 change waits for the server[\s\S]*The last sync failed/); + + machine(home); // the tunnel is back + await sync.pull(); + assert.equal(sync.waiting(), 0); + assert.deepEqual((await serverDb()).ideas.map((i) => i.text), ['written on a train']); + assert.deepEqual(store.load().ideas.map((i) => i.text), ['written on a train']); +}); + +test('the server applies a change only once, also when it comes twice', async () => { + const op = { op: 'add', texts: ['once'], oid: 'same-oid' }; + const post = async () => (await fetch(`${server.url}api/ops`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ ops: [op, { op: 'update', id: 1, patch: { note: 'n' }, oid: 'note-oid' }] }) })).json(); + const first = await post(); + const second = await post(); // the answer to the first got lost, and the machine sent it again + assert.deepEqual(second.results, first.results); + assert.equal(second.db.ideas.length, 1); + assert.deepEqual(second.db.ideas[0].notes.map((n) => n.text), ['n']); + const bad = await (await fetch(`${server.url}api/ops`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ ops: [{ op: 'nope', oid: 'x' }, { op: 'remove', ids: [9], oid: 'y' }] }) })).json(); + assert.deepEqual(bad.results, [{ ok: false, error: 'unknown change "nope"' }, { ok: false, error: 'no idea #9' }]); +}); + +test('a slow answer with an older archive does not undo a newer copy', async () => { + await archive.add(['one']); + await archive.add(['two']); + const newer = store.load(); + const stateNow = sync.readState(); + // The answer to an earlier request arrives last: revision 1 of the same archive. + const old = { ...newer, rev: 1, ideas: newer.ideas.slice(0, 1) }; + const original = globalThis.fetch; + globalThis.fetch = async () => new Response(JSON.stringify({ ok: true, db: old }), { status: 200 }); + try { + await sync.pull(); + } finally { + globalThis.fetch = original; + } + assert.deepEqual(store.load().ideas.map((i) => i.text), ['one', 'two']); + assert.equal(sync.readState().rev, stateNow.rev); +}); + +test('kick() starts a background sync when work waits or the copy is old, and waits after a failure', async () => { + let started = 0; + const kick = (now = Date.now()) => sync.kick({ start: () => started++, now }); + assert.equal(kick(), true); // never synced + await sync.pull(); + assert.equal(kick(), false); // fresh, nothing waits + assert.equal(kick(Date.now() + 61000), true); // the copy is old + sync.logPrompt({ prompt: 'hello', session: 's1', cwd: '/w' }); + assert.equal(kick(), false); // a prompt waits, but the last sync was a moment ago + assert.equal(kick(Date.now() + 21000), true); // a prompt waits, and the last sync is 21 s old + machine(store.home(), deadUrl); + await archive.add(['queued']); + assert.equal(kick(), false); // the sync failed a moment ago + delete process.env.IDEAMINE_SYNC_URL; + assert.equal(kick(Date.now() + 3600000), false); // sync is off + assert.equal(started, 3); +}); + +test('turning sync on keeps the archive that was here in a backup file', async () => { + delete process.env.IDEAMINE_SYNC_URL; + store.addIdeas(['local only']); + const backup = sync.setServer(server.url); + assert.match(path.basename(backup), /^ideas\.before-sync-\d{4}-\d{2}-\d{2}\.json$/); + assert.equal(JSON.parse(fs.readFileSync(backup, 'utf8')).ideas[0].text, 'local only'); + await archive.fresh(); + assert.deepEqual(store.load().ideas, []); // the server's archive, which is empty + assert.throws(() => sync.setServer('ftp://x'), /not an http/); + sync.setServer(''); +}); + +test('the prompt log sends prompts once, cuts a huge paste, and imports older transcripts', async () => { + process.env.IDEAMINE_PROMPT_LOG = 'on'; + const transcripts = path.join(store.home(), 'claude', 'projects', 'C--work-app'); + fs.mkdirSync(transcripts, { recursive: true }); + process.env.CLAUDE_CONFIG_DIR = path.join(store.home(), 'claude'); + const line = (o) => JSON.stringify({ sessionId: 's0', cwd: 'C:\\work\\app', ...o }); + fs.writeFileSync(path.join(transcripts, 's0.jsonl'), [ + line({ type: 'user', timestamp: '2026-01-05T10:00:00.000Z', message: { role: 'user', content: '<system-reminder>\nNotes of the app.\n</system-reminder>\nfix the tests' } }), + line({ type: 'user', timestamp: '2026-01-05T09:59:00.000Z', message: { role: 'user', content: [{ type: 'text', text: '<system-reminder>only a note</system-reminder>' }] } }), + line({ type: 'user', timestamp: '2026-01-05T10:01:00.000Z', message: { role: 'user', content: '<command-name>/ideas</command-name>\n<command-args>go 3</command-args>' } }), + line({ type: 'user', timestamp: '2026-01-05T10:02:00.000Z', message: { role: 'user', content: [{ type: 'tool_result', content: 'ok' }] }, toolUseResult: {} }), + line({ type: 'user', timestamp: '2026-01-05T10:03:00.000Z', isMeta: true, message: { role: 'user', content: 'Caveat: a note of Claude Code' } }), + line({ type: 'user', timestamp: '2026-01-05T10:04:00.000Z', isSidechain: true, message: { role: 'user', content: 'a prompt of a subagent' } }), + line({ type: 'user', timestamp: '2026-01-05T10:05:00.000Z', message: { role: 'user', content: '[Request interrupted by user]' } }), + line({ type: 'assistant', timestamp: '2026-01-05T10:06:00.000Z', message: { role: 'assistant', content: 'hi' } }), + line({ type: 'user', timestamp: '2026-02-01T10:00:00.000Z', message: { role: 'user', content: 'after the live log started' } }), + 'not json', + ].join('\n')); + sync.logPrompt({ prompt: 'x'.repeat(150000), session: 's1', cwd: '/w', at: '2026-01-10T00:00:00.000Z' }); + assert.equal(sync.importPrompts(), 2); // older than the first live prompt only + assert.equal(sync.importPrompts(), 2); // the same ids again: the server skips them + await sync.pull({ prompts: true }); + const { prompts } = await (await fetch(`${server.url}api/prompts`)).json(); + assert.deepEqual(prompts.map((p) => p.prompt.slice(0, 20)), ['fix the tests', '/ideas go 3', 'x'.repeat(20)]); + assert.match(prompts[2].prompt, /… \(cut: 50000 more characters\)$/); + assert.equal(prompts[0].cwd, 'C:\\work\\app'); + assert.deepEqual((await (await fetch(`${server.url}api/prompts?days=1`)).json()).prompts, []); // all older than a day + delete process.env.CLAUDE_CONFIG_DIR; +}); + +// A model of two machines that share the server, with outages. Each idea text is unique, so the +// test can follow an idea without its id. +const command = fc.oneof( + fc.record({ c: fc.constant('add'), who: fc.constantFrom('A', 'B') }), + fc.record({ c: fc.constantFrom('done', 'note', 'rm'), who: fc.constantFrom('A', 'B'), pick: fc.nat(20) }), + fc.record({ c: fc.constantFrom('down', 'up', 'pull'), who: fc.constantFrom('A', 'B') }), +); + +test('two machines with outages end with one archive: no idea lost, none twice, no note twice', async () => { + await fc.assert( + fc.asyncProperty(fc.array(command, { maxLength: 18 }), async (commands) => { + resetServer(); + const homes = { A: newHome('a'), B: newHome('b') }; + const up = { A: true, B: true }; + const added = []; + const removed = new Set(); + let n = 0; + const use = (who) => machine(homes[who], up[who] ? server.url : deadUrl); + for (const cmd of commands) { + if (cmd.c === 'down' || cmd.c === 'up') { + up[cmd.who] = cmd.c === 'up'; + continue; + } + use(cmd.who); + if (cmd.c === 'pull') { + await sync.pull().catch((e) => assert.ok(e instanceof sync.SyncError, e)); + continue; + } + if (cmd.c === 'add') { + const text = `idea ${++n}`; + added.push(text); + await archive.add([text]); + continue; + } + const ideas = store.load().ideas; + if (!ideas.length) continue; + const idea = ideas[cmd.pick % ideas.length]; + if (cmd.c === 'rm') removed.add(idea.text); + const run = cmd.c === 'rm' ? archive.remove([idea.id]) : archive.update(idea.id, cmd.c === 'done' ? { status: 'done' } : { note: `note ${++n}` }); + // The other machine may have deleted the idea already: then the server refuses the change. + await run.catch((e) => assert.match(e.message, /^no idea #\d+$/)); + } + for (const who of ['A', 'B', 'A']) { + machine(homes[who]); + await sync.pull(); + } + const onServer = await serverDb(); + const texts = onServer.ideas.map((i) => i.text).sort(); + assert.deepEqual(texts, added.filter((t) => !removed.has(t)).sort()); + for (const idea of onServer.ideas) { + const notes = idea.notes.map((x) => x.text); + assert.equal(new Set(notes).size, notes.length, `a note came twice on #${idea.id}`); + } + for (const who of ['A', 'B']) { + machine(homes[who]); + assert.deepEqual(store.load().ideas, onServer.ideas); + assert.equal(sync.waiting(), 0); + } + }), + { numRuns: 40 }, + ); +}); + +test('a Windows path from a machine stays as it is on a POSIX server, and names show on any machine', () => { + assert.equal(store.absolute('C:\\Users\\me\\app', 'linux'), 'C:\\Users\\me\\app'); + assert.equal(store.absolute('\\\\nas\\share\\app', 'linux'), '\\\\nas\\share\\app'); + assert.equal(store.absolute('/home/me/app', 'linux'), path.resolve('/home/me/app')); + assert.equal(store.baseName('C:\\Users\\me\\app\\'), 'app'); + assert.equal(store.baseName('/home/me/app'), 'app'); +});