From d8e0542e6d6e3394a596d210a374cd49b6ec0a2f Mon Sep 17 00:00:00 2001 From: AkitaOnRails Date: Sat, 23 May 2026 23:57:10 -0300 Subject: [PATCH] release prep: MIT license, wiki migration framework, release scaffolding - LICENSE (MIT) + workspace license = MIT (down from MIT OR Apache-2.0) - Wiki-structure migration framework: V06 wiki_migrations table, WikiMigration trait, registry, and run_pending runner (registry starts empty; v1 ships per-project layout natively) - CHANGELOG.md, SECURITY.md, CONTRIBUTING.md - bin/release (fmt/clippy/test/deny/audit -> bump -> tag; never pushes) - .github release workflow + issue/PR templates - README: bootstrap example defers to default workspace/project - evals: inherit version/edition/rust-version from workspace Co-Authored-By: Claude Opus 4.7 --- .github/ISSUE_TEMPLATE/bug.md | 28 ++ .github/ISSUE_TEMPLATE/feature.md | 17 ++ .github/pull_request_template.md | 18 ++ .github/workflows/release.yml | 63 ++++ CHANGELOG.md | 83 ++++++ CONTRIBUTING.md | 76 +++++ Cargo.lock | 1 + Cargo.toml | 2 +- LICENSE | 21 ++ README.md | 10 +- SECURITY.md | 63 ++++ bin/release | 197 +++++++++++++ .../migrations/V06__wiki_migrations.sql | 7 + crates/ai-memory-store/src/ops.rs | 17 ++ crates/ai-memory-store/src/reader.rs | 18 ++ crates/ai-memory-store/src/writer.rs | 35 +++ crates/ai-memory-wiki/Cargo.toml | 1 + crates/ai-memory-wiki/src/lib.rs | 2 + crates/ai-memory-wiki/src/migrations/mod.rs | 102 +++++++ .../ai-memory-wiki/src/migrations/runner.rs | 273 ++++++++++++++++++ docs/wiki-migrations.md | 171 +++++++++++ evals/Cargo.toml | 6 +- 22 files changed, 1204 insertions(+), 7 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug.md create mode 100644 .github/ISSUE_TEMPLATE/feature.md create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/release.yml create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 SECURITY.md create mode 100755 bin/release create mode 100644 crates/ai-memory-store/migrations/V06__wiki_migrations.sql create mode 100644 crates/ai-memory-wiki/src/migrations/mod.rs create mode 100644 crates/ai-memory-wiki/src/migrations/runner.rs create mode 100644 docs/wiki-migrations.md diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md new file mode 100644 index 00000000..2d36f16a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -0,0 +1,28 @@ +--- +name: Bug report +about: Something is broken or behaving unexpectedly +labels: bug +--- + +**Version** +Output of `ai-memory --version`: + +**What happened** + + +**What I expected** + + +**Steps to reproduce** +1. +2. +3. + +**Environment** +- OS: +- Docker? (yes / no, and image tag if yes): +- Agent (Claude Code / Codex / OpenCode / other): +- Transport (stdio / http): + +**Relevant logs** + diff --git a/.github/ISSUE_TEMPLATE/feature.md b/.github/ISSUE_TEMPLATE/feature.md new file mode 100644 index 00000000..8860b1bd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest a new capability or improvement +labels: enhancement +--- + +**Problem this would solve** + + +**Proposed solution** + + +**Alternatives considered** + + +**Additional context** + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..bcb657ed --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,18 @@ +## What changed + + + +## Why + + + +## Test plan + +- [ ] `cargo fmt --all -- --check` passes +- [ ] `cargo clippy --workspace --all-targets -- -D warnings` passes +- [ ] `cargo test --workspace` passes +- [ ] Manual test: + +## Notes for reviewers + + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..1b0dabd5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,63 @@ +name: release + +# Triggered when a version tag is pushed, e.g. git push origin v0.2.0. +# Builds the Docker image and pushes it to Docker Hub with both the +# version tag and `latest`. +# +# Required secrets (set in repo Settings → Secrets and variables → Actions): +# DOCKERHUB_USERNAME — Docker Hub account name +# DOCKERHUB_TOKEN — Docker Hub access token (read/write) +# +# The CI workflow already validates the binary (fmt / clippy / test). +# This workflow assumes CI passed on the same commit; it only builds +# and pushes the image. + +on: + push: + tags: + - 'v*.*.*' + +env: + CARGO_TERM_COLOR: always + +jobs: + docker: + name: docker build + push + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + + - uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # Extract the version from the tag (strips the leading "v"). + - name: Extract version + id: version + run: echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: docker/Dockerfile + push: true + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/ai-memory:latest + ${{ secrets.DOCKERHUB_USERNAME }}/ai-memory:${{ steps.version.outputs.VERSION }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Smoke test the published image + run: | + docker pull "${{ secrets.DOCKERHUB_USERNAME }}/ai-memory:${{ steps.version.outputs.VERSION }}" + docker run --rm \ + "${{ secrets.DOCKERHUB_USERNAME }}/ai-memory:${{ steps.version.outputs.VERSION }}" \ + --version diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..ac08cc1b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,83 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- Wiki-structure migration framework: `wiki_migrations` SQL table (V06), + `WikiMigration` trait, migration registry, and `run_pending` runner + invoked at server startup before the watcher starts. +- MCP read tools (`memory_query`, `memory_recent`, `memory_status`, + `memory_briefing`, `memory_explore`) accept an optional `project` + argument to target a specific project on a shared server. + +### Fixed +- OpenCode hook events (`tool.execute.*`, `session.*`) were rejected with + "missing session_id" because OpenCode sends `sessionID` (capital `ID`) + and the extractor only matched `sessionId`. All spellings are now + accepted ([#1]). +- MCP read tools were locked to the server's static `--project` (default + `scratch`), so on a shared HTTP server they returned empty memory even + while hooks populated the correct per-cwd project. The hook router now + publishes the active project to a shared pointer that the read tools + use as their default; an explicit `project` argument overrides it ([#2]). + +## [0.1.0] - 2026-XX-XX + +### Added +- Per-project UUID-namespaced wiki layout: pages live at + `///`. Rename is now + a single column update; purge is `remove_dir_all` on the project dir. +- CLI becomes a thin HTTP client: `bootstrap`, `status`, `search`, + `reorg`, `lint`, `forget-sweep`, `embed`, `commit`, `backup`, + `write-page` all delegate to the running server via `/admin/*` routes. + The server is the sole writer of wiki + SQLite. +- `purge-project` command with cascade-delete indexes and per-project + isolation guard (refuses to delete files claimed by sibling projects). +- `rename-project` command: column-only rename, no file moves. +- `memory_install_self_routing` MCP tool: installs the agent-routing + snippet into CLAUDE.md / AGENTS.md / `.cursorrules` in one call. +- Read-only HTTP wiki browser (`/web`) with project tree, page view, + and full-text search. +- Bearer token auth (`AI_MEMORY_AUTH_TOKEN` / `generate-auth-token`), + Host-header allowlist, and 10 MB body cap for the HTTP server. +- `backup` / `restore` commands using `.tar.gz` archives with live-process + guard (refuses to run if another `ai-memory` is active on the same data dir). +- Per-cwd project routing in hooks: observations route to the project + matching the agent's working directory, not the server default. +- `opencode` / `openclaw` aliases for the OpenCode MCP client. +- Dockerised CLI wrapper (`bin/ai-memory`) with auto-restart for the + local container and nudge for remote upgrades. +- `bootstrap` serialises parallel runs to prevent duplicate project creation + and handles the case where the CWD has no git repo. +- Monthly log-md rotation to keep `log.md` from growing unbounded. +- `memory_consolidate` PreCompact checkpointing falls back to rule-based + summarisation when no LLM is configured. +- `docs/lifecycle-ops.md`: safety matrix for state-touching commands + (reset, restore, purge-project, rename-project). +- `docs/wiki-migrations.md`: when and how to write a wiki migration. + +### Changed +- `bin/ai-memory` forwards `AI_MEMORY_SERVER_URL` and no longer creates + `-w` mount-conflict directories. +- `bootstrap` resolves the repo root via `libgit2`, removing the + `git` binary dependency. +- Admin routes consolidated: dry-run support, correct status codes, + deduplicated handlers. +- Host-header allowlist sourced from `Config.allowed_hosts`; logged at + startup so operators can verify the effective list. + +### Fixed +- `AI_MEMORY_HOST_CWD` handling and dry-run no-project side effects. +- Web page view: strip leading H1 from body to prevent title duplication. +- `install-mcp` Codex config key was `bearer_token`, not + `http_headers` / `headers`. +- Consolidator used server startup default project instead of the + session's actual project. + +[Unreleased]: https://github.com/akitaonrails/ai-memory/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/akitaonrails/ai-memory/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..545116ac --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,76 @@ +# Contributing to ai-memory + +## Dev setup + +```bash +git clone https://github.com/akitaonrails/ai-memory +cd ai-memory +cargo build --workspace +cargo test --workspace +``` + +Rust 1.95 is required (pinned in `rust-toolchain.toml`). The build is +self-contained: SQLite is bundled via `rusqlite`'s `bundled` feature, and +`libgit2` is vendored via `git2`'s `vendored-libgit2` feature. No system +libraries need installing beyond a standard C toolchain. + +## Required gates before every PR + +All four must pass — the CI workflow enforces them and so does the `bin/release` +script: + +```bash +cargo fmt --all -- --check # formatting +cargo clippy --workspace --all-targets -- -D warnings # lints +cargo test --workspace # tests +cargo deny check # dependency policy +``` + +If `cargo-deny` or `cargo-audit` are not installed: + +```bash +cargo install cargo-deny cargo-audit +``` + +## Workflow rules (condensed from CLAUDE.md) + +The full authoritative rules are in [`CLAUDE.md`](CLAUDE.md). Short version: + +1. Work milestone by milestone. Do not start M(n+1) until every "Done when" + bullet in M(n) passes (see `docs/design-decisions.md`). +2. No dead code, no half-built features. Stubs are documented with + `// M TODO` in the module doc-comment. +3. Write tests before claiming done. Parsers, ID derivation, and + retention/decay math especially. +4. Do not refactor outside the milestone. Only touch what the current + milestone requires. +5. Comments explain *why*, never *what*. No comments that restate the line + above them. + +## Cross-cutting invariants + +Never violate any of the invariants in `CLAUDE.md §Cross-cutting invariants`. +Highlights for contributors: + +- All SQLite writes go through the single writer actor (`WriterHandle`). +- Config is read once at startup; never call `std::env::var` outside `Config::load`. +- Atomic file writes only: tmp + rename + fsync; never write in-place. +- Every wiki page is namespaced by `(workspace_id, project_id)`. +- The CLI is always a thin HTTP client to the running server — it never + opens the SQLite file or the wiki directory directly. + +## Versioning and deprecation policy + +This project follows [Semantic Versioning](https://semver.org/): + +- **Patch** (`x.y.Z`): bug fixes that do not change public API or + on-disk format. +- **Minor** (`x.Y.0`): additive changes; new CLI subcommands, new MCP + tools, new config keys. Existing behaviour is preserved. +- **Major** (`X.0.0`): breaking changes. This includes on-disk format + changes that are not handled by a migration, removal of CLI subcommands, + or changes to the MCP tool schema that would break existing agents. + +Breaking changes only ship in major releases. Deprecated items are +documented in the CHANGELOG under `### Deprecated` and removed no sooner +than the following major release. diff --git a/Cargo.lock b/Cargo.lock index bc3fc048..b07cbfd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -237,6 +237,7 @@ dependencies = [ "ai-memory-llm", "ai-memory-store", "anyhow", + "async-trait", "git2", "jiff", "notify", diff --git a/Cargo.toml b/Cargo.toml index ba866d43..7df794d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ members = [ version = "0.1.0" edition = "2024" rust-version = "1.95" -license = "MIT OR Apache-2.0" +license = "MIT" repository = "https://github.com/akitaonrails/ai-memory" authors = ["Fabio Akita "] diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..746f9681 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Fabio Akita + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 026c71ff..60829ff4 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ [![status: v0.2 milestones complete](https://img.shields.io/badge/status-v0.2--complete-green)](docs/ARCHITECTURE.md) [![Rust](https://img.shields.io/badge/rust-1.95+-blue)](rust-toolchain.toml) -[![License](https://img.shields.io/badge/license-MIT%20OR%20Apache--2.0-blue)](#license) +[![License](https://img.shields.io/badge/license-MIT-blue)](LICENSE) ## What it is @@ -381,9 +381,13 @@ solves that by LLM-summarising your existing `git log`, README, # Requires an LLM provider configured on the server. Budget caps at # 50k input tokens (~$0.05 with Claude Haiku 4.5). export AI_MEMORY_SERVER_URL="http://localhost:49374" -ai-memory bootstrap --workspace homelab --project myproj +ai-memory bootstrap ``` +The workspace defaults to `default` and the project defaults to the +current directory's basename — that's almost always what you want, so +omit `--workspace` / `--project` unless you're deliberately overriding. + Bootstrap produces a per-project `bootstrap.md` manifest (under `///`) listing every page generated + a one-paragraph rationale. Run with `--dry-run` first to preview which @@ -706,7 +710,7 @@ data-flow diagram + crate breakdown + cross-cutting invariants. ## License -Dual-licensed under MIT OR Apache-2.0. +MIT — see [LICENSE](LICENSE). ## Acknowledgements diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..8321e229 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,63 @@ +# Security Policy + +## Reporting a vulnerability + +Please **do not open a public GitHub issue** for security vulnerabilities. + +Report security issues by opening a [private security advisory](https://github.com/akitaonrails/ai-memory/security/advisories/new) +on GitHub. You will receive a response within 7 days. If the issue is confirmed +we will aim to release a patch within 30 days and credit you in the changelog +(unless you prefer to remain anonymous). + +## Threat model + +ai-memory is a **single-user, homelab tool**. The following describes what +the project is and is not designed to defend against. + +### In scope + +- **Local data confidentiality.** Wiki files and the SQLite database live + under a single data directory controlled by the operating-system user who + runs the server. We rely on filesystem permissions; no additional + encryption at rest is provided in v1. + +- **Network exposure when binding to non-loopback addresses.** If you run + `ai-memory serve --bind 0.0.0.0:…` you are exposing the MCP and admin + routes to your local network. Protect this with: + - `AI_MEMORY_AUTH_TOKEN` / `ai-memory generate-auth-token` (bearer token + checked on every request). + - Firewall rules or a reverse proxy with TLS. + + The server logs a loud warning if it detects a non-loopback bind without a + configured auth token. + +- **Host-header DNS rebinding.** The HTTP server enforces an + `AI_MEMORY_ALLOWED_HOSTS` allowlist (defaulting to `127.0.0.1` and + `localhost`). Requests with a `Host` header not in the list are rejected + with 403. + +- **Request body size.** Inbound HTTP bodies are capped at 10 MB to prevent + trivial memory exhaustion. + +- **Per-project isolation.** Wiki files and SQLite rows are namespaced by + `(workspace_id, project_id)`. A purge operation for project A cannot + delete files that also belong to project B. + +### Out of scope for v1 + +- **Multi-tenant authentication and authorisation.** There is one bearer + token (or none). There are no per-user roles or per-project ACLs. +- **Encryption at rest.** The data directory is a plain filesystem tree. +- **Remote sync security.** If you push the wiki git repository to a remote, + securing that channel is your responsibility (SSH keys, GitHub access + controls, etc.). +- **MCP tool-call injection via agent output.** The privacy strip + (`Sanitizer`) removes obvious credential patterns from hook payloads, but + it is not a comprehensive injection fence. +- **Denial of service.** The server is not hardened against a malicious local + actor hammering it with requests. + +## Supported versions + +Only the latest release receives security fixes. We do not backport to older +minor versions. diff --git a/bin/release b/bin/release new file mode 100755 index 00000000..9c5fde1c --- /dev/null +++ b/bin/release @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# Cut a release of ai-memory. +# +# Usage: bin/release +# +# Where is a semver string without the leading "v", e.g. "0.2.0". +# +# What this script does: +# 1. Validates the version string. +# 2. Runs fmt / clippy / test gates. +# 3. Runs cargo-deny and cargo-audit (warn-only — a known advisory should +# not block a release; the operator reviews the output). +# 4. Updates [workspace.package] version in Cargo.toml. +# 5. Moves CHANGELOG.md [Unreleased] entries to a new [X.Y.Z] section +# dated today. +# 6. Commits both files as "release: vX.Y.Z". +# 7. Creates an annotated git tag vX.Y.Z whose message is the changelog +# entry for this version. +# 8. Prints next-step instructions: push the commit + tag, then deploy. +# +# The script intentionally does NOT push to git remote or to Docker Hub. +# Releases are operator-driven: review the tag locally before pushing. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# ── argument validation ──────────────────────────────────────────────────── + +if [[ $# -ne 1 ]]; then + echo "Usage: bin/release (e.g. bin/release 0.2.0)" >&2 + exit 64 +fi + +VERSION="$1" + +# Basic semver: X.Y.Z with optional pre-release / build metadata. +# We allow 0.1.0-rc.1 etc. but reject obviously wrong strings early. +if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+([.-][A-Za-z0-9.+_-]*)?$ ]]; then + echo "ERROR: '$VERSION' does not look like a semver string (expected X.Y.Z)" >&2 + exit 1 +fi + +TAG="v${VERSION}" +TODAY="$(date -u +%Y-%m-%d)" +CARGO_TOML="$REPO_ROOT/Cargo.toml" +CHANGELOG="$REPO_ROOT/CHANGELOG.md" + +echo "==> Releasing ${TAG} (dated ${TODAY})" + +# ── quality gates ────────────────────────────────────────────────────────── + +cd "$REPO_ROOT" + +echo "==> cargo fmt --check" +cargo fmt --all -- --check + +echo "==> cargo clippy" +cargo clippy --workspace --all-targets -- -D warnings + +echo "==> cargo test" +cargo test --workspace + +echo "==> cargo deny check (warn-only)" +if command -v cargo-deny &>/dev/null; then + cargo deny check || echo " [warn] cargo deny reported issues — review before pushing" +else + echo " [warn] cargo-deny not installed; skipping (install with: cargo install cargo-deny)" +fi + +echo "==> cargo audit (warn-only)" +if command -v cargo-audit &>/dev/null; then + cargo audit || echo " [warn] cargo audit reported advisories — review before pushing" +else + echo " [warn] cargo-audit not installed; skipping (install with: cargo install cargo-audit)" +fi + +# ── update Cargo.toml version ────────────────────────────────────────────── + +echo "==> Updating Cargo.toml workspace version to ${VERSION}" + +python3 - <$VERSION\g<2>', + text, + count=1, + flags=re.MULTILINE | re.DOTALL, +) +if new_text == text: + sys.exit("ERROR: could not find [workspace.package] version in Cargo.toml") + +# 2. Bump version constraints in [workspace.dependencies] for the intra-crate +# path deps (ai-memory-core, ai-memory-store, etc.). These always use the +# same version as the workspace package itself. +new_text = re.sub( + r'(ai-memory-\w+\s*=\s*\{[^}]*version\s*=\s*")[^"]+(")', + r'\g<1>$VERSION\g<2>', + new_text, + flags=re.MULTILINE, +) + +path.write_text(new_text) +print(f" version -> $VERSION") +PYEOF + +# ── update CHANGELOG.md ──────────────────────────────────────────────────── + +echo "==> Updating CHANGELOG.md" + +python3 - < Committing release files" +git add "$CARGO_TOML" "$CHANGELOG" Cargo.lock +git commit -m "release: ${TAG}" + +echo "==> Extracting changelog entry for tag message" +CHANGELOG_ENTRY="$(python3 - < Creating annotated tag ${TAG}" +git tag -a "${TAG}" -m "$(printf '%s\n\n%s' "Release ${TAG}" "${CHANGELOG_ENTRY}")" + +# ── next steps ───────────────────────────────────────────────────────────── + +cat < StoreResult<()> { + conn.execute( + "INSERT OR IGNORE INTO wiki_migrations (name, applied_at) VALUES (?1, ?2)", + params![name, applied_at], + )?; + Ok(()) +} + /// Delete a project and all its data inside one transaction. /// /// Execution order: diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs index 1d3b7ab1..b01f6aa9 100644 --- a/crates/ai-memory-store/src/reader.rs +++ b/crates/ai-memory-store/src/reader.rs @@ -1431,6 +1431,24 @@ impl ReaderPool { }) .await } + + /// Return all migration names recorded in the `wiki_migrations` table. + /// + /// Used by the wiki migration runner to determine which migrations have + /// already been applied to this data directory. + /// + /// # Errors + /// Propagates any SQL or pool error. + pub async fn wiki_migration_names(&self) -> StoreResult> { + self.with_conn(|conn| { + let mut stmt = conn.prepare("SELECT name FROM wiki_migrations ORDER BY name")?; + let names = stmt + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + Ok(names) + }) + .await + } } fn page_meta_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result> { diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rs index 5593bc3e..14bbc8c6 100644 --- a/crates/ai-memory-store/src/writer.rs +++ b/crates/ai-memory-store/src/writer.rs @@ -109,6 +109,13 @@ pub(crate) enum WriteCmd { new_name: String, reply: oneshot::Sender>, }, + /// Record a successfully-applied wiki-structure migration. + InsertWikiMigration { + name: String, + /// Unix microseconds UTC. + applied_at: i64, + reply: oneshot::Sender>, + }, Shutdown, } @@ -373,6 +380,26 @@ impl WriterHandle { rx.await.map_err(|_| StoreError::WriterClosed)? } + /// Record a wiki-structure migration as successfully applied. + /// + /// Called by the wiki migration runner immediately after [`WikiMigration::up`] + /// returns `Ok`. `applied_at` is unix microseconds UTC. If the name is + /// already present the call is a no-op (idempotent insert-or-ignore). + /// + /// # Errors + /// Returns [`StoreError::WriterClosed`] if the actor has shut down, or + /// propagates the SQL error. + pub async fn insert_wiki_migration(&self, name: String, applied_at: i64) -> StoreResult<()> { + let (tx, rx) = oneshot::channel(); + self.send(WriteCmd::InsertWikiMigration { + name, + applied_at, + reply: tx, + }) + .await?; + rx.await.map_err(|_| StoreError::WriterClosed)? + } + /// Retro-fit sessions and their observations to per-cwd projects and /// graveyard any mash-up pages. The `plan` slice contains /// `(session_id, new_project_id)` pairs. Everything runs in one @@ -566,6 +593,14 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver) { let result = ops::rename_project(&mut conn, &workspace_id, &project_id, &new_name); send_or_warn(reply, result, "rename_project"); } + WriteCmd::InsertWikiMigration { + name, + applied_at, + reply, + } => { + let result = ops::insert_wiki_migration(&mut conn, &name, applied_at); + send_or_warn(reply, result, "insert_wiki_migration"); + } } } tracing::debug!("writer thread exiting cleanly"); diff --git a/crates/ai-memory-wiki/Cargo.toml b/crates/ai-memory-wiki/Cargo.toml index 6505e99f..aa72330c 100644 --- a/crates/ai-memory-wiki/Cargo.toml +++ b/crates/ai-memory-wiki/Cargo.toml @@ -24,6 +24,7 @@ tokio.workspace = true notify.workspace = true notify-debouncer-full.workspace = true git2.workspace = true +async-trait.workspace = true [dev-dependencies] diff --git a/crates/ai-memory-wiki/src/lib.rs b/crates/ai-memory-wiki/src/lib.rs index 1aceaf4b..1f1481f2 100644 --- a/crates/ai-memory-wiki/src/lib.rs +++ b/crates/ai-memory-wiki/src/lib.rs @@ -9,11 +9,13 @@ mod atomic; mod error; mod git; mod markdown; +pub mod migrations; mod watcher; mod wiki; pub use error::{WikiError, WikiResult}; pub use git::{COMMIT_AUTHOR_EMAIL, COMMIT_AUTHOR_NAME, GitAdapter}; pub use markdown::{Markdown, derive_title, emit, parse}; +pub use migrations::run_pending as run_wiki_migrations; pub use watcher::{DEBOUNCE_WINDOW, RECONCILE_INTERVAL, WatcherHandle}; pub use wiki::{Wiki, WritePageRequest}; diff --git a/crates/ai-memory-wiki/src/migrations/mod.rs b/crates/ai-memory-wiki/src/migrations/mod.rs new file mode 100644 index 00000000..6069602b --- /dev/null +++ b/crates/ai-memory-wiki/src/migrations/mod.rs @@ -0,0 +1,102 @@ +//! Wiki-structure migration framework. +//! +//! SQL schema migrations are handled by `refinery` and run before this +//! layer is invoked. Wiki migrations handle *filesystem* changes — file +//! moves, path rewrites, directory renames — that are required when the +//! on-disk wiki layout changes between versions. +//! +//! ## Adding a new migration +//! +//! 1. Create a struct in a new submodule (e.g. `crates/ai-memory-wiki/src/migrations/m2026_05_24_per_project_layout.rs`). +//! 2. Implement [`WikiMigration`] for it. +//! 3. Add an instance to the `vec![]` in [`registry`]. **Always append at the +//! end — never reorder or remove entries.** +//! 4. Add a test in your module that exercises the migration against a tmp +//! wiki directory. +//! +//! ## Naming convention +//! +//! Migration names use `YYYY_MM_DDTHH_MM_`. The +//! timestamp is UTC and chosen at authoring time, not applied time. Using a +//! timestamp rather than a sequence number avoids merge conflicts when two +//! contributors add migrations in parallel. +//! +//! ## Idempotency +//! +//! Every implementation of [`WikiMigration::up`] must be idempotent: if the +//! work is already done (files already in the target layout, target dir +//! already absent, etc.), the migration exits with `Ok(())` without touching +//! anything. The runner marks it applied on first success and never re-runs +//! it, but a correct implementation guards against the invariant anyway. +//! +//! ## What NOT to do +//! +//! - **No destructive deletes without a graveyard step.** Move the file to +//! `/_graveyard//` before deleting so +//! data is recoverable for at least one release cycle. +//! - **No LLM calls.** Migrations run on every server start; they must be +//! fast and free. +//! - **No SQL outside the writer actor.** Use [`WriterHandle`] methods so all +//! writes go through the single-writer channel (invariant #2). + +mod runner; + +use std::path::Path; + +use ai_memory_store::WriterHandle; + +pub use runner::run_pending; + +use crate::error::WikiResult; + +/// A single wiki-structure migration. +/// +/// Implementors describe one idempotent filesystem (and optional SQL) +/// transformation. The runner calls [`up`](WikiMigration::up) exactly once +/// per data directory, tracking completion in the `wiki_migrations` table. +#[async_trait::async_trait] +pub trait WikiMigration: Send + Sync { + /// Unique, sortable name. Convention: `YYYY_MM_DDTHH_MM_`. + /// + /// This value is stored in the `wiki_migrations` table as the primary key. + /// Once chosen and shipped it must never change. + fn name(&self) -> &'static str; + + /// One-line description shown in server logs. + fn description(&self) -> &'static str; + + /// Apply the migration. + /// + /// The `writer` handle is provided so any SQL updates that accompany the + /// file moves go through the single-writer actor (no race with hooks). + /// `wiki_root` is the on-disk root of the wiki directory + /// (`/wiki/`). + /// + /// Implementations **must** be idempotent: if the work has already been + /// done they return `Ok(())` immediately. + /// + /// # Errors + /// + /// Any error returned here causes the server to bail at startup. The + /// migration row is NOT inserted into `wiki_migrations`, so the next start + /// will retry. + async fn up(&self, writer: &WriterHandle, wiki_root: &Path) -> WikiResult<()>; +} + +/// The canonical migration registry. +/// +/// Migrations run sequentially in the order they appear. **Always append at +/// the end; never reorder or remove entries.** The runner uses this order +/// together with the `wiki_migrations` table to determine which entries are +/// still pending. +/// +/// v1 ships with no pending migrations: the per-project UUID-namespaced +/// layout is the native format from day one and requires no transformation +/// of pre-existing data. +#[must_use] +pub fn registry() -> Vec> { + // Keep this empty until a structural change actually needs migrating + // existing installs. Appending to an empty vec is a zero-diff PR for + // future contributors. + vec![] +} diff --git a/crates/ai-memory-wiki/src/migrations/runner.rs b/crates/ai-memory-wiki/src/migrations/runner.rs new file mode 100644 index 00000000..ee9f9bf8 --- /dev/null +++ b/crates/ai-memory-wiki/src/migrations/runner.rs @@ -0,0 +1,273 @@ +//! Runner that applies pending wiki-structure migrations in order. + +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +use ai_memory_store::{ReaderPool, WriterHandle}; +use tracing::{error, info}; + +use super::WikiMigration; +use crate::error::{WikiError, WikiResult}; + +/// Read the set of already-applied migration names from the database. +async fn applied_names(reader: &ReaderPool) -> WikiResult> { + reader + .wiki_migration_names() + .await + .map_err(WikiError::Store) +} + +/// Record one migration as successfully applied. +async fn mark_applied(writer: &WriterHandle, name: &str) -> WikiResult<()> { + let micros = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_micros() as i64) + .unwrap_or(0); + writer + .insert_wiki_migration(name.to_owned(), micros) + .await + .map_err(WikiError::Store) +} + +/// Run all pending wiki-structure migrations. +/// +/// Reads the `wiki_migrations` table to determine which names from +/// `registry` have not yet been applied, then runs each pending +/// migration in registration order. +/// +/// A migration that returns [`Err`] causes this function to bail +/// immediately. The failed migration's name is **not** inserted into +/// `wiki_migrations`, so the next server start will retry it. +/// +/// # Errors +/// +/// Returns the error from the first failing migration, or any database +/// error that prevents reading/writing the `wiki_migrations` table. +pub async fn run_pending( + writer: &WriterHandle, + reader: &ReaderPool, + wiki_root: &Path, + registry: &[Box], +) -> WikiResult<()> { + if registry.is_empty() { + return Ok(()); + } + + let applied = applied_names(reader).await?; + + for migration in registry { + let name = migration.name(); + if applied.iter().any(|n| n == name) { + continue; + } + + info!( + migration = name, + description = migration.description(), + "running wiki migration" + ); + + if let Err(e) = migration.up(writer, wiki_root).await { + error!( + migration = name, + error = %e, + "wiki migration failed — server cannot start until this is resolved" + ); + return Err(e); + } + + mark_applied(writer, name).await?; + info!(migration = name, "applied wiki migration"); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use tempfile::TempDir; + + use super::*; + use crate::error::WikiResult; + use crate::migrations::WikiMigration; + + // ── helpers ────────────────────────────────────────────────────────────── + + fn open_store(dir: &TempDir) -> ai_memory_store::Store { + ai_memory_store::Store::open(dir.path()).expect("store open") + } + + // ── synthetic migrations ────────────────────────────────────────────────── + + struct CountingMigration { + name: &'static str, + run_count: Arc>, + } + + #[async_trait::async_trait] + impl WikiMigration for CountingMigration { + fn name(&self) -> &'static str { + self.name + } + fn description(&self) -> &'static str { + "counting migration for tests" + } + async fn up(&self, _writer: &WriterHandle, _wiki_root: &Path) -> WikiResult<()> { + *self.run_count.lock().unwrap() += 1; + Ok(()) + } + } + + struct FailingMigration; + + #[async_trait::async_trait] + impl WikiMigration for FailingMigration { + fn name(&self) -> &'static str { + "2026_01_01T00_00_failing" + } + fn description(&self) -> &'static str { + "always fails" + } + async fn up(&self, _writer: &WriterHandle, _wiki_root: &Path) -> WikiResult<()> { + Err(WikiError::Io(std::io::Error::other( + "synthetic migration failure", + ))) + } + } + + // ── tests ───────────────────────────────────────────────────────────────── + + /// Empty registry → no-op, no rows inserted. + #[tokio::test] + async fn empty_registry_is_noop() { + let dir = TempDir::new().unwrap(); + let store = open_store(&dir); + let wiki_root = dir.path().join("wiki"); + std::fs::create_dir_all(&wiki_root).unwrap(); + + let result = run_pending(&store.writer, &store.reader, &wiki_root, &[]).await; + assert!(result.is_ok()); + + let names = store.reader.wiki_migration_names().await.unwrap(); + assert!(names.is_empty()); + } + + /// Migration runs once; second call is a no-op. + #[tokio::test] + async fn runs_once_then_skips() { + let dir = TempDir::new().unwrap(); + let store = open_store(&dir); + let wiki_root = dir.path().join("wiki"); + std::fs::create_dir_all(&wiki_root).unwrap(); + + let count = Arc::new(Mutex::new(0u32)); + let migration: Box = Box::new(CountingMigration { + name: "2026_01_01T00_00_counting", + run_count: count.clone(), + }); + let registry: Vec> = vec![migration]; + + run_pending(&store.writer, &store.reader, &wiki_root, ®istry) + .await + .unwrap(); + assert_eq!(*count.lock().unwrap(), 1, "ran once"); + + // Rebuild registry (can't clone Box) and run again. + let migration2: Box = Box::new(CountingMigration { + name: "2026_01_01T00_00_counting", + run_count: count.clone(), + }); + let registry2: Vec> = vec![migration2]; + + run_pending(&store.writer, &store.reader, &wiki_root, ®istry2) + .await + .unwrap(); + assert_eq!(*count.lock().unwrap(), 1, "still one — not re-run"); + } + + /// Failing migration → Err returned, row NOT inserted, next call retries. + #[tokio::test] + async fn failing_migration_not_marked_applied() { + let dir = TempDir::new().unwrap(); + let store = open_store(&dir); + let wiki_root = dir.path().join("wiki"); + std::fs::create_dir_all(&wiki_root).unwrap(); + + let registry: Vec> = vec![Box::new(FailingMigration)]; + + let result = run_pending(&store.writer, &store.reader, &wiki_root, ®istry).await; + assert!(result.is_err(), "must propagate the error"); + + let names = store.reader.wiki_migration_names().await.unwrap(); + assert!( + names.is_empty(), + "failed migration must not be marked applied" + ); + + // Re-run → still errors, still not applied. + let registry2: Vec> = vec![Box::new(FailingMigration)]; + let result2 = run_pending(&store.writer, &store.reader, &wiki_root, ®istry2).await; + assert!(result2.is_err()); + let names2 = store.reader.wiki_migration_names().await.unwrap(); + assert!(names2.is_empty()); + } + + /// Two migrations in registry → both run in order. + #[tokio::test] + async fn two_migrations_run_in_order() { + let dir = TempDir::new().unwrap(); + let store = open_store(&dir); + let wiki_root = dir.path().join("wiki"); + std::fs::create_dir_all(&wiki_root).unwrap(); + + let order: Arc>> = Arc::new(Mutex::new(vec![])); + + struct OrderMigration { + name: &'static str, + log: Arc>>, + } + + #[async_trait::async_trait] + impl WikiMigration for OrderMigration { + fn name(&self) -> &'static str { + self.name + } + fn description(&self) -> &'static str { + "order test" + } + async fn up(&self, _writer: &WriterHandle, _wiki_root: &Path) -> WikiResult<()> { + self.log.lock().unwrap().push(self.name); + Ok(()) + } + } + + let registry: Vec> = vec![ + Box::new(OrderMigration { + name: "2026_01_01T00_00_first", + log: order.clone(), + }), + Box::new(OrderMigration { + name: "2026_01_01T00_01_second", + log: order.clone(), + }), + ]; + + run_pending(&store.writer, &store.reader, &wiki_root, ®istry) + .await + .unwrap(); + + let ran = order.lock().unwrap().clone(); + assert_eq!( + ran, + vec!["2026_01_01T00_00_first", "2026_01_01T00_01_second"], + "must run in registration order" + ); + + // Both names persisted. + let mut names = store.reader.wiki_migration_names().await.unwrap(); + names.sort(); + assert_eq!(names.len(), 2); + } +} diff --git a/docs/wiki-migrations.md b/docs/wiki-migrations.md new file mode 100644 index 00000000..64883032 --- /dev/null +++ b/docs/wiki-migrations.md @@ -0,0 +1,171 @@ +# Wiki-structure migrations + +SQL schema migrations are handled automatically by `refinery` at server +startup. This document covers the parallel mechanism for *filesystem-level* +changes to the wiki directory. + +## When to write a wiki migration + +Write a wiki migration any time a new version of ai-memory requires an +on-disk wiki directory that was created by an older version to be +restructured. Examples that require a migration: + +- The path scheme changes (e.g. `/.md` → `///.md`). +- A directory is renamed, split, or merged. +- Every page of a certain kind gets a new required frontmatter field added. +- Log rotation changes the filename pattern for `log.md` backups. + +Do **not** write a migration for changes that are purely additive and +backward-compatible (e.g. a new optional frontmatter field that defaults to +`null`). + +## How to write a wiki migration + +### 1. Create the migration file + +Add a new file in `crates/ai-memory-wiki/src/migrations/`. Use the naming +convention: + +``` +m__
__.rs +``` + +For example: `m2026_06_01_1200_rename_logs_dir.rs`. + +### 2. Implement the `WikiMigration` trait + +```rust +use std::path::Path; +use ai_memory_store::WriterHandle; +use crate::error::WikiResult; +use crate::migrations::WikiMigration; + +pub struct RenameLogs2026; + +#[async_trait::async_trait] +impl WikiMigration for RenameLogs2026 { + fn name(&self) -> &'static str { + // Must be unique and sortable. Choose once and never change. + "2026_06_01T12_00_rename_logs_dir" + } + + fn description(&self) -> &'static str { + "rename _logs/ to _log/ for consistency with log.md" + } + + async fn up(&self, _writer: &WriterHandle, wiki_root: &Path) -> WikiResult<()> { + let old = wiki_root.join("_logs"); + let new = wiki_root.join("_log"); + + // Idempotency: if the work is already done, return Ok immediately. + if !old.exists() { + return Ok(()); + } + + std::fs::rename(&old, &new)?; + Ok(()) + } +} +``` + +### 3. Register it + +Open `crates/ai-memory-wiki/src/migrations/mod.rs` and append to the +`registry()` function: + +```rust +pub fn registry() -> Vec> { + vec![ + // existing entries... + Box::new(super::m2026_06_01_1200_rename_logs_dir::RenameLogs2026), + ] +} +``` + +Also add `mod m2026_06_01_1200_rename_logs_dir;` near the top of `mod.rs`. + +**Never reorder or remove entries.** The runner uses the registration order +together with the `wiki_migrations` table. + +### 4. Add a unit test + +Every migration module must include a `#[cfg(test)]` block that: + +- Exercises the migration against a `tempfile::TempDir`. +- Verifies the pre-condition (old layout present), post-condition (new layout + present, old absent), and idempotency (running twice is a no-op). + +```rust +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + use ai_memory_store::Store; + + #[tokio::test] + async fn renames_logs_dir() { + let dir = TempDir::new().unwrap(); + let store = Store::open(dir.path()).unwrap(); + let wiki_root = dir.path().join("wiki"); + std::fs::create_dir_all(&wiki_root).unwrap(); + + // Pre-condition. + std::fs::create_dir(wiki_root.join("_logs")).unwrap(); + + let m = RenameLogs2026; + m.up(&store.writer, &wiki_root).await.unwrap(); + + assert!(wiki_root.join("_log").exists()); + assert!(!wiki_root.join("_logs").exists()); + + // Idempotent: running again does not error. + m.up(&store.writer, &wiki_root).await.unwrap(); + } +} +``` + +## How NOT to write a migration + +### No destructive deletes without a graveyard step + +If a migration removes files (not just moves them), copy or move them to +`/_graveyard//` first. This lets +operators recover accidentally-deleted data for at least one release cycle. + +```rust +// BAD — data is gone forever on upgrade. +std::fs::remove_dir_all(wiki_root.join("_tmp"))?; + +// GOOD — data lands in the graveyard, recoverable. +let graveyard = wiki_root.join("_graveyard").join(self.name()); +std::fs::create_dir_all(&graveyard)?; +std::fs::rename(wiki_root.join("_tmp"), graveyard.join("_tmp"))?; +``` + +### No LLM calls + +Migrations run on every server start. They must be fast and free. Any +transformation that requires a language model belongs in a one-time CLI +command or a consolidation job, not a migration. + +### No direct SQL outside the writer actor + +If a migration needs to update the SQLite index alongside the file moves, use +`WriterHandle` methods. Never open a second `Connection`; never call +`ops::*` directly from a migration. This upholds invariant #2 (single-writer +actor) from `CLAUDE.md`. + +## Tracking + +Applied migrations are recorded in the `wiki_migrations` SQLite table: + +```sql +SELECT name, datetime(applied_at / 1000000, 'unixepoch') AS applied +FROM wiki_migrations +ORDER BY name; +``` + +The table is created by `V06__wiki_migrations.sql` (a `refinery` migration +that runs before any wiki migrations). The server bails with a clear error +message if a migration fails; re-starting the server retries the failed +migration automatically. diff --git a/evals/Cargo.toml b/evals/Cargo.toml index 80af7b43..490f866f 100644 --- a/evals/Cargo.toml +++ b/evals/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "ai-memory-eval" -version = "0.1.0" -edition = "2024" -rust-version = "1.95" +version.workspace = true +edition.workspace = true +rust-version.workspace = true license = "MIT OR Apache-2.0" publish = false