mirror of
https://github.com/earendil-works/pi.git
synced 2026-10-02 08:44:38 +08:00
feat: add durable SQLite storage backend
Add the portable SQLite storage core, Node adapter, versioned migrations, conformance coverage, and storage benchmarks.
This commit is contained in:
Mario Zechner
@@ -8,6 +8,35 @@ This package contains the Pico runtime. Its current public API provides the dura
|
||||
import { MemoryStorage, ROOT_CONVERSATION_ID } from "@earendil-works/pi-durable";
|
||||
```
|
||||
|
||||
The root export is runtime-neutral. Storage implementations also have explicit subpaths:
|
||||
|
||||
```ts
|
||||
import { MemoryStorage } from "@earendil-works/pi-durable/storage/memory";
|
||||
```
|
||||
|
||||
Node applications can open file-backed SQLite through its Node-only subpath:
|
||||
|
||||
```ts
|
||||
import { openNodeSqliteStorage } from "@earendil-works/pi-durable/storage/sqlite/node";
|
||||
|
||||
const storage = await openNodeSqliteStorage("./session.sqlite");
|
||||
```
|
||||
|
||||
The portable SQLite core, minimal database facade, and ordered schema migrations are exported from `@earendil-works/pi-durable/storage/sqlite`. Adapters for synchronous SQLite environments such as Bun and Cloudflare Durable Objects can implement that facade without importing Node APIs. Remote asynchronous APIs such as Cloudflare D1 cannot implement this synchronous facade; they require a dedicated `Storage` backend.
|
||||
|
||||
The Node adapter uses WAL mode with `synchronous = NORMAL` and checkpoints the WAL on close. Acknowledged commits survive process crashes, but the newest commits may be lost after a power or host failure. One `SqliteStorage` owner must serialize writes to a database file; cross-process ID allocation is not supported.
|
||||
|
||||
## Storage benchmarks
|
||||
|
||||
From this package directory:
|
||||
|
||||
```sh
|
||||
npm run bench:storage
|
||||
npm run bench:storage:memory
|
||||
```
|
||||
|
||||
The timing suite compares memory and file-backed SQLite across representative commits, indexed reads, pagination, fork traversal, document replay, historical reads, and SQLite reopen. The memory suite measures each backend in a separate process at 1k and 10k scales and reports heap, RSS, external memory, and SQLite file/page metrics. These deterministic synthetic workloads are baselines for regression analysis, not production capacity limits or CI pass/fail thresholds.
|
||||
|
||||
The normative design and implementation sequence are in:
|
||||
|
||||
- [`docs/pico-v5.md`](docs/pico-v5.md)
|
||||
|
||||
@@ -11,7 +11,7 @@ facades, membranes, document routing, view projection, events, or clone chains.
|
||||
|
||||
- Obsolete `pico` and `pico4` prototypes were removed.
|
||||
- `pico3` remains.
|
||||
- Packages 1 and 2 are implemented in `packages/durable`; later Pico5 runtime packages remain.
|
||||
- Packages 1–3 are implemented in `packages/durable`; later Pico5 runtime packages remain.
|
||||
|
||||
## 1. Records, cursors, and memory tables
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ import type {
|
||||
} from "@earendil-works/pi-ai";
|
||||
|
||||
type JsonObject = { [key: string]: JsonValue };
|
||||
type StoredError = { message: string; detail?: JsonValue };
|
||||
type TaskOutcomeError = { message: string; detail?: JsonValue };
|
||||
```
|
||||
|
||||
Pico5 targets the transcript `SystemMessage` contract from pi-ai PR
|
||||
@@ -70,6 +70,7 @@ these contracts.
|
||||
|
||||
```ts
|
||||
type Id = number;
|
||||
/** Strictly increases between commits; gaps are permitted. */
|
||||
type Seq = number;
|
||||
const ROOT_CONVERSATION_ID: Id = 1;
|
||||
|
||||
@@ -804,11 +805,11 @@ interface Tx {
|
||||
scanEntries(query: EntryQuery): Promise<readonly EntryRecord[]>;
|
||||
scanTasks(query: TaskQuery): Promise<readonly TaskRecord<JsonValue, JsonValue, JsonValue>[]>;
|
||||
|
||||
createConversation(value: Omit<ConversationRecord, "id">): ConversationRecord;
|
||||
appendEntry(conversationId: Id, value: EntryDraft): EntryRecord;
|
||||
createConversation(value: Omit<ConversationRecord, "id">): Promise<ConversationRecord>;
|
||||
appendEntry(conversationId: Id, value: EntryDraft): Promise<EntryRecord>;
|
||||
createTask<I, S extends { phase: string }, R, H extends object>(
|
||||
task: Task<I, S, R, H>, input: I, options?: TaskOptions,
|
||||
): TaskRef<R>;
|
||||
): Promise<TaskRef<R>>;
|
||||
setTask(value: TaskRecord<JsonValue, JsonValue, JsonValue>): void;
|
||||
|
||||
doc<T extends JsonObject>(token: SessionDocToken<T>): Promise<Draft<T>>;
|
||||
@@ -827,7 +828,7 @@ interface Tx {
|
||||
}
|
||||
```
|
||||
|
||||
Only public typed `tx.doc()` is get-or-create. Internal fork copying may create
|
||||
ID-creating transaction methods are asynchronous because remote storage may allocate globally unique IDs durably. Only public typed `tx.doc()` is get-or-create. Internal fork copying may create
|
||||
new incarnations directly from stored values without a definition. `tx.doc()`
|
||||
receives the definition token that supplies
|
||||
its static type, initializer, migration, and checkpoint policy. Scope-preserving
|
||||
@@ -1069,7 +1070,7 @@ await session.commit(async tx => {
|
||||
const task = await tx.task(taskId); // table read
|
||||
const live = await tx.doc(LiveDoc, conversationId);
|
||||
|
||||
tx.appendEntry(conversationId, message); // first table write
|
||||
await tx.appendEntry(conversationId, message); // first table write
|
||||
delete live.message; // document mutation remains valid
|
||||
tx.setTask(nextTask(task));
|
||||
}, context);
|
||||
@@ -1104,10 +1105,10 @@ Storage ownership:
|
||||
```ts
|
||||
type TaskOutcome<R> =
|
||||
| { readonly status: "completed"; readonly result: R }
|
||||
| { readonly status: "failed"; readonly error: StoredError; readonly result?: R }
|
||||
| { readonly status: "failed"; readonly error: TaskOutcomeError; readonly result?: R }
|
||||
| { readonly status: "aborted"; readonly reason?: string; readonly result?: R }
|
||||
| { readonly status: "orphaned"; readonly reason: string }
|
||||
| { readonly status: "faulted"; readonly error: StoredError };
|
||||
| { readonly status: "faulted"; readonly error: TaskOutcomeError };
|
||||
|
||||
type TaskState<S, R> =
|
||||
| { readonly status: "pending"; readonly checkpoint: S }
|
||||
@@ -1993,11 +1994,12 @@ type StorageWrite =
|
||||
* Trusts the owning Session to supply semantically valid records, references,
|
||||
* ancestry, and transitions. Enforces atomicity, global ID ownership, immutable
|
||||
* conversation/entry creation, document record consistency, and detached
|
||||
* values; Session serializes commits.
|
||||
* values; Session serializes commits. Sequences strictly increase but may have
|
||||
* gaps. Once commit() resolves, later reads through that Storage observe it.
|
||||
*/
|
||||
interface Storage {
|
||||
commit(writes: readonly StorageWrite[], context: Context): Promise<Seq>;
|
||||
mintId(): Id;
|
||||
mintId(): Promise<Id>;
|
||||
|
||||
conversation(id: Id, context: Context): Promise<ConversationRecord | undefined>;
|
||||
scanConversations(cursor: Cursor | undefined, limit: number, context: Context): Promise<Page<ConversationRecord, Cursor>>;
|
||||
@@ -2042,8 +2044,10 @@ ascending incarnation IDs. There is no ordinary open-time all-document scan.
|
||||
Task queries support conversation, kind, live/terminal status, abort mark, and
|
||||
background status.
|
||||
|
||||
`document(id, at)` internally selects the newest applicable base, applies its
|
||||
ordered Chord delta tail, and returns the detached materialized value plus stored
|
||||
`document(id, at)` materializes one specific incarnation and never follows a
|
||||
replacement at the same logical address. Callers resolve an address with
|
||||
`findDocument()` when they do not already hold an incarnation ID. It selects the
|
||||
newest applicable base, applies its ordered Chord delta tail, and returns the detached materialized value plus stored
|
||||
definition version. Base/delta records are backend-private. The lookup never
|
||||
scans unrelated documents. An unknown ID returns `undefined`. At `"current"`, a
|
||||
retired incarnation returns `undefined`. A numeric lookup of a rewindable
|
||||
|
||||
@@ -11,6 +11,21 @@
|
||||
"types": "./dist/index.d.ts",
|
||||
"import": "./dist/index.js"
|
||||
},
|
||||
"./storage/memory": {
|
||||
"source": "./src/storage/memory.ts",
|
||||
"types": "./dist/storage/memory.d.ts",
|
||||
"import": "./dist/storage/memory.js"
|
||||
},
|
||||
"./storage/sqlite": {
|
||||
"source": "./src/storage/sqlite/index.ts",
|
||||
"types": "./dist/storage/sqlite/index.d.ts",
|
||||
"import": "./dist/storage/sqlite/index.js"
|
||||
},
|
||||
"./storage/sqlite/node": {
|
||||
"source": "./src/storage/sqlite/node.ts",
|
||||
"types": "./dist/storage/sqlite/node.d.ts",
|
||||
"import": "./dist/storage/sqlite/node.js"
|
||||
},
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
"sideEffects": false,
|
||||
@@ -23,6 +38,8 @@
|
||||
"clean": "shx rm -rf dist",
|
||||
"build": "tsgo -p tsconfig.build.json",
|
||||
"test": "vitest --run",
|
||||
"bench:storage": "vitest bench --config vitest.benchmark.config.ts",
|
||||
"bench:storage:memory": "node --conditions=source --experimental-strip-types test/storage-memory.ts",
|
||||
"prepublishOnly": "npm run clean && npm run build"
|
||||
},
|
||||
"keywords": [
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export { MemoryStorage } from "./memory-storage.ts";
|
||||
export { MemoryStorage } from "./storage/memory.ts";
|
||||
export type {
|
||||
ContextEdit,
|
||||
ConversationRecord,
|
||||
@@ -19,10 +19,10 @@ export type {
|
||||
Storage,
|
||||
StorageWrite,
|
||||
StoredDocument,
|
||||
StoredError,
|
||||
SubmissionCreate,
|
||||
SubmissionRecord,
|
||||
TaskOutcome,
|
||||
TaskOutcomeError,
|
||||
TaskQuery,
|
||||
TaskRecord,
|
||||
TaskState,
|
||||
|
||||
@@ -21,7 +21,7 @@ import type {
|
||||
SubmissionRecord,
|
||||
TaskQuery,
|
||||
TaskRecord,
|
||||
} from "./types.ts";
|
||||
} from "../types.ts";
|
||||
|
||||
type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
|
||||
type TaskStatus = StoredTask["state"]["status"];
|
||||
@@ -268,7 +268,7 @@ export class MemoryStorage implements Storage {
|
||||
return seq;
|
||||
}
|
||||
|
||||
mintId(): Id {
|
||||
async mintId(): Promise<Id> {
|
||||
this.assertOpen();
|
||||
if (!Number.isSafeInteger(this.nextId)) throw new Error("ID space is exhausted");
|
||||
return this.nextId++;
|
||||
@@ -0,0 +1,27 @@
|
||||
/** Values supported by the portable SQLite storage core. */
|
||||
export type SqliteValue = null | number | bigint | string | Uint8Array;
|
||||
|
||||
/**
|
||||
* A prepared synchronous SQLite statement.
|
||||
* Implementations must support repeated execution with new bindings across transactions.
|
||||
*/
|
||||
export interface SqliteStatement {
|
||||
run(...params: SqliteValue[]): void;
|
||||
get<T extends object>(...params: SqliteValue[]): T | undefined;
|
||||
all<T extends object>(...params: SqliteValue[]): T[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal database facade required by `SqliteStorage`.
|
||||
*
|
||||
* Queries and transaction callbacks are synchronous so the same storage core can
|
||||
* run on Node, Bun, and Cloudflare Durable Object SQLite. An adapter may return a
|
||||
* promise from `transaction` while it waits for the transaction to settle.
|
||||
* Callers must not close the database while a returned settlement is pending.
|
||||
*/
|
||||
export interface SqliteDatabase {
|
||||
exec(sql: string): void;
|
||||
prepare(sql: string): SqliteStatement;
|
||||
transaction<T>(callback: () => T): T | Promise<T>;
|
||||
close(): void | Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
export type { SqliteDatabase, SqliteStatement, SqliteValue } from "./database.ts";
|
||||
export {
|
||||
applySqliteMigrations,
|
||||
CURRENT_SQLITE_SCHEMA_VERSION,
|
||||
SQLITE_MIGRATIONS,
|
||||
type SqliteMigration,
|
||||
} from "./migrations.ts";
|
||||
export { SqliteStorage } from "./storage.ts";
|
||||
@@ -0,0 +1,118 @@
|
||||
import type { SqliteDatabase } from "./database.ts";
|
||||
|
||||
export type SqliteMigration = {
|
||||
readonly version: number;
|
||||
readonly statements: readonly string[];
|
||||
};
|
||||
|
||||
// next_id is TEXT because node:sqlite rejects INTEGER results outside JavaScript's safe integer range.
|
||||
const INITIAL_SCHEMA: readonly string[] = [
|
||||
`CREATE TABLE durable_metadata (
|
||||
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
||||
next_id TEXT NOT NULL,
|
||||
next_seq INTEGER NOT NULL
|
||||
) STRICT`,
|
||||
`INSERT INTO durable_metadata (singleton, next_id, next_seq) VALUES (1, '2', 1)`,
|
||||
`CREATE TABLE record_ids (
|
||||
id INTEGER PRIMARY KEY,
|
||||
record_type TEXT NOT NULL CHECK (record_type IN ('conversation', 'entry', 'task', 'submission', 'document'))
|
||||
) STRICT`,
|
||||
`CREATE TABLE conversations (
|
||||
id INTEGER PRIMARY KEY,
|
||||
record TEXT NOT NULL CHECK (json_valid(record))
|
||||
) STRICT`,
|
||||
`CREATE TABLE entries (
|
||||
id INTEGER PRIMARY KEY,
|
||||
conversation_id INTEGER NOT NULL,
|
||||
head INTEGER,
|
||||
commit_seq INTEGER NOT NULL,
|
||||
record TEXT NOT NULL CHECK (json_valid(record))
|
||||
) STRICT`,
|
||||
"CREATE INDEX entries_by_conversation ON entries (conversation_id, id DESC)",
|
||||
"CREATE INDEX entry_heads_by_conversation ON entries (conversation_id, id DESC) WHERE head IS NOT NULL",
|
||||
`CREATE TABLE tasks (
|
||||
id INTEGER PRIMARY KEY,
|
||||
conversation_id INTEGER NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
status TEXT NOT NULL CHECK (status IN ('pending', 'running', 'terminal')),
|
||||
abort_requested INTEGER NOT NULL CHECK (abort_requested IN (0, 1)),
|
||||
background INTEGER NOT NULL CHECK (background IN (0, 1)),
|
||||
record TEXT NOT NULL CHECK (json_valid(record))
|
||||
) STRICT`,
|
||||
"CREATE INDEX tasks_by_status ON tasks (status, id)",
|
||||
"CREATE INDEX tasks_by_conversation ON tasks (conversation_id, id)",
|
||||
"CREATE INDEX tasks_by_kind ON tasks (kind, id)",
|
||||
"CREATE INDEX tasks_by_abort_requested ON tasks (abort_requested, id)",
|
||||
"CREATE INDEX tasks_by_background ON tasks (background, id)",
|
||||
`CREATE TABLE submissions (
|
||||
id INTEGER PRIMARY KEY,
|
||||
conversation_id INTEGER NOT NULL,
|
||||
request_id TEXT,
|
||||
record TEXT NOT NULL CHECK (json_valid(record))
|
||||
) STRICT`,
|
||||
"CREATE INDEX submissions_by_request ON submissions (conversation_id, request_id)",
|
||||
`CREATE TABLE documents (
|
||||
id INTEGER PRIMARY KEY,
|
||||
kind TEXT NOT NULL,
|
||||
family INTEGER NOT NULL CHECK (family IN (0, 1)),
|
||||
key_value TEXT NOT NULL,
|
||||
scope_kind TEXT NOT NULL CHECK (scope_kind IN ('session', 'conversation', 'task')),
|
||||
owner_id INTEGER NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
retired_at INTEGER,
|
||||
record TEXT NOT NULL CHECK (json_valid(record))
|
||||
) STRICT`,
|
||||
`CREATE INDEX documents_by_address
|
||||
ON documents (kind, scope_kind, owner_id, family, key_value, created_at DESC, retired_at)`,
|
||||
"CREATE INDEX documents_by_scope ON documents (scope_kind, owner_id, id)",
|
||||
"CREATE INDEX documents_by_scope_kind ON documents (scope_kind, owner_id, kind, id)",
|
||||
`CREATE TABLE document_revisions (
|
||||
document_id INTEGER NOT NULL,
|
||||
seq INTEGER NOT NULL,
|
||||
kind TEXT NOT NULL CHECK (kind IN ('base', 'delta')),
|
||||
version INTEGER NOT NULL,
|
||||
content TEXT NOT NULL CHECK (json_valid(content)),
|
||||
PRIMARY KEY (document_id, seq)
|
||||
) STRICT`,
|
||||
"CREATE INDEX document_revisions_by_kind ON document_revisions (document_id, kind, seq DESC)",
|
||||
];
|
||||
|
||||
/** Immutable, ordered schema history. Append new migrations; never edit released ones. */
|
||||
export const SQLITE_MIGRATIONS: readonly SqliteMigration[] = [{ version: 1, statements: INITIAL_SCHEMA }];
|
||||
|
||||
export const CURRENT_SQLITE_SCHEMA_VERSION = SQLITE_MIGRATIONS.at(-1)?.version ?? 0;
|
||||
|
||||
type SchemaRow = { readonly version: number };
|
||||
|
||||
/** Apply all pending schema migrations atomically. */
|
||||
export async function applySqliteMigrations(
|
||||
database: SqliteDatabase,
|
||||
migrations: readonly SqliteMigration[] = SQLITE_MIGRATIONS,
|
||||
): Promise<void> {
|
||||
for (let index = 0; index < migrations.length; index++) {
|
||||
if (migrations[index]?.version !== index + 1) {
|
||||
throw new Error("Durable SQLite migrations must have contiguous versions starting at 1");
|
||||
}
|
||||
}
|
||||
|
||||
await database.transaction(() => {
|
||||
database.exec(`CREATE TABLE IF NOT EXISTS durable_schema (
|
||||
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
||||
version INTEGER NOT NULL CHECK (version >= 0)
|
||||
) STRICT`);
|
||||
database.prepare("INSERT OR IGNORE INTO durable_schema (singleton, version) VALUES (1, 0)").run();
|
||||
const row = database.prepare("SELECT version FROM durable_schema WHERE singleton = 1").get<SchemaRow>();
|
||||
if (row === undefined) throw new Error("Durable SQLite schema metadata is missing");
|
||||
const currentVersion = migrations.at(-1)?.version ?? 0;
|
||||
if (row.version > currentVersion) {
|
||||
throw new Error(
|
||||
`Durable SQLite schema version ${row.version} is newer than supported version ${currentVersion}`,
|
||||
);
|
||||
}
|
||||
for (const migration of migrations) {
|
||||
if (migration.version <= row.version) continue;
|
||||
for (const statement of migration.statements) database.exec(statement);
|
||||
database.prepare("UPDATE durable_schema SET version = ? WHERE singleton = 1").run(migration.version);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import { dirname } from "node:path";
|
||||
import type { SQLInputValue, StatementSync } from "node:sqlite";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import type { SqliteDatabase, SqliteStatement, SqliteValue } from "./database.ts";
|
||||
import { SqliteStorage } from "./storage.ts";
|
||||
|
||||
/** Node SQLite connection settings for a durable storage file. */
|
||||
export type NodeSqliteStorageOptions = {
|
||||
/** SQLite WAL auto-checkpoint threshold. SQLite and this adapter default to 1,000 pages; 0 disables it. */
|
||||
readonly walAutoCheckpointPages?: number;
|
||||
/** Time SQLite waits for a competing file lock. SQLite defaults to 0; this adapter defaults to 5,000 ms. */
|
||||
readonly busyTimeoutMs?: number;
|
||||
};
|
||||
|
||||
const DEFAULT_WAL_AUTO_CHECKPOINT_PAGES = 1_000;
|
||||
const DEFAULT_BUSY_TIMEOUT_MS = 5_000;
|
||||
|
||||
class NodeSqliteStatement implements SqliteStatement {
|
||||
private readonly statement: StatementSync;
|
||||
|
||||
constructor(statement: StatementSync) {
|
||||
this.statement = statement;
|
||||
}
|
||||
|
||||
run(...params: SqliteValue[]): void {
|
||||
this.statement.run(...(params as SQLInputValue[]));
|
||||
}
|
||||
|
||||
get<T extends object>(...params: SqliteValue[]): T | undefined {
|
||||
return this.statement.get(...(params as SQLInputValue[])) as T | undefined;
|
||||
}
|
||||
|
||||
all<T extends object>(...params: SqliteValue[]): T[] {
|
||||
return this.statement.all(...(params as SQLInputValue[])) as T[];
|
||||
}
|
||||
}
|
||||
|
||||
/** `SqliteDatabase` adapter backed by Node's built-in `node:sqlite`. */
|
||||
export class NodeSqliteDatabase implements SqliteDatabase {
|
||||
private readonly database: DatabaseSync;
|
||||
private closed = false;
|
||||
|
||||
constructor(database: DatabaseSync) {
|
||||
this.database = database;
|
||||
}
|
||||
|
||||
exec(sql: string): void {
|
||||
this.database.exec(sql);
|
||||
}
|
||||
|
||||
prepare(sql: string): SqliteStatement {
|
||||
return new NodeSqliteStatement(this.database.prepare(sql));
|
||||
}
|
||||
|
||||
transaction<T>(callback: () => T): T {
|
||||
this.database.exec("BEGIN IMMEDIATE");
|
||||
try {
|
||||
const result = callback();
|
||||
if (
|
||||
result !== null &&
|
||||
(typeof result === "object" || typeof result === "function") &&
|
||||
typeof Reflect.get(result, "then") === "function"
|
||||
) {
|
||||
throw new TypeError("SQLite transaction callbacks must be synchronous");
|
||||
}
|
||||
this.database.exec("COMMIT");
|
||||
return result;
|
||||
} catch (error) {
|
||||
try {
|
||||
this.database.exec("ROLLBACK");
|
||||
} catch {
|
||||
// Preserve the error that caused the transaction to fail.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
try {
|
||||
this.database.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
||||
} finally {
|
||||
this.database.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Open and configure a Node-backed SQLite database facade. */
|
||||
export async function openNodeSqliteDatabase(
|
||||
path: string,
|
||||
options: NodeSqliteStorageOptions = {},
|
||||
): Promise<NodeSqliteDatabase> {
|
||||
const checkpointPages = options.walAutoCheckpointPages ?? DEFAULT_WAL_AUTO_CHECKPOINT_PAGES;
|
||||
const timeout = options.busyTimeoutMs ?? DEFAULT_BUSY_TIMEOUT_MS;
|
||||
if (path !== ":memory:") await mkdir(dirname(path), { recursive: true });
|
||||
const database = new DatabaseSync(path, { timeout });
|
||||
const adapter = new NodeSqliteDatabase(database);
|
||||
try {
|
||||
adapter.exec("PRAGMA journal_mode = WAL");
|
||||
adapter.exec("PRAGMA synchronous = NORMAL");
|
||||
adapter.exec(`PRAGMA wal_autocheckpoint = ${checkpointPages}`);
|
||||
return adapter;
|
||||
} catch (error) {
|
||||
try {
|
||||
adapter.close();
|
||||
} catch {
|
||||
// Preserve the configuration failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Open or create file-backed durable storage using Node's built-in SQLite. */
|
||||
export async function openNodeSqliteStorage(
|
||||
path: string,
|
||||
options: NodeSqliteStorageOptions = {},
|
||||
): Promise<SqliteStorage> {
|
||||
return SqliteStorage.open(await openNodeSqliteDatabase(path, options));
|
||||
}
|
||||
@@ -0,0 +1,704 @@
|
||||
import type { Context, JsonValue } from "@earendil-works/chord";
|
||||
import { applyImmutable, type Op } from "@earendil-works/chord/delta";
|
||||
import type {
|
||||
ConversationRecord,
|
||||
Cursor,
|
||||
DocumentAddress,
|
||||
DocumentContent,
|
||||
DocumentCreate,
|
||||
DocumentPoint,
|
||||
DocumentQuery,
|
||||
DocumentRecord,
|
||||
EntryQuery,
|
||||
EntryRecord,
|
||||
Id,
|
||||
JsonObject,
|
||||
Page,
|
||||
Seq,
|
||||
Storage,
|
||||
StorageWrite,
|
||||
StoredDocument,
|
||||
SubmissionRecord,
|
||||
TaskQuery,
|
||||
TaskRecord,
|
||||
} from "../../types.ts";
|
||||
import type { SqliteDatabase, SqliteStatement, SqliteValue } from "./database.ts";
|
||||
import { applySqliteMigrations } from "./migrations.ts";
|
||||
|
||||
type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
|
||||
type TableName = "conversation" | "entry" | "task" | "submission" | "document";
|
||||
type RecordIdRow = { readonly record_type: TableName };
|
||||
type JsonRow = { readonly record: string };
|
||||
type EntryJsonRow = { readonly record: string; readonly commit_seq: number };
|
||||
type RevisionRow = {
|
||||
readonly seq: number;
|
||||
readonly kind: DocumentContent["kind"];
|
||||
readonly version: number;
|
||||
readonly content: string;
|
||||
};
|
||||
type IdRow = { readonly id: number };
|
||||
type MetadataRow = { readonly next_id: string; readonly next_seq: number };
|
||||
type DocumentAction = {
|
||||
create?: DocumentCreate;
|
||||
content?: DocumentContent;
|
||||
retire: boolean;
|
||||
};
|
||||
type ScopeColumns = {
|
||||
readonly scopeKind: DocumentRecord["scope"]["kind"];
|
||||
readonly ownerId: number;
|
||||
};
|
||||
|
||||
const parseJson = <T>(value: string): T => JSON.parse(value) as T;
|
||||
const encodeJson = (value: unknown): string => JSON.stringify(value) as string;
|
||||
// Some SQLite bindings replace lone UTF-16 surrogates. JSON encoding keeps indexed identities lossless.
|
||||
const encodeIndexedString = (value: string): string => JSON.stringify(value);
|
||||
|
||||
const getRow = <T extends object>(statement: SqliteStatement, ...params: SqliteValue[]): T | undefined =>
|
||||
statement.get<T>(...params);
|
||||
|
||||
const allRows = <T extends object>(statement: SqliteStatement, ...params: SqliteValue[]): T[] =>
|
||||
statement.all<T>(...params);
|
||||
|
||||
const cursorId = (cursor: Cursor | undefined): Id | undefined => cursor?.after as Id | undefined;
|
||||
|
||||
const page = <T extends { readonly id: Id }>(values: readonly T[], limit: number): Page<T, Cursor> => {
|
||||
const items = values.slice(0, limit);
|
||||
if (values.length <= limit) return { items };
|
||||
return { items, next: { after: items.at(-1)!.id } };
|
||||
};
|
||||
|
||||
const scopeColumns = (scope: DocumentRecord["scope"]): ScopeColumns => {
|
||||
switch (scope.kind) {
|
||||
case "session":
|
||||
return { scopeKind: "session", ownerId: 0 };
|
||||
case "conversation":
|
||||
return { scopeKind: "conversation", ownerId: scope.conversationId };
|
||||
case "task":
|
||||
return { scopeKind: "task", ownerId: scope.taskId };
|
||||
}
|
||||
};
|
||||
|
||||
const addressParts = (address: DocumentAddress | DocumentCreate | DocumentRecord) => {
|
||||
const scope = scopeColumns(address.scope);
|
||||
return {
|
||||
kind: encodeIndexedString(address.kind),
|
||||
...scope,
|
||||
family: address.key === undefined ? 0 : 1,
|
||||
keyValue: encodeIndexedString(address.key ?? ""),
|
||||
};
|
||||
};
|
||||
|
||||
const addressKey = (address: DocumentAddress | DocumentCreate | DocumentRecord): string => {
|
||||
const parts = addressParts(address);
|
||||
return JSON.stringify([parts.kind, parts.scopeKind, parts.ownerId, parts.family, parts.keyValue]);
|
||||
};
|
||||
|
||||
const isAliveAt = (record: DocumentRecord, at: DocumentPoint): boolean => {
|
||||
if (at === "current") return record.retiredAt === undefined;
|
||||
return record.createdAt <= at && (record.retiredAt === undefined || at < record.retiredAt);
|
||||
};
|
||||
|
||||
const isCurrentOnly = (record: DocumentRecord): boolean =>
|
||||
record.scope.kind !== "conversation" || record.history === "latest";
|
||||
|
||||
const writeId = (write: StorageWrite): Id | undefined => {
|
||||
switch (write.type) {
|
||||
case "conversation":
|
||||
case "entry":
|
||||
case "task":
|
||||
case "submission":
|
||||
return write.value.id;
|
||||
case "document.create":
|
||||
return write.record.id;
|
||||
case "document.change":
|
||||
case "document.retire":
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
class StatementCachingDatabase implements SqliteDatabase {
|
||||
private readonly database: SqliteDatabase;
|
||||
private readonly statements = new Map<string, SqliteStatement>();
|
||||
|
||||
constructor(database: SqliteDatabase) {
|
||||
this.database = database;
|
||||
}
|
||||
|
||||
exec(sql: string): void {
|
||||
this.database.exec(sql);
|
||||
}
|
||||
|
||||
prepare(sql: string): SqliteStatement {
|
||||
let statement = this.statements.get(sql);
|
||||
if (statement === undefined) {
|
||||
statement = this.database.prepare(sql);
|
||||
this.statements.set(sql, statement);
|
||||
}
|
||||
return statement;
|
||||
}
|
||||
|
||||
transaction<T>(callback: () => T): T | Promise<T> {
|
||||
return this.database.transaction(callback);
|
||||
}
|
||||
|
||||
close(): void | Promise<void> {
|
||||
this.statements.clear();
|
||||
return this.database.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Portable SQLite implementation of the Pico storage contract. */
|
||||
export class SqliteStorage implements Storage {
|
||||
private readonly db: SqliteDatabase;
|
||||
private nextId: Id;
|
||||
private closed = false;
|
||||
|
||||
private constructor(db: SqliteDatabase, nextId: Id) {
|
||||
this.db = new StatementCachingDatabase(db);
|
||||
this.nextId = nextId;
|
||||
}
|
||||
|
||||
/** Initialize storage over an owned SQLite database facade. */
|
||||
static async open(db: SqliteDatabase): Promise<SqliteStorage> {
|
||||
try {
|
||||
await applySqliteMigrations(db);
|
||||
const metadata = getRow<MetadataRow>(
|
||||
db.prepare("SELECT next_id, next_seq FROM durable_metadata WHERE singleton = 1"),
|
||||
);
|
||||
if (metadata === undefined) throw new Error("Durable SQLite metadata is missing");
|
||||
return new SqliteStorage(db, Number(metadata.next_id));
|
||||
} catch (error) {
|
||||
try {
|
||||
await db.close();
|
||||
} catch {
|
||||
// Preserve the initialization failure.
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async commit(writes: readonly StorageWrite[], _context: Context): Promise<Seq> {
|
||||
this.assertOpen();
|
||||
const documentActions = this.prepareDocumentActions(writes);
|
||||
const candidateNextId = this.candidateNextId(writes);
|
||||
const seq = await this.db.transaction(() => {
|
||||
const metadata = getRow<MetadataRow>(
|
||||
this.db.prepare("SELECT next_id, next_seq FROM durable_metadata WHERE singleton = 1"),
|
||||
);
|
||||
if (metadata === undefined) throw new Error("Durable SQLite metadata is missing");
|
||||
const committedSeq = metadata.next_seq;
|
||||
this.checkGlobalIds(writes);
|
||||
this.checkDocumentActions(documentActions);
|
||||
for (const write of writes) this.applyTableWrite(write, committedSeq);
|
||||
this.applyDocumentActions(documentActions, committedSeq);
|
||||
this.db
|
||||
.prepare("UPDATE durable_metadata SET next_id = ?, next_seq = ? WHERE singleton = 1")
|
||||
.run(String(Math.max(Number(metadata.next_id), candidateNextId)), committedSeq + 1);
|
||||
return committedSeq;
|
||||
});
|
||||
this.nextId = Math.max(this.nextId, candidateNextId);
|
||||
return seq;
|
||||
}
|
||||
|
||||
async mintId(): Promise<Id> {
|
||||
this.assertOpen();
|
||||
if (!Number.isSafeInteger(this.nextId)) throw new Error("ID space is exhausted");
|
||||
return this.nextId++;
|
||||
}
|
||||
|
||||
async conversation(id: Id, _context: Context): Promise<ConversationRecord | undefined> {
|
||||
this.assertOpen();
|
||||
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM conversations WHERE id = ?"), id);
|
||||
return row === undefined ? undefined : parseJson<ConversationRecord>(row.record);
|
||||
}
|
||||
|
||||
async scanConversations(
|
||||
cursor: Cursor | undefined,
|
||||
limit: number,
|
||||
_context: Context,
|
||||
): Promise<Page<ConversationRecord, Cursor>> {
|
||||
this.assertOpen();
|
||||
const after = cursorId(cursor) ?? -1;
|
||||
const rows = allRows<JsonRow>(
|
||||
this.db.prepare("SELECT record FROM conversations WHERE id > ? ORDER BY id LIMIT ?"),
|
||||
after,
|
||||
limit + 1,
|
||||
);
|
||||
return page(
|
||||
rows.map((row) => parseJson<ConversationRecord>(row.record)),
|
||||
limit,
|
||||
);
|
||||
}
|
||||
|
||||
async entry(
|
||||
id: Id,
|
||||
_context: Context,
|
||||
): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined> {
|
||||
this.assertOpen();
|
||||
const row = getRow<EntryJsonRow>(this.db.prepare("SELECT record, commit_seq FROM entries WHERE id = ?"), id);
|
||||
return row === undefined ? undefined : { entry: parseJson<EntryRecord>(row.record), commitSeq: row.commit_seq };
|
||||
}
|
||||
|
||||
async findLatestHeadMarker(
|
||||
conversationId: Id,
|
||||
atOrBeforeEntryId: Id | undefined,
|
||||
_context: Context,
|
||||
): Promise<(EntryRecord & { readonly head: Id }) | undefined> {
|
||||
this.assertOpen();
|
||||
let conversation = this.readConversation(conversationId);
|
||||
if (conversation === undefined) throw new Error(`Unknown conversation: ${conversationId}`);
|
||||
let upper = atOrBeforeEntryId;
|
||||
while (true) {
|
||||
const row =
|
||||
upper === undefined
|
||||
? getRow<JsonRow>(
|
||||
this.db.prepare(
|
||||
"SELECT record FROM entries WHERE conversation_id = ? AND head IS NOT NULL ORDER BY id DESC LIMIT 1",
|
||||
),
|
||||
conversation.id,
|
||||
)
|
||||
: getRow<JsonRow>(
|
||||
this.db.prepare(
|
||||
"SELECT record FROM entries WHERE conversation_id = ? AND head IS NOT NULL AND id <= ? ORDER BY id DESC LIMIT 1",
|
||||
),
|
||||
conversation.id,
|
||||
upper,
|
||||
);
|
||||
if (row !== undefined) return parseJson<EntryRecord & { readonly head: Id }>(row.record);
|
||||
if (conversation.parent === undefined) return undefined;
|
||||
upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at);
|
||||
conversation = this.readConversation(conversation.parent.conversationId)!;
|
||||
}
|
||||
}
|
||||
|
||||
async scanEntries(
|
||||
query: EntryQuery,
|
||||
cursor: Cursor | undefined,
|
||||
limit: number,
|
||||
_context: Context,
|
||||
): Promise<Page<EntryRecord, Cursor>> {
|
||||
this.assertOpen();
|
||||
let conversation = this.readConversation(query.conversationId);
|
||||
if (conversation === undefined) throw new Error(`Unknown conversation: ${query.conversationId}`);
|
||||
const after = cursorId(cursor);
|
||||
let upper = query.maxEntryId;
|
||||
if (after !== undefined) upper = Math.min(upper ?? Number.MAX_SAFE_INTEGER, after - 1);
|
||||
const values: EntryRecord[] = [];
|
||||
while (true) {
|
||||
const clauses = ["conversation_id = ?"];
|
||||
const params: SqliteValue[] = [conversation.id];
|
||||
if (query.minEntryId !== undefined) {
|
||||
clauses.push("id >= ?");
|
||||
params.push(query.minEntryId);
|
||||
}
|
||||
if (upper !== undefined) {
|
||||
clauses.push("id <= ?");
|
||||
params.push(upper);
|
||||
}
|
||||
params.push(limit + 1 - values.length);
|
||||
const rows = allRows<JsonRow>(
|
||||
this.db.prepare(`SELECT record FROM entries WHERE ${clauses.join(" AND ")} ORDER BY id DESC LIMIT ?`),
|
||||
...params,
|
||||
);
|
||||
values.push(...rows.map((row) => parseJson<EntryRecord>(row.record)));
|
||||
if (values.length > limit || conversation.parent === undefined) break;
|
||||
upper = upper === undefined ? conversation.parent.at : Math.min(upper, conversation.parent.at);
|
||||
if (query.minEntryId !== undefined && upper < query.minEntryId) break;
|
||||
conversation = this.readConversation(conversation.parent.conversationId)!;
|
||||
}
|
||||
return page(values, limit);
|
||||
}
|
||||
|
||||
async task(id: Id, _context: Context): Promise<StoredTask | undefined> {
|
||||
this.assertOpen();
|
||||
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM tasks WHERE id = ?"), id);
|
||||
return row === undefined ? undefined : parseJson<StoredTask>(row.record);
|
||||
}
|
||||
|
||||
async scanTasks(
|
||||
query: TaskQuery,
|
||||
cursor: Cursor | undefined,
|
||||
limit: number,
|
||||
_context: Context,
|
||||
): Promise<Page<StoredTask, Cursor>> {
|
||||
this.assertOpen();
|
||||
const clauses = ["id > ?"];
|
||||
const params: SqliteValue[] = [cursorId(cursor) ?? -1];
|
||||
if (query.conversationId !== undefined) {
|
||||
clauses.push("conversation_id = ?");
|
||||
params.push(query.conversationId);
|
||||
}
|
||||
if (query.kind !== undefined) {
|
||||
clauses.push("kind = ?");
|
||||
params.push(encodeIndexedString(query.kind));
|
||||
}
|
||||
if (query.status !== undefined) {
|
||||
clauses.push("status = ?");
|
||||
params.push(query.status);
|
||||
}
|
||||
if (query.abortRequested !== undefined) {
|
||||
clauses.push("abort_requested = ?");
|
||||
params.push(query.abortRequested ? 1 : 0);
|
||||
}
|
||||
if (query.background !== undefined) {
|
||||
clauses.push("background = ?");
|
||||
params.push(query.background ? 1 : 0);
|
||||
}
|
||||
params.push(limit + 1);
|
||||
const rows = allRows<JsonRow>(
|
||||
this.db.prepare(`SELECT record FROM tasks WHERE ${clauses.join(" AND ")} ORDER BY id LIMIT ?`),
|
||||
...params,
|
||||
);
|
||||
return page(
|
||||
rows.map((row) => parseJson<StoredTask>(row.record)),
|
||||
limit,
|
||||
);
|
||||
}
|
||||
|
||||
async submission(id: Id, _context: Context): Promise<SubmissionRecord | undefined> {
|
||||
this.assertOpen();
|
||||
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM submissions WHERE id = ?"), id);
|
||||
return row === undefined ? undefined : parseJson<SubmissionRecord>(row.record);
|
||||
}
|
||||
|
||||
async submissionByRequest(
|
||||
conversationId: Id,
|
||||
requestId: string,
|
||||
_context: Context,
|
||||
): Promise<SubmissionRecord | undefined> {
|
||||
this.assertOpen();
|
||||
const row = getRow<JsonRow>(
|
||||
this.db.prepare("SELECT record FROM submissions WHERE conversation_id = ? AND request_id = ?"),
|
||||
conversationId,
|
||||
encodeIndexedString(requestId),
|
||||
);
|
||||
return row === undefined ? undefined : parseJson<SubmissionRecord>(row.record);
|
||||
}
|
||||
|
||||
async findDocument(
|
||||
address: DocumentAddress,
|
||||
at: DocumentPoint,
|
||||
_context: Context,
|
||||
): Promise<DocumentRecord | undefined> {
|
||||
this.assertOpen();
|
||||
const parts = addressParts(address);
|
||||
const statement =
|
||||
at === "current"
|
||||
? this.db.prepare(`SELECT record FROM documents
|
||||
WHERE kind = ? AND scope_kind = ? AND owner_id = ? AND family = ? AND key_value = ?
|
||||
AND retired_at IS NULL ORDER BY created_at DESC LIMIT 1`)
|
||||
: this.db.prepare(`SELECT record FROM documents
|
||||
WHERE kind = ? AND scope_kind = ? AND owner_id = ? AND family = ? AND key_value = ?
|
||||
AND created_at <= ? AND (retired_at IS NULL OR retired_at > ?)
|
||||
ORDER BY created_at DESC LIMIT 1`);
|
||||
const params: SqliteValue[] = [parts.kind, parts.scopeKind, parts.ownerId, parts.family, parts.keyValue];
|
||||
if (at !== "current") params.push(at, at);
|
||||
const row = getRow<JsonRow>(statement, ...params);
|
||||
return row === undefined ? undefined : parseJson<DocumentRecord>(row.record);
|
||||
}
|
||||
|
||||
async document(id: Id, at: DocumentPoint, _context: Context): Promise<StoredDocument | undefined> {
|
||||
this.assertOpen();
|
||||
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM documents WHERE id = ?"), id);
|
||||
if (row === undefined) return undefined;
|
||||
const record = parseJson<DocumentRecord>(row.record);
|
||||
if (at !== "current" && isCurrentOnly(record)) {
|
||||
throw new Error(`Document ${id} does not retain historical content`);
|
||||
}
|
||||
if (!isAliveAt(record, at)) return undefined;
|
||||
const upper = at === "current" ? Number.MAX_SAFE_INTEGER : at;
|
||||
const base = getRow<RevisionRow>(
|
||||
this.db.prepare(`SELECT seq, kind, version, content FROM document_revisions
|
||||
WHERE document_id = ? AND kind = 'base' AND seq <= ? ORDER BY seq DESC LIMIT 1`),
|
||||
id,
|
||||
upper,
|
||||
);
|
||||
if (base === undefined) throw new Error(`Document ${id} is missing a required base`);
|
||||
let value = parseJson<JsonObject>(base.content);
|
||||
const tail = allRows<RevisionRow>(
|
||||
this.db.prepare(`SELECT seq, kind, version, content FROM document_revisions
|
||||
WHERE document_id = ? AND seq > ? AND seq <= ? ORDER BY seq`),
|
||||
id,
|
||||
base.seq,
|
||||
upper,
|
||||
);
|
||||
for (const revision of tail) {
|
||||
if (revision.kind !== "delta" || revision.version !== base.version) {
|
||||
throw new Error(`Document ${id} crosses a stored version boundary without a base`);
|
||||
}
|
||||
value = applyImmutable(value, parseJson<readonly Op[]>(revision.content)) as JsonObject;
|
||||
}
|
||||
return { record, version: base.version, value };
|
||||
}
|
||||
|
||||
async scanDocuments(
|
||||
query: DocumentQuery,
|
||||
cursor: Cursor | undefined,
|
||||
limit: number,
|
||||
_context: Context,
|
||||
): Promise<Page<DocumentRecord, Cursor>> {
|
||||
this.assertOpen();
|
||||
const scope = scopeColumns(query.scope);
|
||||
const clauses = ["scope_kind = ?", "owner_id = ?", "id > ?"];
|
||||
const params: SqliteValue[] = [scope.scopeKind, scope.ownerId, cursorId(cursor) ?? -1];
|
||||
if (query.kind !== undefined) {
|
||||
clauses.push("kind = ?");
|
||||
params.push(encodeIndexedString(query.kind));
|
||||
}
|
||||
if (query.at === "current") {
|
||||
clauses.push("retired_at IS NULL");
|
||||
} else {
|
||||
clauses.push("created_at <= ?", "(retired_at IS NULL OR retired_at > ?)");
|
||||
params.push(query.at, query.at);
|
||||
}
|
||||
params.push(limit + 1);
|
||||
const rows = allRows<JsonRow>(
|
||||
this.db.prepare(`SELECT record FROM documents WHERE ${clauses.join(" AND ")} ORDER BY id LIMIT ?`),
|
||||
...params,
|
||||
);
|
||||
return page(
|
||||
rows.map((row) => parseJson<DocumentRecord>(row.record)),
|
||||
limit,
|
||||
);
|
||||
}
|
||||
|
||||
async close(_context: Context): Promise<void> {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
await this.db.close();
|
||||
}
|
||||
|
||||
private readConversation(id: Id): ConversationRecord | undefined {
|
||||
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM conversations WHERE id = ?"), id);
|
||||
return row === undefined ? undefined : parseJson<ConversationRecord>(row.record);
|
||||
}
|
||||
|
||||
private candidateNextId(writes: readonly StorageWrite[]): Id {
|
||||
let nextId = this.nextId;
|
||||
for (const write of writes) {
|
||||
const id = writeId(write);
|
||||
if (id !== undefined) nextId = Math.max(nextId, id + 1);
|
||||
}
|
||||
return nextId;
|
||||
}
|
||||
|
||||
private checkGlobalIds(writes: readonly StorageWrite[]): void {
|
||||
const claimed = new Map<Id, TableName>();
|
||||
const lookup = this.db.prepare("SELECT record_type FROM record_ids WHERE id = ?");
|
||||
for (const write of writes) {
|
||||
if (write.type === "document.change" || write.type === "document.retire") continue;
|
||||
const table: TableName = write.type === "document.create" ? "document" : write.type;
|
||||
const id = write.type === "document.create" ? write.record.id : write.value.id;
|
||||
const existing = getRow<RecordIdRow>(lookup, id)?.record_type;
|
||||
const earlier = claimed.get(id);
|
||||
if (table === "conversation" || table === "entry" || table === "document") {
|
||||
if (existing !== undefined) throw new Error(`ID ${id} already belongs to ${existing}`);
|
||||
if (earlier !== undefined) throw new Error(`ID ${id} is written more than once`);
|
||||
} else {
|
||||
if (existing !== undefined && existing !== table)
|
||||
throw new Error(`ID ${id} already belongs to ${existing}`);
|
||||
if (earlier !== undefined && earlier !== table) throw new Error(`ID ${id} is written as two record types`);
|
||||
}
|
||||
claimed.set(id, table);
|
||||
}
|
||||
}
|
||||
|
||||
private prepareDocumentActions(writes: readonly StorageWrite[]): Map<Id, DocumentAction> {
|
||||
const actions = new Map<Id, DocumentAction>();
|
||||
for (const write of writes) {
|
||||
if (write.type !== "document.create" && write.type !== "document.change" && write.type !== "document.retire") {
|
||||
continue;
|
||||
}
|
||||
const id = write.type === "document.create" ? write.record.id : write.id;
|
||||
let action = actions.get(id);
|
||||
if (action === undefined) {
|
||||
action = { retire: false };
|
||||
actions.set(id, action);
|
||||
}
|
||||
switch (write.type) {
|
||||
case "document.create":
|
||||
if (action.create !== undefined || action.content !== undefined) {
|
||||
throw new Error(`Document ${id} has more than one content command`);
|
||||
}
|
||||
action.create = write.record;
|
||||
action.content = write.content;
|
||||
break;
|
||||
case "document.change":
|
||||
if (action.content !== undefined) throw new Error(`Document ${id} has more than one content command`);
|
||||
action.content = write.content;
|
||||
break;
|
||||
case "document.retire":
|
||||
if (action.retire) throw new Error(`Document ${id} is retired more than once`);
|
||||
action.retire = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return actions;
|
||||
}
|
||||
|
||||
private checkDocumentActions(actions: ReadonlyMap<Id, DocumentAction>): void {
|
||||
const liveCounts = new Map<string, number>();
|
||||
for (const [id, action] of actions) {
|
||||
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM documents WHERE id = ?"), id);
|
||||
const existing = row === undefined ? undefined : parseJson<DocumentRecord>(row.record);
|
||||
if (action.create === undefined && existing === undefined) throw new Error(`Unknown document: ${id}`);
|
||||
if (action.create !== undefined && existing !== undefined) throw new Error(`Document ${id} already exists`);
|
||||
if (existing?.retiredAt !== undefined) throw new Error(`Document ${id} is retired`);
|
||||
if (action.create !== undefined && action.content?.kind !== "base") {
|
||||
throw new Error(`Document ${id} creation requires a base`);
|
||||
}
|
||||
if (action.content?.kind === "delta") {
|
||||
const previous = getRow<{ readonly version: number }>(
|
||||
this.db.prepare(
|
||||
"SELECT version FROM document_revisions WHERE document_id = ? ORDER BY seq DESC LIMIT 1",
|
||||
),
|
||||
id,
|
||||
);
|
||||
if (previous === undefined) throw new Error(`Document ${id} delta has no base`);
|
||||
if (previous.version !== action.content.version) {
|
||||
throw new Error(`Document ${id} version transition requires a base`);
|
||||
}
|
||||
}
|
||||
const record = action.create ?? existing!;
|
||||
const key = addressKey(record);
|
||||
let live = liveCounts.get(key);
|
||||
if (live === undefined) live = this.currentDocumentId(record) === undefined ? 0 : 1;
|
||||
if (action.retire && existing !== undefined) live--;
|
||||
if (action.create !== undefined && !action.retire) live++;
|
||||
liveCounts.set(key, live);
|
||||
}
|
||||
for (const live of liveCounts.values()) {
|
||||
if (live > 1) throw new Error("Document address already has a current incarnation");
|
||||
}
|
||||
}
|
||||
|
||||
private currentDocumentId(address: DocumentAddress | DocumentCreate | DocumentRecord): Id | undefined {
|
||||
const parts = addressParts(address);
|
||||
return getRow<IdRow>(
|
||||
this.db.prepare(`SELECT id FROM documents
|
||||
WHERE kind = ? AND scope_kind = ? AND owner_id = ? AND family = ? AND key_value = ? AND retired_at IS NULL
|
||||
LIMIT 1`),
|
||||
parts.kind,
|
||||
parts.scopeKind,
|
||||
parts.ownerId,
|
||||
parts.family,
|
||||
parts.keyValue,
|
||||
)?.id;
|
||||
}
|
||||
|
||||
private applyTableWrite(write: StorageWrite, seq: Seq): void {
|
||||
switch (write.type) {
|
||||
case "conversation":
|
||||
this.claimId(write.value.id, "conversation");
|
||||
this.db
|
||||
.prepare("INSERT INTO conversations (id, record) VALUES (?, ?)")
|
||||
.run(write.value.id, encodeJson(write.value));
|
||||
break;
|
||||
case "entry":
|
||||
this.claimId(write.value.id, "entry");
|
||||
this.db
|
||||
.prepare("INSERT INTO entries (id, conversation_id, head, commit_seq, record) VALUES (?, ?, ?, ?, ?)")
|
||||
.run(write.value.id, write.value.conversationId, write.value.head ?? null, seq, encodeJson(write.value));
|
||||
break;
|
||||
case "task":
|
||||
this.claimId(write.value.id, "task");
|
||||
this.db
|
||||
.prepare(`INSERT INTO tasks (id, conversation_id, kind, status, abort_requested, background, record)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET conversation_id = excluded.conversation_id, kind = excluded.kind,
|
||||
status = excluded.status, abort_requested = excluded.abort_requested,
|
||||
background = excluded.background, record = excluded.record`)
|
||||
.run(
|
||||
write.value.id,
|
||||
write.value.conversationId,
|
||||
encodeIndexedString(write.value.kind),
|
||||
write.value.state.status,
|
||||
write.value.abortRequested ? 1 : 0,
|
||||
write.value.background ? 1 : 0,
|
||||
encodeJson(write.value),
|
||||
);
|
||||
break;
|
||||
case "submission":
|
||||
this.claimId(write.value.id, "submission");
|
||||
this.db
|
||||
.prepare(`INSERT INTO submissions (id, conversation_id, request_id, record) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET conversation_id = excluded.conversation_id,
|
||||
request_id = excluded.request_id, record = excluded.record`)
|
||||
.run(
|
||||
write.value.id,
|
||||
write.value.conversationId,
|
||||
write.value.requestId === undefined ? null : encodeIndexedString(write.value.requestId),
|
||||
encodeJson(write.value),
|
||||
);
|
||||
break;
|
||||
case "document.create":
|
||||
case "document.change":
|
||||
case "document.retire":
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private claimId(id: Id, table: TableName): void {
|
||||
this.db.prepare("INSERT OR IGNORE INTO record_ids (id, record_type) VALUES (?, ?)").run(id, table);
|
||||
}
|
||||
|
||||
private applyDocumentActions(actions: ReadonlyMap<Id, DocumentAction>, seq: Seq): void {
|
||||
for (const [id, action] of actions) {
|
||||
let record: DocumentRecord;
|
||||
if (action.create !== undefined) {
|
||||
record = {
|
||||
...action.create,
|
||||
createdAt: seq,
|
||||
...(action.retire ? { retiredAt: seq } : {}),
|
||||
};
|
||||
const parts = addressParts(record);
|
||||
this.claimId(id, "document");
|
||||
this.db
|
||||
.prepare(`INSERT INTO documents
|
||||
(id, kind, family, key_value, scope_kind, owner_id, created_at, retired_at, record)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
.run(
|
||||
id,
|
||||
parts.kind,
|
||||
parts.family,
|
||||
parts.keyValue,
|
||||
parts.scopeKind,
|
||||
parts.ownerId,
|
||||
seq,
|
||||
action.retire ? seq : null,
|
||||
encodeJson(record),
|
||||
);
|
||||
} else {
|
||||
const row = getRow<JsonRow>(this.db.prepare("SELECT record FROM documents WHERE id = ?"), id)!;
|
||||
record = parseJson<DocumentRecord>(row.record);
|
||||
}
|
||||
|
||||
if (action.content !== undefined) {
|
||||
if (action.content.kind === "base" && isCurrentOnly(record)) {
|
||||
this.db.prepare("DELETE FROM document_revisions WHERE document_id = ?").run(id);
|
||||
}
|
||||
const encodedContent =
|
||||
action.content.kind === "base" ? encodeJson(action.content.value) : encodeJson(action.content.ops);
|
||||
this.db
|
||||
.prepare(
|
||||
"INSERT INTO document_revisions (document_id, seq, kind, version, content) VALUES (?, ?, ?, ?, ?)",
|
||||
)
|
||||
.run(id, seq, action.content.kind, action.content.version, encodedContent);
|
||||
}
|
||||
|
||||
if (action.retire) {
|
||||
if (action.create === undefined) {
|
||||
record = { ...record, retiredAt: seq };
|
||||
this.db
|
||||
.prepare("UPDATE documents SET retired_at = ?, record = ? WHERE id = ?")
|
||||
.run(seq, encodeJson(record), id);
|
||||
}
|
||||
if (isCurrentOnly(record)) this.db.prepare("DELETE FROM document_revisions WHERE document_id = ?").run(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private assertOpen(): void {
|
||||
if (this.closed) throw new Error("SqliteStorage is closed");
|
||||
}
|
||||
}
|
||||
@@ -8,19 +8,12 @@ export type JsonObject = { [key: string]: JsonValue };
|
||||
/** Session-global identifier shared by every durable record table. */
|
||||
export type Id = number;
|
||||
|
||||
/** Monotonic sequence assigned to one atomic storage commit. */
|
||||
/** Strictly increasing sequence assigned to one atomic storage commit; gaps are permitted. */
|
||||
export type Seq = number;
|
||||
|
||||
/** The root conversation always uses this reserved ID. */
|
||||
export const ROOT_CONVERSATION_ID: Id = 1;
|
||||
|
||||
/** JSON-safe error snapshot persisted instead of a runtime `Error` object. */
|
||||
export type StoredError = {
|
||||
readonly message: string;
|
||||
/** Optional structured diagnostic data for inspection or recovery. */
|
||||
readonly detail?: JsonValue;
|
||||
};
|
||||
|
||||
/** Immutable identity, history ancestry, and task ownership of a transcript scope. */
|
||||
export type ConversationRecord = {
|
||||
readonly id: Id;
|
||||
@@ -158,6 +151,13 @@ export type SubmissionCreate = SubmissionRecord extends infer Record
|
||||
: never
|
||||
: never;
|
||||
|
||||
/** JSON-safe error snapshot persisted instead of a runtime `Error` object. */
|
||||
export type TaskOutcomeError = {
|
||||
readonly message: string;
|
||||
/** Optional structured diagnostic data for inspection or recovery. */
|
||||
readonly detail?: JsonValue;
|
||||
};
|
||||
|
||||
/** Durable reason and optional result recorded when a task becomes terminal. */
|
||||
export type TaskOutcome<R> =
|
||||
| {
|
||||
@@ -169,7 +169,7 @@ export type TaskOutcome<R> =
|
||||
/** Expected task or domain failure explicitly committed by its implementation. */
|
||||
| {
|
||||
readonly status: "failed";
|
||||
readonly error: StoredError;
|
||||
readonly error: TaskOutcomeError;
|
||||
readonly result?: R;
|
||||
readonly reason?: never;
|
||||
}
|
||||
@@ -190,7 +190,7 @@ export type TaskOutcome<R> =
|
||||
/** Runtime-detected contract failure, such as an uncaught throw or no durable progress. */
|
||||
| {
|
||||
readonly status: "faulted";
|
||||
readonly error: StoredError;
|
||||
readonly error: TaskOutcomeError;
|
||||
readonly result?: never;
|
||||
readonly reason?: never;
|
||||
};
|
||||
@@ -388,11 +388,13 @@ export type StorageWrite =
|
||||
* Session serializes commits.
|
||||
*/
|
||||
export interface Storage {
|
||||
/** Atomically persist one batch and return the sequence assigned to that commit. */
|
||||
/**
|
||||
* Atomically persist one batch and return its sequence. Once resolved, later reads through this storage observe it.
|
||||
*/
|
||||
commit(writes: readonly StorageWrite[], context: Context): Promise<Seq>;
|
||||
|
||||
/** Return a fresh candidate from the Session-global record ID namespace. */
|
||||
mintId(): Id;
|
||||
mintId(): Promise<Id>;
|
||||
|
||||
/** Look up one conversation by exact ID. */
|
||||
conversation(id: Id, context: Context): Promise<ConversationRecord | undefined>;
|
||||
@@ -445,7 +447,7 @@ export interface Storage {
|
||||
/** Resolve the incarnation occupying one exact logical address at the selected point. */
|
||||
findDocument(address: DocumentAddress, at: DocumentPoint, context: Context): Promise<DocumentRecord | undefined>;
|
||||
|
||||
/** Materialize one incarnation at the selected point without scanning unrelated documents. */
|
||||
/** Materialize one specific incarnation by ID at the selected point without following a replacement at its address. */
|
||||
document(id: Id, at: DocumentPoint, context: Context): Promise<StoredDocument | undefined>;
|
||||
|
||||
/** Scan incarnations alive in one exact scope at the selected point. */
|
||||
|
||||
@@ -1,967 +1,4 @@
|
||||
import type { JsonValue } from "@earendil-works/chord";
|
||||
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
||||
import type { Op } from "@earendil-works/chord/delta";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { MemoryStorage } from "../src/memory-storage.ts";
|
||||
import {
|
||||
type DocumentCreate,
|
||||
type EntryRecord,
|
||||
type Id,
|
||||
type JsonObject,
|
||||
ROOT_CONVERSATION_ID,
|
||||
type StorageWrite,
|
||||
type SubmissionRecord,
|
||||
type TaskRecord,
|
||||
} from "../src/types.ts";
|
||||
import { MemoryStorage } from "../src/storage/memory.ts";
|
||||
import { registerStorageConformance } from "./storage-conformance.ts";
|
||||
|
||||
const context = BACKGROUND_CONTEXT;
|
||||
type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
|
||||
|
||||
async function createRoot(storage: MemoryStorage): Promise<Id> {
|
||||
await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context);
|
||||
return ROOT_CONVERSATION_ID;
|
||||
}
|
||||
|
||||
function pendingTask(id: Id, conversationId: Id, phase = "ready") {
|
||||
return {
|
||||
id,
|
||||
conversationId,
|
||||
kind: "test.task",
|
||||
version: 1,
|
||||
input: { value: id },
|
||||
state: { status: "pending", checkpoint: { phase } },
|
||||
after: [],
|
||||
background: false,
|
||||
abortRequested: false,
|
||||
} satisfies StoredTask;
|
||||
}
|
||||
|
||||
function entry(id: Id, conversationId: Id, kind = "message", extra: Partial<EntryRecord> = {}): EntryRecord {
|
||||
return { id, conversationId, kind, ...extra };
|
||||
}
|
||||
|
||||
describe("Pico MemoryStorage", () => {
|
||||
it("reserves ID 1 for the immutable root conversation", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
expect(storage.mintId()).toBe(2);
|
||||
await expect(createRoot(storage)).resolves.toBe(ROOT_CONVERSATION_ID);
|
||||
expect(await storage.conversation(ROOT_CONVERSATION_ID, context)).toEqual({ id: ROOT_CONVERSATION_ID });
|
||||
await expect(
|
||||
storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context),
|
||||
).rejects.toThrow(`ID ${ROOT_CONVERSATION_ID} already belongs to conversation`);
|
||||
});
|
||||
|
||||
it("commits mixed table writes atomically and rolls all of them back on failure", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const entryId = storage.mintId();
|
||||
const taskId = storage.mintId();
|
||||
const submissionId = storage.mintId();
|
||||
const task = pendingTask(taskId, rootId);
|
||||
const input: SubmissionRecord = {
|
||||
id: submissionId,
|
||||
conversationId: rootId,
|
||||
requestId: "request-1",
|
||||
type: "input",
|
||||
status: "placed",
|
||||
entry: entryId,
|
||||
};
|
||||
const initialSeq = await storage.commit(
|
||||
[
|
||||
{ type: "entry", value: entry(entryId, rootId, "user", { data: { text: "hello" } }) },
|
||||
{ type: "task", value: task },
|
||||
{ type: "submission", value: input },
|
||||
],
|
||||
context,
|
||||
);
|
||||
|
||||
expect(await storage.entry(entryId, context)).toEqual({
|
||||
entry: entry(entryId, rootId, "user", { data: { text: "hello" } }),
|
||||
commitSeq: initialSeq,
|
||||
});
|
||||
expect(await storage.task(taskId, context)).toEqual(task);
|
||||
expect(await storage.submission(submissionId, context)).toEqual(input);
|
||||
|
||||
const transientEntryId = storage.mintId();
|
||||
const runningTask: StoredTask = {
|
||||
...task,
|
||||
state: { status: "running", checkpoint: { phase: "effect" } },
|
||||
};
|
||||
const doneInput: SubmissionRecord = { ...input, status: "done", answer: transientEntryId };
|
||||
await expect(
|
||||
storage.commit(
|
||||
[
|
||||
{ type: "task", value: runningTask },
|
||||
{ type: "submission", value: doneInput },
|
||||
{ type: "entry", value: entry(transientEntryId, rootId, "assistant") },
|
||||
{ type: "conversation", value: { id: rootId } },
|
||||
],
|
||||
context,
|
||||
),
|
||||
).rejects.toThrow(`ID ${rootId} already belongs to conversation`);
|
||||
|
||||
expect(await storage.task(taskId, context)).toEqual(task);
|
||||
expect(await storage.submission(submissionId, context)).toEqual(input);
|
||||
expect(await storage.entry(transientEntryId, context)).toBeUndefined();
|
||||
expect(
|
||||
await storage.commit([{ type: "entry", value: entry(storage.mintId(), rootId, "after-rollback") }], context),
|
||||
).toBe(initialSeq + 1);
|
||||
});
|
||||
|
||||
it("detaches retained writes and every returned record", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const entryId = storage.mintId();
|
||||
const taskId = storage.mintId();
|
||||
const submissionId = storage.mintId();
|
||||
const entryData = { nested: [1, 2] };
|
||||
const checkpoint = { phase: "ready", nested: { count: 1 } };
|
||||
const detail = { codes: ["initial"] };
|
||||
const storedEntry = entry(entryId, rootId, "note", { data: entryData });
|
||||
const storedTask: StoredTask = {
|
||||
...pendingTask(taskId, rootId),
|
||||
state: { status: "pending", checkpoint },
|
||||
};
|
||||
const storedInput: SubmissionRecord = {
|
||||
id: submissionId,
|
||||
conversationId: rootId,
|
||||
type: "input",
|
||||
status: "unanswered",
|
||||
reason: "failed",
|
||||
detail,
|
||||
};
|
||||
await storage.commit(
|
||||
[
|
||||
{ type: "entry", value: storedEntry },
|
||||
{ type: "task", value: storedTask },
|
||||
{ type: "submission", value: storedInput },
|
||||
],
|
||||
context,
|
||||
);
|
||||
|
||||
entryData.nested.push(3);
|
||||
checkpoint.nested.count = 2;
|
||||
detail.codes.push("mutated");
|
||||
expect((await storage.entry(entryId, context))?.entry.data).toEqual({ nested: [1, 2] });
|
||||
expect((await storage.task(taskId, context))?.state).toEqual({
|
||||
status: "pending",
|
||||
checkpoint: { phase: "ready", nested: { count: 1 } },
|
||||
});
|
||||
expect((await storage.submission(submissionId, context))?.detail).toEqual({ codes: ["initial"] });
|
||||
|
||||
const readEntry = (await storage.entry(entryId, context))!.entry;
|
||||
(readEntry.data as { nested: number[] }).nested.push(9);
|
||||
const readTask = (await storage.task(taskId, context))!;
|
||||
if (readTask.state.status !== "terminal") {
|
||||
(readTask.state.checkpoint as { phase: string; nested: { count: number } }).nested.count = 9;
|
||||
}
|
||||
const readInput = (await storage.submission(submissionId, context))!;
|
||||
(readInput.detail as { codes: string[] }).codes.push("read mutation");
|
||||
|
||||
expect((await storage.entry(entryId, context))?.entry.data).toEqual({ nested: [1, 2] });
|
||||
expect((await storage.task(taskId, context))?.state).toEqual({
|
||||
status: "pending",
|
||||
checkpoint: { phase: "ready", nested: { count: 1 } },
|
||||
});
|
||||
expect((await storage.submission(submissionId, context))?.detail).toEqual({ codes: ["initial"] });
|
||||
});
|
||||
|
||||
it("detaches prototype-like JSON keys without changing object prototypes", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const entryId = storage.mintId();
|
||||
const data = JSON.parse(
|
||||
'{"__proto__":{"polluted":false},"constructor":{"label":"stored"},"toString":"value"}',
|
||||
) as Record<string, JsonValue>;
|
||||
await storage.commit([{ type: "entry", value: entry(entryId, rootId, "note", { data }) }], context);
|
||||
|
||||
(Reflect.get(data, "__proto__") as Record<string, JsonValue>).polluted = true;
|
||||
(Reflect.get(data, "constructor") as Record<string, JsonValue>).label = "mutated";
|
||||
const firstRead = (await storage.entry(entryId, context))!.entry.data as Record<string, JsonValue>;
|
||||
expect(Object.getPrototypeOf(firstRead)).toBe(Object.prototype);
|
||||
expect(Object.hasOwn(firstRead, "__proto__")).toBe(true);
|
||||
expect(Reflect.get(firstRead, "__proto__")).toEqual({ polluted: false });
|
||||
expect(Reflect.get(firstRead, "constructor")).toEqual({ label: "stored" });
|
||||
expect(Reflect.get(firstRead, "toString")).toBe("value");
|
||||
expect(({} as { polluted?: boolean }).polluted).toBeUndefined();
|
||||
|
||||
(Reflect.get(firstRead, "__proto__") as Record<string, JsonValue>).polluted = true;
|
||||
const secondRead = (await storage.entry(entryId, context))!.entry.data as Record<string, JsonValue>;
|
||||
expect(Reflect.get(secondRead, "__proto__")).toEqual({ polluted: false });
|
||||
expect(Reflect.get(secondRead, "constructor")).toEqual({ label: "stored" });
|
||||
expect(Reflect.get(secondRead, "toString")).toBe("value");
|
||||
});
|
||||
|
||||
it("indexes entries committed out of ID order", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
await storage.commit(
|
||||
[
|
||||
{ type: "entry", value: entry(30, rootId) },
|
||||
{ type: "entry", value: entry(10, rootId) },
|
||||
{ type: "entry", value: entry(20, rootId, "marker", { head: 10 }) },
|
||||
],
|
||||
context,
|
||||
);
|
||||
|
||||
expect(
|
||||
(await storage.scanEntries({ conversationId: rootId }, undefined, 10, context)).items.map(({ id }) => id),
|
||||
).toEqual([30, 20, 10]);
|
||||
expect((await storage.findLatestHeadMarker(rootId, undefined, context))?.id).toBe(20);
|
||||
});
|
||||
|
||||
it("continues an entry cursor below its last item after a newer commit", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const oldestId = storage.mintId();
|
||||
const middleId = storage.mintId();
|
||||
const newestId = storage.mintId();
|
||||
await storage.commit(
|
||||
[
|
||||
{ type: "entry", value: entry(oldestId, rootId) },
|
||||
{ type: "entry", value: entry(middleId, rootId) },
|
||||
{ type: "entry", value: entry(newestId, rootId) },
|
||||
],
|
||||
context,
|
||||
);
|
||||
|
||||
const first = await storage.scanEntries({ conversationId: rootId }, undefined, 2, context);
|
||||
expect(first.items.map(({ id }) => id)).toEqual([newestId, middleId]);
|
||||
const appendedId = storage.mintId();
|
||||
await storage.commit([{ type: "entry", value: entry(appendedId, rootId) }], context);
|
||||
const second = await storage.scanEntries({ conversationId: rootId }, first.next, 2, context);
|
||||
expect(second.items.map(({ id }) => id)).toEqual([oldestId]);
|
||||
expect(second.next).toBeUndefined();
|
||||
});
|
||||
|
||||
it("paginates conversations by opaque cursor in ascending ID order", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const secondId = storage.mintId();
|
||||
const thirdId = storage.mintId();
|
||||
await storage.commit(
|
||||
[
|
||||
{ type: "conversation", value: { id: thirdId } },
|
||||
{ type: "conversation", value: { id: secondId } },
|
||||
],
|
||||
context,
|
||||
);
|
||||
|
||||
const first = await storage.scanConversations(undefined, 2, context);
|
||||
expect(first.items.map(({ id }) => id)).toEqual([rootId, secondId]);
|
||||
expect(first.next).toEqual({ after: secondId });
|
||||
const second = await storage.scanConversations(first.next, 2, context);
|
||||
expect(second.items.map(({ id }) => id)).toEqual([thirdId]);
|
||||
expect(second.next).toBeUndefined();
|
||||
});
|
||||
|
||||
it("scans deep fork history newest-first through every ancestor cap", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const rootFirst = storage.mintId();
|
||||
const rootForkPoint = storage.mintId();
|
||||
const rootExcludedSameCommit = storage.mintId();
|
||||
const rootEntriesSeq = await storage.commit(
|
||||
[
|
||||
{ type: "entry", value: entry(rootFirst, rootId) },
|
||||
{
|
||||
type: "entry",
|
||||
value: entry(rootForkPoint, rootId, "marker", { head: rootFirst }),
|
||||
},
|
||||
{ type: "entry", value: entry(rootExcludedSameCommit, rootId) },
|
||||
],
|
||||
context,
|
||||
);
|
||||
const childId = storage.mintId();
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "conversation",
|
||||
value: { id: childId, parent: { conversationId: rootId, at: rootForkPoint } },
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
const childForkPoint = storage.mintId();
|
||||
const childExcluded = storage.mintId();
|
||||
await storage.commit(
|
||||
[
|
||||
{ type: "entry", value: entry(childForkPoint, childId, "note") },
|
||||
{ type: "entry", value: entry(childExcluded, childId) },
|
||||
],
|
||||
context,
|
||||
);
|
||||
const rootExcludedLater = storage.mintId();
|
||||
await storage.commit([{ type: "entry", value: entry(rootExcludedLater, rootId) }], context);
|
||||
const grandchildId = storage.mintId();
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "conversation",
|
||||
value: { id: grandchildId, parent: { conversationId: childId, at: childForkPoint } },
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
const grandchildHead = storage.mintId();
|
||||
const grandchildTail = storage.mintId();
|
||||
const grandchildEntriesSeq = await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "entry",
|
||||
value: entry(grandchildHead, grandchildId, "marker", { head: grandchildHead }),
|
||||
},
|
||||
{ type: "entry", value: entry(grandchildTail, grandchildId) },
|
||||
],
|
||||
context,
|
||||
);
|
||||
const childExcludedLater = storage.mintId();
|
||||
await storage.commit([{ type: "entry", value: entry(childExcludedLater, childId) }], context);
|
||||
|
||||
const first = await storage.scanEntries({ conversationId: grandchildId }, undefined, 2, context);
|
||||
expect(first.items.map(({ id }) => id)).toEqual([grandchildTail, grandchildHead]);
|
||||
const second = await storage.scanEntries({ conversationId: grandchildId }, first.next, 2, context);
|
||||
expect(second.items.map(({ id }) => id)).toEqual([childForkPoint, rootForkPoint]);
|
||||
const third = await storage.scanEntries({ conversationId: grandchildId }, second.next, 2, context);
|
||||
expect(third.items.map(({ id }) => id)).toEqual([rootFirst]);
|
||||
expect(third.next).toBeUndefined();
|
||||
|
||||
const currentMarker = await storage.findLatestHeadMarker(grandchildId, undefined, context);
|
||||
expect(currentMarker?.id).toBe(grandchildHead);
|
||||
expect(currentMarker?.head).toBe(grandchildHead);
|
||||
const historicalMarker = await storage.findLatestHeadMarker(grandchildId, childForkPoint, context);
|
||||
expect(historicalMarker?.id).toBe(rootForkPoint);
|
||||
expect(historicalMarker?.head).toBe(rootFirst);
|
||||
expect(await storage.findLatestHeadMarker(grandchildId, rootFirst, context)).toBeUndefined();
|
||||
|
||||
const activeFirst = await storage.scanEntries(
|
||||
{ conversationId: grandchildId, minEntryId: currentMarker?.head },
|
||||
undefined,
|
||||
1,
|
||||
context,
|
||||
);
|
||||
expect(activeFirst.items.map(({ id }) => id)).toEqual([grandchildTail]);
|
||||
expect(activeFirst.next).toBeDefined();
|
||||
const activeSecond = await storage.scanEntries(
|
||||
{ conversationId: grandchildId, minEntryId: currentMarker?.head },
|
||||
activeFirst.next,
|
||||
1,
|
||||
context,
|
||||
);
|
||||
expect(activeSecond.items.map(({ id }) => id)).toEqual([grandchildHead]);
|
||||
expect(activeSecond.next).toBeUndefined();
|
||||
|
||||
expect(
|
||||
(
|
||||
await storage.scanEntries(
|
||||
{
|
||||
conversationId: grandchildId,
|
||||
minEntryId: historicalMarker?.head,
|
||||
maxEntryId: childForkPoint,
|
||||
},
|
||||
undefined,
|
||||
10,
|
||||
context,
|
||||
)
|
||||
).items.map(({ id }) => id),
|
||||
).toEqual([childForkPoint, rootForkPoint, rootFirst]);
|
||||
|
||||
expect(await storage.entry(rootFirst, context)).toEqual({
|
||||
entry: entry(rootFirst, rootId),
|
||||
commitSeq: rootEntriesSeq,
|
||||
});
|
||||
expect((await storage.entry(rootForkPoint, context))?.commitSeq).toBe(rootEntriesSeq);
|
||||
expect((await storage.entry(grandchildHead, context))?.commitSeq).toBe(grandchildEntriesSeq);
|
||||
expect((await storage.entry(grandchildTail, context))?.commitSeq).toBe(grandchildEntriesSeq);
|
||||
expect(await storage.entry(999_999, context)).toBeUndefined();
|
||||
await expect(storage.scanEntries({ conversationId: 999_999 }, undefined, 10, context)).rejects.toThrow(
|
||||
"Unknown conversation",
|
||||
);
|
||||
});
|
||||
|
||||
it("replaces complete task records and pages filtered task scans", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const firstId = storage.mintId();
|
||||
const secondId = storage.mintId();
|
||||
const thirdId = storage.mintId();
|
||||
const first = { ...pendingTask(firstId, rootId), memos: { winner: "first" } } satisfies StoredTask;
|
||||
const second = { ...pendingTask(secondId, rootId), background: true } satisfies StoredTask;
|
||||
const third = { ...pendingTask(thirdId, rootId), abortRequested: true } satisfies StoredTask;
|
||||
await storage.commit(
|
||||
[
|
||||
{ type: "task", value: first },
|
||||
{ type: "task", value: second },
|
||||
{ type: "task", value: third },
|
||||
],
|
||||
context,
|
||||
);
|
||||
|
||||
const running: StoredTask = {
|
||||
...first,
|
||||
state: { status: "running", checkpoint: { phase: "effect", attempt: 1 } },
|
||||
abortRequested: true,
|
||||
};
|
||||
await storage.commit([{ type: "task", value: running }], context);
|
||||
expect(await storage.task(firstId, context)).toEqual(running);
|
||||
const terminal: StoredTask = {
|
||||
id: firstId,
|
||||
conversationId: rootId,
|
||||
kind: first.kind,
|
||||
version: first.version,
|
||||
input: first.input,
|
||||
state: { status: "terminal", outcome: { status: "completed", result: { entryId: 99 } } },
|
||||
after: [],
|
||||
background: false,
|
||||
abortRequested: true,
|
||||
};
|
||||
await storage.commit([{ type: "task", value: terminal }], context);
|
||||
expect(await storage.task(firstId, context)).toEqual(terminal);
|
||||
|
||||
const pendingPage = await storage.scanTasks({ status: "pending" }, undefined, 1, context);
|
||||
expect(pendingPage.items.map(({ id }) => id)).toEqual([secondId]);
|
||||
expect(pendingPage.next).toBeDefined();
|
||||
expect(
|
||||
(await storage.scanTasks({ status: "pending" }, pendingPage.next, 1, context)).items.map(({ id }) => id),
|
||||
).toEqual([thirdId]);
|
||||
expect(
|
||||
(await storage.scanTasks({ status: "terminal", abortRequested: true }, undefined, 10, context)).items,
|
||||
).toEqual([terminal]);
|
||||
expect((await storage.scanTasks({ background: true }, undefined, 10, context)).items.map(({ id }) => id)).toEqual(
|
||||
[secondId],
|
||||
);
|
||||
});
|
||||
|
||||
it("indexes request IDs per conversation and replaces complete submission records", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const secondConversationId = storage.mintId();
|
||||
await storage.commit([{ type: "conversation", value: { id: secondConversationId } }], context);
|
||||
const firstId = storage.mintId();
|
||||
const secondId = storage.mintId();
|
||||
const otherConversationId = storage.mintId();
|
||||
const first: SubmissionRecord = {
|
||||
id: firstId,
|
||||
conversationId: rootId,
|
||||
requestId: "same",
|
||||
type: "input",
|
||||
status: "queued",
|
||||
};
|
||||
const second: SubmissionRecord = {
|
||||
id: secondId,
|
||||
conversationId: rootId,
|
||||
requestId: "other",
|
||||
type: "input",
|
||||
status: "queued",
|
||||
};
|
||||
const otherConversation: SubmissionRecord = {
|
||||
id: otherConversationId,
|
||||
conversationId: secondConversationId,
|
||||
requestId: "same",
|
||||
type: "input",
|
||||
status: "queued",
|
||||
};
|
||||
await storage.commit(
|
||||
[
|
||||
{ type: "submission", value: first },
|
||||
{ type: "submission", value: second },
|
||||
{ type: "submission", value: otherConversation },
|
||||
],
|
||||
context,
|
||||
);
|
||||
expect(await storage.submissionByRequest(rootId, "same", context)).toEqual(first);
|
||||
expect(await storage.submissionByRequest(secondConversationId, "same", context)).toEqual(otherConversation);
|
||||
|
||||
const placedSecond: SubmissionRecord = { ...second, status: "placed", entry: storage.mintId() };
|
||||
await storage.commit([{ type: "submission", value: placedSecond }], context);
|
||||
expect(await storage.submission(secondId, context)).toEqual(placedSecond);
|
||||
expect(await storage.submissionByRequest(rootId, "other", context)).toEqual(placedSecond);
|
||||
});
|
||||
|
||||
it("stores passive write submissions without input-only lifecycle states", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const doneId = storage.mintId();
|
||||
const failedId = storage.mintId();
|
||||
const queuedDone: SubmissionRecord = {
|
||||
id: doneId,
|
||||
conversationId: rootId,
|
||||
requestId: "passive-done",
|
||||
type: "write",
|
||||
status: "queued",
|
||||
};
|
||||
const queuedFailed: SubmissionRecord = {
|
||||
id: failedId,
|
||||
conversationId: rootId,
|
||||
requestId: "passive-failed",
|
||||
type: "write",
|
||||
status: "queued",
|
||||
};
|
||||
await storage.commit(
|
||||
[
|
||||
{ type: "submission", value: queuedDone },
|
||||
{ type: "submission", value: queuedFailed },
|
||||
],
|
||||
context,
|
||||
);
|
||||
|
||||
const done: SubmissionRecord = { ...queuedDone, status: "done", entry: storage.mintId() };
|
||||
const unanswered: SubmissionRecord = {
|
||||
...queuedFailed,
|
||||
status: "unanswered",
|
||||
reason: "closed",
|
||||
detail: { retryable: false },
|
||||
};
|
||||
await storage.commit(
|
||||
[
|
||||
{ type: "submission", value: done },
|
||||
{ type: "submission", value: unanswered },
|
||||
],
|
||||
context,
|
||||
);
|
||||
expect(await storage.submission(doneId, context)).toEqual(done);
|
||||
expect(await storage.submissionByRequest(rootId, "passive-done", context)).toEqual(done);
|
||||
expect(await storage.submission(failedId, context)).toEqual(unanswered);
|
||||
expect(await storage.submissionByRequest(rootId, "passive-failed", context)).toEqual(unanswered);
|
||||
});
|
||||
|
||||
it("reconstructs rewindable documents and preserves half-open incarnations", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const firstId = storage.mintId();
|
||||
const firstRecord = {
|
||||
id: firstId,
|
||||
kind: "conversation.notes",
|
||||
scope: { kind: "conversation", conversationId: rootId },
|
||||
history: "rewindable",
|
||||
fork: "asOf",
|
||||
} satisfies DocumentCreate;
|
||||
const initial: JsonObject = { items: ["a"], nested: { count: 1 } };
|
||||
const createdAt = await storage.commit(
|
||||
[{ type: "document.create", record: firstRecord, content: { kind: "base", version: 1, value: initial } }],
|
||||
context,
|
||||
);
|
||||
const appended = ["b"];
|
||||
const ops: Op[] = [
|
||||
["p", ["items"], 1, 0, appended],
|
||||
["s", ["nested", "count"], 2],
|
||||
];
|
||||
const changedAt = await storage.commit(
|
||||
[{ type: "document.change", id: firstId, content: { kind: "delta", version: 1, ops } }],
|
||||
context,
|
||||
);
|
||||
|
||||
(initial.items as string[]).push("caller mutation");
|
||||
appended.push("caller mutation");
|
||||
expect(await storage.document(firstId, createdAt, context)).toMatchObject({
|
||||
version: 1,
|
||||
value: { items: ["a"], nested: { count: 1 } },
|
||||
});
|
||||
const changed = (await storage.document(firstId, changedAt, context))!;
|
||||
expect(changed.value).toEqual({ items: ["a", "b"], nested: { count: 2 } });
|
||||
(changed.value.items as string[]).push("read mutation");
|
||||
expect((await storage.document(firstId, "current", context))?.value).toEqual({
|
||||
items: ["a", "b"],
|
||||
nested: { count: 2 },
|
||||
});
|
||||
|
||||
const checkpointAt = await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.change",
|
||||
id: firstId,
|
||||
content: { kind: "base", version: 2, value: { items: ["checkpoint"], nested: { count: 3 } } },
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
const replacedAt = await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.change",
|
||||
id: firstId,
|
||||
content: {
|
||||
kind: "delta",
|
||||
version: 2,
|
||||
ops: [["r", { items: ["replacement"], nested: { count: 4 } }]],
|
||||
},
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
expect(await storage.document(firstId, changedAt, context)).toMatchObject({
|
||||
version: 1,
|
||||
value: { items: ["a", "b"], nested: { count: 2 } },
|
||||
});
|
||||
expect(await storage.document(firstId, checkpointAt, context)).toMatchObject({
|
||||
version: 2,
|
||||
value: { items: ["checkpoint"], nested: { count: 3 } },
|
||||
});
|
||||
expect((await storage.document(firstId, replacedAt, context))?.value).toEqual({
|
||||
items: ["replacement"],
|
||||
nested: { count: 4 },
|
||||
});
|
||||
|
||||
const secondId = storage.mintId();
|
||||
const retiredAt = await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.create",
|
||||
record: { ...firstRecord, id: secondId },
|
||||
content: { kind: "base", version: 1, value: { items: ["new"] } },
|
||||
},
|
||||
{ type: "document.retire", id: firstId },
|
||||
{
|
||||
type: "document.change",
|
||||
id: firstId,
|
||||
content: { kind: "delta", version: 2, ops: [["s", ["retiring"], true]] },
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
const address = { kind: firstRecord.kind, scope: firstRecord.scope };
|
||||
expect((await storage.findDocument(address, changedAt, context))?.id).toBe(firstId);
|
||||
expect(await storage.findDocument(address, retiredAt, context)).toMatchObject({
|
||||
id: secondId,
|
||||
createdAt: retiredAt,
|
||||
});
|
||||
expect(
|
||||
(await storage.scanDocuments({ scope: firstRecord.scope, at: changedAt }, undefined, 10, context)).items.map(
|
||||
({ id }) => id,
|
||||
),
|
||||
).toEqual([firstId]);
|
||||
expect(
|
||||
(await storage.scanDocuments({ scope: firstRecord.scope, at: retiredAt }, undefined, 10, context)).items.map(
|
||||
({ id }) => id,
|
||||
),
|
||||
).toEqual([secondId]);
|
||||
expect(await storage.document(firstId, retiredAt, context)).toBeUndefined();
|
||||
expect((await storage.document(secondId, "current", context))?.value).toEqual({ items: ["new"] });
|
||||
});
|
||||
|
||||
it("uses bases for version transitions and rejects historical reads of current-only documents", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
await createRoot(storage);
|
||||
const id = storage.mintId();
|
||||
const record = {
|
||||
id,
|
||||
kind: "session.settings",
|
||||
scope: { kind: "session" },
|
||||
} satisfies DocumentCreate;
|
||||
await storage.commit(
|
||||
[{ type: "document.create", record, content: { kind: "base", version: 1, value: { count: 1 } } }],
|
||||
context,
|
||||
);
|
||||
await storage.commit(
|
||||
[{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["count"], 2]] } }],
|
||||
context,
|
||||
);
|
||||
const migratedAt = await storage.commit(
|
||||
[{ type: "document.change", id, content: { kind: "base", version: 2, value: { count: 3 } } }],
|
||||
context,
|
||||
);
|
||||
expect(await storage.document(id, "current", context)).toMatchObject({ version: 2, value: { count: 3 } });
|
||||
await expect(storage.document(id, migratedAt, context)).rejects.toThrow("does not retain historical content");
|
||||
|
||||
await expect(
|
||||
storage.commit(
|
||||
[{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["count"], 4]] } }],
|
||||
context,
|
||||
),
|
||||
).rejects.toThrow("version transition requires a base");
|
||||
expect((await storage.document(id, "current", context))?.value).toEqual({ count: 3 });
|
||||
await storage.commit([{ type: "document.retire", id }], context);
|
||||
expect(await storage.document(id, "current", context)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("indexes logical addresses and exact-scope scans independently", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const firstId = storage.mintId();
|
||||
const secondId = storage.mintId();
|
||||
const conversationId = storage.mintId();
|
||||
const taskId = storage.mintId();
|
||||
const taskSingletonId = storage.mintId();
|
||||
const taskFamilyId = storage.mintId();
|
||||
const taskOtherKindId = storage.mintId();
|
||||
const createdAt = await storage.commit(
|
||||
[
|
||||
{ type: "task", value: pendingTask(taskId, rootId) },
|
||||
{
|
||||
type: "document.create",
|
||||
record: {
|
||||
id: firstId,
|
||||
kind: "cache",
|
||||
scope: { kind: "session" },
|
||||
key: "__proto__",
|
||||
},
|
||||
content: { kind: "base", version: 1, value: { owner: "first" } },
|
||||
},
|
||||
{
|
||||
type: "document.create",
|
||||
record: {
|
||||
id: secondId,
|
||||
kind: "cache",
|
||||
scope: { kind: "session" },
|
||||
key: "constructor",
|
||||
},
|
||||
content: { kind: "base", version: 1, value: { owner: "second" } },
|
||||
},
|
||||
{
|
||||
type: "document.create",
|
||||
record: {
|
||||
id: conversationId,
|
||||
kind: "cache",
|
||||
scope: { kind: "conversation", conversationId: rootId },
|
||||
history: "latest",
|
||||
fork: "current",
|
||||
key: "__proto__",
|
||||
},
|
||||
content: { kind: "base", version: 1, value: { owner: "conversation" } },
|
||||
},
|
||||
{
|
||||
type: "document.create",
|
||||
record: {
|
||||
id: taskSingletonId,
|
||||
kind: "task.cache",
|
||||
scope: { kind: "task", taskId },
|
||||
},
|
||||
content: { kind: "base", version: 1, value: { owner: "singleton" } },
|
||||
},
|
||||
{
|
||||
type: "document.create",
|
||||
record: {
|
||||
id: taskFamilyId,
|
||||
kind: "task.cache",
|
||||
scope: { kind: "task", taskId },
|
||||
key: "member",
|
||||
},
|
||||
content: { kind: "base", version: 1, value: { owner: "family" } },
|
||||
},
|
||||
{
|
||||
type: "document.create",
|
||||
record: {
|
||||
id: taskOtherKindId,
|
||||
kind: "task.other",
|
||||
scope: { kind: "task", taskId },
|
||||
},
|
||||
content: { kind: "base", version: 1, value: { owner: "other" } },
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
|
||||
expect(
|
||||
(
|
||||
await storage.findDocument(
|
||||
{ kind: "cache", scope: { kind: "session" }, key: "__proto__" },
|
||||
"current",
|
||||
context,
|
||||
)
|
||||
)?.id,
|
||||
).toBe(firstId);
|
||||
expect(
|
||||
(await storage.scanDocuments({ scope: { kind: "session" }, at: "current" }, undefined, 1, context)).items,
|
||||
).toHaveLength(1);
|
||||
const first = await storage.scanDocuments({ scope: { kind: "session" }, at: "current" }, undefined, 1, context);
|
||||
const second = await storage.scanDocuments({ scope: { kind: "session" }, at: "current" }, first.next, 1, context);
|
||||
expect([...first.items, ...second.items].map(({ id }) => id)).toEqual([firstId, secondId]);
|
||||
expect(
|
||||
(
|
||||
await storage.scanDocuments(
|
||||
{ scope: { kind: "conversation", conversationId: rootId }, at: "current" },
|
||||
undefined,
|
||||
10,
|
||||
context,
|
||||
)
|
||||
).items.map(({ id }) => id),
|
||||
).toEqual([conversationId]);
|
||||
expect(
|
||||
(await storage.findDocument({ kind: "task.cache", scope: { kind: "task", taskId } }, "current", context))?.id,
|
||||
).toBe(taskSingletonId);
|
||||
expect(
|
||||
(
|
||||
await storage.findDocument(
|
||||
{ kind: "task.cache", scope: { kind: "task", taskId }, key: "member" },
|
||||
"current",
|
||||
context,
|
||||
)
|
||||
)?.id,
|
||||
).toBe(taskFamilyId);
|
||||
expect(
|
||||
(
|
||||
await storage.scanDocuments(
|
||||
{ scope: { kind: "task", taskId }, at: "current", kind: "task.cache" },
|
||||
undefined,
|
||||
10,
|
||||
context,
|
||||
)
|
||||
).items.map(({ id }) => id),
|
||||
).toEqual([taskSingletonId, taskFamilyId]);
|
||||
await expect(storage.document(taskSingletonId, createdAt, context)).rejects.toThrow(
|
||||
"does not retain historical content",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps document lifecycle failures atomic and gives create-plus-retire an empty lifetime", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const firstId = storage.mintId();
|
||||
const secondId = storage.mintId();
|
||||
const record = {
|
||||
id: firstId,
|
||||
kind: "singleton",
|
||||
scope: { kind: "session" },
|
||||
} satisfies DocumentCreate;
|
||||
await storage.commit(
|
||||
[{ type: "document.create", record, content: { kind: "base", version: 1, value: { value: 1 } } }],
|
||||
context,
|
||||
);
|
||||
await expect(
|
||||
storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.create",
|
||||
record: { ...record, id: secondId },
|
||||
content: { kind: "base", version: 1, value: { value: 2 } },
|
||||
},
|
||||
{ type: "document.change", id: firstId, content: { kind: "delta", version: 1, ops: [] } },
|
||||
],
|
||||
context,
|
||||
),
|
||||
).rejects.toThrow("already has a current incarnation");
|
||||
expect((await storage.document(firstId, "current", context))?.value).toEqual({ value: 1 });
|
||||
expect(await storage.document(secondId, "current", context)).toBeUndefined();
|
||||
|
||||
const emptyId = storage.mintId();
|
||||
const emptyAt = await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.create",
|
||||
record: {
|
||||
id: emptyId,
|
||||
kind: record.kind,
|
||||
key: "empty",
|
||||
scope: { kind: "conversation", conversationId: rootId },
|
||||
history: "rewindable",
|
||||
fork: "initial",
|
||||
},
|
||||
content: { kind: "base", version: 1, value: {} },
|
||||
},
|
||||
{ type: "document.retire", id: emptyId },
|
||||
],
|
||||
context,
|
||||
);
|
||||
expect(await storage.document(emptyId, "current", context)).toBeUndefined();
|
||||
expect(await storage.document(emptyId, emptyAt, context)).toBeUndefined();
|
||||
expect(
|
||||
await storage.findDocument(
|
||||
{
|
||||
kind: record.kind,
|
||||
scope: { kind: "conversation", conversationId: rootId },
|
||||
key: "empty",
|
||||
},
|
||||
emptyAt,
|
||||
context,
|
||||
),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rolls back record tables and secondary indexes when a document command fails", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const taskId = storage.mintId();
|
||||
const submissionId = storage.mintId();
|
||||
const documentId = storage.mintId();
|
||||
const task = pendingTask(taskId, rootId);
|
||||
const submission: SubmissionRecord = {
|
||||
id: submissionId,
|
||||
conversationId: rootId,
|
||||
requestId: "atomic",
|
||||
type: "input",
|
||||
status: "queued",
|
||||
};
|
||||
const record = {
|
||||
id: documentId,
|
||||
kind: "atomic",
|
||||
scope: { kind: "session" },
|
||||
} satisfies DocumentCreate;
|
||||
const baselineSeq = await storage.commit(
|
||||
[
|
||||
{ type: "task", value: task },
|
||||
{ type: "submission", value: submission },
|
||||
{ type: "document.create", record, content: { kind: "base", version: 1, value: { count: 1 } } },
|
||||
],
|
||||
context,
|
||||
);
|
||||
|
||||
const entryId = storage.mintId();
|
||||
const conflictingDocumentId = storage.mintId();
|
||||
await expect(
|
||||
storage.commit(
|
||||
[
|
||||
{
|
||||
type: "task",
|
||||
value: { ...task, state: { status: "running", checkpoint: { phase: "effect" } } },
|
||||
},
|
||||
{
|
||||
type: "submission",
|
||||
value: { ...submission, status: "unanswered", reason: "failed" },
|
||||
},
|
||||
{ type: "entry", value: entry(entryId, rootId, "transient") },
|
||||
{
|
||||
type: "document.create",
|
||||
record: { ...record, id: conflictingDocumentId },
|
||||
content: { kind: "base", version: 1, value: { count: 2 } },
|
||||
},
|
||||
],
|
||||
context,
|
||||
),
|
||||
).rejects.toThrow("already has a current incarnation");
|
||||
|
||||
expect(await storage.task(taskId, context)).toEqual(task);
|
||||
expect((await storage.scanTasks({ status: "pending" }, undefined, 10, context)).items).toEqual([task]);
|
||||
expect(await storage.submissionByRequest(rootId, "atomic", context)).toEqual(submission);
|
||||
expect(await storage.entry(entryId, context)).toBeUndefined();
|
||||
expect(await storage.document(conflictingDocumentId, "current", context)).toBeUndefined();
|
||||
expect((await storage.findDocument({ kind: record.kind, scope: record.scope }, "current", context))?.id).toBe(
|
||||
documentId,
|
||||
);
|
||||
expect(
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.change",
|
||||
id: documentId,
|
||||
content: { kind: "delta", version: 1, ops: [["s", ["count"], 3]] },
|
||||
},
|
||||
],
|
||||
context,
|
||||
),
|
||||
).toBe(baselineSeq + 1);
|
||||
});
|
||||
|
||||
it("keeps one global record ID namespace and rejects exhausted ID minting", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
const rootId = await createRoot(storage);
|
||||
const explicitEntryId = 100;
|
||||
await storage.commit([{ type: "entry", value: entry(explicitEntryId, rootId) }], context);
|
||||
expect(storage.mintId()).toBe(101);
|
||||
await expect(
|
||||
storage.commit([{ type: "task", value: pendingTask(explicitEntryId, rootId) }], context),
|
||||
).rejects.toThrow(`ID ${explicitEntryId} already belongs to entry`);
|
||||
|
||||
await storage.commit([{ type: "entry", value: entry(Number.MAX_SAFE_INTEGER, rootId, "last-id") }], context);
|
||||
expect(() => storage.mintId()).toThrow("ID space is exhausted");
|
||||
expect(() => storage.mintId()).toThrow("ID space is exhausted");
|
||||
});
|
||||
|
||||
it("rejects every operation after close", async () => {
|
||||
const storage = new MemoryStorage();
|
||||
await createRoot(storage);
|
||||
await storage.close(context);
|
||||
await expect(storage.conversation(ROOT_CONVERSATION_ID, context)).rejects.toThrow("closed");
|
||||
await expect(storage.commit([] satisfies StorageWrite[], context)).rejects.toThrow("closed");
|
||||
expect(() => storage.mintId()).toThrow("closed");
|
||||
});
|
||||
});
|
||||
registerStorageConformance("Pico MemoryStorage", () => new MemoryStorage());
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { SqliteDatabase, SqliteStatement } from "../src/storage/sqlite/index.ts";
|
||||
import { SqliteStorage } from "../src/storage/sqlite/index.ts";
|
||||
import { type NodeSqliteDatabase, openNodeSqliteDatabase } from "../src/storage/sqlite/node.ts";
|
||||
|
||||
type SettlementMode = "immediate" | "delay" | "reject";
|
||||
|
||||
class ControlledSettlementDatabase implements SqliteDatabase {
|
||||
private readonly delegate: NodeSqliteDatabase;
|
||||
private mode: SettlementMode = "immediate";
|
||||
private pendingSettlement: (() => void) | undefined;
|
||||
private readonly prepareCounts = new Map<string, number>();
|
||||
|
||||
constructor(delegate: NodeSqliteDatabase) {
|
||||
this.delegate = delegate;
|
||||
}
|
||||
|
||||
exec(sql: string): void {
|
||||
this.delegate.exec(sql);
|
||||
}
|
||||
|
||||
prepare(sql: string): SqliteStatement {
|
||||
this.prepareCounts.set(sql, (this.prepareCounts.get(sql) ?? 0) + 1);
|
||||
return this.delegate.prepare(sql);
|
||||
}
|
||||
|
||||
transaction<T>(callback: () => T): T | Promise<T> {
|
||||
const mode = this.mode;
|
||||
this.mode = "immediate";
|
||||
if (mode === "immediate") return this.delegate.transaction(callback);
|
||||
try {
|
||||
const result = this.delegate.transaction(() => {
|
||||
const value = callback();
|
||||
if (mode === "reject") throw new Error("controlled settlement rejection");
|
||||
return value;
|
||||
});
|
||||
return new Promise<T>((resolve) => {
|
||||
this.pendingSettlement = () => resolve(result);
|
||||
});
|
||||
} catch (error) {
|
||||
return new Promise<T>((_resolve, reject) => {
|
||||
this.pendingSettlement = () => reject(error);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.delegate.close();
|
||||
}
|
||||
|
||||
prepareCount(sql: string): number {
|
||||
return this.prepareCounts.get(sql) ?? 0;
|
||||
}
|
||||
|
||||
controlNextSettlement(mode: Exclude<SettlementMode, "immediate">): void {
|
||||
if (this.pendingSettlement !== undefined) throw new Error("A settlement is already pending");
|
||||
this.mode = mode;
|
||||
}
|
||||
|
||||
settle(): void {
|
||||
const settle = this.pendingSettlement;
|
||||
if (settle === undefined) throw new Error("No settlement is pending");
|
||||
this.pendingSettlement = undefined;
|
||||
settle();
|
||||
}
|
||||
}
|
||||
|
||||
describe("portable SQLite facade settlement", () => {
|
||||
it("prepares each storage statement once and rebinds it across commits", async () => {
|
||||
const database = new ControlledSettlementDatabase(await openNodeSqliteDatabase(":memory:"));
|
||||
const storage = await SqliteStorage.open(database);
|
||||
await storage.commit([{ type: "conversation", value: { id: 1 } }], BACKGROUND_CONTEXT);
|
||||
await storage.commit(
|
||||
Array.from({ length: 100 }, (_, index) => ({
|
||||
type: "entry" as const,
|
||||
value: { id: index + 2, conversationId: 1, kind: "cached" },
|
||||
})),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await storage.commit(
|
||||
[{ type: "entry", value: { id: 102, conversationId: 1, kind: "cached-again" } }],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect((await storage.entry(2, BACKGROUND_CONTEXT))?.entry.kind).toBe("cached");
|
||||
expect((await storage.entry(102, BACKGROUND_CONTEXT))?.entry.kind).toBe("cached-again");
|
||||
await expect(storage.commit([{ type: "conversation", value: { id: 1 } }], BACKGROUND_CONTEXT)).rejects.toThrow(
|
||||
"ID 1 already belongs to conversation",
|
||||
);
|
||||
expect((await storage.entry(2, BACKGROUND_CONTEXT))?.entry.kind).toBe("cached");
|
||||
expect(database.prepareCount("SELECT record, commit_seq FROM entries WHERE id = ?")).toBe(1);
|
||||
expect(database.prepareCount("INSERT OR IGNORE INTO record_ids (id, record_type) VALUES (?, ?)")).toBe(1);
|
||||
expect(
|
||||
database.prepareCount(
|
||||
"INSERT INTO entries (id, conversation_id, head, commit_seq, record) VALUES (?, ?, ?, ?, ?)",
|
||||
),
|
||||
).toBe(1);
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("rejects asynchronous Node transaction callbacks and closes idempotently", async () => {
|
||||
const database = await openNodeSqliteDatabase(":memory:");
|
||||
expect(() => database.transaction(() => Promise.resolve())).toThrow(
|
||||
"SQLite transaction callbacks must be synchronous",
|
||||
);
|
||||
database.close();
|
||||
database.close();
|
||||
});
|
||||
|
||||
it("awaits async transaction settlement and adopts IDs only after success", async () => {
|
||||
const database = new ControlledSettlementDatabase(await openNodeSqliteDatabase(":memory:"));
|
||||
database.controlNextSettlement("delay");
|
||||
const opening = SqliteStorage.open(database);
|
||||
let opened = false;
|
||||
void opening.then(() => {
|
||||
opened = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(opened).toBe(false);
|
||||
database.settle();
|
||||
const storage = await opening;
|
||||
|
||||
database.controlNextSettlement("delay");
|
||||
const committing = storage.commit(
|
||||
[{ type: "entry", value: { id: 100, conversationId: 1, kind: "settled" } }],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
let committed = false;
|
||||
void committing.then(() => {
|
||||
committed = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(committed).toBe(false);
|
||||
expect(await storage.mintId()).toBe(2);
|
||||
database.settle();
|
||||
await expect(committing).resolves.toBe(1);
|
||||
expect(await storage.mintId()).toBe(101);
|
||||
|
||||
database.controlNextSettlement("reject");
|
||||
const rejected = storage.commit(
|
||||
[{ type: "entry", value: { id: 200, conversationId: 1, kind: "rejected" } }],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect(await storage.mintId()).toBe(102);
|
||||
database.settle();
|
||||
await expect(rejected).rejects.toThrow("controlled settlement rejection");
|
||||
expect(await storage.mintId()).toBe(103);
|
||||
expect(await storage.entry(200, BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,158 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
applySqliteMigrations,
|
||||
CURRENT_SQLITE_SCHEMA_VERSION,
|
||||
SQLITE_MIGRATIONS,
|
||||
type SqliteMigration,
|
||||
} from "../src/storage/sqlite/index.ts";
|
||||
import { openNodeSqliteDatabase, openNodeSqliteStorage } from "../src/storage/sqlite/node.ts";
|
||||
import { ROOT_CONVERSATION_ID } from "../src/types.ts";
|
||||
|
||||
const directories = new Set<string>();
|
||||
|
||||
async function databasePath(): Promise<string> {
|
||||
const directory = await mkdtemp(join(tmpdir(), "pi-durable-migrations-"));
|
||||
directories.add(directory);
|
||||
return join(directory, "storage.sqlite");
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const directory of directories) await rm(directory, { recursive: true, force: true });
|
||||
directories.clear();
|
||||
});
|
||||
|
||||
describe("durable SQLite migrations", () => {
|
||||
it("creates the current schema and can be applied repeatedly", async () => {
|
||||
const database = await openNodeSqliteDatabase(await databasePath());
|
||||
try {
|
||||
await applySqliteMigrations(database);
|
||||
await applySqliteMigrations(database);
|
||||
expect(database.prepare("SELECT version FROM durable_schema WHERE singleton = 1").get()).toEqual({
|
||||
version: CURRENT_SQLITE_SCHEMA_VERSION,
|
||||
});
|
||||
expect(database.prepare("SELECT next_id, next_seq FROM durable_metadata WHERE singleton = 1").get()).toEqual({
|
||||
next_id: "2",
|
||||
next_seq: 1,
|
||||
});
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a database newer than the portable core", async () => {
|
||||
const path = await databasePath();
|
||||
const database = await openNodeSqliteDatabase(path);
|
||||
await applySqliteMigrations(database);
|
||||
database
|
||||
.prepare("UPDATE durable_schema SET version = ? WHERE singleton = 1")
|
||||
.run(CURRENT_SQLITE_SCHEMA_VERSION + 1);
|
||||
database.close();
|
||||
|
||||
await expect(openNodeSqliteStorage(path)).rejects.toThrow("is newer than supported version");
|
||||
});
|
||||
|
||||
it("rolls initial bootstrap and every pending migration back together", async () => {
|
||||
const database = await openNodeSqliteDatabase(await databasePath());
|
||||
try {
|
||||
const failed: readonly SqliteMigration[] = [
|
||||
{
|
||||
version: 1,
|
||||
statements: [
|
||||
"CREATE TABLE migration_first (value TEXT) STRICT",
|
||||
"INSERT INTO migration_first (value) VALUES ('retained')",
|
||||
],
|
||||
},
|
||||
{
|
||||
version: 2,
|
||||
statements: ["CREATE TABLE migration_second (value TEXT) STRICT", "THIS IS NOT SQL"],
|
||||
},
|
||||
];
|
||||
await expect(applySqliteMigrations(database, failed)).rejects.toThrow();
|
||||
expect(
|
||||
database
|
||||
.prepare(
|
||||
"SELECT count(*) AS count FROM sqlite_schema WHERE name IN ('durable_schema', 'migration_first', 'migration_second')",
|
||||
)
|
||||
.get(),
|
||||
).toEqual({ count: 0 });
|
||||
|
||||
await applySqliteMigrations(database, [
|
||||
failed[0],
|
||||
{ version: 2, statements: ["CREATE TABLE migration_second (value TEXT) STRICT"] },
|
||||
]);
|
||||
expect(database.prepare("SELECT version FROM durable_schema WHERE singleton = 1").get()).toEqual({
|
||||
version: 2,
|
||||
});
|
||||
expect(database.prepare("SELECT value FROM migration_first").get()).toEqual({ value: "retained" });
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rolls a failed migration back and preserves stored data for a successful retry", async () => {
|
||||
const path = await databasePath();
|
||||
const storage = await openNodeSqliteStorage(path);
|
||||
await storage.commit(
|
||||
[
|
||||
{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } },
|
||||
{
|
||||
type: "entry",
|
||||
value: {
|
||||
id: 2,
|
||||
conversationId: ROOT_CONVERSATION_ID,
|
||||
kind: "retained",
|
||||
data: { retained: true },
|
||||
},
|
||||
},
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
|
||||
const database = await openNodeSqliteDatabase(path);
|
||||
const nextVersion = CURRENT_SQLITE_SCHEMA_VERSION + 1;
|
||||
const failedMigrations: readonly SqliteMigration[] = [
|
||||
...SQLITE_MIGRATIONS,
|
||||
{
|
||||
version: nextVersion,
|
||||
statements: ["CREATE TABLE migration_probe (value TEXT) STRICT", "THIS IS NOT SQL"],
|
||||
},
|
||||
];
|
||||
await expect(applySqliteMigrations(database, failedMigrations)).rejects.toThrow();
|
||||
expect(database.prepare("SELECT version FROM durable_schema WHERE singleton = 1").get()).toEqual({
|
||||
version: CURRENT_SQLITE_SCHEMA_VERSION,
|
||||
});
|
||||
expect(
|
||||
database
|
||||
.prepare("SELECT count(*) AS count FROM sqlite_schema WHERE type = 'table' AND name = 'migration_probe'")
|
||||
.get(),
|
||||
).toEqual({ count: 0 });
|
||||
|
||||
const successfulMigrations: readonly SqliteMigration[] = [
|
||||
...SQLITE_MIGRATIONS,
|
||||
{ version: nextVersion, statements: ["CREATE TABLE migration_probe (value TEXT) STRICT"] },
|
||||
];
|
||||
await applySqliteMigrations(database, successfulMigrations);
|
||||
expect(database.prepare("SELECT version FROM durable_schema WHERE singleton = 1").get()).toEqual({
|
||||
version: nextVersion,
|
||||
});
|
||||
expect(database.prepare("SELECT record, commit_seq FROM entries WHERE id = 2").get()).toEqual({
|
||||
record: JSON.stringify({
|
||||
id: 2,
|
||||
conversationId: ROOT_CONVERSATION_ID,
|
||||
kind: "retained",
|
||||
data: { retained: true },
|
||||
}),
|
||||
commit_seq: 1,
|
||||
});
|
||||
expect(database.prepare("SELECT next_id, next_seq FROM durable_metadata WHERE singleton = 1").get()).toEqual({
|
||||
next_id: "3",
|
||||
next_seq: 2,
|
||||
});
|
||||
database.close();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,499 @@
|
||||
import { mkdtemp, rm, stat } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import type { JsonValue } from "@earendil-works/chord";
|
||||
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
||||
import { afterEach, describe, expect, it, onTestFinished } from "vitest";
|
||||
import type { SqliteStorage } from "../src/storage/sqlite/index.ts";
|
||||
import { type NodeSqliteStorageOptions, openNodeSqliteStorage } from "../src/storage/sqlite/node.ts";
|
||||
import type { DocumentCreate, EntryRecord, Seq, Storage, StorageWrite, TaskRecord } from "../src/types.ts";
|
||||
import { ROOT_CONVERSATION_ID } from "../src/types.ts";
|
||||
import { registerStorageConformance } from "./storage-conformance.ts";
|
||||
|
||||
const context = BACKGROUND_CONTEXT;
|
||||
const openStorages = new Set<SqliteStorage>();
|
||||
const tempDirectories = new Set<string>();
|
||||
|
||||
afterEach(async () => {
|
||||
for (const storage of openStorages) await storage.close(context);
|
||||
openStorages.clear();
|
||||
for (const directory of tempDirectories) await rm(directory, { recursive: true, force: true });
|
||||
tempDirectories.clear();
|
||||
});
|
||||
|
||||
async function createSqliteStorage(options: { readonly walAutoCheckpointPages?: number } = {}) {
|
||||
const directory = await mkdtemp(join(tmpdir(), "pi-durable-sqlite-"));
|
||||
tempDirectories.add(directory);
|
||||
const path = join(directory, "storage.sqlite");
|
||||
const storage = await openNodeSqliteStorage(path, options);
|
||||
openStorages.add(storage);
|
||||
return { storage, path };
|
||||
}
|
||||
|
||||
class ReopeningStorage implements Storage {
|
||||
private current: SqliteStorage;
|
||||
private readonly path: string;
|
||||
private readonly options: NodeSqliteStorageOptions;
|
||||
private closed = false;
|
||||
|
||||
constructor(current: SqliteStorage, path: string, options: NodeSqliteStorageOptions = {}) {
|
||||
this.current = current;
|
||||
this.path = path;
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
async commit(writes: readonly StorageWrite[], commitContext: Parameters<Storage["commit"]>[1]): Promise<Seq> {
|
||||
if (this.closed) return this.current.commit(writes, commitContext);
|
||||
try {
|
||||
return await this.current.commit(writes, commitContext);
|
||||
} finally {
|
||||
await this.current.close(context);
|
||||
this.current = await openNodeSqliteStorage(this.path, this.options);
|
||||
}
|
||||
}
|
||||
|
||||
mintId: Storage["mintId"] = () => this.current.mintId();
|
||||
conversation: Storage["conversation"] = (id, readContext) => this.current.conversation(id, readContext);
|
||||
scanConversations: Storage["scanConversations"] = (cursor, limit, readContext) =>
|
||||
this.current.scanConversations(cursor, limit, readContext);
|
||||
entry: Storage["entry"] = (id, readContext) => this.current.entry(id, readContext);
|
||||
findLatestHeadMarker: Storage["findLatestHeadMarker"] = (conversationId, at, readContext) =>
|
||||
this.current.findLatestHeadMarker(conversationId, at, readContext);
|
||||
scanEntries: Storage["scanEntries"] = (query, cursor, limit, readContext) =>
|
||||
this.current.scanEntries(query, cursor, limit, readContext);
|
||||
task: Storage["task"] = (id, readContext) => this.current.task(id, readContext);
|
||||
scanTasks: Storage["scanTasks"] = (query, cursor, limit, readContext) =>
|
||||
this.current.scanTasks(query, cursor, limit, readContext);
|
||||
submission: Storage["submission"] = (id, readContext) => this.current.submission(id, readContext);
|
||||
submissionByRequest: Storage["submissionByRequest"] = (conversationId, requestId, readContext) =>
|
||||
this.current.submissionByRequest(conversationId, requestId, readContext);
|
||||
findDocument: Storage["findDocument"] = (address, at, readContext) =>
|
||||
this.current.findDocument(address, at, readContext);
|
||||
document: Storage["document"] = (id, at, readContext) => this.current.document(id, at, readContext);
|
||||
scanDocuments: Storage["scanDocuments"] = (query, cursor, limit, readContext) =>
|
||||
this.current.scanDocuments(query, cursor, limit, readContext);
|
||||
|
||||
async close(closeContext: Parameters<Storage["close"]>[0]): Promise<void> {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
await this.current.close(closeContext);
|
||||
}
|
||||
}
|
||||
|
||||
registerStorageConformance("Pico SqliteStorage conformance", async () => (await createSqliteStorage()).storage);
|
||||
|
||||
registerStorageConformance("Pico SqliteStorage conformance across reopen", async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), "pi-durable-sqlite-conformance-"));
|
||||
const path = join(directory, "storage.sqlite");
|
||||
const created = await openNodeSqliteStorage(path);
|
||||
await created.close(context);
|
||||
const storage = new ReopeningStorage(await openNodeSqliteStorage(path), path);
|
||||
onTestFinished(async () => {
|
||||
await storage.close(context);
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
return storage;
|
||||
});
|
||||
|
||||
function entry(id: number, conversationId: number, data?: JsonValue): EntryRecord {
|
||||
return { id, conversationId, kind: "message", ...(data === undefined ? {} : { data }) };
|
||||
}
|
||||
|
||||
async function createRoot(storage: Storage): Promise<number> {
|
||||
return storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context);
|
||||
}
|
||||
|
||||
function pendingTask(id: number): TaskRecord<JsonValue, JsonValue, JsonValue> {
|
||||
return {
|
||||
id,
|
||||
conversationId: ROOT_CONVERSATION_ID,
|
||||
kind: "test.task",
|
||||
version: 1,
|
||||
input: null,
|
||||
state: { status: "pending", checkpoint: { phase: "ready" } },
|
||||
after: [],
|
||||
background: false,
|
||||
abortRequested: false,
|
||||
};
|
||||
}
|
||||
|
||||
function scalar(db: DatabaseSync, sql: string): number {
|
||||
const row = db.prepare(sql).get() as { readonly value: number };
|
||||
return row.value;
|
||||
}
|
||||
|
||||
function revisionCount(path: string, documentId: number): number {
|
||||
const db = new DatabaseSync(path, { readOnly: true });
|
||||
try {
|
||||
const row = db
|
||||
.prepare("SELECT count(*) AS count FROM document_revisions WHERE document_id = ?")
|
||||
.get(documentId) as { readonly count: number };
|
||||
return row.count;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
describe("Pico SqliteStorage", () => {
|
||||
it("persists records, sequence allocation, and global ID allocation across reopen", async () => {
|
||||
const { storage, path } = await createSqliteStorage();
|
||||
expect(await createRoot(storage)).toBe(1);
|
||||
const entryId = await storage.mintId();
|
||||
expect(await storage.commit([{ type: "entry", value: entry(entryId, ROOT_CONVERSATION_ID) }], context)).toBe(2);
|
||||
await storage.close(context);
|
||||
openStorages.delete(storage);
|
||||
|
||||
const reopened = await openNodeSqliteStorage(path);
|
||||
openStorages.add(reopened);
|
||||
expect(await reopened.entry(entryId, context)).toEqual({
|
||||
entry: entry(entryId, ROOT_CONVERSATION_ID),
|
||||
commitSeq: 2,
|
||||
});
|
||||
expect(await reopened.mintId()).toBe(entryId + 1);
|
||||
expect(await reopened.commit([{ type: "task", value: pendingTask(await reopened.mintId()) }], context)).toBe(3);
|
||||
});
|
||||
|
||||
it("rejects persisted metadata corruption on reopen", async () => {
|
||||
const { storage, path } = await createSqliteStorage();
|
||||
await storage.close(context);
|
||||
openStorages.delete(storage);
|
||||
const database = new DatabaseSync(path);
|
||||
try {
|
||||
database.exec("DELETE FROM durable_metadata");
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
await expect(openNodeSqliteStorage(path)).rejects.toThrow("Durable SQLite metadata is missing");
|
||||
});
|
||||
|
||||
it("rejects a document whose required base is missing", async () => {
|
||||
const { storage, path } = await createSqliteStorage();
|
||||
await createRoot(storage);
|
||||
const id = await storage.mintId();
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.create",
|
||||
record: { id, kind: "corrupt", scope: { kind: "session" } },
|
||||
content: { kind: "base", version: 1, value: { retained: true } },
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
const database = new DatabaseSync(path);
|
||||
try {
|
||||
database.prepare("DELETE FROM document_revisions WHERE document_id = ?").run(id);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
await expect(storage.document(id, "current", context)).rejects.toThrow(
|
||||
`Document ${id} is missing a required base`,
|
||||
);
|
||||
});
|
||||
|
||||
it("rolls SQL rows and sequence allocation back as one transaction", async () => {
|
||||
const { storage, path } = await createSqliteStorage();
|
||||
await createRoot(storage);
|
||||
const transientId = await storage.mintId();
|
||||
const transientTaskId = await storage.mintId();
|
||||
const circular: { self?: unknown } = {};
|
||||
circular.self = circular;
|
||||
await expect(
|
||||
storage.commit(
|
||||
[
|
||||
{ type: "entry", value: entry(transientId, ROOT_CONVERSATION_ID) },
|
||||
{
|
||||
type: "task",
|
||||
value: { ...pendingTask(transientTaskId), input: circular as unknown as JsonValue },
|
||||
},
|
||||
],
|
||||
context,
|
||||
),
|
||||
).rejects.toThrow("circular structure");
|
||||
expect(await storage.entry(transientId, context)).toBeUndefined();
|
||||
expect(await storage.task(transientTaskId, context)).toBeUndefined();
|
||||
const committedId = await storage.mintId();
|
||||
expect(await storage.commit([{ type: "entry", value: entry(committedId, ROOT_CONVERSATION_ID) }], context)).toBe(
|
||||
2,
|
||||
);
|
||||
|
||||
const db = new DatabaseSync(path, { readOnly: true });
|
||||
try {
|
||||
expect(scalar(db, "SELECT count(*) AS value FROM entries")).toBe(1);
|
||||
expect(scalar(db, "SELECT next_seq AS value FROM durable_metadata WHERE singleton = 1")).toBe(3);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("reconstructs recent and ancient rewindable points after reopen", async () => {
|
||||
const { storage, path } = await createSqliteStorage();
|
||||
await createRoot(storage);
|
||||
const id = await storage.mintId();
|
||||
const record = {
|
||||
id,
|
||||
kind: "history",
|
||||
scope: { kind: "conversation", conversationId: ROOT_CONVERSATION_ID },
|
||||
history: "rewindable",
|
||||
fork: "asOf",
|
||||
} satisfies DocumentCreate;
|
||||
const createdAt = await storage.commit(
|
||||
[{ type: "document.create", record, content: { kind: "base", version: 1, value: { count: 0 } } }],
|
||||
context,
|
||||
);
|
||||
let ancientAt = createdAt;
|
||||
let recentAt = createdAt;
|
||||
for (let count = 1; count <= 40; count++) {
|
||||
recentAt = await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.change",
|
||||
id,
|
||||
content:
|
||||
count === 20
|
||||
? { kind: "base", version: 1, value: { count } }
|
||||
: { kind: "delta", version: 1, ops: [["s", ["count"], count]] },
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
if (count === 5) ancientAt = recentAt;
|
||||
}
|
||||
await storage.close(context);
|
||||
openStorages.delete(storage);
|
||||
const reopened = await openNodeSqliteStorage(path);
|
||||
openStorages.add(reopened);
|
||||
expect((await reopened.document(id, ancientAt, context))?.value).toEqual({ count: 5 });
|
||||
expect((await reopened.document(id, recentAt, context))?.value).toEqual({ count: 40 });
|
||||
});
|
||||
|
||||
it("uses indexes for exact addresses, exact scopes, entry history, and document revision tails", async () => {
|
||||
const { storage, path } = await createSqliteStorage();
|
||||
await storage.close(context);
|
||||
openStorages.delete(storage);
|
||||
const db = new DatabaseSync(path, { readOnly: true });
|
||||
try {
|
||||
const plans = [
|
||||
db
|
||||
.prepare(`EXPLAIN QUERY PLAN SELECT record FROM documents
|
||||
WHERE kind = ? AND scope_kind = ? AND owner_id = ? AND family = ? AND key_value = ?
|
||||
AND retired_at IS NULL ORDER BY created_at DESC LIMIT 1`)
|
||||
.all("kind", "session", 0, 0, ""),
|
||||
db
|
||||
.prepare(`EXPLAIN QUERY PLAN SELECT record FROM documents
|
||||
WHERE kind = ? AND scope_kind = ? AND owner_id = ? AND family = ? AND key_value = ?
|
||||
AND created_at <= ? AND (retired_at IS NULL OR retired_at > ?)
|
||||
ORDER BY created_at DESC LIMIT 1`)
|
||||
.all("kind", "conversation", 1, 0, "", 10, 10),
|
||||
db
|
||||
.prepare(`EXPLAIN QUERY PLAN SELECT record FROM documents
|
||||
WHERE scope_kind = ? AND owner_id = ? AND kind = ? AND id > ? ORDER BY id LIMIT ?`)
|
||||
.all("task", 1, "kind", 0, 10),
|
||||
db
|
||||
.prepare(`EXPLAIN QUERY PLAN SELECT record FROM entries
|
||||
WHERE conversation_id = ? AND id <= ? ORDER BY id DESC LIMIT ?`)
|
||||
.all(1, 10, 10),
|
||||
db
|
||||
.prepare(`EXPLAIN QUERY PLAN SELECT record FROM entries
|
||||
WHERE conversation_id = ? AND head IS NOT NULL AND id <= ? ORDER BY id DESC LIMIT 1`)
|
||||
.all(1, 10),
|
||||
db
|
||||
.prepare("EXPLAIN QUERY PLAN SELECT record FROM tasks WHERE status = ? AND id > ? ORDER BY id LIMIT ?")
|
||||
.all("pending", 0, 10),
|
||||
db
|
||||
.prepare(`EXPLAIN QUERY PLAN SELECT seq, kind, version, content FROM document_revisions
|
||||
WHERE document_id = ? AND kind = 'base' AND seq <= ? ORDER BY seq DESC LIMIT 1`)
|
||||
.all(1, 10),
|
||||
db
|
||||
.prepare(`EXPLAIN QUERY PLAN SELECT seq, kind, version, content FROM document_revisions
|
||||
WHERE document_id = ? AND seq > ? AND seq <= ? ORDER BY seq`)
|
||||
.all(1, 5, 10),
|
||||
];
|
||||
const details = plans.map((plan) => plan.map((row) => (row as { readonly detail: string }).detail).join("\n"));
|
||||
expect(details[0]).toContain("documents_by_address");
|
||||
expect(details[1]).toContain("documents_by_address");
|
||||
expect(details[2]).toContain("documents_by_scope_kind");
|
||||
expect(details[3]).toContain("entries_by_conversation");
|
||||
expect(details[4]).toContain("entry_heads_by_conversation");
|
||||
expect(details[5]).toContain("tasks_by_status");
|
||||
expect(details[6]).toContain("document_revisions_by_kind");
|
||||
expect(details[7]).toContain("sqlite_autoindex_document_revisions_1");
|
||||
for (const detail of details.slice(0, 3)) expect(detail).not.toContain("SCAN documents");
|
||||
expect(details[7]).not.toContain("SCAN document_revisions");
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("reclaims current-only revisions only after a base or retirement", async () => {
|
||||
const { storage, path } = await createSqliteStorage();
|
||||
await createRoot(storage);
|
||||
const id = await storage.mintId();
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.create",
|
||||
record: { id, kind: "latest", scope: { kind: "session" } },
|
||||
content: { kind: "base", version: 1, value: { count: 0 } },
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
for (let count = 1; count <= 10; count++) {
|
||||
await storage.commit(
|
||||
[{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] } }],
|
||||
context,
|
||||
);
|
||||
}
|
||||
expect(revisionCount(path, id)).toBe(11);
|
||||
const revisions = new DatabaseSync(path, { readOnly: true });
|
||||
try {
|
||||
const row = revisions
|
||||
.prepare(
|
||||
"SELECT content FROM document_revisions WHERE document_id = ? AND kind = 'delta' ORDER BY seq DESC LIMIT 1",
|
||||
)
|
||||
.get(id) as { readonly content: string };
|
||||
expect(JSON.parse(row.content)).toEqual([["s", ["count"], 10]]);
|
||||
} finally {
|
||||
revisions.close();
|
||||
}
|
||||
await storage.commit(
|
||||
[{ type: "document.change", id, content: { kind: "base", version: 1, value: { count: 11 } } }],
|
||||
context,
|
||||
);
|
||||
expect(revisionCount(path, id)).toBe(1);
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.change",
|
||||
id,
|
||||
content: { kind: "delta", version: 1, ops: [["r", { count: 12 }]] },
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
expect(revisionCount(path, id)).toBe(2);
|
||||
await storage.commit([{ type: "document.retire", id }], context);
|
||||
expect(revisionCount(path, id)).toBe(0);
|
||||
});
|
||||
|
||||
it("auto-checkpoints WAL frames and truncates the WAL on close", async () => {
|
||||
const { storage, path } = await createSqliteStorage({ walAutoCheckpointPages: 1 });
|
||||
await createRoot(storage);
|
||||
for (let index = 0; index < 20; index++) {
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "entry",
|
||||
value: entry(await storage.mintId(), ROOT_CONVERSATION_ID, { text: "x".repeat(32 * 1024), index }),
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
}
|
||||
const walPath = `${path}-wal`;
|
||||
expect((await stat(walPath)).size).toBeLessThan(512 * 1024);
|
||||
const observer = new DatabaseSync(path, { readOnly: true });
|
||||
try {
|
||||
expect(scalar(observer, "SELECT count(*) AS value FROM entries")).toBe(20);
|
||||
await storage.close(context);
|
||||
openStorages.delete(storage);
|
||||
expect((await stat(walPath)).size).toBe(0);
|
||||
} finally {
|
||||
observer.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("reuses pages released by current-only checkpoints", async () => {
|
||||
const { storage, path } = await createSqliteStorage();
|
||||
await createRoot(storage);
|
||||
const id = await storage.mintId();
|
||||
const large = "x".repeat(512 * 1024);
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.create",
|
||||
record: { id, kind: "reuse", scope: { kind: "session" } },
|
||||
content: { kind: "base", version: 1, value: { text: large } },
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
await storage.commit(
|
||||
[{ type: "document.change", id, content: { kind: "base", version: 1, value: { text: "small" } } }],
|
||||
context,
|
||||
);
|
||||
const before = new DatabaseSync(path, { readOnly: true });
|
||||
let pagesAfterDelete: number;
|
||||
let freeAfterDelete: number;
|
||||
try {
|
||||
pagesAfterDelete = scalar(before, "SELECT page_count AS value FROM pragma_page_count() ");
|
||||
freeAfterDelete = scalar(before, "SELECT freelist_count AS value FROM pragma_freelist_count() ");
|
||||
} finally {
|
||||
before.close();
|
||||
}
|
||||
expect(freeAfterDelete).toBeGreaterThan(0);
|
||||
await storage.commit(
|
||||
[{ type: "document.change", id, content: { kind: "base", version: 1, value: { text: large } } }],
|
||||
context,
|
||||
);
|
||||
const after = new DatabaseSync(path, { readOnly: true });
|
||||
try {
|
||||
const pagesAfterReuse = scalar(after, "SELECT page_count AS value FROM pragma_page_count() ");
|
||||
const freeAfterReuse = scalar(after, "SELECT freelist_count AS value FROM pragma_freelist_count() ");
|
||||
expect(pagesAfterReuse).toBeLessThanOrEqual(pagesAfterDelete + 2);
|
||||
expect(freeAfterReuse).toBeLessThan(freeAfterDelete);
|
||||
} finally {
|
||||
after.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps representative row and document storage bounded", async () => {
|
||||
const { storage, path } = await createSqliteStorage();
|
||||
await createRoot(storage);
|
||||
for (let index = 0; index < 100; index++) {
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "entry",
|
||||
value: entry(await storage.mintId(), ROOT_CONVERSATION_ID, { index, text: "x".repeat(1_024) }),
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
}
|
||||
const documentId = await storage.mintId();
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.create",
|
||||
record: {
|
||||
id: documentId,
|
||||
kind: "size.history",
|
||||
scope: { kind: "conversation", conversationId: ROOT_CONVERSATION_ID },
|
||||
history: "rewindable",
|
||||
fork: "asOf",
|
||||
},
|
||||
content: { kind: "base", version: 1, value: { count: 0 } },
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
for (let count = 1; count <= 100; count++) {
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.change",
|
||||
id: documentId,
|
||||
content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] },
|
||||
},
|
||||
],
|
||||
context,
|
||||
);
|
||||
}
|
||||
await storage.close(context);
|
||||
openStorages.delete(storage);
|
||||
expect((await stat(path)).size).toBeLessThan(1024 * 1024);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,460 @@
|
||||
import type { JsonValue } from "@earendil-works/chord";
|
||||
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
||||
import type { DocumentCreate, Id, Seq, Storage, StorageWrite, TaskRecord } from "../src/types.ts";
|
||||
import { ROOT_CONVERSATION_ID } from "../src/types.ts";
|
||||
|
||||
export const STORAGE_BENCHMARK_BACKENDS = ["memory", "sqlite"] as const;
|
||||
export type StorageBenchmarkBackend = (typeof STORAGE_BENCHMARK_BACKENDS)[number];
|
||||
|
||||
export type StorageBenchmarkScale = {
|
||||
readonly name: string;
|
||||
readonly entryCount: number;
|
||||
readonly taskCount: number;
|
||||
readonly documentCount: number;
|
||||
};
|
||||
|
||||
export const STORAGE_MEMORY_SCALES: readonly StorageBenchmarkScale[] = [
|
||||
{ name: "1k", entryCount: 1_000, taskCount: 200, documentCount: 200 },
|
||||
{ name: "10k", entryCount: 10_000, taskCount: 2_000, documentCount: 2_000 },
|
||||
];
|
||||
|
||||
export const TIMING_SCALE: StorageBenchmarkScale = {
|
||||
name: "timing",
|
||||
entryCount: 1_000,
|
||||
taskCount: 300,
|
||||
documentCount: 300,
|
||||
};
|
||||
|
||||
const REPLAY_TAILS = [0, 16, 128, 1_024] as const;
|
||||
const HISTORY_SEGMENT_LENGTH = 128;
|
||||
const FORK_DEPTH = 8;
|
||||
const ENTRIES_PER_FORK = 32;
|
||||
const BATCH_SIZE = 100;
|
||||
|
||||
export function storageBenchmarkPrimaryRecordCount(scale: StorageBenchmarkScale): number {
|
||||
return (
|
||||
1 +
|
||||
scale.entryCount +
|
||||
scale.taskCount +
|
||||
scale.documentCount +
|
||||
REPLAY_TAILS.length +
|
||||
1 +
|
||||
FORK_DEPTH * (1 + ENTRIES_PER_FORK)
|
||||
);
|
||||
}
|
||||
|
||||
type StoredTask = TaskRecord<JsonValue, JsonValue, JsonValue>;
|
||||
|
||||
export type StorageBenchmarkDataset = {
|
||||
readonly firstEntryId: Id;
|
||||
readonly filteredTaskCount: number;
|
||||
readonly exactDocumentId: Id;
|
||||
readonly exactDocumentKey: string;
|
||||
readonly replayDocumentIds: Readonly<Record<(typeof REPLAY_TAILS)[number], Id>>;
|
||||
readonly historicalDocumentId: Id;
|
||||
readonly ancientAt: Seq;
|
||||
readonly recentAt: Seq;
|
||||
readonly deepestConversationId: Id;
|
||||
readonly ancestorHeadEntryId: Id;
|
||||
};
|
||||
|
||||
function task(id: Id, index: number): StoredTask {
|
||||
const statuses = ["pending", "running", "terminal"] as const;
|
||||
const status = statuses[index % statuses.length];
|
||||
const common = {
|
||||
id,
|
||||
conversationId: ROOT_CONVERSATION_ID,
|
||||
kind: index % 4 === 0 ? "benchmark.filtered" : "benchmark.other",
|
||||
version: 1,
|
||||
input: { index },
|
||||
after: [],
|
||||
background: index % 5 === 0,
|
||||
abortRequested: index % 7 === 0,
|
||||
};
|
||||
if (status === "terminal") {
|
||||
return { ...common, state: { status, outcome: { status: "completed", result: { index } } } };
|
||||
}
|
||||
return { ...common, state: { status, checkpoint: { index, payload: "x".repeat(64) } } };
|
||||
}
|
||||
|
||||
/** Seed deterministic representative data through only the public `Storage` contract. */
|
||||
export async function seedStorageBenchmark(
|
||||
storage: Storage,
|
||||
scale: StorageBenchmarkScale = TIMING_SCALE,
|
||||
): Promise<StorageBenchmarkDataset> {
|
||||
await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT);
|
||||
|
||||
let firstEntryId = 0;
|
||||
for (let start = 0; start < scale.entryCount; start += BATCH_SIZE) {
|
||||
const writes: StorageWrite[] = [];
|
||||
for (let index = start; index < Math.min(start + BATCH_SIZE, scale.entryCount); index++) {
|
||||
const id = await storage.mintId();
|
||||
if (index === 0) firstEntryId = id;
|
||||
writes.push({
|
||||
type: "entry",
|
||||
value: {
|
||||
id,
|
||||
conversationId: ROOT_CONVERSATION_ID,
|
||||
kind: "benchmark.entry",
|
||||
...(index === 0 ? { head: id } : {}),
|
||||
data: { index, text: `entry-${index}-${"x".repeat(96)}` },
|
||||
},
|
||||
});
|
||||
}
|
||||
await storage.commit(writes, BACKGROUND_CONTEXT);
|
||||
}
|
||||
|
||||
for (let start = 0; start < scale.taskCount; start += BATCH_SIZE) {
|
||||
const writes: StorageWrite[] = [];
|
||||
for (let index = start; index < Math.min(start + BATCH_SIZE, scale.taskCount); index++) {
|
||||
writes.push({ type: "task", value: task(await storage.mintId(), index) });
|
||||
}
|
||||
await storage.commit(writes, BACKGROUND_CONTEXT);
|
||||
}
|
||||
|
||||
let exactDocumentId = 0;
|
||||
for (let start = 0; start < scale.documentCount; start += BATCH_SIZE) {
|
||||
const writes: StorageWrite[] = [];
|
||||
for (let index = start; index < Math.min(start + BATCH_SIZE, scale.documentCount); index++) {
|
||||
const id = await storage.mintId();
|
||||
exactDocumentId = id;
|
||||
writes.push({
|
||||
type: "document.create",
|
||||
record: { id, kind: "benchmark.family", key: `key-${index}`, scope: { kind: "session" } },
|
||||
content: { kind: "base", version: 1, value: { index, text: "x".repeat(128) } },
|
||||
});
|
||||
}
|
||||
await storage.commit(writes, BACKGROUND_CONTEXT);
|
||||
}
|
||||
|
||||
const replayEntries = await Promise.all(
|
||||
REPLAY_TAILS.map(async (tail) => {
|
||||
const id = await storage.mintId();
|
||||
return { tail, id };
|
||||
}),
|
||||
);
|
||||
await storage.commit(
|
||||
replayEntries.map(({ tail, id }) => ({
|
||||
type: "document.create" as const,
|
||||
record: {
|
||||
id,
|
||||
kind: "benchmark.replay",
|
||||
key: String(tail),
|
||||
scope: { kind: "conversation" as const, conversationId: ROOT_CONVERSATION_ID },
|
||||
history: "rewindable" as const,
|
||||
fork: "asOf" as const,
|
||||
},
|
||||
content: { kind: "base" as const, version: 1, value: { count: 0, text: "x".repeat(64) } },
|
||||
})),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
for (let count = 1; count <= REPLAY_TAILS.at(-1)!; count++) {
|
||||
await storage.commit(
|
||||
replayEntries
|
||||
.filter(({ tail }) => count <= tail)
|
||||
.map(({ id }) => ({
|
||||
type: "document.change" as const,
|
||||
id,
|
||||
content: { kind: "delta" as const, version: 1, ops: [["s", ["count"], count] as const] },
|
||||
})),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
}
|
||||
|
||||
const historicalDocumentId = await storage.mintId();
|
||||
const historicalRecord = {
|
||||
id: historicalDocumentId,
|
||||
kind: "benchmark.history",
|
||||
scope: { kind: "conversation", conversationId: ROOT_CONVERSATION_ID },
|
||||
history: "rewindable",
|
||||
fork: "asOf",
|
||||
} satisfies DocumentCreate;
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.create",
|
||||
record: historicalRecord,
|
||||
content: { kind: "base", version: 1, value: { count: 0 } },
|
||||
},
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
let ancientAt = 0;
|
||||
for (let count = 1; count <= HISTORY_SEGMENT_LENGTH; count++) {
|
||||
ancientAt = await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.change",
|
||||
id: historicalDocumentId,
|
||||
content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] },
|
||||
},
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
}
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.change",
|
||||
id: historicalDocumentId,
|
||||
content: { kind: "base", version: 1, value: { count: HISTORY_SEGMENT_LENGTH } },
|
||||
},
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
let recentAt = ancientAt;
|
||||
for (let count = HISTORY_SEGMENT_LENGTH + 1; count <= HISTORY_SEGMENT_LENGTH * 2; count++) {
|
||||
recentAt = await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "document.change",
|
||||
id: historicalDocumentId,
|
||||
content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] },
|
||||
},
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
}
|
||||
|
||||
let parentConversationId = ROOT_CONVERSATION_ID;
|
||||
let parentAt = firstEntryId;
|
||||
let deepestConversationId = ROOT_CONVERSATION_ID;
|
||||
for (let depth = 0; depth < FORK_DEPTH; depth++) {
|
||||
const conversationId = await storage.mintId();
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "conversation",
|
||||
value: { id: conversationId, parent: { conversationId: parentConversationId, at: parentAt } },
|
||||
},
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const ids = await Promise.all(Array.from({ length: ENTRIES_PER_FORK }, () => storage.mintId()));
|
||||
await storage.commit(
|
||||
ids.map((id, index) => ({
|
||||
type: "entry" as const,
|
||||
value: {
|
||||
id,
|
||||
conversationId,
|
||||
kind: "benchmark.fork",
|
||||
data: { depth, index },
|
||||
},
|
||||
})),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
parentConversationId = conversationId;
|
||||
parentAt = ids.at(-1)!;
|
||||
deepestConversationId = conversationId;
|
||||
}
|
||||
|
||||
return {
|
||||
firstEntryId,
|
||||
filteredTaskCount: Math.min(50, Math.ceil(scale.taskCount / 60)),
|
||||
exactDocumentId,
|
||||
exactDocumentKey: `key-${scale.documentCount - 1}`,
|
||||
replayDocumentIds: Object.fromEntries(replayEntries.map(({ tail, id }) => [tail, id])) as Record<
|
||||
(typeof REPLAY_TAILS)[number],
|
||||
Id
|
||||
>,
|
||||
historicalDocumentId,
|
||||
ancientAt,
|
||||
recentAt,
|
||||
deepestConversationId,
|
||||
ancestorHeadEntryId: firstEntryId,
|
||||
};
|
||||
}
|
||||
|
||||
export type StorageReadBenchmark = {
|
||||
readonly name: string;
|
||||
run(storage: Storage, dataset: StorageBenchmarkDataset): Promise<number>;
|
||||
expected(dataset: StorageBenchmarkDataset): number;
|
||||
};
|
||||
|
||||
export const STORAGE_READ_BENCHMARKS: readonly StorageReadBenchmark[] = [
|
||||
{
|
||||
name: "exact entry lookup",
|
||||
async run(storage, dataset) {
|
||||
return (await storage.entry(dataset.firstEntryId, BACKGROUND_CONTEXT))?.entry.id ?? -1;
|
||||
},
|
||||
expected: ({ firstEntryId }) => firstEntryId,
|
||||
},
|
||||
{
|
||||
name: "entry page scan (100)",
|
||||
async run(storage) {
|
||||
return (
|
||||
await storage.scanEntries({ conversationId: ROOT_CONVERSATION_ID }, undefined, 100, BACKGROUND_CONTEXT)
|
||||
).items.length;
|
||||
},
|
||||
expected: () => 100,
|
||||
},
|
||||
{
|
||||
name: "filtered task scan (50)",
|
||||
async run(storage) {
|
||||
return (
|
||||
await storage.scanTasks(
|
||||
{ kind: "benchmark.filtered", status: "pending", background: true },
|
||||
undefined,
|
||||
50,
|
||||
BACKGROUND_CONTEXT,
|
||||
)
|
||||
).items.length;
|
||||
},
|
||||
expected: ({ filteredTaskCount }) => filteredTaskCount,
|
||||
},
|
||||
{
|
||||
name: "exact document address among many",
|
||||
async run(storage, dataset) {
|
||||
return (
|
||||
(
|
||||
await storage.findDocument(
|
||||
{ kind: "benchmark.family", key: dataset.exactDocumentKey, scope: { kind: "session" } },
|
||||
"current",
|
||||
BACKGROUND_CONTEXT,
|
||||
)
|
||||
)?.id ?? -1
|
||||
);
|
||||
},
|
||||
expected: ({ exactDocumentId }) => exactDocumentId,
|
||||
},
|
||||
...REPLAY_TAILS.map(
|
||||
(tail): StorageReadBenchmark => ({
|
||||
name: `document replay tail (${tail})`,
|
||||
async run(storage, dataset) {
|
||||
return Number(
|
||||
(await storage.document(dataset.replayDocumentIds[tail], "current", BACKGROUND_CONTEXT))?.value.count,
|
||||
);
|
||||
},
|
||||
expected: () => tail,
|
||||
}),
|
||||
),
|
||||
{
|
||||
name: "ancient historical read before newer base",
|
||||
async run(storage, dataset) {
|
||||
return Number(
|
||||
(await storage.document(dataset.historicalDocumentId, dataset.ancientAt, BACKGROUND_CONTEXT))?.value.count,
|
||||
);
|
||||
},
|
||||
expected: () => HISTORY_SEGMENT_LENGTH,
|
||||
},
|
||||
{
|
||||
name: "recent historical read after newer base",
|
||||
async run(storage, dataset) {
|
||||
return Number(
|
||||
(await storage.document(dataset.historicalDocumentId, dataset.recentAt, BACKGROUND_CONTEXT))?.value.count,
|
||||
);
|
||||
},
|
||||
expected: () => HISTORY_SEGMENT_LENGTH * 2,
|
||||
},
|
||||
{
|
||||
name: "fork-depth history scan (100)",
|
||||
async run(storage, dataset) {
|
||||
return (
|
||||
await storage.scanEntries(
|
||||
{ conversationId: dataset.deepestConversationId },
|
||||
undefined,
|
||||
100,
|
||||
BACKGROUND_CONTEXT,
|
||||
)
|
||||
).items.length;
|
||||
},
|
||||
expected: () => 100,
|
||||
},
|
||||
{
|
||||
name: "fork-depth head lookup",
|
||||
async run(storage, dataset) {
|
||||
return (
|
||||
(await storage.findLatestHeadMarker(dataset.deepestConversationId, undefined, BACKGROUND_CONTEXT))?.id ?? -1
|
||||
);
|
||||
},
|
||||
expected: ({ ancestorHeadEntryId }) => ancestorHeadEntryId,
|
||||
},
|
||||
];
|
||||
|
||||
export type StorageWriteBenchmark = {
|
||||
readonly name: string;
|
||||
readonly expected: number;
|
||||
run(storage: Storage): Promise<number>;
|
||||
};
|
||||
|
||||
export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [
|
||||
{
|
||||
name: "commit one entry",
|
||||
expected: 1,
|
||||
async run(storage) {
|
||||
const id = await storage.mintId();
|
||||
await storage.commit(
|
||||
[
|
||||
{
|
||||
type: "entry",
|
||||
value: {
|
||||
id,
|
||||
conversationId: ROOT_CONVERSATION_ID,
|
||||
kind: "benchmark.write",
|
||||
data: { text: "x".repeat(128) },
|
||||
},
|
||||
},
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
return 1;
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "commit 100 entries",
|
||||
expected: 100,
|
||||
async run(storage) {
|
||||
const writes = await Promise.all(
|
||||
Array.from(
|
||||
{ length: 100 },
|
||||
async (_, index): Promise<StorageWrite> => ({
|
||||
type: "entry",
|
||||
value: {
|
||||
id: await storage.mintId(),
|
||||
conversationId: ROOT_CONVERSATION_ID,
|
||||
kind: "benchmark.write",
|
||||
data: { index, text: "x".repeat(128) },
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
await storage.commit(writes, BACKGROUND_CONTEXT);
|
||||
return writes.length;
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "commit mixed entry/task/submission/document",
|
||||
expected: 4,
|
||||
async run(storage) {
|
||||
const entryId = await storage.mintId();
|
||||
const taskId = await storage.mintId();
|
||||
const submissionId = await storage.mintId();
|
||||
const documentId = await storage.mintId();
|
||||
const writes: readonly StorageWrite[] = [
|
||||
{
|
||||
type: "entry",
|
||||
value: { id: entryId, conversationId: ROOT_CONVERSATION_ID, kind: "benchmark.mixed" },
|
||||
},
|
||||
{ type: "task", value: task(taskId, taskId) },
|
||||
{
|
||||
type: "submission",
|
||||
value: {
|
||||
id: submissionId,
|
||||
conversationId: ROOT_CONVERSATION_ID,
|
||||
requestId: `benchmark-${submissionId}`,
|
||||
type: "write",
|
||||
status: "done",
|
||||
entry: entryId,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "document.create",
|
||||
record: { id: documentId, kind: "benchmark.mixed", key: String(documentId), scope: { kind: "session" } },
|
||||
content: { kind: "base", version: 1, value: { entryId, taskId } },
|
||||
},
|
||||
];
|
||||
await storage.commit(writes, BACKGROUND_CONTEXT);
|
||||
return writes.length;
|
||||
},
|
||||
},
|
||||
];
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,187 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdtemp, rm, stat } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
||||
import { MemoryStorage } from "../src/storage/memory.ts";
|
||||
import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts";
|
||||
import type { Storage } from "../src/types.ts";
|
||||
import {
|
||||
STORAGE_BENCHMARK_BACKENDS,
|
||||
STORAGE_MEMORY_SCALES,
|
||||
STORAGE_READ_BENCHMARKS,
|
||||
type StorageBenchmarkBackend,
|
||||
type StorageBenchmarkScale,
|
||||
seedStorageBenchmark,
|
||||
storageBenchmarkPrimaryRecordCount,
|
||||
} from "./storage-benchmark.ts";
|
||||
|
||||
type MemorySnapshot = {
|
||||
readonly heapUsed: number;
|
||||
readonly rss: number;
|
||||
readonly external: number;
|
||||
};
|
||||
|
||||
type SqliteFootprint = {
|
||||
readonly mainBytes: number;
|
||||
readonly walBytes: number;
|
||||
readonly pageCount: number;
|
||||
readonly freelistCount: number;
|
||||
};
|
||||
|
||||
type StorageMemoryResult = {
|
||||
readonly backend: StorageBenchmarkBackend;
|
||||
readonly scale: string;
|
||||
readonly recordCount: number;
|
||||
readonly baseline: MemorySnapshot;
|
||||
readonly postSeed: MemorySnapshot;
|
||||
readonly postRead: MemorySnapshot;
|
||||
readonly sqlite?: SqliteFootprint;
|
||||
};
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
function collectGarbage(): void {
|
||||
if (globalThis.gc === undefined) throw new Error("Storage memory measurement requires Node.js --expose-gc");
|
||||
for (let index = 0; index < 3; index++) globalThis.gc();
|
||||
}
|
||||
|
||||
function snapshot(): MemorySnapshot {
|
||||
const usage = process.memoryUsage();
|
||||
return { heapUsed: usage.heapUsed, rss: usage.rss, external: usage.external };
|
||||
}
|
||||
|
||||
async function fileSize(path: string): Promise<number> {
|
||||
try {
|
||||
return (await stat(path)).size;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return 0;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function sqliteMetrics(path: string): SqliteFootprint {
|
||||
const database = new DatabaseSync(path, { readOnly: true });
|
||||
try {
|
||||
const pageCount = database.prepare("SELECT page_count AS value FROM pragma_page_count()").get() as {
|
||||
readonly value: number;
|
||||
};
|
||||
const freelistCount = database.prepare("SELECT freelist_count AS value FROM pragma_freelist_count()").get() as {
|
||||
readonly value: number;
|
||||
};
|
||||
return { mainBytes: 0, walBytes: 0, pageCount: pageCount.value, freelistCount: freelistCount.value };
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runWorker(backend: StorageBenchmarkBackend, scale: StorageBenchmarkScale): Promise<void> {
|
||||
let storage: Storage;
|
||||
let directory: string | undefined;
|
||||
let path: string | undefined;
|
||||
if (backend === "memory") {
|
||||
storage = new MemoryStorage();
|
||||
} else {
|
||||
directory = await mkdtemp(join(tmpdir(), "pi-durable-memory-"));
|
||||
path = join(directory, "storage.sqlite");
|
||||
storage = await openNodeSqliteStorage(path);
|
||||
}
|
||||
|
||||
try {
|
||||
collectGarbage();
|
||||
const baseline = snapshot();
|
||||
const dataset = await seedStorageBenchmark(storage, scale);
|
||||
collectGarbage();
|
||||
const postSeed = snapshot();
|
||||
let checksum = 0;
|
||||
for (const scenario of STORAGE_READ_BENCHMARKS) {
|
||||
const result = await scenario.run(storage, dataset);
|
||||
if (result !== scenario.expected(dataset)) throw new Error(`Invalid benchmark result: ${scenario.name}`);
|
||||
checksum += result;
|
||||
}
|
||||
for (let round = 1; round < 10; round++) {
|
||||
for (const scenario of STORAGE_READ_BENCHMARKS) checksum += await scenario.run(storage, dataset);
|
||||
}
|
||||
if (!Number.isFinite(checksum)) throw new Error("Storage memory read checksum is invalid");
|
||||
collectGarbage();
|
||||
const postRead = snapshot();
|
||||
let sqlite: SqliteFootprint | undefined;
|
||||
if (path !== undefined) {
|
||||
sqlite = sqliteMetrics(path);
|
||||
sqlite = {
|
||||
...sqlite,
|
||||
mainBytes: await fileSize(path),
|
||||
walBytes: await fileSize(`${path}-wal`),
|
||||
};
|
||||
}
|
||||
const recordCount = storageBenchmarkPrimaryRecordCount(scale);
|
||||
console.log(JSON.stringify({ backend, scale: scale.name, recordCount, baseline, postSeed, postRead, sqlite }));
|
||||
} finally {
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
if (directory !== undefined) await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function delta(after: MemorySnapshot, before: MemorySnapshot, field: keyof MemorySnapshot): number {
|
||||
return after[field] - before[field];
|
||||
}
|
||||
|
||||
function mebibytes(bytes: number): string {
|
||||
return (bytes / 1024 / 1024).toFixed(2);
|
||||
}
|
||||
|
||||
async function runDriver(): Promise<void> {
|
||||
const workerPath = fileURLToPath(import.meta.url);
|
||||
const results: StorageMemoryResult[] = [];
|
||||
for (const backend of STORAGE_BENCHMARK_BACKENDS) {
|
||||
for (const scale of STORAGE_MEMORY_SCALES) {
|
||||
const { stdout } = await execFileAsync(
|
||||
process.execPath,
|
||||
[
|
||||
"--conditions=source",
|
||||
"--expose-gc",
|
||||
"--experimental-strip-types",
|
||||
workerPath,
|
||||
"--worker",
|
||||
backend,
|
||||
scale.name,
|
||||
],
|
||||
{ cwd: fileURLToPath(new URL("..", import.meta.url)), maxBuffer: 1024 * 1024 },
|
||||
);
|
||||
results.push(JSON.parse(stdout) as StorageMemoryResult);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("Storage footprint after deterministic synthetic workloads; values are process deltas, not limits.");
|
||||
console.table(
|
||||
results.map((result) => ({
|
||||
backend: result.backend,
|
||||
scale: result.scale,
|
||||
"heap after seed MiB": mebibytes(delta(result.postSeed, result.baseline, "heapUsed")),
|
||||
"RSS after seed MiB": mebibytes(delta(result.postSeed, result.baseline, "rss")),
|
||||
"external after seed MiB": mebibytes(delta(result.postSeed, result.baseline, "external")),
|
||||
"heap after reads MiB": mebibytes(delta(result.postRead, result.postSeed, "heapUsed")),
|
||||
"JS heap bytes/primary record": Math.round(
|
||||
delta(result.postSeed, result.baseline, "heapUsed") / result.recordCount,
|
||||
),
|
||||
"live SQLite main MiB": result.sqlite === undefined ? "-" : mebibytes(result.sqlite.mainBytes),
|
||||
"live SQLite WAL MiB": result.sqlite === undefined ? "-" : mebibytes(result.sqlite.walBytes),
|
||||
"SQLite pages/free":
|
||||
result.sqlite === undefined ? "-" : `${result.sqlite.pageCount}/${result.sqlite.freelistCount}`,
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
const [mode, backendName, scaleName] = process.argv.slice(2);
|
||||
if (mode === "--worker") {
|
||||
const backend = STORAGE_BENCHMARK_BACKENDS.find((candidate) => candidate === backendName);
|
||||
if (backend === undefined) throw new Error(`Unknown storage benchmark backend: ${backendName}`);
|
||||
const scale = STORAGE_MEMORY_SCALES.find((candidate) => candidate.name === scaleName);
|
||||
if (scale === undefined) throw new Error(`Unknown storage benchmark scale: ${scaleName}`);
|
||||
await runWorker(backend, scale);
|
||||
} else {
|
||||
await runDriver();
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const sourceRoot = fileURLToPath(new URL("../src", import.meta.url));
|
||||
|
||||
async function sourceGraph(entry: string): Promise<Set<string>> {
|
||||
const visited = new Set<string>();
|
||||
const pending = [resolve(sourceRoot, entry)];
|
||||
while (pending.length > 0) {
|
||||
const path = pending.pop()!;
|
||||
if (visited.has(path)) continue;
|
||||
visited.add(path);
|
||||
const source = await readFile(path, "utf8");
|
||||
expect(source, `${path} imports a Node built-in`).not.toMatch(/(?:from\s+|import\s*)["']node:/);
|
||||
for (const match of source.matchAll(/(?:from\s+|import\s*)["'](\.[^"']+)["']/g)) {
|
||||
pending.push(resolve(dirname(path), match[1]));
|
||||
}
|
||||
}
|
||||
return visited;
|
||||
}
|
||||
|
||||
describe("durable storage runtime boundaries", () => {
|
||||
it("keeps the package root free of SQLite and Node imports", async () => {
|
||||
const graph = await sourceGraph("index.ts");
|
||||
expect([...graph].some((path) => path.includes("/storage/sqlite/"))).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps the portable SQLite subpath free of Node imports", async () => {
|
||||
const graph = await sourceGraph("storage/sqlite/index.ts");
|
||||
expect([...graph].some((path) => path.endsWith("/storage/sqlite/node.ts"))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,161 @@
|
||||
import { strictEqual } from "node:assert/strict";
|
||||
import { copyFile, mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
|
||||
import { afterAll, bench, describe } from "vitest";
|
||||
import { MemoryStorage } from "../src/storage/memory.ts";
|
||||
import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts";
|
||||
import type { Storage } from "../src/types.ts";
|
||||
import {
|
||||
STORAGE_BENCHMARK_BACKENDS,
|
||||
STORAGE_READ_BENCHMARKS,
|
||||
STORAGE_WRITE_BENCHMARKS,
|
||||
type StorageBenchmarkBackend,
|
||||
seedStorageBenchmark,
|
||||
} from "./storage-benchmark.ts";
|
||||
|
||||
const READ_OPTIONS = { time: 300, iterations: 10, warmupTime: 75, warmupIterations: 3 } as const;
|
||||
const WRITE_OPTIONS = { time: 0, iterations: 20, warmupTime: 0, warmupIterations: 5 } as const;
|
||||
const REOPEN_OPTIONS = { time: 0, iterations: 20, warmupTime: 0, warmupIterations: 3 } as const;
|
||||
|
||||
type Fixture = {
|
||||
readonly backend: StorageBenchmarkBackend;
|
||||
readonly storage: Storage;
|
||||
readonly path?: string;
|
||||
};
|
||||
|
||||
const fixtures: Fixture[] = [];
|
||||
const directories: string[] = [];
|
||||
|
||||
async function createFixture(backend: StorageBenchmarkBackend): Promise<Fixture> {
|
||||
if (backend === "memory") {
|
||||
const fixture = { backend, storage: new MemoryStorage() } satisfies Fixture;
|
||||
fixtures.push(fixture);
|
||||
return fixture;
|
||||
}
|
||||
const directory = await mkdtemp(join(tmpdir(), "pi-durable-benchmark-"));
|
||||
directories.push(directory);
|
||||
const path = join(directory, "storage.sqlite");
|
||||
const fixture = { backend, storage: await openNodeSqliteStorage(path), path } satisfies Fixture;
|
||||
fixtures.push(fixture);
|
||||
return fixture;
|
||||
}
|
||||
|
||||
const readFixtures = await Promise.all(STORAGE_BENCHMARK_BACKENDS.map(createFixture));
|
||||
const readDatasets = await Promise.all(readFixtures.map(({ storage }) => seedStorageBenchmark(storage)));
|
||||
for (let index = 0; index < readFixtures.length; index++) {
|
||||
for (const scenario of STORAGE_READ_BENCHMARKS) {
|
||||
strictEqual(
|
||||
await scenario.run(readFixtures[index].storage, readDatasets[index]),
|
||||
scenario.expected(readDatasets[index]),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const scenario of STORAGE_READ_BENCHMARKS) {
|
||||
describe(scenario.name, () => {
|
||||
for (let index = 0; index < readFixtures.length; index++) {
|
||||
const fixture = readFixtures[index];
|
||||
const dataset = readDatasets[index];
|
||||
bench(
|
||||
fixture.backend,
|
||||
async () => {
|
||||
await scenario.run(fixture.storage, dataset);
|
||||
},
|
||||
READ_OPTIONS,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function createWriteFixture(backend: StorageBenchmarkBackend): Promise<Fixture> {
|
||||
const fixture = await createFixture(backend);
|
||||
await fixture.storage.commit([{ type: "conversation", value: { id: 1 } }], BACKGROUND_CONTEXT);
|
||||
await fixture.storage.commit(
|
||||
await Promise.all(
|
||||
Array.from({ length: 100 }, async (_, index) => ({
|
||||
type: "entry" as const,
|
||||
value: {
|
||||
id: await fixture.storage.mintId(),
|
||||
conversationId: 1,
|
||||
kind: "benchmark.baseline",
|
||||
data: { index },
|
||||
},
|
||||
})),
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
return fixture;
|
||||
}
|
||||
|
||||
const writePools = new Map<string, Fixture[]>();
|
||||
for (const scenario of STORAGE_WRITE_BENCHMARKS) {
|
||||
for (const backend of STORAGE_BENCHMARK_BACKENDS) {
|
||||
const validation = await createWriteFixture(backend);
|
||||
strictEqual(await scenario.run(validation.storage), scenario.expected);
|
||||
const pool = await Promise.all(
|
||||
Array.from({ length: WRITE_OPTIONS.iterations + WRITE_OPTIONS.warmupIterations }, () =>
|
||||
createWriteFixture(backend),
|
||||
),
|
||||
);
|
||||
writePools.set(`${scenario.name}:${backend}`, pool);
|
||||
}
|
||||
describe(scenario.name, () => {
|
||||
for (const backend of STORAGE_BENCHMARK_BACKENDS) {
|
||||
const pool = writePools.get(`${scenario.name}:${backend}`)!;
|
||||
bench(
|
||||
backend,
|
||||
async () => {
|
||||
const fixture = pool.shift();
|
||||
if (fixture === undefined) throw new Error("Write benchmark fixture pool was exhausted");
|
||||
await scenario.run(fixture.storage);
|
||||
},
|
||||
WRITE_OPTIONS,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const reopenDirectory = await mkdtemp(join(tmpdir(), "pi-durable-reopen-benchmark-"));
|
||||
directories.push(reopenDirectory);
|
||||
const reopenPath = join(reopenDirectory, "storage.sqlite");
|
||||
const reopenSeed = await openNodeSqliteStorage(reopenPath);
|
||||
const reopenDataset = await seedStorageBenchmark(reopenSeed);
|
||||
await reopenSeed.close(BACKGROUND_CONTEXT);
|
||||
const reopenPaths = await Promise.all(
|
||||
Array.from({ length: REOPEN_OPTIONS.iterations + REOPEN_OPTIONS.warmupIterations }, async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), "pi-durable-reopen-sample-"));
|
||||
directories.push(directory);
|
||||
const path = join(directory, "storage.sqlite");
|
||||
await copyFile(reopenPath, path);
|
||||
return path;
|
||||
}),
|
||||
);
|
||||
const reopenedStorages: Storage[] = [];
|
||||
async function reopenAndRead(path: string): Promise<{ readonly id: number; readonly storage: Storage }> {
|
||||
const storage = await openNodeSqliteStorage(path);
|
||||
const id = (await storage.entry(reopenDataset.firstEntryId, BACKGROUND_CONTEXT))?.entry.id ?? -1;
|
||||
return { id, storage };
|
||||
}
|
||||
const reopenValidation = await reopenAndRead(reopenPath);
|
||||
strictEqual(reopenValidation.id, reopenDataset.firstEntryId);
|
||||
await reopenValidation.storage.close(BACKGROUND_CONTEXT);
|
||||
describe("SQLite reopen and first exact read", () => {
|
||||
bench(
|
||||
"sqlite",
|
||||
async () => {
|
||||
const path = reopenPaths.shift();
|
||||
if (path === undefined) throw new Error("Reopen benchmark fixture pool was exhausted");
|
||||
const result = await reopenAndRead(path);
|
||||
reopenedStorages.push(result.storage);
|
||||
},
|
||||
REOPEN_OPTIONS,
|
||||
);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
for (const storage of reopenedStorages) await storage.close(BACKGROUND_CONTEXT);
|
||||
for (const fixture of fixtures) await fixture.storage.close(BACKGROUND_CONTEXT);
|
||||
for (const directory of directories) await rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
@@ -0,0 +1,13 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
environment: "node",
|
||||
benchmark: {
|
||||
include: ["test/**/*.bench.ts"],
|
||||
reporters: ["verbose"],
|
||||
},
|
||||
},
|
||||
resolve: { conditions: ["source"] },
|
||||
ssr: { resolve: { conditions: ["source"] } },
|
||||
});
|
||||
Reference in New Issue
Block a user