Files
wikideck/server/wikimedia/cache.ts
T
claudeandClaude Opus 5.5 f544019bfe Polite Wikimedia client, mint-path fix, disk cache and mining prefetch pool (#62)
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket
  (1 req/s, burst 3), 1 in flight per host, mint > interactive > background
  lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After
  and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and
  a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP).
- server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match,
  stale-while-revalidate on the background lane. Mint reads never use it.
- Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh()
  (template 503, miner retries) instead of minting attention: null forever.
- server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for
  /api/mining/template, about 3 requests per block regardless of miner count.
- User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or
  PUBLIC_ORIGIN) and a start-up warning when it is unset.
- Mint request parameters are byte-identical apart from maxlag (golden test).
  No consensus, block or protocol changes. Tests use fake fetch only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 21:51:20 -05:00

75 lines
3.5 KiB
TypeScript

/**
* On-disk HTTP cache for off-chain Wikimedia reads (`$WIKIDECK_DATA_DIR/wikimedia-cache.sqlite`).
* It stores the raw body with its ETag / Last-Modified so stale entries can be revalidated with
* conditional requests. Mint-path requests never use it (see client.ts).
*/
import { DatabaseSync } from 'node:sqlite';
export type StoredEntry = {
status: number; body: Buffer; contentType: string; etag: string | null; lastModified: string | null;
expiresAt: number; storedAt: number;
};
export type CacheEntry = StoredEntry & { toResponse(): Response };
/** Canonical key: origin + path + sorted query, without `maxlag` (it changes only error behaviour). */
export function cacheKey(url: URL): string {
const params = [...url.searchParams].filter(([name]) => name !== 'maxlag')
.sort(([a, x], [b, y]) => a < b ? -1 : a > b ? 1 : x < y ? -1 : x > y ? 1 : 0);
return `${url.origin}${url.pathname}?${new URLSearchParams(params).toString()}`;
}
function withResponse(entry: StoredEntry): CacheEntry {
return { ...entry, toResponse: () => new Response(new Uint8Array(entry.body), { status: entry.status,
headers: { 'Content-Type': entry.contentType, 'Content-Length': String(entry.body.byteLength),
...(entry.etag ? { ETag: entry.etag } : {}) } }) };
}
type Row = { status: number; body: Uint8Array; content_type: string; etag: string | null;
last_modified: string | null; expires_at: number; stored_at: number };
export class WikimediaCache {
private readonly db: DatabaseSync;
private readonly maxRows: number;
private writes = 0;
/** `path` may be ':memory:' (tests). */
constructor(path: string, options: { maxRows?: number } = {}) {
this.maxRows = options.maxRows ?? 100_000;
this.db = new DatabaseSync(path);
this.db.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;
CREATE TABLE IF NOT EXISTS http_cache (key TEXT PRIMARY KEY, status INTEGER NOT NULL, body BLOB NOT NULL,
content_type TEXT NOT NULL, etag TEXT, last_modified TEXT, expires_at INTEGER NOT NULL, stored_at INTEGER NOT NULL);
CREATE INDEX IF NOT EXISTS http_cache_stored ON http_cache(stored_at);`);
}
get(key: string): CacheEntry | undefined {
const row = this.db.prepare('SELECT * FROM http_cache WHERE key = ?').get(key) as Row | undefined;
if (!row) return undefined;
return withResponse({ status: row.status, body: Buffer.from(row.body), contentType: row.content_type,
etag: row.etag, lastModified: row.last_modified, expiresAt: row.expires_at, storedAt: row.stored_at });
}
put(key: string, entry: StoredEntry): CacheEntry {
this.db.prepare(`INSERT OR REPLACE INTO http_cache (key, status, body, content_type, etag, last_modified, expires_at, stored_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(key, entry.status, entry.body, entry.contentType, entry.etag,
entry.lastModified, entry.expiresAt, entry.storedAt);
if (++this.writes % 500 === 0) this.prune();
return withResponse(entry);
}
touch(key: string, expiresAt: number): void {
this.db.prepare('UPDATE http_cache SET expires_at = ? WHERE key = ?').run(expiresAt, key);
}
delete(key: string): void { this.db.prepare('DELETE FROM http_cache WHERE key = ?').run(key); }
size(): number { return (this.db.prepare('SELECT COUNT(*) AS n FROM http_cache').get() as { n: number }).n; }
prune(): void {
this.db.prepare(`DELETE FROM http_cache WHERE key IN (SELECT key FROM http_cache ORDER BY stored_at DESC LIMIT -1 OFFSET ?)`)
.run(this.maxRows);
}
close(): void { this.db.close(); }
}