- server/wikimedia/client.ts: one process-wide client. Per-host token bucket (1 req/s, burst 3), 1 in flight per host, mint > interactive > background lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP). - server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match, stale-while-revalidate on the background lane. Mint reads never use it. - Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh() (template 503, miner retries) instead of minting attention: null forever. - server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for /api/mining/template, about 3 requests per block regardless of miner count. - User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or PUBLIC_ORIGIN) and a start-up warning when it is unset. - Mint request parameters are byte-identical apart from maxlag (golden test). No consensus, block or protocol changes. Tests use fake fetch only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
75 lines
3.5 KiB
TypeScript
75 lines
3.5 KiB
TypeScript
/**
|
|
* On-disk HTTP cache for off-chain Wikimedia reads (`$WIKIDECK_DATA_DIR/wikimedia-cache.sqlite`).
|
|
* It stores the raw body with its ETag / Last-Modified so stale entries can be revalidated with
|
|
* conditional requests. Mint-path requests never use it (see client.ts).
|
|
*/
|
|
import { DatabaseSync } from 'node:sqlite';
|
|
|
|
export type StoredEntry = {
|
|
status: number; body: Buffer; contentType: string; etag: string | null; lastModified: string | null;
|
|
expiresAt: number; storedAt: number;
|
|
};
|
|
export type CacheEntry = StoredEntry & { toResponse(): Response };
|
|
|
|
/** Canonical key: origin + path + sorted query, without `maxlag` (it changes only error behaviour). */
|
|
export function cacheKey(url: URL): string {
|
|
const params = [...url.searchParams].filter(([name]) => name !== 'maxlag')
|
|
.sort(([a, x], [b, y]) => a < b ? -1 : a > b ? 1 : x < y ? -1 : x > y ? 1 : 0);
|
|
return `${url.origin}${url.pathname}?${new URLSearchParams(params).toString()}`;
|
|
}
|
|
|
|
function withResponse(entry: StoredEntry): CacheEntry {
|
|
return { ...entry, toResponse: () => new Response(new Uint8Array(entry.body), { status: entry.status,
|
|
headers: { 'Content-Type': entry.contentType, 'Content-Length': String(entry.body.byteLength),
|
|
...(entry.etag ? { ETag: entry.etag } : {}) } }) };
|
|
}
|
|
|
|
type Row = { status: number; body: Uint8Array; content_type: string; etag: string | null;
|
|
last_modified: string | null; expires_at: number; stored_at: number };
|
|
|
|
export class WikimediaCache {
|
|
private readonly db: DatabaseSync;
|
|
private readonly maxRows: number;
|
|
private writes = 0;
|
|
|
|
/** `path` may be ':memory:' (tests). */
|
|
constructor(path: string, options: { maxRows?: number } = {}) {
|
|
this.maxRows = options.maxRows ?? 100_000;
|
|
this.db = new DatabaseSync(path);
|
|
this.db.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;
|
|
CREATE TABLE IF NOT EXISTS http_cache (key TEXT PRIMARY KEY, status INTEGER NOT NULL, body BLOB NOT NULL,
|
|
content_type TEXT NOT NULL, etag TEXT, last_modified TEXT, expires_at INTEGER NOT NULL, stored_at INTEGER NOT NULL);
|
|
CREATE INDEX IF NOT EXISTS http_cache_stored ON http_cache(stored_at);`);
|
|
}
|
|
|
|
get(key: string): CacheEntry | undefined {
|
|
const row = this.db.prepare('SELECT * FROM http_cache WHERE key = ?').get(key) as Row | undefined;
|
|
if (!row) return undefined;
|
|
return withResponse({ status: row.status, body: Buffer.from(row.body), contentType: row.content_type,
|
|
etag: row.etag, lastModified: row.last_modified, expiresAt: row.expires_at, storedAt: row.stored_at });
|
|
}
|
|
|
|
put(key: string, entry: StoredEntry): CacheEntry {
|
|
this.db.prepare(`INSERT OR REPLACE INTO http_cache (key, status, body, content_type, etag, last_modified, expires_at, stored_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(key, entry.status, entry.body, entry.contentType, entry.etag,
|
|
entry.lastModified, entry.expiresAt, entry.storedAt);
|
|
if (++this.writes % 500 === 0) this.prune();
|
|
return withResponse(entry);
|
|
}
|
|
|
|
touch(key: string, expiresAt: number): void {
|
|
this.db.prepare('UPDATE http_cache SET expires_at = ? WHERE key = ?').run(expiresAt, key);
|
|
}
|
|
|
|
delete(key: string): void { this.db.prepare('DELETE FROM http_cache WHERE key = ?').run(key); }
|
|
|
|
size(): number { return (this.db.prepare('SELECT COUNT(*) AS n FROM http_cache').get() as { n: number }).n; }
|
|
|
|
prune(): void {
|
|
this.db.prepare(`DELETE FROM http_cache WHERE key IN (SELECT key FROM http_cache ORDER BY stored_at DESC LIMIT -1 OFFSET ?)`)
|
|
.run(this.maxRows);
|
|
}
|
|
|
|
close(): void { this.db.close(); }
|
|
}
|