Files
wikideck/server/wikimedia/index.ts
claudeandClaude Opus 5.5 f544019bfe Polite Wikimedia client, mint-path fix, disk cache and mining prefetch pool (#62)
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket
  (1 req/s, burst 3), 1 in flight per host, mint > interactive > background
  lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After
  and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and
  a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP).
- server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match,
  stale-while-revalidate on the background lane. Mint reads never use it.
- Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh()
  (template 503, miner retries) instead of minting attention: null forever.
- server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for
  /api/mining/template, about 3 requests per block regardless of miner count.
- User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or
  PUBLIC_ORIGIN) and a start-up warning when it is unset.
- Mint request parameters are byte-identical apart from maxlag (golden test).
  No consensus, block or protocol changes. Tests use fake fetch only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 21:51:20 -05:00

37 lines
1.5 KiB
TypeScript

/**
* Node start-up wiring for the polite Wikimedia client: the disk cache, the User-Agent warning,
* the mining prefetch pool and an ops status snapshot. See docs/night2/WIKIMEDIA.md.
*/
import { resolve } from 'node:path';
import { WikimediaCache } from './cache.ts';
import { wikimediaClient } from './client.ts';
import { MintTemplatePool } from './pool.ts';
import { warnIfNoContact } from '../wikimedia-user-agent.ts';
import { wikipediaArticles } from '../wiki.ts';
import { wikipediaEnrichment } from '../enrichment.ts';
export { wikimediaClient } from './client.ts';
/** Serves every mining template; shared by all miners of this node. */
export const mintPool = new MintTemplatePool({ articles: wikipediaArticles, enrichment: wikipediaEnrichment });
let cache: WikimediaCache | undefined;
/** Call once at start-up. Without it the client still works, with in-memory caching only. */
export function configureWikimedia(options: { dataDir: string }): void {
warnIfNoContact();
if (cache) return;
try {
cache = new WikimediaCache(resolve(options.dataDir, 'wikimedia-cache.sqlite'));
wikimediaClient.attachCache(cache);
} catch (error) {
console.warn(`[wikimedia] disk cache disabled: ${(error as Error).message}`);
}
}
export function closeWikimedia(): void { wikimediaClient.attachCache(undefined); cache?.close(); cache = undefined; }
export function wikimediaStatus() {
return { client: wikimediaClient.status(), cacheEntries: cache?.size() ?? null, mintPool: mintPool.status() };
}