forked from WikiDeck/wikideck
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket (1 req/s, burst 3), 1 in flight per host, mint > interactive > background lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP). - server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match, stale-while-revalidate on the background lane. Mint reads never use it. - Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh() (template 503, miner retries) instead of minting attention: null forever. - server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for /api/mining/template, about 3 requests per block regardless of miner count. - User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or PUBLIC_ORIGIN) and a start-up warning when it is unset. - Mint request parameters are byte-identical apart from maxlag (golden test). No consensus, block or protocol changes. Tests use fake fetch only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
37 lines
1.5 KiB
TypeScript
37 lines
1.5 KiB
TypeScript
/**
|
|
* Node start-up wiring for the polite Wikimedia client: the disk cache, the User-Agent warning,
|
|
* the mining prefetch pool and an ops status snapshot. See docs/night2/WIKIMEDIA.md.
|
|
*/
|
|
import { resolve } from 'node:path';
|
|
import { WikimediaCache } from './cache.ts';
|
|
import { wikimediaClient } from './client.ts';
|
|
import { MintTemplatePool } from './pool.ts';
|
|
import { warnIfNoContact } from '../wikimedia-user-agent.ts';
|
|
import { wikipediaArticles } from '../wiki.ts';
|
|
import { wikipediaEnrichment } from '../enrichment.ts';
|
|
|
|
export { wikimediaClient } from './client.ts';
|
|
|
|
/** Serves every mining template; shared by all miners of this node. */
|
|
export const mintPool = new MintTemplatePool({ articles: wikipediaArticles, enrichment: wikipediaEnrichment });
|
|
|
|
let cache: WikimediaCache | undefined;
|
|
|
|
/** Call once at start-up. Without it the client still works, with in-memory caching only. */
|
|
export function configureWikimedia(options: { dataDir: string }): void {
|
|
warnIfNoContact();
|
|
if (cache) return;
|
|
try {
|
|
cache = new WikimediaCache(resolve(options.dataDir, 'wikimedia-cache.sqlite'));
|
|
wikimediaClient.attachCache(cache);
|
|
} catch (error) {
|
|
console.warn(`[wikimedia] disk cache disabled: ${(error as Error).message}`);
|
|
}
|
|
}
|
|
|
|
export function closeWikimedia(): void { wikimediaClient.attachCache(undefined); cache?.close(); cache = undefined; }
|
|
|
|
export function wikimediaStatus() {
|
|
return { client: wikimediaClient.status(), cacheEntries: cache?.size() ?? null, mintPool: mintPool.status() };
|
|
}
|