Files
wikideck/server/wikimedia/client.ts
T
claudeanderic dbb89ce9c5 Night 2: 40-card Magic-style duel, PvP with owned decks, polite Wikimedia client, Android mining (#23)
Night 2 (2026-11-17), from the `claude/wikideck` fork's `night/2026-11-17` branch. It follows Night 1 (#22).

## What's in it
- **Duel engine** (fork #72, #80): 40-card decks of cards you own, Notability/Citations, keywords, effects T1–T12, Connected, a hidden-information-safe `viewFor`/`logFor`, and replay. Balance v1 tuning (#84) gives the first player about 51%.
- **Card profiles** (#70, #82): Figure, Location, Event, Institution or Doctrine, plus stats, keywords and rules text, all derived from real article data. Templates only.
- **Design and story** (#66, #69, #83): DUEL_DESIGN.md, plus the match log and the encyclopedic "Battle of X (2026)" recap.
- **PvP** (#74, #77, #81, #76, #85):
  - The server checks ownership, and 40-card eligibility is enforced.
  - Modes: Classic (ranked), Draft and Trending.
  - Invites, quick match, Elo, a ladder and recap pages. Online matches render on the duel board.
- **Duel board UI** (#79): MTG-style card faces and combat declaration, built for 360 px first.
- **Polite Wikimedia client** (#73, #75): throttled, cached and identified requests, the mint-path fix, and a prefetch pool.
- **Android mining** (#78): native RandomX, as a sideload build. The emulator mined a testnet block.
- **Testnet ops** (#67, #68).

PR numbers refer to `claude/wikideck`.

## Status
- **Tests:** 423, of which 422 pass, 0 fail and 1 is skipped. The build is OK (`node:24-alpine`, `--network none`).
- **Testnet:** running this commit (036489c) on r350.
- **Untouched:** no changes to `server/testnet/consensus.ts`, `randomx.ts`, `native/` or the mainnet compose.
- **Review:** night-review reviewed fork #66–#79. #80–#85 came from the agents' unfinished branches and were not reviewed.
- **Known gaps:**
  - The PvP end-to-end test hasn't been run (it needs two eom.dev accounts).
  - Bot matches aren't reachable from Play.
  - Card identity pre-bake hasn't been done.
  - The RandomX verifier latch (`server/testnet/randomx.ts:140`) is still open.

This PR is for Eric to merge.

🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #23

Co-authored-by: Claude <claude@eom.dev>
2026-11-18 18:12:24 +00:00

442 lines
20 KiB
TypeScript

/**
* One process-wide, polite Wikimedia HTTP client (engineer review §4, data review §5).
*
* Every server-side Wikimedia request goes through `WikimediaClient.request()`:
* - a token bucket per upstream host (default 1 req/s, burst 3) and 1 request in flight per host;
* - priority lanes: `mint` before `interactive` before `background`, with a bounded queue;
* - `maxlag=5` on Action API requests (it changes only error behaviour, never content);
* - `Retry-After` honoured on 429/503/maxlag by pausing the whole host, with exponential backoff;
* - a circuit breaker: 3 consecutive 429s stop all requests to that host for 10 minutes;
* - a daily request cap, and a budget shared by the nodes behind one egress IP;
* - an optional on-disk cache (see cache.ts) with ETag / If-None-Match and stale-while-revalidate.
*
* The client is transport only. It never rewrites request parameters other than adding `maxlag`,
* and requests made with `cache: undefined` (all mint-path requests) never read or write the cache.
* Nothing here is reachable from block validation, so it cannot affect consensus.
*/
import { WikimediaCache, cacheKey } from './cache.ts';
import { wikimediaUserAgent, defaultUserAgent } from '../wikimedia-user-agent.ts';
export type Lane = 'mint' | 'interactive' | 'background';
const LANES: Lane[] = ['mint', 'interactive', 'background'];
export type Clock = {
now(): number;
setTimeout(fn: () => void, ms: number): unknown;
clearTimeout(handle: unknown): void;
};
const realClock: Clock = {
now: () => Date.now(),
setTimeout: (fn, ms) => { const t = setTimeout(fn, ms); t.unref?.(); return t; },
clearTimeout: handle => clearTimeout(handle as ReturnType<typeof setTimeout>),
};
export type ClientConfig = {
/** Sustained requests per second per upstream host, for this whole egress IP. */
requestsPerSecond: number;
burst: number;
concurrencyPerHost: number;
/** Requests per UTC day for this whole egress IP. */
dailyCap: number;
/** Nodes sharing one egress IP; the rate and daily cap are divided between them. */
nodesPerIp: number;
queueLimit: number;
/** Longest wait in the queue before a request fails, per lane. */
maxWaitMs: Record<Lane, number>;
breakerThreshold: number;
breakerMs: number;
maxBackoffMs: number;
maxlag: number;
/** Retries of a single request after a maxlag reply (HTTP 200/503 with MediaWiki-API-Error: maxlag). */
maxlagRetries: number;
};
export const DEFAULT_CONFIG: ClientConfig = {
requestsPerSecond: 1, burst: 3, concurrencyPerHost: 1, dailyCap: 20_000, nodesPerIp: 1, queueLimit: 64,
maxWaitMs: { mint: 10_000, interactive: 5_000, background: 120_000 },
breakerThreshold: 3, breakerMs: 10 * 60_000, maxBackoffMs: 10 * 60_000, maxlag: 5, maxlagRetries: 2,
};
/** Unthrottled settings for tests and tools that inject their own fetch. */
export const UNTHROTTLED: Partial<ClientConfig> = {
requestsPerSecond: 1e9, burst: 1e9, concurrencyPerHost: 1e9, dailyCap: Number.MAX_SAFE_INTEGER, queueLimit: 1e9,
breakerThreshold: Number.MAX_SAFE_INTEGER, maxlagRetries: 0, maxBackoffMs: 0,
};
function positive(value: string | undefined, fallback: number, integer = false): number {
if (value === undefined || value.trim() === '') return fallback;
const n = Number(value);
if (!Number.isFinite(n) || n <= 0 || (integer && !Number.isSafeInteger(n))) {
console.warn(`[wikimedia] ignoring invalid setting ${JSON.stringify(value)}; using ${fallback}`);
return fallback;
}
return n;
}
/** Operator settings. Documented in docs/night2/WIKIMEDIA.md. */
export function configFromEnv(env: Record<string, string | undefined> = process.env): ClientConfig {
return { ...DEFAULT_CONFIG,
requestsPerSecond: positive(env.WIKIDECK_WIKIMEDIA_RPS, DEFAULT_CONFIG.requestsPerSecond),
burst: positive(env.WIKIDECK_WIKIMEDIA_BURST, DEFAULT_CONFIG.burst),
concurrencyPerHost: positive(env.WIKIDECK_WIKIMEDIA_CONCURRENCY, DEFAULT_CONFIG.concurrencyPerHost, true),
dailyCap: positive(env.WIKIDECK_WIKIMEDIA_DAILY_CAP, DEFAULT_CONFIG.dailyCap, true),
nodesPerIp: positive(env.WIKIDECK_WIKIMEDIA_NODES_PER_IP, DEFAULT_CONFIG.nodesPerIp, true),
};
}
/** Thrown for every refusal or transport failure: queue full, cap reached, circuit open, timeout, network. */
export class WikimediaUnavailable extends Error {
readonly reason: string;
readonly status?: number;
constructor(reason: string, status?: number) {
super(`Wikimedia unavailable: ${reason}`);
this.name = 'WikimediaUnavailable';
this.reason = reason; this.status = status;
}
}
export type CachePolicy = {
ttlMs: number;
/** Serve a stale entry at once and revalidate it on the background lane. */
staleWhileRevalidate?: boolean;
/** Largest body stored, in bytes. */
maxBytes?: number;
};
export type RequestOptions = {
lane: Lane;
/** Starts when the request is dispatched, not while it waits in the queue. */
timeoutMs: number;
headers?: Record<string, string>;
/** Omit for mint data: the request then bypasses the cache completely. */
cache?: CachePolicy;
};
export type ClientResponse = { response: Response; signal: AbortSignal; fromCache: boolean };
type Waiter = {
lane: Lane; host: string; enqueuedAt: number;
run: () => void; fail: (error: Error) => void; timer?: unknown;
};
type HostState = {
tokens: number; refilledAt: number; inFlight: number;
pausedUntil: number; breakerUntil: number; consecutive429: number; backoffExponent: number;
};
export type ClientStatus = {
day: string; requestsToday: number; dailyCap: number; requestsPerSecond: number; queued: number;
counters: Record<string, number>;
hosts: Record<string, { inFlight: number; pausedForMs: number; circuitOpen: boolean; consecutive429: number }>;
};
function parseRetryAfter(value: string | null, now: number): number | undefined {
if (!value) return undefined;
if (/^\d+$/.test(value.trim())) return Number(value.trim()) * 1000;
const at = Date.parse(value);
return Number.isNaN(at) ? undefined : Math.max(0, at - now);
}
function isActionApi(url: URL): boolean {
return url.pathname === '/w/api.php' && /(^|\.)(wikipedia|wikidata|wikimedia)\.org$/.test(url.hostname);
}
export class WikimediaClient {
readonly config: ClientConfig;
private readonly fetcher: typeof globalThis.fetch;
private readonly clock: Clock;
private readonly userAgent: () => string;
private cache: WikimediaCache | undefined;
private readonly hosts = new Map<string, HostState>();
private readonly queues = new Map<Lane, Waiter[]>(LANES.map(lane => [lane, []]));
private readonly revalidating = new Set<string>();
private pumpTimer: unknown;
private pumpAt = Infinity;
private day = '';
private requestsToday = 0;
readonly counters: Record<string, number> = {};
constructor(options: { fetch?: typeof globalThis.fetch; config?: Partial<ClientConfig>; clock?: Clock;
cache?: WikimediaCache; userAgent?: () => string } = {}) {
this.fetcher = options.fetch ?? ((...args) => globalThis.fetch(...args));
this.config = { ...DEFAULT_CONFIG, ...options.config,
maxWaitMs: { ...DEFAULT_CONFIG.maxWaitMs, ...options.config?.maxWaitMs } };
this.clock = options.clock ?? realClock;
this.userAgent = options.userAgent ?? (() => wikimediaUserAgent(defaultUserAgent()));
this.cache = options.cache;
}
/** A pass-through client for injected fetchers (tests): no rate limit, no breaker, no cache. */
static unthrottled(fetcher: typeof globalThis.fetch): WikimediaClient {
return new WikimediaClient({ fetch: fetcher, config: UNTHROTTLED });
}
attachCache(cache: WikimediaCache | undefined): void { this.cache = cache; }
private count(name: string): void { this.counters[name] = (this.counters[name] ?? 0) + 1; }
private get rate(): number { return this.config.requestsPerSecond / this.config.nodesPerIp; }
private get cap(): number { return Math.max(1, Math.floor(this.config.dailyCap / this.config.nodesPerIp)); }
private host(name: string): HostState {
let state = this.hosts.get(name);
if (!state) {
state = { tokens: this.config.burst, refilledAt: this.clock.now(), inFlight: 0,
pausedUntil: 0, breakerUntil: 0, consecutive429: 0, backoffExponent: 0 };
this.hosts.set(name, state);
}
return state;
}
private refill(state: HostState, now: number): void {
state.tokens = Math.min(this.config.burst, state.tokens + (now - state.refilledAt) / 1000 * this.rate);
state.refilledAt = now;
}
/** Daily budget by lane: background stops at 80 %, interactive at 100 %, mint has a 10 % reserve. */
private overCap(lane: Lane): boolean {
const today = new Date(this.clock.now()).toISOString().slice(0, 10);
if (today !== this.day) { this.day = today; this.requestsToday = 0; }
const limit = lane === 'background' ? this.cap * 0.8 : lane === 'interactive' ? this.cap : this.cap * 1.1;
return this.requestsToday >= limit;
}
status(): ClientStatus {
const now = this.clock.now();
return { day: this.day, requestsToday: this.requestsToday, dailyCap: this.cap, requestsPerSecond: this.rate,
queued: LANES.reduce((n, lane) => n + this.queues.get(lane)!.length, 0), counters: { ...this.counters },
hosts: Object.fromEntries([...this.hosts].map(([name, s]) => [name,
{ inFlight: s.inFlight, pausedForMs: Math.max(0, s.pausedUntil - now), circuitOpen: s.breakerUntil > now,
consecutive429: s.consecutive429 }])) };
}
async request(input: URL, options: RequestOptions): Promise<ClientResponse> {
const url = new URL(input);
if (url.protocol !== 'https:') throw new RangeError('Wikimedia requests must use HTTPS');
if (isActionApi(url) && !url.searchParams.has('maxlag')) url.searchParams.append('maxlag', String(this.config.maxlag));
const key = options.cache && this.cache ? cacheKey(url) : undefined;
const entry = key ? this.cache?.get(key) : undefined;
const now = this.clock.now();
if (entry && entry.expiresAt > now) {
this.count('cache.hit');
return { response: entry.toResponse(), signal: new AbortController().signal, fromCache: true };
}
if (entry && options.cache?.staleWhileRevalidate) {
this.count('cache.stale');
this.revalidate(url, key!, options);
return { response: entry.toResponse(), signal: new AbortController().signal, fromCache: true };
}
try {
return await this.network(url, options, key);
} catch (error) {
// Prefer a stale copy over an error for anything that is allowed to use the cache.
if (entry && error instanceof WikimediaUnavailable) {
this.count('cache.staleOnError');
return { response: entry.toResponse(), signal: new AbortController().signal, fromCache: true };
}
throw error;
}
}
private revalidate(url: URL, key: string, options: RequestOptions): void {
if (this.revalidating.has(key)) return;
this.revalidating.add(key);
void this.network(url, { ...options, lane: 'background' }, key)
.then(({ response }) => response.body?.cancel(), () => this.count('cache.revalidateFailed'))
.finally(() => this.revalidating.delete(key));
}
private async network(url: URL, options: RequestOptions, key: string | undefined): Promise<ClientResponse> {
for (let attempt = 0; ; attempt++) {
const result = await this.dispatch(url, options, key);
if (result !== 'maxlag') return result;
if (attempt >= this.config.maxlagRetries) throw new WikimediaUnavailable('maxlag');
}
}
/** Wait for a slot, send one request, and classify the reply. */
private async dispatch(url: URL, options: RequestOptions, key: string | undefined): Promise<ClientResponse | 'maxlag'> {
const hostName = url.hostname;
const wait = this.acquire(hostName, options.lane);
if (wait) await wait; // No await on the fast path: an idle host dispatches synchronously.
const state = this.host(hostName);
const headers = new Headers({ Accept: 'application/json', 'Accept-Encoding': 'gzip', ...options.headers,
'User-Agent': this.userAgent() });
const entry = key ? this.cache?.get(key) : undefined;
if (entry?.etag) headers.set('If-None-Match', entry.etag);
if (entry?.lastModified) headers.set('If-Modified-Since', entry.lastModified);
const signal = AbortSignal.timeout(options.timeoutMs);
this.requestsToday++;
this.count(`lane.${options.lane}`);
let response: Response;
try {
response = await this.fetcher(url, { signal, redirect: 'error', headers });
} catch (error) {
this.release(state);
this.count('error.network');
throw new WikimediaUnavailable(signal.aborted ? 'timeout' : `network: ${(error as Error).message}`);
}
const now = this.clock.now();
const retryAfter = parseRetryAfter(response.headers.get('retry-after'), now);
const apiError = response.headers.get('mediawiki-api-error');
if (apiError === 'maxlag') {
await response.body?.cancel();
this.count('maxlag');
this.pause(state, Math.min(this.config.maxBackoffMs, retryAfter ?? 5000), false);
this.release(state);
return 'maxlag';
}
if (response.status === 429 || response.status === 503) {
await response.body?.cancel();
this.count(`http.${response.status}`);
if (response.status === 429) state.consecutive429++;
const backoff = Math.min(this.config.maxBackoffMs, 2 ** state.backoffExponent * 1000 * (1 + Math.random() * 0.25));
state.backoffExponent = Math.min(state.backoffExponent + 1, 20);
if (state.consecutive429 >= this.config.breakerThreshold) {
this.pause(state, this.config.breakerMs, true);
this.count('breaker.open');
} else {
this.pause(state, Math.min(this.config.maxBackoffMs, Math.max(retryAfter ?? 0, backoff)), false);
}
this.release(state);
throw new WikimediaUnavailable(`HTTP ${response.status}`, response.status);
}
state.consecutive429 = 0;
state.backoffExponent = 0;
this.release(state);
if (response.status === 304 && entry && key) {
this.count('cache.revalidated');
this.cache?.touch(key, now + options.cache!.ttlMs);
return { response: entry.toResponse(), signal, fromCache: true };
}
if (key && this.cache && response.status === 200 && !response.redirected && options.cache &&
/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '')) {
// Store the raw body; callers re-validate it exactly as they would a network reply.
const body = await this.readLimited(response, options.cache.maxBytes ?? 1024 * 1024, signal);
const stored = this.cache.put(key, { status: response.status, body,
contentType: response.headers.get('content-type') ?? '', etag: response.headers.get('etag'),
lastModified: response.headers.get('last-modified'), expiresAt: now + options.cache.ttlMs, storedAt: now });
return { response: stored.toResponse(), signal, fromCache: false };
}
if (response.status >= 500) this.count(`http.${response.status}`);
return { response, signal, fromCache: false };
}
private async readLimited(response: Response, maxBytes: number, signal: AbortSignal): Promise<Buffer> {
const length = response.headers.get('content-length');
if (length !== null && (!/^\d+$/.test(length) || Number(length) > maxBytes) || !response.body) {
await response.body?.cancel();
throw new WikimediaUnavailable('oversized or empty response');
}
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let size = 0;
try {
for (;;) {
if (signal.aborted) throw new WikimediaUnavailable('timeout');
const { done, value } = await reader.read();
if (done) break;
size += value.byteLength;
if (size > maxBytes) throw new WikimediaUnavailable('oversized response');
chunks.push(value);
}
return Buffer.concat(chunks);
} catch (error) {
throw error instanceof WikimediaUnavailable ? error : new WikimediaUnavailable('read failed');
} finally {
await reader.cancel().catch(() => {});
reader.releaseLock();
}
}
private pause(state: HostState, ms: number, breaker: boolean): void {
const until = this.clock.now() + ms;
if (until > state.pausedUntil) {
if (state.pausedUntil <= this.clock.now()) {
console.warn(`[wikimedia] ${breaker ? 'circuit open' : 'pausing'} for ${Math.ceil(ms / 1000)} s`);
}
state.pausedUntil = until;
}
if (breaker) { state.consecutive429 = 0; state.breakerUntil = until; }
}
private release(state: HostState): void {
state.inFlight--;
this.pump();
}
/** Undefined when a slot was taken at once; otherwise a promise for the slot. Throws on refusal. */
private acquire(hostName: string, lane: Lane): Promise<void> | undefined {
const state = this.host(hostName);
const now = this.clock.now();
if (this.overCap(lane)) { this.count('refused.cap'); throw new WikimediaUnavailable('daily cap reached'); }
// A paused host refuses at once when the pause outlasts this lane's patience.
if (state.pausedUntil - now > this.config.maxWaitMs[lane]) {
this.count('refused.paused');
throw new WikimediaUnavailable(state.breakerUntil > now ? 'circuit open' : 'host paused');
}
this.refill(state, now);
if (state.pausedUntil <= now && state.inFlight < this.config.concurrencyPerHost && state.tokens >= 1 &&
!LANES.some(l => this.queues.get(l)!.some(w => w.host === hostName))) {
state.tokens -= 1;
state.inFlight++;
return undefined;
}
const queued = LANES.reduce((n, l) => n + this.queues.get(l)!.length, 0);
if (queued >= this.config.queueLimit) {
// Background work is shed first; a mint request evicts the newest background waiter.
const background = this.queues.get('background')!;
if (lane === 'background' || background.length === 0) {
this.count('refused.queue');
throw new WikimediaUnavailable('queue full');
}
const evicted = background.pop()!;
this.clock.clearTimeout(evicted.timer);
evicted.fail(new WikimediaUnavailable('queue full'));
}
return new Promise<void>((resolve, reject) => {
const waiter: Waiter = { lane, host: hostName, enqueuedAt: now,
run: () => { this.clock.clearTimeout(waiter.timer); resolve(); },
fail: error => reject(error) };
waiter.timer = this.clock.setTimeout(() => {
const queue = this.queues.get(lane)!;
const index = queue.indexOf(waiter);
if (index >= 0) { queue.splice(index, 1); this.count('refused.wait'); reject(new WikimediaUnavailable('queue wait')); }
}, this.config.maxWaitMs[lane]);
this.queues.get(lane)!.push(waiter);
this.pump();
});
}
/** Start every waiter that may run now, highest lane first; otherwise arm a timer for the next slot. */
private pump(): void {
const now = this.clock.now();
let next = Infinity;
for (const lane of LANES) {
const queue = this.queues.get(lane)!;
for (let i = 0; i < queue.length;) {
const waiter = queue[i];
// A host is served strictly by priority: lower lanes wait while a higher lane waits for the same host.
if (LANES.slice(0, LANES.indexOf(lane)).some(l => this.queues.get(l)!.some(w => w.host === waiter.host))) { i++; continue; }
const state = this.host(waiter.host);
this.refill(state, now);
if (state.pausedUntil > now) { next = Math.min(next, state.pausedUntil); i++; continue; }
if (state.inFlight >= this.config.concurrencyPerHost) { i++; continue; }
if (state.tokens < 1) { next = Math.min(next, now + (1 - state.tokens) / this.rate * 1000); i++; continue; }
state.tokens -= 1;
state.inFlight++;
queue.splice(i, 1);
waiter.run();
}
}
if (next < Infinity && next < this.pumpAt) {
if (this.pumpTimer !== undefined) this.clock.clearTimeout(this.pumpTimer);
this.pumpAt = next;
this.pumpTimer = this.clock.setTimeout(() => { this.pumpTimer = undefined; this.pumpAt = Infinity; this.pump(); },
Math.max(1, Math.ceil(next - now)));
}
}
}
/** The process-wide client. `configureWikimedia()` (server start-up) attaches the disk cache. */
export const wikimediaClient = new WikimediaClient({ config: configFromEnv() });