Night 2 (2026-11-17), from the `claude/wikideck` fork's `night/2026-11-17` branch. It follows Night 1 (#22).
## What's in it
- **Duel engine** (fork #72, #80): 40-card decks of cards you own, Notability/Citations, keywords, effects T1–T12, Connected, a hidden-information-safe `viewFor`/`logFor`, and replay. Balance v1 tuning (#84) gives the first player about 51%.
- **Card profiles** (#70, #82): Figure, Location, Event, Institution or Doctrine, plus stats, keywords and rules text, all derived from real article data. Templates only.
- **Design and story** (#66, #69, #83): DUEL_DESIGN.md, plus the match log and the encyclopedic "Battle of X (2026)" recap.
- **PvP** (#74, #77, #81, #76, #85):
- The server checks ownership, and 40-card eligibility is enforced.
- Modes: Classic (ranked), Draft and Trending.
- Invites, quick match, Elo, a ladder and recap pages. Online matches render on the duel board.
- **Duel board UI** (#79): MTG-style card faces and combat declaration, built for 360 px first.
- **Polite Wikimedia client** (#73, #75): throttled, cached and identified requests, the mint-path fix, and a prefetch pool.
- **Android mining** (#78): native RandomX, as a sideload build. The emulator mined a testnet block.
- **Testnet ops** (#67, #68).
PR numbers refer to `claude/wikideck`.
## Status
- **Tests:** 423, of which 422 pass, 0 fail and 1 is skipped. The build is OK (`node:24-alpine`, `--network none`).
- **Testnet:** running this commit (036489c) on r350.
- **Untouched:** no changes to `server/testnet/consensus.ts`, `randomx.ts`, `native/` or the mainnet compose.
- **Review:** night-review reviewed fork #66–#79. #80–#85 came from the agents' unfinished branches and were not reviewed.
- **Known gaps:**
- The PvP end-to-end test hasn't been run (it needs two eom.dev accounts).
- Bot matches aren't reachable from Play.
- Card identity pre-bake hasn't been done.
- The RandomX verifier latch (`server/testnet/randomx.ts:140`) is still open.
This PR is for Eric to merge.
🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: #23
Co-authored-by: Claude <claude@eom.dev>
442 lines
20 KiB
TypeScript
442 lines
20 KiB
TypeScript
/**
|
|
* One process-wide, polite Wikimedia HTTP client (engineer review §4, data review §5).
|
|
*
|
|
* Every server-side Wikimedia request goes through `WikimediaClient.request()`:
|
|
* - a token bucket per upstream host (default 1 req/s, burst 3) and 1 request in flight per host;
|
|
* - priority lanes: `mint` before `interactive` before `background`, with a bounded queue;
|
|
* - `maxlag=5` on Action API requests (it changes only error behaviour, never content);
|
|
* - `Retry-After` honoured on 429/503/maxlag by pausing the whole host, with exponential backoff;
|
|
* - a circuit breaker: 3 consecutive 429s stop all requests to that host for 10 minutes;
|
|
* - a daily request cap, and a budget shared by the nodes behind one egress IP;
|
|
* - an optional on-disk cache (see cache.ts) with ETag / If-None-Match and stale-while-revalidate.
|
|
*
|
|
* The client is transport only. It never rewrites request parameters other than adding `maxlag`,
|
|
* and requests made with `cache: undefined` (all mint-path requests) never read or write the cache.
|
|
* Nothing here is reachable from block validation, so it cannot affect consensus.
|
|
*/
|
|
import { WikimediaCache, cacheKey } from './cache.ts';
|
|
import { wikimediaUserAgent, defaultUserAgent } from '../wikimedia-user-agent.ts';
|
|
|
|
export type Lane = 'mint' | 'interactive' | 'background';
|
|
const LANES: Lane[] = ['mint', 'interactive', 'background'];
|
|
|
|
export type Clock = {
|
|
now(): number;
|
|
setTimeout(fn: () => void, ms: number): unknown;
|
|
clearTimeout(handle: unknown): void;
|
|
};
|
|
const realClock: Clock = {
|
|
now: () => Date.now(),
|
|
setTimeout: (fn, ms) => { const t = setTimeout(fn, ms); t.unref?.(); return t; },
|
|
clearTimeout: handle => clearTimeout(handle as ReturnType<typeof setTimeout>),
|
|
};
|
|
|
|
export type ClientConfig = {
|
|
/** Sustained requests per second per upstream host, for this whole egress IP. */
|
|
requestsPerSecond: number;
|
|
burst: number;
|
|
concurrencyPerHost: number;
|
|
/** Requests per UTC day for this whole egress IP. */
|
|
dailyCap: number;
|
|
/** Nodes sharing one egress IP; the rate and daily cap are divided between them. */
|
|
nodesPerIp: number;
|
|
queueLimit: number;
|
|
/** Longest wait in the queue before a request fails, per lane. */
|
|
maxWaitMs: Record<Lane, number>;
|
|
breakerThreshold: number;
|
|
breakerMs: number;
|
|
maxBackoffMs: number;
|
|
maxlag: number;
|
|
/** Retries of a single request after a maxlag reply (HTTP 200/503 with MediaWiki-API-Error: maxlag). */
|
|
maxlagRetries: number;
|
|
};
|
|
|
|
export const DEFAULT_CONFIG: ClientConfig = {
|
|
requestsPerSecond: 1, burst: 3, concurrencyPerHost: 1, dailyCap: 20_000, nodesPerIp: 1, queueLimit: 64,
|
|
maxWaitMs: { mint: 10_000, interactive: 5_000, background: 120_000 },
|
|
breakerThreshold: 3, breakerMs: 10 * 60_000, maxBackoffMs: 10 * 60_000, maxlag: 5, maxlagRetries: 2,
|
|
};
|
|
|
|
/** Unthrottled settings for tests and tools that inject their own fetch. */
|
|
export const UNTHROTTLED: Partial<ClientConfig> = {
|
|
requestsPerSecond: 1e9, burst: 1e9, concurrencyPerHost: 1e9, dailyCap: Number.MAX_SAFE_INTEGER, queueLimit: 1e9,
|
|
breakerThreshold: Number.MAX_SAFE_INTEGER, maxlagRetries: 0, maxBackoffMs: 0,
|
|
};
|
|
|
|
function positive(value: string | undefined, fallback: number, integer = false): number {
|
|
if (value === undefined || value.trim() === '') return fallback;
|
|
const n = Number(value);
|
|
if (!Number.isFinite(n) || n <= 0 || (integer && !Number.isSafeInteger(n))) {
|
|
console.warn(`[wikimedia] ignoring invalid setting ${JSON.stringify(value)}; using ${fallback}`);
|
|
return fallback;
|
|
}
|
|
return n;
|
|
}
|
|
|
|
/** Operator settings. Documented in docs/night2/WIKIMEDIA.md. */
|
|
export function configFromEnv(env: Record<string, string | undefined> = process.env): ClientConfig {
|
|
return { ...DEFAULT_CONFIG,
|
|
requestsPerSecond: positive(env.WIKIDECK_WIKIMEDIA_RPS, DEFAULT_CONFIG.requestsPerSecond),
|
|
burst: positive(env.WIKIDECK_WIKIMEDIA_BURST, DEFAULT_CONFIG.burst),
|
|
concurrencyPerHost: positive(env.WIKIDECK_WIKIMEDIA_CONCURRENCY, DEFAULT_CONFIG.concurrencyPerHost, true),
|
|
dailyCap: positive(env.WIKIDECK_WIKIMEDIA_DAILY_CAP, DEFAULT_CONFIG.dailyCap, true),
|
|
nodesPerIp: positive(env.WIKIDECK_WIKIMEDIA_NODES_PER_IP, DEFAULT_CONFIG.nodesPerIp, true),
|
|
};
|
|
}
|
|
|
|
/** Thrown for every refusal or transport failure: queue full, cap reached, circuit open, timeout, network. */
|
|
export class WikimediaUnavailable extends Error {
|
|
readonly reason: string;
|
|
readonly status?: number;
|
|
constructor(reason: string, status?: number) {
|
|
super(`Wikimedia unavailable: ${reason}`);
|
|
this.name = 'WikimediaUnavailable';
|
|
this.reason = reason; this.status = status;
|
|
}
|
|
}
|
|
|
|
export type CachePolicy = {
|
|
ttlMs: number;
|
|
/** Serve a stale entry at once and revalidate it on the background lane. */
|
|
staleWhileRevalidate?: boolean;
|
|
/** Largest body stored, in bytes. */
|
|
maxBytes?: number;
|
|
};
|
|
|
|
export type RequestOptions = {
|
|
lane: Lane;
|
|
/** Starts when the request is dispatched, not while it waits in the queue. */
|
|
timeoutMs: number;
|
|
headers?: Record<string, string>;
|
|
/** Omit for mint data: the request then bypasses the cache completely. */
|
|
cache?: CachePolicy;
|
|
};
|
|
|
|
export type ClientResponse = { response: Response; signal: AbortSignal; fromCache: boolean };
|
|
|
|
type Waiter = {
|
|
lane: Lane; host: string; enqueuedAt: number;
|
|
run: () => void; fail: (error: Error) => void; timer?: unknown;
|
|
};
|
|
type HostState = {
|
|
tokens: number; refilledAt: number; inFlight: number;
|
|
pausedUntil: number; breakerUntil: number; consecutive429: number; backoffExponent: number;
|
|
};
|
|
|
|
export type ClientStatus = {
|
|
day: string; requestsToday: number; dailyCap: number; requestsPerSecond: number; queued: number;
|
|
counters: Record<string, number>;
|
|
hosts: Record<string, { inFlight: number; pausedForMs: number; circuitOpen: boolean; consecutive429: number }>;
|
|
};
|
|
|
|
function parseRetryAfter(value: string | null, now: number): number | undefined {
|
|
if (!value) return undefined;
|
|
if (/^\d+$/.test(value.trim())) return Number(value.trim()) * 1000;
|
|
const at = Date.parse(value);
|
|
return Number.isNaN(at) ? undefined : Math.max(0, at - now);
|
|
}
|
|
|
|
function isActionApi(url: URL): boolean {
|
|
return url.pathname === '/w/api.php' && /(^|\.)(wikipedia|wikidata|wikimedia)\.org$/.test(url.hostname);
|
|
}
|
|
|
|
export class WikimediaClient {
|
|
readonly config: ClientConfig;
|
|
private readonly fetcher: typeof globalThis.fetch;
|
|
private readonly clock: Clock;
|
|
private readonly userAgent: () => string;
|
|
private cache: WikimediaCache | undefined;
|
|
private readonly hosts = new Map<string, HostState>();
|
|
private readonly queues = new Map<Lane, Waiter[]>(LANES.map(lane => [lane, []]));
|
|
private readonly revalidating = new Set<string>();
|
|
private pumpTimer: unknown;
|
|
private pumpAt = Infinity;
|
|
private day = '';
|
|
private requestsToday = 0;
|
|
readonly counters: Record<string, number> = {};
|
|
|
|
constructor(options: { fetch?: typeof globalThis.fetch; config?: Partial<ClientConfig>; clock?: Clock;
|
|
cache?: WikimediaCache; userAgent?: () => string } = {}) {
|
|
this.fetcher = options.fetch ?? ((...args) => globalThis.fetch(...args));
|
|
this.config = { ...DEFAULT_CONFIG, ...options.config,
|
|
maxWaitMs: { ...DEFAULT_CONFIG.maxWaitMs, ...options.config?.maxWaitMs } };
|
|
this.clock = options.clock ?? realClock;
|
|
this.userAgent = options.userAgent ?? (() => wikimediaUserAgent(defaultUserAgent()));
|
|
this.cache = options.cache;
|
|
}
|
|
|
|
/** A pass-through client for injected fetchers (tests): no rate limit, no breaker, no cache. */
|
|
static unthrottled(fetcher: typeof globalThis.fetch): WikimediaClient {
|
|
return new WikimediaClient({ fetch: fetcher, config: UNTHROTTLED });
|
|
}
|
|
|
|
attachCache(cache: WikimediaCache | undefined): void { this.cache = cache; }
|
|
|
|
private count(name: string): void { this.counters[name] = (this.counters[name] ?? 0) + 1; }
|
|
|
|
private get rate(): number { return this.config.requestsPerSecond / this.config.nodesPerIp; }
|
|
private get cap(): number { return Math.max(1, Math.floor(this.config.dailyCap / this.config.nodesPerIp)); }
|
|
|
|
private host(name: string): HostState {
|
|
let state = this.hosts.get(name);
|
|
if (!state) {
|
|
state = { tokens: this.config.burst, refilledAt: this.clock.now(), inFlight: 0,
|
|
pausedUntil: 0, breakerUntil: 0, consecutive429: 0, backoffExponent: 0 };
|
|
this.hosts.set(name, state);
|
|
}
|
|
return state;
|
|
}
|
|
|
|
private refill(state: HostState, now: number): void {
|
|
state.tokens = Math.min(this.config.burst, state.tokens + (now - state.refilledAt) / 1000 * this.rate);
|
|
state.refilledAt = now;
|
|
}
|
|
|
|
/** Daily budget by lane: background stops at 80 %, interactive at 100 %, mint has a 10 % reserve. */
|
|
private overCap(lane: Lane): boolean {
|
|
const today = new Date(this.clock.now()).toISOString().slice(0, 10);
|
|
if (today !== this.day) { this.day = today; this.requestsToday = 0; }
|
|
const limit = lane === 'background' ? this.cap * 0.8 : lane === 'interactive' ? this.cap : this.cap * 1.1;
|
|
return this.requestsToday >= limit;
|
|
}
|
|
|
|
status(): ClientStatus {
|
|
const now = this.clock.now();
|
|
return { day: this.day, requestsToday: this.requestsToday, dailyCap: this.cap, requestsPerSecond: this.rate,
|
|
queued: LANES.reduce((n, lane) => n + this.queues.get(lane)!.length, 0), counters: { ...this.counters },
|
|
hosts: Object.fromEntries([...this.hosts].map(([name, s]) => [name,
|
|
{ inFlight: s.inFlight, pausedForMs: Math.max(0, s.pausedUntil - now), circuitOpen: s.breakerUntil > now,
|
|
consecutive429: s.consecutive429 }])) };
|
|
}
|
|
|
|
async request(input: URL, options: RequestOptions): Promise<ClientResponse> {
|
|
const url = new URL(input);
|
|
if (url.protocol !== 'https:') throw new RangeError('Wikimedia requests must use HTTPS');
|
|
if (isActionApi(url) && !url.searchParams.has('maxlag')) url.searchParams.append('maxlag', String(this.config.maxlag));
|
|
const key = options.cache && this.cache ? cacheKey(url) : undefined;
|
|
const entry = key ? this.cache?.get(key) : undefined;
|
|
const now = this.clock.now();
|
|
if (entry && entry.expiresAt > now) {
|
|
this.count('cache.hit');
|
|
return { response: entry.toResponse(), signal: new AbortController().signal, fromCache: true };
|
|
}
|
|
if (entry && options.cache?.staleWhileRevalidate) {
|
|
this.count('cache.stale');
|
|
this.revalidate(url, key!, options);
|
|
return { response: entry.toResponse(), signal: new AbortController().signal, fromCache: true };
|
|
}
|
|
try {
|
|
return await this.network(url, options, key);
|
|
} catch (error) {
|
|
// Prefer a stale copy over an error for anything that is allowed to use the cache.
|
|
if (entry && error instanceof WikimediaUnavailable) {
|
|
this.count('cache.staleOnError');
|
|
return { response: entry.toResponse(), signal: new AbortController().signal, fromCache: true };
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
private revalidate(url: URL, key: string, options: RequestOptions): void {
|
|
if (this.revalidating.has(key)) return;
|
|
this.revalidating.add(key);
|
|
void this.network(url, { ...options, lane: 'background' }, key)
|
|
.then(({ response }) => response.body?.cancel(), () => this.count('cache.revalidateFailed'))
|
|
.finally(() => this.revalidating.delete(key));
|
|
}
|
|
|
|
private async network(url: URL, options: RequestOptions, key: string | undefined): Promise<ClientResponse> {
|
|
for (let attempt = 0; ; attempt++) {
|
|
const result = await this.dispatch(url, options, key);
|
|
if (result !== 'maxlag') return result;
|
|
if (attempt >= this.config.maxlagRetries) throw new WikimediaUnavailable('maxlag');
|
|
}
|
|
}
|
|
|
|
/** Wait for a slot, send one request, and classify the reply. */
|
|
private async dispatch(url: URL, options: RequestOptions, key: string | undefined): Promise<ClientResponse | 'maxlag'> {
|
|
const hostName = url.hostname;
|
|
const wait = this.acquire(hostName, options.lane);
|
|
if (wait) await wait; // No await on the fast path: an idle host dispatches synchronously.
|
|
const state = this.host(hostName);
|
|
const headers = new Headers({ Accept: 'application/json', 'Accept-Encoding': 'gzip', ...options.headers,
|
|
'User-Agent': this.userAgent() });
|
|
const entry = key ? this.cache?.get(key) : undefined;
|
|
if (entry?.etag) headers.set('If-None-Match', entry.etag);
|
|
if (entry?.lastModified) headers.set('If-Modified-Since', entry.lastModified);
|
|
const signal = AbortSignal.timeout(options.timeoutMs);
|
|
this.requestsToday++;
|
|
this.count(`lane.${options.lane}`);
|
|
let response: Response;
|
|
try {
|
|
response = await this.fetcher(url, { signal, redirect: 'error', headers });
|
|
} catch (error) {
|
|
this.release(state);
|
|
this.count('error.network');
|
|
throw new WikimediaUnavailable(signal.aborted ? 'timeout' : `network: ${(error as Error).message}`);
|
|
}
|
|
const now = this.clock.now();
|
|
const retryAfter = parseRetryAfter(response.headers.get('retry-after'), now);
|
|
const apiError = response.headers.get('mediawiki-api-error');
|
|
if (apiError === 'maxlag') {
|
|
await response.body?.cancel();
|
|
this.count('maxlag');
|
|
this.pause(state, Math.min(this.config.maxBackoffMs, retryAfter ?? 5000), false);
|
|
this.release(state);
|
|
return 'maxlag';
|
|
}
|
|
if (response.status === 429 || response.status === 503) {
|
|
await response.body?.cancel();
|
|
this.count(`http.${response.status}`);
|
|
if (response.status === 429) state.consecutive429++;
|
|
const backoff = Math.min(this.config.maxBackoffMs, 2 ** state.backoffExponent * 1000 * (1 + Math.random() * 0.25));
|
|
state.backoffExponent = Math.min(state.backoffExponent + 1, 20);
|
|
if (state.consecutive429 >= this.config.breakerThreshold) {
|
|
this.pause(state, this.config.breakerMs, true);
|
|
this.count('breaker.open');
|
|
} else {
|
|
this.pause(state, Math.min(this.config.maxBackoffMs, Math.max(retryAfter ?? 0, backoff)), false);
|
|
}
|
|
this.release(state);
|
|
throw new WikimediaUnavailable(`HTTP ${response.status}`, response.status);
|
|
}
|
|
state.consecutive429 = 0;
|
|
state.backoffExponent = 0;
|
|
this.release(state);
|
|
if (response.status === 304 && entry && key) {
|
|
this.count('cache.revalidated');
|
|
this.cache?.touch(key, now + options.cache!.ttlMs);
|
|
return { response: entry.toResponse(), signal, fromCache: true };
|
|
}
|
|
if (key && this.cache && response.status === 200 && !response.redirected && options.cache &&
|
|
/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '')) {
|
|
// Store the raw body; callers re-validate it exactly as they would a network reply.
|
|
const body = await this.readLimited(response, options.cache.maxBytes ?? 1024 * 1024, signal);
|
|
const stored = this.cache.put(key, { status: response.status, body,
|
|
contentType: response.headers.get('content-type') ?? '', etag: response.headers.get('etag'),
|
|
lastModified: response.headers.get('last-modified'), expiresAt: now + options.cache.ttlMs, storedAt: now });
|
|
return { response: stored.toResponse(), signal, fromCache: false };
|
|
}
|
|
if (response.status >= 500) this.count(`http.${response.status}`);
|
|
return { response, signal, fromCache: false };
|
|
}
|
|
|
|
private async readLimited(response: Response, maxBytes: number, signal: AbortSignal): Promise<Buffer> {
|
|
const length = response.headers.get('content-length');
|
|
if (length !== null && (!/^\d+$/.test(length) || Number(length) > maxBytes) || !response.body) {
|
|
await response.body?.cancel();
|
|
throw new WikimediaUnavailable('oversized or empty response');
|
|
}
|
|
const reader = response.body.getReader();
|
|
const chunks: Uint8Array[] = [];
|
|
let size = 0;
|
|
try {
|
|
for (;;) {
|
|
if (signal.aborted) throw new WikimediaUnavailable('timeout');
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
size += value.byteLength;
|
|
if (size > maxBytes) throw new WikimediaUnavailable('oversized response');
|
|
chunks.push(value);
|
|
}
|
|
return Buffer.concat(chunks);
|
|
} catch (error) {
|
|
throw error instanceof WikimediaUnavailable ? error : new WikimediaUnavailable('read failed');
|
|
} finally {
|
|
await reader.cancel().catch(() => {});
|
|
reader.releaseLock();
|
|
}
|
|
}
|
|
|
|
private pause(state: HostState, ms: number, breaker: boolean): void {
|
|
const until = this.clock.now() + ms;
|
|
if (until > state.pausedUntil) {
|
|
if (state.pausedUntil <= this.clock.now()) {
|
|
console.warn(`[wikimedia] ${breaker ? 'circuit open' : 'pausing'} for ${Math.ceil(ms / 1000)} s`);
|
|
}
|
|
state.pausedUntil = until;
|
|
}
|
|
if (breaker) { state.consecutive429 = 0; state.breakerUntil = until; }
|
|
}
|
|
|
|
private release(state: HostState): void {
|
|
state.inFlight--;
|
|
this.pump();
|
|
}
|
|
|
|
/** Undefined when a slot was taken at once; otherwise a promise for the slot. Throws on refusal. */
|
|
private acquire(hostName: string, lane: Lane): Promise<void> | undefined {
|
|
const state = this.host(hostName);
|
|
const now = this.clock.now();
|
|
if (this.overCap(lane)) { this.count('refused.cap'); throw new WikimediaUnavailable('daily cap reached'); }
|
|
// A paused host refuses at once when the pause outlasts this lane's patience.
|
|
if (state.pausedUntil - now > this.config.maxWaitMs[lane]) {
|
|
this.count('refused.paused');
|
|
throw new WikimediaUnavailable(state.breakerUntil > now ? 'circuit open' : 'host paused');
|
|
}
|
|
this.refill(state, now);
|
|
if (state.pausedUntil <= now && state.inFlight < this.config.concurrencyPerHost && state.tokens >= 1 &&
|
|
!LANES.some(l => this.queues.get(l)!.some(w => w.host === hostName))) {
|
|
state.tokens -= 1;
|
|
state.inFlight++;
|
|
return undefined;
|
|
}
|
|
const queued = LANES.reduce((n, l) => n + this.queues.get(l)!.length, 0);
|
|
if (queued >= this.config.queueLimit) {
|
|
// Background work is shed first; a mint request evicts the newest background waiter.
|
|
const background = this.queues.get('background')!;
|
|
if (lane === 'background' || background.length === 0) {
|
|
this.count('refused.queue');
|
|
throw new WikimediaUnavailable('queue full');
|
|
}
|
|
const evicted = background.pop()!;
|
|
this.clock.clearTimeout(evicted.timer);
|
|
evicted.fail(new WikimediaUnavailable('queue full'));
|
|
}
|
|
return new Promise<void>((resolve, reject) => {
|
|
const waiter: Waiter = { lane, host: hostName, enqueuedAt: now,
|
|
run: () => { this.clock.clearTimeout(waiter.timer); resolve(); },
|
|
fail: error => reject(error) };
|
|
waiter.timer = this.clock.setTimeout(() => {
|
|
const queue = this.queues.get(lane)!;
|
|
const index = queue.indexOf(waiter);
|
|
if (index >= 0) { queue.splice(index, 1); this.count('refused.wait'); reject(new WikimediaUnavailable('queue wait')); }
|
|
}, this.config.maxWaitMs[lane]);
|
|
this.queues.get(lane)!.push(waiter);
|
|
this.pump();
|
|
});
|
|
}
|
|
|
|
/** Start every waiter that may run now, highest lane first; otherwise arm a timer for the next slot. */
|
|
private pump(): void {
|
|
const now = this.clock.now();
|
|
let next = Infinity;
|
|
for (const lane of LANES) {
|
|
const queue = this.queues.get(lane)!;
|
|
for (let i = 0; i < queue.length;) {
|
|
const waiter = queue[i];
|
|
// A host is served strictly by priority: lower lanes wait while a higher lane waits for the same host.
|
|
if (LANES.slice(0, LANES.indexOf(lane)).some(l => this.queues.get(l)!.some(w => w.host === waiter.host))) { i++; continue; }
|
|
const state = this.host(waiter.host);
|
|
this.refill(state, now);
|
|
if (state.pausedUntil > now) { next = Math.min(next, state.pausedUntil); i++; continue; }
|
|
if (state.inFlight >= this.config.concurrencyPerHost) { i++; continue; }
|
|
if (state.tokens < 1) { next = Math.min(next, now + (1 - state.tokens) / this.rate * 1000); i++; continue; }
|
|
state.tokens -= 1;
|
|
state.inFlight++;
|
|
queue.splice(i, 1);
|
|
waiter.run();
|
|
}
|
|
}
|
|
if (next < Infinity && next < this.pumpAt) {
|
|
if (this.pumpTimer !== undefined) this.clock.clearTimeout(this.pumpTimer);
|
|
this.pumpAt = next;
|
|
this.pumpTimer = this.clock.setTimeout(() => { this.pumpTimer = undefined; this.pumpAt = Infinity; this.pump(); },
|
|
Math.max(1, Math.ceil(next - now)));
|
|
}
|
|
}
|
|
}
|
|
|
|
/** The process-wide client. `configureWikimedia()` (server start-up) attaches the disk cache. */
|
|
export const wikimediaClient = new WikimediaClient({ config: configFromEnv() });
|