forked from WikiDeck/wikideck
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket (1 req/s, burst 3), 1 in flight per host, mint > interactive > background lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP). - server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match, stale-while-revalidate on the background lane. Mint reads never use it. - Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh() (template 503, miner retries) instead of minting attention: null forever. - server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for /api/mining/template, about 3 requests per block regardless of miner count. - User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or PUBLIC_ORIGIN) and a start-up warning when it is unset. - Mint request parameters are byte-identical apart from maxlag (golden test). No consensus, block or protocol changes. Tests use fake fetch only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
49 lines
2.5 KiB
TypeScript
49 lines
2.5 KiB
TypeScript
// User-Agent for Wikimedia API requests. Wikimedia's User-Agent policy asks automated clients for a
|
|
// descriptive agent with contact details: `Tool/version (+https://site; contact) lib/version`.
|
|
// Operators set WIKIDECK_OPERATOR_CONTACT (an email address or URL); PUBLIC_ORIGIN is the fallback.
|
|
// WIKIDECK_USER_AGENT still overrides the whole string.
|
|
import { readFileSync } from 'node:fs';
|
|
|
|
export const PROJECT_URL = 'https://gitea.eom.dev/WikiDeck/wikideck';
|
|
const MAX_LENGTH = 256;
|
|
const SAFE = /^[\x20-\x7e]+$/;
|
|
|
|
function packageVersion(): string {
|
|
try {
|
|
const raw = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')) as { version?: unknown };
|
|
return typeof raw.version === 'string' && /^[\w.+-]{1,32}$/.test(raw.version) ? raw.version : '0';
|
|
} catch { return '0'; }
|
|
}
|
|
const VERSION = packageVersion();
|
|
|
|
/** The operator contact, or undefined when none (or only an unsafe value) is configured. */
|
|
export function operatorContact(env: Record<string, string | undefined> = process.env): string | undefined {
|
|
for (const value of [env.WIKIDECK_OPERATOR_CONTACT, env.PUBLIC_ORIGIN]) {
|
|
const trimmed = value?.trim();
|
|
if (trimmed && trimmed.length <= 128 && SAFE.test(trimmed) && !/[();]/.test(trimmed)) return trimmed;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
/** `WikiDeck/<version> (+<project URL>; <contact>) node/<version>`. */
|
|
export function defaultUserAgent(env: Record<string, string | undefined> = process.env): string {
|
|
const contact = operatorContact(env);
|
|
return `WikiDeck/${VERSION} (+${PROJECT_URL}${contact ? `; ${contact}` : ''}) node/${process.versions.node}`;
|
|
}
|
|
|
|
export function wikimediaUserAgent(fallback: string, env: Record<string, string | undefined> = process.env): string {
|
|
const configured = env.WIKIDECK_USER_AGENT?.trim();
|
|
// Reject header-injection/control characters and oversized values; fall back to the default.
|
|
if (!configured || configured.length > MAX_LENGTH || !SAFE.test(configured)) return fallback;
|
|
return configured;
|
|
}
|
|
|
|
/** Start-up check: Wikimedia throttles or blocks clients without contact details first. */
|
|
export function warnIfNoContact(env: Record<string, string | undefined> = process.env,
|
|
warn: (message: string) => void = console.warn): boolean {
|
|
if (wikimediaUserAgent('', env) || operatorContact(env)) return false;
|
|
warn('[wikimedia] WARNING: no operator contact in the Wikimedia User-Agent. Set WIKIDECK_OPERATOR_CONTACT '
|
|
+ '(an email address or URL) or PUBLIC_ORIGIN; Wikimedia policy requires it and may block this node.');
|
|
return true;
|
|
}
|