Files
wikideck/server/wikimedia-user-agent.ts
claudeandClaude Opus 5.5 f544019bfe Polite Wikimedia client, mint-path fix, disk cache and mining prefetch pool (#62)
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket
  (1 req/s, burst 3), 1 in flight per host, mint > interactive > background
  lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After
  and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and
  a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP).
- server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match,
  stale-while-revalidate on the background lane. Mint reads never use it.
- Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh()
  (template 503, miner retries) instead of minting attention: null forever.
- server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for
  /api/mining/template, about 3 requests per block regardless of miner count.
- User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or
  PUBLIC_ORIGIN) and a start-up warning when it is unset.
- Mint request parameters are byte-identical apart from maxlag (golden test).
  No consensus, block or protocol changes. Tests use fake fetch only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 21:51:20 -05:00

49 lines
2.5 KiB
TypeScript

// User-Agent for Wikimedia API requests. Wikimedia's User-Agent policy asks automated clients for a
// descriptive agent with contact details: `Tool/version (+https://site; contact) lib/version`.
// Operators set WIKIDECK_OPERATOR_CONTACT (an email address or URL); PUBLIC_ORIGIN is the fallback.
// WIKIDECK_USER_AGENT still overrides the whole string.
import { readFileSync } from 'node:fs';
export const PROJECT_URL = 'https://gitea.eom.dev/WikiDeck/wikideck';
const MAX_LENGTH = 256;
const SAFE = /^[\x20-\x7e]+$/;
function packageVersion(): string {
try {
const raw = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')) as { version?: unknown };
return typeof raw.version === 'string' && /^[\w.+-]{1,32}$/.test(raw.version) ? raw.version : '0';
} catch { return '0'; }
}
const VERSION = packageVersion();
/** The operator contact, or undefined when none (or only an unsafe value) is configured. */
export function operatorContact(env: Record<string, string | undefined> = process.env): string | undefined {
for (const value of [env.WIKIDECK_OPERATOR_CONTACT, env.PUBLIC_ORIGIN]) {
const trimmed = value?.trim();
if (trimmed && trimmed.length <= 128 && SAFE.test(trimmed) && !/[();]/.test(trimmed)) return trimmed;
}
return undefined;
}
/** `WikiDeck/<version> (+<project URL>; <contact>) node/<version>`. */
export function defaultUserAgent(env: Record<string, string | undefined> = process.env): string {
const contact = operatorContact(env);
return `WikiDeck/${VERSION} (+${PROJECT_URL}${contact ? `; ${contact}` : ''}) node/${process.versions.node}`;
}
export function wikimediaUserAgent(fallback: string, env: Record<string, string | undefined> = process.env): string {
const configured = env.WIKIDECK_USER_AGENT?.trim();
// Reject header-injection/control characters and oversized values; fall back to the default.
if (!configured || configured.length > MAX_LENGTH || !SAFE.test(configured)) return fallback;
return configured;
}
/** Start-up check: Wikimedia throttles or blocks clients without contact details first. */
export function warnIfNoContact(env: Record<string, string | undefined> = process.env,
warn: (message: string) => void = console.warn): boolean {
if (wikimediaUserAgent('', env) || operatorContact(env)) return false;
warn('[wikimedia] WARNING: no operator contact in the Wikimedia User-Agent. Set WIKIDECK_OPERATOR_CONTACT '
+ '(an email address or URL) or PUBLIC_ORIGIN; Wikimedia policy requires it and may block this node.');
return true;
}