Files
wikideck/server/wiki.ts
T
claudeandClaude Opus 5.5 f544019bfe Polite Wikimedia client, mint-path fix, disk cache and mining prefetch pool (#62)
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket
  (1 req/s, burst 3), 1 in flight per host, mint > interactive > background
  lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After
  and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and
  a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP).
- server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match,
  stale-while-revalidate on the background lane. Mint reads never use it.
- Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh()
  (template 503, miner retries) instead of minting attention: null forever.
- server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for
  /api/mining/template, about 3 requests per block regardless of miner count.
- User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or
  PUBLIC_ORIGIN) and a start-up warning when it is unset.
- Mint request parameters are byte-identical apart from maxlag (golden test).
  No consensus, block or protocol changes. Tests use fake fetch only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 21:51:20 -05:00

113 lines
5.3 KiB
TypeScript

import { z } from 'zod';
import type { Article } from '../shared/protocol.ts';
import { WikimediaClient, WikimediaUnavailable, wikimediaClient } from './wikimedia/client.ts';
const pageSchema = z.object({
pageid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), ns: z.literal(0),
title: z.string().trim().min(1).max(200), extract: z.string().max(4096).optional(),
lastrevid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
thumbnail: z.object({ source: z.string().url().max(1500) }).optional(),
categories: z.array(z.object({ title: z.string().min(1).max(300) })).max(50).optional(),
});
const responseSchema = z.object({ query: z.object({ pages: z.array(z.unknown()).max(10) }) });
const MAX_RESPONSE_BYTES = 512 * 1024;
const MAX_ATTEMPTS = 3;
const imageOrigins = new Set(['https://upload.wikimedia.org', 'https://thumb.wikimedia.org']);
function category(title: string, categories: string[]): string {
const text = `${title} ${categories.join(' ')}`.toLowerCase();
if (/astronom|planet|space|galax|moon|star\b/.test(text)) return 'Space';
if (/animal|plant|species|ocean|biolog|mollusc/.test(text)) return 'Nature';
if (/births|deaths|people|scientist/.test(text)) return 'People';
if (/physic|chem|math|science|technolog/.test(text)) return 'Science';
if (/histor|ancient|archaeolog|empire/.test(text)) return 'History';
return 'Culture';
}
async function readJson(response: Response): Promise<unknown> {
const length = response.headers.get('content-length');
if (!response.ok || response.redirected ||
!/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') ||
(length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_RESPONSE_BYTES)) || !response.body) {
await response.body?.cancel();
throw new Error('Invalid Wikipedia API response');
}
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let size = 0;
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
size += value.byteLength;
if (size > MAX_RESPONSE_BYTES) throw new Error('Wikipedia response too large');
chunks.push(value);
}
return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)));
} finally {
await reader.cancel();
reader.releaseLock();
}
}
function pictureUrl(value: string | undefined): string | undefined {
if (!value) return undefined;
try {
const url = new URL(value);
return url.protocol === 'https:' && imageOrigins.has(url.origin) && !url.username && !url.password && !url.hash
? url.toString() : undefined;
} catch { return undefined; }
}
/** The exact W1 request. Its parameters decide minted article content: keep them byte-identical. */
export function randomArticlesUrl(): URL {
const url = new URL('https://en.wikipedia.org/w/api.php');
url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
generator: 'random', grnnamespace: '0', grnlimit: '10', prop: 'extracts|pageimages|info|categories',
exintro: '1', explaintext: '1', exchars: '900', piprop: 'thumbnail', pithumbsize: '800', cllimit: '20' }).toString();
return url;
}
export class WikipediaArticleSource {
private readonly client: WikimediaClient;
private active = 0;
/** A bare fetch (tests) gets an unthrottled pass-through client; production uses the shared client. */
constructor(source: typeof globalThis.fetch | WikimediaClient = wikimediaClient) {
this.client = source instanceof WikimediaClient ? source : WikimediaClient.unthrottled(source);
}
async random(): Promise<Article> {
if (this.active >= 4) throw new Error('Wikipedia article service is busy');
this.active++;
try {
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
try {
// Mint lane, never cached: random articles become immutable chain data.
const { response } = await this.client.request(randomArticlesUrl(), { lane: 'mint', timeoutMs: 3500 });
const data = responseSchema.parse(await readJson(response));
for (const rawPage of data.query.pages) {
const parsedPage = pageSchema.safeParse(rawPage);
if (!parsedPage.success) continue;
const page = parsedPage.data;
const image = pictureUrl(page.thumbnail?.source);
const extract = page.extract?.trim();
if (!image || !extract || extract.length < 40) continue;
return { pageId: page.pageid, title: page.title, extract: extract.slice(0, 900), image,
url: `https://en.wikipedia.org/?curid=${page.pageid}`, revisionId: page.lastrevid,
source: 'live', category: category(page.title, page.categories?.map(c => c.title) ?? []) };
}
} catch (error) {
// Throttled, paused or failing upstream: stop now rather than hammer it with retries.
if (error instanceof WikimediaUnavailable) throw error;
/* Otherwise try another random API response, but keep the number of attempts bounded. */
}
}
throw new Error('Wikipedia did not return an article with a picture after three attempts');
} finally {
this.active--;
}
}
}
export const wikipediaArticles = new WikipediaArticleSource();