- server/wikimedia/client.ts: one process-wide client. Per-host token bucket (1 req/s, burst 3), 1 in flight per host, mint > interactive > background lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP). - server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match, stale-while-revalidate on the background lane. Mint reads never use it. - Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh() (template 503, miner retries) instead of minting attention: null forever. - server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for /api/mining/template, about 3 requests per block regardless of miner count. - User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or PUBLIC_ORIGIN) and a start-up warning when it is unset. - Mint request parameters are byte-identical apart from maxlag (golden test). No consensus, block or protocol changes. Tests use fake fetch only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
113 lines
5.3 KiB
TypeScript
113 lines
5.3 KiB
TypeScript
import { z } from 'zod';
|
|
import type { Article } from '../shared/protocol.ts';
|
|
import { WikimediaClient, WikimediaUnavailable, wikimediaClient } from './wikimedia/client.ts';
|
|
|
|
const pageSchema = z.object({
|
|
pageid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), ns: z.literal(0),
|
|
title: z.string().trim().min(1).max(200), extract: z.string().max(4096).optional(),
|
|
lastrevid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
|
|
thumbnail: z.object({ source: z.string().url().max(1500) }).optional(),
|
|
categories: z.array(z.object({ title: z.string().min(1).max(300) })).max(50).optional(),
|
|
});
|
|
const responseSchema = z.object({ query: z.object({ pages: z.array(z.unknown()).max(10) }) });
|
|
const MAX_RESPONSE_BYTES = 512 * 1024;
|
|
const MAX_ATTEMPTS = 3;
|
|
const imageOrigins = new Set(['https://upload.wikimedia.org', 'https://thumb.wikimedia.org']);
|
|
|
|
function category(title: string, categories: string[]): string {
|
|
const text = `${title} ${categories.join(' ')}`.toLowerCase();
|
|
if (/astronom|planet|space|galax|moon|star\b/.test(text)) return 'Space';
|
|
if (/animal|plant|species|ocean|biolog|mollusc/.test(text)) return 'Nature';
|
|
if (/births|deaths|people|scientist/.test(text)) return 'People';
|
|
if (/physic|chem|math|science|technolog/.test(text)) return 'Science';
|
|
if (/histor|ancient|archaeolog|empire/.test(text)) return 'History';
|
|
return 'Culture';
|
|
}
|
|
async function readJson(response: Response): Promise<unknown> {
|
|
const length = response.headers.get('content-length');
|
|
if (!response.ok || response.redirected ||
|
|
!/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') ||
|
|
(length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_RESPONSE_BYTES)) || !response.body) {
|
|
await response.body?.cancel();
|
|
throw new Error('Invalid Wikipedia API response');
|
|
}
|
|
const reader = response.body.getReader();
|
|
const chunks: Uint8Array[] = [];
|
|
let size = 0;
|
|
try {
|
|
for (;;) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
size += value.byteLength;
|
|
if (size > MAX_RESPONSE_BYTES) throw new Error('Wikipedia response too large');
|
|
chunks.push(value);
|
|
}
|
|
return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)));
|
|
} finally {
|
|
await reader.cancel();
|
|
reader.releaseLock();
|
|
}
|
|
}
|
|
|
|
function pictureUrl(value: string | undefined): string | undefined {
|
|
if (!value) return undefined;
|
|
try {
|
|
const url = new URL(value);
|
|
return url.protocol === 'https:' && imageOrigins.has(url.origin) && !url.username && !url.password && !url.hash
|
|
? url.toString() : undefined;
|
|
} catch { return undefined; }
|
|
}
|
|
|
|
/** The exact W1 request. Its parameters decide minted article content: keep them byte-identical. */
|
|
export function randomArticlesUrl(): URL {
|
|
const url = new URL('https://en.wikipedia.org/w/api.php');
|
|
url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
|
|
generator: 'random', grnnamespace: '0', grnlimit: '10', prop: 'extracts|pageimages|info|categories',
|
|
exintro: '1', explaintext: '1', exchars: '900', piprop: 'thumbnail', pithumbsize: '800', cllimit: '20' }).toString();
|
|
return url;
|
|
}
|
|
|
|
export class WikipediaArticleSource {
|
|
private readonly client: WikimediaClient;
|
|
private active = 0;
|
|
|
|
/** A bare fetch (tests) gets an unthrottled pass-through client; production uses the shared client. */
|
|
constructor(source: typeof globalThis.fetch | WikimediaClient = wikimediaClient) {
|
|
this.client = source instanceof WikimediaClient ? source : WikimediaClient.unthrottled(source);
|
|
}
|
|
|
|
async random(): Promise<Article> {
|
|
if (this.active >= 4) throw new Error('Wikipedia article service is busy');
|
|
this.active++;
|
|
try {
|
|
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
try {
|
|
// Mint lane, never cached: random articles become immutable chain data.
|
|
const { response } = await this.client.request(randomArticlesUrl(), { lane: 'mint', timeoutMs: 3500 });
|
|
const data = responseSchema.parse(await readJson(response));
|
|
for (const rawPage of data.query.pages) {
|
|
const parsedPage = pageSchema.safeParse(rawPage);
|
|
if (!parsedPage.success) continue;
|
|
const page = parsedPage.data;
|
|
const image = pictureUrl(page.thumbnail?.source);
|
|
const extract = page.extract?.trim();
|
|
if (!image || !extract || extract.length < 40) continue;
|
|
return { pageId: page.pageid, title: page.title, extract: extract.slice(0, 900), image,
|
|
url: `https://en.wikipedia.org/?curid=${page.pageid}`, revisionId: page.lastrevid,
|
|
source: 'live', category: category(page.title, page.categories?.map(c => c.title) ?? []) };
|
|
}
|
|
} catch (error) {
|
|
// Throttled, paused or failing upstream: stop now rather than hammer it with retries.
|
|
if (error instanceof WikimediaUnavailable) throw error;
|
|
/* Otherwise try another random API response, but keep the number of attempts bounded. */
|
|
}
|
|
}
|
|
throw new Error('Wikipedia did not return an article with a picture after three attempts');
|
|
} finally {
|
|
this.active--;
|
|
}
|
|
}
|
|
}
|
|
|
|
export const wikipediaArticles = new WikipediaArticleSource();
|