import { z } from 'zod'; import type { Article } from '../shared/protocol.ts'; const pageSchema = z.object({ pageid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), ns: z.literal(0), title: z.string().trim().min(1).max(200), extract: z.string().max(4096).optional(), lastrevid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), thumbnail: z.object({ source: z.string().url().max(1500) }).optional(), categories: z.array(z.object({ title: z.string().min(1).max(300) })).max(50).optional(), }); const responseSchema = z.object({ query: z.object({ pages: z.array(z.unknown()).max(10) }) }); const MAX_RESPONSE_BYTES = 512 * 1024; const MAX_ATTEMPTS = 3; const imageOrigins = new Set(['https://upload.wikimedia.org', 'https://thumb.wikimedia.org']); function category(title: string, categories: string[]): string { const text = `${title} ${categories.join(' ')}`.toLowerCase(); if (/astronom|planet|space|galax|moon|star\b/.test(text)) return 'Space'; if (/animal|plant|species|ocean|biolog|mollusc/.test(text)) return 'Nature'; if (/births|deaths|people|scientist/.test(text)) return 'People'; if (/physic|chem|math|science|technolog/.test(text)) return 'Science'; if (/histor|ancient|archaeolog|empire/.test(text)) return 'History'; return 'Culture'; } async function readJson(response: Response): Promise { const length = response.headers.get('content-length'); if (!response.ok || response.redirected || !/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') || (length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_RESPONSE_BYTES)) || !response.body) { await response.body?.cancel(); throw new Error('Invalid Wikipedia API response'); } const reader = response.body.getReader(); const chunks: Uint8Array[] = []; let size = 0; try { for (;;) { const { done, value } = await reader.read(); if (done) break; size += value.byteLength; if (size > MAX_RESPONSE_BYTES) throw new Error('Wikipedia response too large'); chunks.push(value); } return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks))); } finally { await reader.cancel(); reader.releaseLock(); } } function pictureUrl(value: string | undefined): string | undefined { if (!value) return undefined; try { const url = new URL(value); return url.protocol === 'https:' && imageOrigins.has(url.origin) && !url.username && !url.password && !url.hash ? url.toString() : undefined; } catch { return undefined; } } export class WikipediaArticleSource { private readonly fetcher: typeof globalThis.fetch; private active = 0; constructor(fetcher: typeof globalThis.fetch = globalThis.fetch) { this.fetcher = fetcher; } async random(): Promise
{ if (this.active >= 4) throw new Error('Wikipedia article service is busy'); this.active++; try { for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { try { const url = new URL('https://en.wikipedia.org/w/api.php'); url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2', generator: 'random', grnnamespace: '0', grnlimit: '10', prop: 'extracts|pageimages|info|categories', exintro: '1', explaintext: '1', exchars: '900', piprop: 'thumbnail', pithumbsize: '800', cllimit: '20' }).toString(); const response = await this.fetcher(url, { signal: AbortSignal.timeout(3500), redirect: 'error', headers: { 'User-Agent': 'Wikideck/0.1 (experimental Wikipedia article cards)', Accept: 'application/json' } }); const data = responseSchema.parse(await readJson(response)); for (const rawPage of data.query.pages) { const parsedPage = pageSchema.safeParse(rawPage); if (!parsedPage.success) continue; const page = parsedPage.data; const image = pictureUrl(page.thumbnail?.source); const extract = page.extract?.trim(); if (!image || !extract || extract.length < 40) continue; return { pageId: page.pageid, title: page.title, extract: extract.slice(0, 900), image, url: `https://en.wikipedia.org/?curid=${page.pageid}`, revisionId: page.lastrevid, source: 'live', category: category(page.title, page.categories?.map(c => c.title) ?? []) }; } } catch { /* Try another random API response, but keep the number of attempts bounded. */ } } throw new Error('Wikipedia did not return an article with a picture after three attempts'); } finally { this.active--; } } } export const wikipediaArticles = new WikipediaArticleSource();