100 lines
4.8 KiB
TypeScript
100 lines
4.8 KiB
TypeScript
import { z } from 'zod';
|
|
import type { Article } from '../shared/protocol.ts';
|
|
|
|
const pageSchema = z.object({
|
|
pageid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), ns: z.literal(0),
|
|
title: z.string().trim().min(1).max(200), extract: z.string().max(4096).optional(),
|
|
lastrevid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
|
|
thumbnail: z.object({ source: z.string().url().max(1500) }).optional(),
|
|
categories: z.array(z.object({ title: z.string().min(1).max(300) })).max(50).optional(),
|
|
});
|
|
const responseSchema = z.object({ query: z.object({ pages: z.array(z.unknown()).max(10) }) });
|
|
const MAX_RESPONSE_BYTES = 512 * 1024;
|
|
const MAX_ATTEMPTS = 3;
|
|
const imageOrigins = new Set(['https://upload.wikimedia.org', 'https://thumb.wikimedia.org']);
|
|
|
|
function category(title: string, categories: string[]): string {
|
|
const text = `${title} ${categories.join(' ')}`.toLowerCase();
|
|
if (/astronom|planet|space|galax|moon|star\b/.test(text)) return 'Space';
|
|
if (/animal|plant|species|ocean|biolog|mollusc/.test(text)) return 'Nature';
|
|
if (/births|deaths|people|scientist/.test(text)) return 'People';
|
|
if (/physic|chem|math|science|technolog/.test(text)) return 'Science';
|
|
if (/histor|ancient|archaeolog|empire/.test(text)) return 'History';
|
|
return 'Culture';
|
|
}
|
|
async function readJson(response: Response): Promise<unknown> {
|
|
const length = response.headers.get('content-length');
|
|
if (!response.ok || response.redirected ||
|
|
!/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') ||
|
|
(length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_RESPONSE_BYTES)) || !response.body) {
|
|
await response.body?.cancel();
|
|
throw new Error('Invalid Wikipedia API response');
|
|
}
|
|
const reader = response.body.getReader();
|
|
const chunks: Uint8Array[] = [];
|
|
let size = 0;
|
|
try {
|
|
for (;;) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
size += value.byteLength;
|
|
if (size > MAX_RESPONSE_BYTES) throw new Error('Wikipedia response too large');
|
|
chunks.push(value);
|
|
}
|
|
return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)));
|
|
} finally {
|
|
await reader.cancel();
|
|
reader.releaseLock();
|
|
}
|
|
}
|
|
|
|
function pictureUrl(value: string | undefined): string | undefined {
|
|
if (!value) return undefined;
|
|
try {
|
|
const url = new URL(value);
|
|
return url.protocol === 'https:' && imageOrigins.has(url.origin) && !url.username && !url.password && !url.hash
|
|
? url.toString() : undefined;
|
|
} catch { return undefined; }
|
|
}
|
|
|
|
export class WikipediaArticleSource {
|
|
private readonly fetcher: typeof globalThis.fetch;
|
|
private active = 0;
|
|
|
|
constructor(fetcher: typeof globalThis.fetch = globalThis.fetch) { this.fetcher = fetcher; }
|
|
|
|
async random(): Promise<Article> {
|
|
if (this.active >= 4) throw new Error('Wikipedia article service is busy');
|
|
this.active++;
|
|
try {
|
|
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
try {
|
|
const url = new URL('https://en.wikipedia.org/w/api.php');
|
|
url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
|
|
generator: 'random', grnnamespace: '0', grnlimit: '10', prop: 'extracts|pageimages|info|categories',
|
|
exintro: '1', explaintext: '1', exchars: '900', piprop: 'thumbnail', pithumbsize: '800', cllimit: '20' }).toString();
|
|
const response = await this.fetcher(url, { signal: AbortSignal.timeout(3500), redirect: 'error',
|
|
headers: { 'User-Agent': 'Wikideck/0.1 (experimental Wikipedia article cards)', Accept: 'application/json' } });
|
|
const data = responseSchema.parse(await readJson(response));
|
|
for (const rawPage of data.query.pages) {
|
|
const parsedPage = pageSchema.safeParse(rawPage);
|
|
if (!parsedPage.success) continue;
|
|
const page = parsedPage.data;
|
|
const image = pictureUrl(page.thumbnail?.source);
|
|
const extract = page.extract?.trim();
|
|
if (!image || !extract || extract.length < 40) continue;
|
|
return { pageId: page.pageid, title: page.title, extract: extract.slice(0, 900), image,
|
|
url: `https://en.wikipedia.org/?curid=${page.pageid}`, revisionId: page.lastrevid,
|
|
source: 'live', category: category(page.title, page.categories?.map(c => c.title) ?? []) };
|
|
}
|
|
} catch { /* Try another random API response, but keep the number of attempts bounded. */ }
|
|
}
|
|
throw new Error('Wikipedia did not return an article with a picture after three attempts');
|
|
} finally {
|
|
this.active--;
|
|
}
|
|
}
|
|
}
|
|
|
|
export const wikipediaArticles = new WikipediaArticleSource();
|