More cards

2026-10-19 18:52:49 -04:00
parent 716c9612cc
commit 0b9061372b
26 changed files with 549 additions and 201 deletions
@@ -1,13 +1,17 @@
 import { z } from 'zod';
 import type { Article } from '../shared/protocol.ts';
-import { STARTER_ARTICLES } from '../shared/catalog.ts';

 const pageSchema = z.object({
-  pageid: z.number().int().positive(), title: z.string(), extract: z.string().default(''),
-  lastrevid: z.number().int().nonnegative().default(0),
-  thumbnail: z.object({ source: z.string().url() }).optional(),
-  categories: z.array(z.object({ title: z.string() })).optional(),
+  pageid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), ns: z.literal(0),
+  title: z.string().trim().min(1).max(200), extract: z.string().max(4096).optional(),
+  lastrevid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
+  thumbnail: z.object({ source: z.string().url().max(1500) }).optional(),
+  categories: z.array(z.object({ title: z.string().min(1).max(300) })).max(50).optional(),
 });
+const responseSchema = z.object({ query: z.object({ pages: z.array(z.unknown()).max(10) }) });
+const MAX_RESPONSE_BYTES = 512 * 1024;
+const MAX_ATTEMPTS = 3;
+const imageOrigins = new Set(['https://upload.wikimedia.org', 'https://thumb.wikimedia.org']);

 function category(title: string, categories: string[]): string {
  const text = `${title} ${categories.join(' ')}`.toLowerCase();
@@ -18,27 +22,78 @@ function category(title: string, categories: string[]): string {
  if (/histor|ancient|archaeolog|empire/.test(text)) return 'History';
  return 'Culture';
 }
-export async function wikipediaArticle(): Promise<Article> {
-  if (process.env.WIKIDECK_OFFLINE === '1') return { ...STARTER_ARTICLES[Math.floor(Math.random() * STARTER_ARTICLES.length)] };
+async function readJson(response: Response): Promise<unknown> {
+  const length = response.headers.get('content-length');
+  if (!response.ok || response.redirected ||
+      !/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') ||
+      (length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_RESPONSE_BYTES)) || !response.body) {
+    await response.body?.cancel();
+    throw new Error('Invalid Wikipedia API response');
+  }
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let size = 0;
  try {
-    const url = new URL('https://en.wikipedia.org/w/api.php');
-    url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
-      generator: 'random', grnnamespace: '0', grnlimit: '6', prop: 'extracts|pageimages|info|categories',
-      exintro: '1', explaintext: '1', exchars: '700', piprop: 'thumbnail', pithumbsize: '900', cllimit: '8' }).toString();
-    const response = await fetch(url, { signal: AbortSignal.timeout(6000), redirect: 'error',
-      headers: { 'User-Agent': 'Wikideck/0.1 (local educational card-game prototype)', Accept: 'application/json' } });
-    if (!response.ok) throw new Error('Wikipedia unavailable');
-    const raw = await response.text();
-    if (raw.length > 1_000_000) throw new Error('Wikipedia response too large');
-    const result = z.object({ query: z.object({ pages: z.array(pageSchema) }) }).parse(JSON.parse(raw));
-    const page = result.query.pages.find(p => p.extract.length > 120 && p.thumbnail) ?? result.query.pages.find(p => p.extract.length > 120);
-    if (!page) throw new Error('No suitable article');
-    const image = page.thumbnail?.source;
-    return { pageId: page.pageid, title: page.title, extract: page.extract.slice(0, 700),
-      image: image && ['https://upload.wikimedia.org', 'https://thumb.wikimedia.org'].includes(new URL(image).origin) ? image : undefined,
-      url: `https://en.wikipedia.org/?curid=${page.pageid}`, revisionId: page.lastrevid,
-      source: 'live', category: category(page.title, page.categories?.map(c => c.title) ?? []) };
-  } catch {
-    return { ...STARTER_ARTICLES[Math.floor(Math.random() * STARTER_ARTICLES.length)] };
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      size += value.byteLength;
+      if (size > MAX_RESPONSE_BYTES) throw new Error('Wikipedia response too large');
+      chunks.push(value);
+    }
+    return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)));
+  } finally {
+    await reader.cancel();
+    reader.releaseLock();
  }
 }
+
+function pictureUrl(value: string | undefined): string | undefined {
+  if (!value) return undefined;
+  try {
+    const url = new URL(value);
+    return url.protocol === 'https:' && imageOrigins.has(url.origin) && !url.username && !url.password && !url.hash
+      ? url.toString() : undefined;
+  } catch { return undefined; }
+}
+
+export class WikipediaArticleSource {
+  private readonly fetcher: typeof globalThis.fetch;
+  private active = 0;
+
+  constructor(fetcher: typeof globalThis.fetch = globalThis.fetch) { this.fetcher = fetcher; }
+
+  async random(): Promise<Article> {
+    if (this.active >= 4) throw new Error('Wikipedia article service is busy');
+    this.active++;
+    try {
+      for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+        try {
+          const url = new URL('https://en.wikipedia.org/w/api.php');
+          url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
+            generator: 'random', grnnamespace: '0', grnlimit: '10', prop: 'extracts|pageimages|info|categories',
+            exintro: '1', explaintext: '1', exchars: '900', piprop: 'thumbnail', pithumbsize: '800', cllimit: '20' }).toString();
+          const response = await this.fetcher(url, { signal: AbortSignal.timeout(3500), redirect: 'error',
+            headers: { User-Agent: 'Wikideck/0.1 (experimental Wikipedia article cards)', Accept: 'application/json' } });
+          const data = responseSchema.parse(await readJson(response));
+          for (const rawPage of data.query.pages) {
+            const parsedPage = pageSchema.safeParse(rawPage);
+            if (!parsedPage.success) continue;
+            const page = parsedPage.data;
+            const image = pictureUrl(page.thumbnail?.source);
+            const extract = page.extract?.trim();
+            if (!image || !extract || extract.length < 40) continue;
+            return { pageId: page.pageid, title: page.title, extract: extract.slice(0, 900), image,
+              url: `https://en.wikipedia.org/?curid=${page.pageid}`, revisionId: page.lastrevid,
+              source: 'live', category: category(page.title, page.categories?.map(c => c.title) ?? []) };
+          }
+        } catch { /* Try another random API response, but keep the number of attempts bounded. */ }
+      }
+      throw new Error('Wikipedia did not return an article with a picture after three attempts');
+    } finally {
+      this.active--;
+    }
+  }
+}
+
+export const wikipediaArticles = new WikipediaArticleSource();