import { z } from 'zod'; import { deriveStats, type PageEnrichment } from '../shared/enrichment.ts'; const MAX_BYTES = 1024 * 1024; const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER); const text = z.string().max(4096); const pageSchema = z.object({ pageid: integer.positive(), ns: z.literal(0), title: text.min(1), missing: z.never().optional(), invalid: z.never().optional(), lastrevid: integer.positive(), length: integer, description: text.optional(), pageprops: z.object({ wikibase_item: z.string().regex(/^Q[1-9]\d{0,19}$/).optional() }).optional(), categories: z.array(z.object({ ns: z.literal(14), title: text.min(1) })).max(50).optional(), langlinks: z.array(z.object({ lang: z.string().min(1).max(64) })).max(500).optional(), }); const metadataSchema = z.object({ query: z.object({ pages: z.array(pageSchema).length(1) }), continue: z.object({ clcontinue: text.optional(), llcontinue: text.optional() }).optional(), }); const viewsSchema = z.object({ items: z.array(z.object({ timestamp: z.string().regex(/^\d{6}0100$/), views: integer, project: z.literal('en.wikipedia'), access: z.literal('all-access'), agent: z.literal('user'), granularity: z.literal('monthly'), })).max(6) }); type CacheEntry = { expires: number; value: PageEnrichment } | { expires: number; error: Error }; type Options = { fetch?: typeof globalThis.fetch; now?: () => Date; ttlMs?: number; maxEntries?: number }; function monthWindow(now: Date): string[] { return Array.from({ length: 6 }, (_, index) => new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 6 + index, 1)).toISOString().slice(0, 7)); } /** Fixed-source read-only enrichment; no consensus, persistence, URL or header inputs. */ export class WikipediaEnrichment { private readonly fetcher: typeof globalThis.fetch; private readonly now: () => Date; private readonly ttlMs: number; private readonly maxEntries: number; private readonly cache = new Map(); private readonly pending = new Map>(); private active = 0; constructor(options: Options = {}) { this.fetcher = options.fetch ?? globalThis.fetch; this.now = options.now ?? (() => new Date()); this.ttlMs = options.ttlMs ?? 6 * 60 * 60 * 1000; this.maxEntries = options.maxEntries ?? 128; if (!Number.isSafeInteger(this.ttlMs) || this.ttlMs <= 0 || !Number.isSafeInteger(this.maxEntries) || this.maxEntries <= 0 || this.maxEntries > 128) { throw new RangeError('Invalid enrichment cache options'); } } async get(pageId: number): Promise { if (!Number.isSafeInteger(pageId) || pageId <= 0) throw new RangeError('Invalid Wikipedia page ID'); const now = this.now(); const cached = this.cache.get(pageId); if (cached && cached.expires > now.getTime() && (!('value' in cached) || cached.value.pageviews.latestMonth === monthWindow(now)[5])) { this.cache.delete(pageId); this.cache.set(pageId, cached); if ('error' in cached) throw cached.error; return structuredClone(cached.value); } this.cache.delete(pageId); const existing = this.pending.get(pageId); if (existing) return structuredClone(await existing); // Reject immediately: no unbounded waiting queue. Four active jobs also keep // the pending-map ceiling of eight conservative (one upstream request/job). if (this.pending.size >= 8 || this.active >= 4) throw new Error('Wikipedia enrichment busy'); this.active++; const job = this.load(pageId, now).then(value => { const ttl = value.pageviews.status === 'available' ? this.ttlMs : Math.min(this.ttlMs, 5 * 60 * 1000); this.remember(pageId, { value, expires: this.now().getTime() + ttl }); return value; }, () => { const error = new Error('Wikipedia page enrichment unavailable'); this.remember(pageId, { error, expires: this.now().getTime() + 30_000 }); throw error; }).finally(() => { this.active--; this.pending.delete(pageId); }); this.pending.set(pageId, job); return structuredClone(await job); } private remember(pageId: number, entry: CacheEntry): void { this.cache.delete(pageId); this.cache.set(pageId, entry); while (this.cache.size > this.maxEntries) this.cache.delete(this.cache.keys().next().value!); } private async json(url: URL): Promise { const signal = AbortSignal.timeout(6000); const response = await this.fetcher(url, { signal, redirect: 'error', headers: { Accept: 'application/json', 'User-Agent': 'Wikideck/0.1 (Wikipedia off-chain page enrichment)' } }); const length = response.headers.get('content-length'); if (!response.ok || response.redirected || !/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') || (length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_BYTES)) || !response.body) { await response.body?.cancel(); throw new Error('Invalid upstream response'); } const reader = response.body.getReader(); const chunks: Uint8Array[] = []; let size = 0; try { for (;;) { signal.throwIfAborted(); const { done, value } = await reader.read(); signal.throwIfAborted(); if (done) break; size += value.byteLength; if (size > MAX_BYTES) throw new Error('Upstream response too large'); chunks.push(value); } return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks))); } finally { await reader.cancel(); reader.releaseLock(); } } private async load(pageId: number, now: Date): Promise { const url = new URL('https://en.wikipedia.org/w/api.php'); url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2', pageids: String(pageId), prop: 'info|categories|pageprops|description|langlinks', clshow: '!hidden', cllimit: '50', lllimit: '500', ppprop: 'wikibase_item' }).toString(); const metadata = metadataSchema.parse(await this.json(url)); const page = metadata.query.pages[0]; if (page.pageid !== pageId) throw new Error('Wikipedia page ID mismatch'); const months = monthWindow(now); const latestMonth = months[5]; let pageviews: PageEnrichment['pageviews'] = { status: 'unavailable', months: [], latestMonth, total: null, dailyAverage: null, trendPercent: null }; try { const title = encodeURIComponent(page.title.replace(/ /g, '_')); const stamp = (month: string) => `${month.replace('-', '')}0100`; const endpoint = new URL(`https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/${title}/monthly/${stamp(months[0])}/${stamp(latestMonth)}`); const result = viewsSchema.parse(await this.json(endpoint)); const byMonth = new Map(); for (const item of result.items) { const month = `${item.timestamp.slice(0, 4)}-${item.timestamp.slice(4, 6)}`; if (!months.includes(month) || byMonth.has(month)) throw new Error('Invalid pageview month'); byMonth.set(month, item.views); } if (byMonth.size !== 6) throw new Error('Incomplete pageviews'); const ordered = months.map(month => ({ month, views: byMonth.get(month)! })); const total = ordered.reduce((sum, item) => sum + item.views, 0); if (!Number.isSafeInteger(total)) throw new Error('Pageview total overflow'); const latest = ordered[5].views, prior = ordered[4].views; const days = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 0)).getUTCDate(); pageviews = { status: 'available', months: ordered, latestMonth, total, dailyAverage: latest / days, trendPercent: prior > 0 ? (latest - prior) / prior * 100 : null }; } catch { /* Metadata remains useful; unknown views must not become fake zeroes. */ } return { pageId, title: page.title, description: page.description ?? null, wikidataId: page.pageprops?.wikibase_item ?? null, revisionId: page.lastrevid, pageBytes: page.length, languageCount: page.langlinks?.length ?? 0, languagesTruncated: metadata.continue?.llcontinue !== undefined, categories: (page.categories ?? []).map(item => item.title.replace(/^Category:/, '')), categoriesTruncated: metadata.continue?.clcontinue !== undefined, fetchedAt: now.toISOString(), pageviews, stats: deriveStats(page.length, page.langlinks?.length ?? 0, pageviews.status === 'available' ? pageviews.months[5].views : null) }; } } export const wikipediaEnrichment = new WikipediaEnrichment();