175 lines
9.0 KiB
TypeScript
175 lines
9.0 KiB
TypeScript
import { z } from 'zod';
|
|
import { deriveStats, type PageEnrichment } from '../shared/enrichment.ts';
|
|
|
|
const MAX_BYTES = 1024 * 1024;
|
|
const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
|
|
const text = z.string().max(4096);
|
|
const pageSchema = z.object({
|
|
pageid: integer.positive(), ns: z.literal(0), title: text.min(1),
|
|
missing: z.never().optional(), invalid: z.never().optional(),
|
|
lastrevid: integer.positive(), length: integer,
|
|
description: text.optional(),
|
|
pageprops: z.object({ wikibase_item: z.string().regex(/^Q[1-9]\d{0,19}$/).optional() }).optional(),
|
|
categories: z.array(z.object({ ns: z.literal(14), title: text.min(1) })).max(50).optional(),
|
|
langlinks: z.array(z.object({ lang: z.string().min(1).max(64) })).max(500).optional(),
|
|
});
|
|
const metadataSchema = z.object({
|
|
query: z.object({ pages: z.array(pageSchema).length(1) }),
|
|
continue: z.object({ clcontinue: text.optional(), llcontinue: text.optional() }).optional(),
|
|
});
|
|
const viewsSchema = z.object({ items: z.array(z.object({
|
|
timestamp: z.string().regex(/^\d{6}0100$/), views: integer,
|
|
project: z.literal('en.wikipedia'), access: z.literal('all-access'),
|
|
agent: z.literal('user'), granularity: z.literal('monthly'),
|
|
})).max(6) });
|
|
|
|
type CacheEntry = { expires: number; value: PageEnrichment } | { expires: number; error: Error };
|
|
type Options = { fetch?: typeof globalThis.fetch; now?: () => Date; ttlMs?: number; maxEntries?: number };
|
|
|
|
function monthWindow(now: Date): string[] {
|
|
return Array.from({ length: 6 }, (_, index) =>
|
|
new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 6 + index, 1)).toISOString().slice(0, 7));
|
|
}
|
|
|
|
/** Fixed-source read-only enrichment; no consensus, persistence, URL or header inputs. */
|
|
export class WikipediaEnrichment {
|
|
private readonly fetcher: typeof globalThis.fetch;
|
|
private readonly now: () => Date;
|
|
private readonly ttlMs: number;
|
|
private readonly maxEntries: number;
|
|
private readonly cache = new Map<number, CacheEntry>();
|
|
private readonly pending = new Map<number, Promise<PageEnrichment>>();
|
|
private active = 0;
|
|
|
|
constructor(options: Options = {}) {
|
|
this.fetcher = options.fetch ?? globalThis.fetch;
|
|
this.now = options.now ?? (() => new Date());
|
|
this.ttlMs = options.ttlMs ?? 6 * 60 * 60 * 1000;
|
|
this.maxEntries = options.maxEntries ?? 128;
|
|
if (!Number.isSafeInteger(this.ttlMs) || this.ttlMs <= 0 ||
|
|
!Number.isSafeInteger(this.maxEntries) || this.maxEntries <= 0 || this.maxEntries > 128) {
|
|
throw new RangeError('Invalid enrichment cache options');
|
|
}
|
|
}
|
|
|
|
async get(pageId: number): Promise<PageEnrichment> {
|
|
return this.loadCached(pageId, false);
|
|
}
|
|
|
|
/** Bypass cached values when the result is being pinned into a newly minted block. */
|
|
async getFresh(pageId: number): Promise<PageEnrichment> {
|
|
return this.loadCached(pageId, true);
|
|
}
|
|
|
|
private async loadCached(pageId: number, fresh: boolean): Promise<PageEnrichment> {
|
|
if (!Number.isSafeInteger(pageId) || pageId <= 0) throw new RangeError('Invalid Wikipedia page ID');
|
|
const now = this.now();
|
|
const cached = this.cache.get(pageId);
|
|
if (!fresh && cached && cached.expires > now.getTime() &&
|
|
(!('value' in cached) || cached.value.pageviews.latestMonth === monthWindow(now)[5])) {
|
|
this.cache.delete(pageId);
|
|
this.cache.set(pageId, cached);
|
|
if ('error' in cached) throw cached.error;
|
|
return structuredClone(cached.value);
|
|
}
|
|
this.cache.delete(pageId);
|
|
const existing = this.pending.get(pageId);
|
|
if (existing) return structuredClone(await existing);
|
|
// Reject immediately: no unbounded waiting queue. Four active jobs also keep
|
|
// the pending-map ceiling of eight conservative (one upstream request/job).
|
|
if (this.pending.size >= 8 || this.active >= 4) throw new Error('Wikipedia enrichment busy');
|
|
this.active++;
|
|
const job = this.load(pageId, now).then(value => {
|
|
const ttl = value.pageviews.status === 'available' ? this.ttlMs : Math.min(this.ttlMs, 5 * 60 * 1000);
|
|
this.remember(pageId, { value, expires: this.now().getTime() + ttl });
|
|
return value;
|
|
}, () => {
|
|
const error = new Error('Wikipedia page enrichment unavailable');
|
|
this.remember(pageId, { error, expires: this.now().getTime() + 30_000 });
|
|
throw error;
|
|
}).finally(() => { this.active--; this.pending.delete(pageId); });
|
|
this.pending.set(pageId, job);
|
|
return structuredClone(await job);
|
|
}
|
|
|
|
private remember(pageId: number, entry: CacheEntry): void {
|
|
this.cache.delete(pageId);
|
|
this.cache.set(pageId, entry);
|
|
while (this.cache.size > this.maxEntries) this.cache.delete(this.cache.keys().next().value!);
|
|
}
|
|
|
|
private async json(url: URL): Promise<unknown> {
|
|
const signal = AbortSignal.timeout(6000);
|
|
const response = await this.fetcher(url, { signal, redirect: 'error',
|
|
headers: { Accept: 'application/json', 'User-Agent': 'Wikideck/0.1 (Wikipedia off-chain page enrichment)' } });
|
|
const length = response.headers.get('content-length');
|
|
if (!response.ok || response.redirected ||
|
|
!/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') ||
|
|
(length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_BYTES)) || !response.body) {
|
|
await response.body?.cancel();
|
|
throw new Error('Invalid upstream response');
|
|
}
|
|
const reader = response.body.getReader();
|
|
const chunks: Uint8Array[] = [];
|
|
let size = 0;
|
|
try {
|
|
for (;;) {
|
|
signal.throwIfAborted();
|
|
const { done, value } = await reader.read();
|
|
signal.throwIfAborted();
|
|
if (done) break;
|
|
size += value.byteLength;
|
|
if (size > MAX_BYTES) throw new Error('Upstream response too large');
|
|
chunks.push(value);
|
|
}
|
|
return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)));
|
|
} finally {
|
|
await reader.cancel();
|
|
reader.releaseLock();
|
|
}
|
|
}
|
|
|
|
private async load(pageId: number, now: Date): Promise<PageEnrichment> {
|
|
const url = new URL('https://en.wikipedia.org/w/api.php');
|
|
url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
|
|
pageids: String(pageId), prop: 'info|categories|pageprops|description|langlinks',
|
|
clshow: '!hidden', cllimit: '50', lllimit: '500', ppprop: 'wikibase_item' }).toString();
|
|
const metadata = metadataSchema.parse(await this.json(url));
|
|
const page = metadata.query.pages[0];
|
|
if (page.pageid !== pageId) throw new Error('Wikipedia page ID mismatch');
|
|
const months = monthWindow(now);
|
|
const latestMonth = months[5];
|
|
let pageviews: PageEnrichment['pageviews'] = { status: 'unavailable', months: [], latestMonth,
|
|
total: null, dailyAverage: null, trendPercent: null };
|
|
try {
|
|
const title = encodeURIComponent(page.title.replace(/ /g, '_'));
|
|
const stamp = (month: string) => `${month.replace('-', '')}0100`;
|
|
const endpoint = new URL(`https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/${title}/monthly/${stamp(months[0])}/${stamp(latestMonth)}`);
|
|
const result = viewsSchema.parse(await this.json(endpoint));
|
|
const byMonth = new Map<string, number>();
|
|
for (const item of result.items) {
|
|
const month = `${item.timestamp.slice(0, 4)}-${item.timestamp.slice(4, 6)}`;
|
|
if (!months.includes(month) || byMonth.has(month)) throw new Error('Invalid pageview month');
|
|
byMonth.set(month, item.views);
|
|
}
|
|
if (byMonth.size !== 6) throw new Error('Incomplete pageviews');
|
|
const ordered = months.map(month => ({ month, views: byMonth.get(month)! }));
|
|
const total = ordered.reduce((sum, item) => sum + item.views, 0);
|
|
if (!Number.isSafeInteger(total)) throw new Error('Pageview total overflow');
|
|
const latest = ordered[5].views, prior = ordered[4].views;
|
|
const days = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 0)).getUTCDate();
|
|
pageviews = { status: 'available', months: ordered, latestMonth, total,
|
|
dailyAverage: latest / days, trendPercent: prior > 0 ? (latest - prior) / prior * 100 : null };
|
|
} catch { /* Metadata remains useful; unknown views must not become fake zeroes. */ }
|
|
return { pageId, title: page.title, description: page.description ?? null,
|
|
wikidataId: page.pageprops?.wikibase_item ?? null, revisionId: page.lastrevid, pageBytes: page.length,
|
|
languageCount: page.langlinks?.length ?? 0, languagesTruncated: metadata.continue?.llcontinue !== undefined,
|
|
categories: (page.categories ?? []).map(item => item.title.replace(/^Category:/, '')),
|
|
categoriesTruncated: metadata.continue?.clcontinue !== undefined, fetchedAt: now.toISOString(), pageviews,
|
|
stats: deriveStats(page.length, page.langlinks?.length ?? 0,
|
|
pageviews.status === 'available' ? pageviews.months[5].views : null) };
|
|
}
|
|
}
|
|
|
|
export const wikipediaEnrichment = new WikipediaEnrichment();
|