Files
wikideck/server/enrichment.ts
claudeandClaude Opus 5.5 f544019bfe Polite Wikimedia client, mint-path fix, disk cache and mining prefetch pool (#62)
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket
  (1 req/s, burst 3), 1 in flight per host, mint > interactive > background
  lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After
  and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and
  a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP).
- server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match,
  stale-while-revalidate on the background lane. Mint reads never use it.
- Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh()
  (template 503, miner retries) instead of minting attention: null forever.
- server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for
  /api/mining/template, about 3 requests per block regardless of miner count.
- User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or
  PUBLIC_ORIGIN) and a start-up warning when it is unset.
- Mint request parameters are byte-identical apart from maxlag (golden test).
  No consensus, block or protocol changes. Tests use fake fetch only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 21:51:20 -05:00

221 lines
12 KiB
TypeScript

import { z } from 'zod';
import { WikimediaClient, WikimediaUnavailable, wikimediaClient, type CachePolicy, type Lane } from './wikimedia/client.ts';
import { deriveStats, type PageEnrichment } from '../shared/enrichment.ts';
const MAX_BYTES = 1024 * 1024;
const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
const text = z.string().max(4096);
const pageSchema = z.object({
pageid: integer.positive(), ns: z.literal(0), title: text.min(1),
missing: z.never().optional(), invalid: z.never().optional(),
lastrevid: integer.positive(), length: integer,
description: text.optional(),
pageprops: z.object({ wikibase_item: z.string().regex(/^Q[1-9]\d{0,19}$/).optional() }).optional(),
categories: z.array(z.object({ ns: z.literal(14), title: text.min(1) })).max(50).optional(),
langlinks: z.array(z.object({ lang: z.string().min(1).max(64) })).max(500).optional(),
});
const metadataSchema = z.object({
query: z.object({ pages: z.array(pageSchema).length(1) }),
continue: z.object({ clcontinue: text.optional(), llcontinue: text.optional() }).optional(),
});
const viewsSchema = z.object({ items: z.array(z.object({
timestamp: z.string().regex(/^\d{6}0100$/), views: integer,
project: z.literal('en.wikipedia'), access: z.literal('all-access'),
agent: z.literal('user'), granularity: z.literal('monthly'),
})).max(6) });
type CacheEntry = { expires: number; value: PageEnrichment } | { expires: number; error: Error };
type Options = { fetch?: typeof globalThis.fetch; client?: WikimediaClient; now?: () => Date; ttlMs?: number; maxEntries?: number };
type Mode = 'mint' | 'display' | 'background';
/** Disk-cache lifetimes (engineer review §4). Mint reads never use the cache. */
const METADATA_CACHE: CachePolicy = { ttlMs: 24 * 3600_000, staleWhileRevalidate: true, maxBytes: MAX_BYTES };
// The URL names its six completed months, which are immutable; a week bounds late corrections.
const PAGEVIEWS_CACHE: CachePolicy = { ttlMs: 7 * 24 * 3600_000, staleWhileRevalidate: true, maxBytes: MAX_BYTES };
const NOT_FOUND = Symbol('not found');
/** The upstream did not answer properly (throttled, 5xx, timeout, network, non-JSON). Not "no data". */
export class UpstreamFailure extends Error {
constructor(message: string) { super(message); this.name = 'UpstreamFailure'; }
}
function transportFailure(error: unknown): boolean {
return error instanceof WikimediaUnavailable || error instanceof UpstreamFailure ||
(error instanceof DOMException && (error.name === 'TimeoutError' || error.name === 'AbortError'));
}
function monthWindow(now: Date): string[] {
return Array.from({ length: 6 }, (_, index) =>
new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 6 + index, 1)).toISOString().slice(0, 7));
}
/** Fixed-source read-only enrichment; no consensus, persistence, URL or header inputs. */
export class WikipediaEnrichment {
private readonly client: WikimediaClient;
private readonly now: () => Date;
private readonly ttlMs: number;
private readonly maxEntries: number;
private readonly cache = new Map<number, CacheEntry>();
private readonly pending = new Map<string, Promise<PageEnrichment>>();
private active = 0;
private mintActive = 0;
constructor(options: Options = {}) {
this.client = options.client ?? (options.fetch ? WikimediaClient.unthrottled(options.fetch) : wikimediaClient);
this.now = options.now ?? (() => new Date());
this.ttlMs = options.ttlMs ?? 6 * 60 * 60 * 1000;
this.maxEntries = options.maxEntries ?? 128;
if (!Number.isSafeInteger(this.ttlMs) || this.ttlMs <= 0 ||
!Number.isSafeInteger(this.maxEntries) || this.maxEntries <= 0 || this.maxEntries > 128) {
throw new RangeError('Invalid enrichment cache options');
}
}
/** Card faces (`/api/wikipedia/:id`): memory, then the disk cache, then the interactive lane. */
async get(pageId: number): Promise<PageEnrichment> {
return this.loadCached(pageId, 'display');
}
/** Like get(), on the background lane: for trends and other work nobody is waiting on. */
async getBackground(pageId: number): Promise<PageEnrichment> {
return this.loadCached(pageId, 'background');
}
/**
* Mint data: bypasses every cache and uses the mint lane. If pageviews fail in transport
* (429, 5xx, timeout, throttled), this rejects so the template fails and the miner retries;
* `unavailable` is recorded only for a genuine 404 or missing data.
*/
async getFresh(pageId: number): Promise<PageEnrichment> {
return this.loadCached(pageId, 'mint');
}
private async loadCached(pageId: number, mode: Mode): Promise<PageEnrichment> {
const fresh = mode === 'mint';
if (!Number.isSafeInteger(pageId) || pageId <= 0) throw new RangeError('Invalid Wikipedia page ID');
const now = this.now();
const cached = this.cache.get(pageId);
if (!fresh && cached && cached.expires > now.getTime() &&
(!('value' in cached) || cached.value.pageviews.latestMonth === monthWindow(now)[5])) {
this.cache.delete(pageId);
this.cache.set(pageId, cached);
if ('error' in cached) throw cached.error;
return structuredClone(cached.value);
}
if (!fresh) this.cache.delete(pageId);
// Mint jobs never join a display job: that one may be served from the disk cache.
const pendingKey = `${fresh ? 'mint' : 'cached'}:${pageId}`;
const existing = this.pending.get(pendingKey);
if (existing) return structuredClone(await existing);
// Reject immediately: no unbounded waiting queue. Four active jobs also keep
// the pending-map ceiling of eight conservative (one upstream request/job).
// Mint jobs have their own small allowance so card faces can never starve a template.
if (fresh ? this.mintActive >= 2 : this.pending.size - this.mintActive >= 8 || this.active >= 4) {
throw new Error('Wikipedia enrichment busy');
}
if (fresh) this.mintActive++; else this.active++;
const job = this.load(pageId, now, mode).then(value => {
const ttl = value.pageviews.status === 'available' ? this.ttlMs : Math.min(this.ttlMs, 5 * 60 * 1000);
this.remember(pageId, { value, expires: this.now().getTime() + ttl });
return value;
}, () => {
const error = new Error('Wikipedia page enrichment unavailable');
// A failed mint read says nothing new about display data; keep any cached value.
if (!fresh) this.remember(pageId, { error, expires: this.now().getTime() + 30_000 });
throw error;
}).finally(() => { if (fresh) this.mintActive--; else this.active--; this.pending.delete(pendingKey); });
this.pending.set(pendingKey, job);
return structuredClone(await job);
}
private remember(pageId: number, entry: CacheEntry): void {
this.cache.delete(pageId);
this.cache.set(pageId, entry);
while (this.cache.size > this.maxEntries) this.cache.delete(this.cache.keys().next().value!);
}
private async json(url: URL, lane: Lane, cache: CachePolicy | undefined, allowNotFound = false): Promise<unknown> {
const { response, signal } = await this.client.request(url, { lane, timeoutMs: 6000, cache });
if (allowNotFound && response.status === 404) { await response.body?.cancel(); return NOT_FOUND; }
if (!response.ok || response.redirected) {
await response.body?.cancel();
throw new UpstreamFailure(`Upstream HTTP ${response.status}`);
}
const length = response.headers.get('content-length');
if (
!/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') ||
(length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_BYTES)) || !response.body) {
await response.body?.cancel();
throw new UpstreamFailure('Invalid upstream response');
}
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let size = 0;
try {
for (;;) {
signal.throwIfAborted();
const { done, value } = await reader.read();
signal.throwIfAborted();
if (done) break;
size += value.byteLength;
if (size > MAX_BYTES) throw new Error('Upstream response too large');
chunks.push(value);
}
return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)));
} finally {
await reader.cancel();
reader.releaseLock();
}
}
private async load(pageId: number, now: Date, mode: Mode): Promise<PageEnrichment> {
const lane: Lane = mode === 'mint' ? 'mint' : mode === 'display' ? 'interactive' : 'background';
const fresh = mode === 'mint';
const url = new URL('https://en.wikipedia.org/w/api.php');
url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
pageids: String(pageId), prop: 'info|categories|pageprops|description|langlinks',
clshow: '!hidden', cllimit: '50', lllimit: '500', ppprop: 'wikibase_item' }).toString();
const metadata = metadataSchema.parse(await this.json(url, lane, fresh ? undefined : METADATA_CACHE));
const page = metadata.query.pages[0];
if (page.pageid !== pageId) throw new Error('Wikipedia page ID mismatch');
const months = monthWindow(now);
const latestMonth = months[5];
let pageviews: PageEnrichment['pageviews'] = { status: 'unavailable', months: [], latestMonth,
total: null, dailyAverage: null, trendPercent: null };
try {
const title = encodeURIComponent(page.title.replace(/ /g, '_'));
const stamp = (month: string) => `${month.replace('-', '')}0100`;
const endpoint = new URL(`https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/${title}/monthly/${stamp(months[0])}/${stamp(latestMonth)}`);
const raw = await this.json(endpoint, lane, fresh ? undefined : PAGEVIEWS_CACHE, true);
if (raw === NOT_FOUND) throw new Error('No pageview data');
const result = viewsSchema.parse(raw);
const byMonth = new Map<string, number>();
for (const item of result.items) {
const month = `${item.timestamp.slice(0, 4)}-${item.timestamp.slice(4, 6)}`;
if (!months.includes(month) || byMonth.has(month)) throw new Error('Invalid pageview month');
byMonth.set(month, item.views);
}
if (byMonth.size !== 6) throw new Error('Incomplete pageviews');
const ordered = months.map(month => ({ month, views: byMonth.get(month)! }));
const total = ordered.reduce((sum, item) => sum + item.views, 0);
if (!Number.isSafeInteger(total)) throw new Error('Pageview total overflow');
const latest = ordered[5].views, prior = ordered[4].views;
const days = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 0)).getUTCDate();
pageviews = { status: 'available', months: ordered, latestMonth, total,
dailyAverage: latest / days, trendPercent: prior > 0 ? (latest - prior) / prior * 100 : null };
} catch (error) {
// Mint: a throttled or failed upstream must not pin `attention: null` into a block forever.
if (fresh && transportFailure(error)) throw error;
/* Metadata remains useful; unknown views must not become fake zeroes. */
}
return { pageId, title: page.title, description: page.description ?? null,
wikidataId: page.pageprops?.wikibase_item ?? null, revisionId: page.lastrevid, pageBytes: page.length,
languageCount: page.langlinks?.length ?? 0, languagesTruncated: metadata.continue?.llcontinue !== undefined,
categories: (page.categories ?? []).map(item => item.title.replace(/^Category:/, '')),
categoriesTruncated: metadata.continue?.clcontinue !== undefined, fetchedAt: now.toISOString(), pageviews,
stats: deriveStats(page.length, page.langlinks?.length ?? 0,
pageviews.status === 'available' ? pageviews.months[5].views : null) };
}
}
export const wikipediaEnrichment = new WikipediaEnrichment();