Files
wikideck/server/enrichment.ts
claudeanderic dbb89ce9c5 Night 2: 40-card Magic-style duel, PvP with owned decks, polite Wikimedia client, Android mining (#23)
Night 2 (2026-11-17), from the `claude/wikideck` fork's `night/2026-11-17` branch. It follows Night 1 (#22).

## What's in it
- **Duel engine** (fork #72, #80): 40-card decks of cards you own, Notability/Citations, keywords, effects T1–T12, Connected, a hidden-information-safe `viewFor`/`logFor`, and replay. Balance v1 tuning (#84) gives the first player about 51%.
- **Card profiles** (#70, #82): Figure, Location, Event, Institution or Doctrine, plus stats, keywords and rules text, all derived from real article data. Templates only.
- **Design and story** (#66, #69, #83): DUEL_DESIGN.md, plus the match log and the encyclopedic "Battle of X (2026)" recap.
- **PvP** (#74, #77, #81, #76, #85):
  - The server checks ownership, and 40-card eligibility is enforced.
  - Modes: Classic (ranked), Draft and Trending.
  - Invites, quick match, Elo, a ladder and recap pages. Online matches render on the duel board.
- **Duel board UI** (#79): MTG-style card faces and combat declaration, built for 360 px first.
- **Polite Wikimedia client** (#73, #75): throttled, cached and identified requests, the mint-path fix, and a prefetch pool.
- **Android mining** (#78): native RandomX, as a sideload build. The emulator mined a testnet block.
- **Testnet ops** (#67, #68).

PR numbers refer to `claude/wikideck`.

## Status
- **Tests:** 423, of which 422 pass, 0 fail and 1 is skipped. The build is OK (`node:24-alpine`, `--network none`).
- **Testnet:** running this commit (036489c) on r350.
- **Untouched:** no changes to `server/testnet/consensus.ts`, `randomx.ts`, `native/` or the mainnet compose.
- **Review:** night-review reviewed fork #66–#79. #80–#85 came from the agents' unfinished branches and were not reviewed.
- **Known gaps:**
  - The PvP end-to-end test hasn't been run (it needs two eom.dev accounts).
  - Bot matches aren't reachable from Play.
  - Card identity pre-bake hasn't been done.
  - The RandomX verifier latch (`server/testnet/randomx.ts:140`) is still open.

This PR is for Eric to merge.

🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: WikiDeck/wikideck#23

Co-authored-by: Claude <claude@eom.dev>
2026-11-18 18:12:24 +00:00

221 lines
12 KiB
TypeScript

import { z } from 'zod';
import { WikimediaClient, WikimediaUnavailable, wikimediaClient, type CachePolicy, type Lane } from './wikimedia/client.ts';
import { deriveStats, type PageEnrichment } from '../shared/enrichment.ts';
const MAX_BYTES = 1024 * 1024;
const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
const text = z.string().max(4096);
const pageSchema = z.object({
pageid: integer.positive(), ns: z.literal(0), title: text.min(1),
missing: z.never().optional(), invalid: z.never().optional(),
lastrevid: integer.positive(), length: integer,
description: text.optional(),
pageprops: z.object({ wikibase_item: z.string().regex(/^Q[1-9]\d{0,19}$/).optional() }).optional(),
categories: z.array(z.object({ ns: z.literal(14), title: text.min(1) })).max(50).optional(),
langlinks: z.array(z.object({ lang: z.string().min(1).max(64) })).max(500).optional(),
});
const metadataSchema = z.object({
query: z.object({ pages: z.array(pageSchema).length(1) }),
continue: z.object({ clcontinue: text.optional(), llcontinue: text.optional() }).optional(),
});
const viewsSchema = z.object({ items: z.array(z.object({
timestamp: z.string().regex(/^\d{6}0100$/), views: integer,
project: z.literal('en.wikipedia'), access: z.literal('all-access'),
agent: z.literal('user'), granularity: z.literal('monthly'),
})).max(6) });
type CacheEntry = { expires: number; value: PageEnrichment } | { expires: number; error: Error };
type Options = { fetch?: typeof globalThis.fetch; client?: WikimediaClient; now?: () => Date; ttlMs?: number; maxEntries?: number };
type Mode = 'mint' | 'display' | 'background';
/** Disk-cache lifetimes (engineer review §4). Mint reads never use the cache. */
const METADATA_CACHE: CachePolicy = { ttlMs: 24 * 3600_000, staleWhileRevalidate: true, maxBytes: MAX_BYTES };
// The URL names its six completed months, which are immutable; a week bounds late corrections.
const PAGEVIEWS_CACHE: CachePolicy = { ttlMs: 7 * 24 * 3600_000, staleWhileRevalidate: true, maxBytes: MAX_BYTES };
const NOT_FOUND = Symbol('not found');
/** The upstream did not answer properly (throttled, 5xx, timeout, network, non-JSON). Not "no data". */
export class UpstreamFailure extends Error {
constructor(message: string) { super(message); this.name = 'UpstreamFailure'; }
}
function transportFailure(error: unknown): boolean {
return error instanceof WikimediaUnavailable || error instanceof UpstreamFailure ||
(error instanceof DOMException && (error.name === 'TimeoutError' || error.name === 'AbortError'));
}
function monthWindow(now: Date): string[] {
return Array.from({ length: 6 }, (_, index) =>
new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 6 + index, 1)).toISOString().slice(0, 7));
}
/** Fixed-source read-only enrichment; no consensus, persistence, URL or header inputs. */
export class WikipediaEnrichment {
private readonly client: WikimediaClient;
private readonly now: () => Date;
private readonly ttlMs: number;
private readonly maxEntries: number;
private readonly cache = new Map<number, CacheEntry>();
private readonly pending = new Map<string, Promise<PageEnrichment>>();
private active = 0;
private mintActive = 0;
constructor(options: Options = {}) {
this.client = options.client ?? (options.fetch ? WikimediaClient.unthrottled(options.fetch) : wikimediaClient);
this.now = options.now ?? (() => new Date());
this.ttlMs = options.ttlMs ?? 6 * 60 * 60 * 1000;
this.maxEntries = options.maxEntries ?? 128;
if (!Number.isSafeInteger(this.ttlMs) || this.ttlMs <= 0 ||
!Number.isSafeInteger(this.maxEntries) || this.maxEntries <= 0 || this.maxEntries > 128) {
throw new RangeError('Invalid enrichment cache options');
}
}
/** Card faces (`/api/wikipedia/:id`): memory, then the disk cache, then the interactive lane. */
async get(pageId: number): Promise<PageEnrichment> {
return this.loadCached(pageId, 'display');
}
/** Like get(), on the background lane: for trends and other work nobody is waiting on. */
async getBackground(pageId: number): Promise<PageEnrichment> {
return this.loadCached(pageId, 'background');
}
/**
* Mint data: bypasses every cache and uses the mint lane. If pageviews fail in transport
* (429, 5xx, timeout, throttled), this rejects so the template fails and the miner retries;
* `unavailable` is recorded only for a genuine 404 or missing data.
*/
async getFresh(pageId: number): Promise<PageEnrichment> {
return this.loadCached(pageId, 'mint');
}
private async loadCached(pageId: number, mode: Mode): Promise<PageEnrichment> {
const fresh = mode === 'mint';
if (!Number.isSafeInteger(pageId) || pageId <= 0) throw new RangeError('Invalid Wikipedia page ID');
const now = this.now();
const cached = this.cache.get(pageId);
if (!fresh && cached && cached.expires > now.getTime() &&
(!('value' in cached) || cached.value.pageviews.latestMonth === monthWindow(now)[5])) {
this.cache.delete(pageId);
this.cache.set(pageId, cached);
if ('error' in cached) throw cached.error;
return structuredClone(cached.value);
}
if (!fresh) this.cache.delete(pageId);
// Mint jobs never join a display job: that one may be served from the disk cache.
const pendingKey = `${fresh ? 'mint' : 'cached'}:${pageId}`;
const existing = this.pending.get(pendingKey);
if (existing) return structuredClone(await existing);
// Reject immediately: no unbounded waiting queue. Four active jobs also keep
// the pending-map ceiling of eight conservative (one upstream request/job).
// Mint jobs have their own small allowance so card faces can never starve a template.
if (fresh ? this.mintActive >= 2 : this.pending.size - this.mintActive >= 8 || this.active >= 4) {
throw new Error('Wikipedia enrichment busy');
}
if (fresh) this.mintActive++; else this.active++;
const job = this.load(pageId, now, mode).then(value => {
const ttl = value.pageviews.status === 'available' ? this.ttlMs : Math.min(this.ttlMs, 5 * 60 * 1000);
this.remember(pageId, { value, expires: this.now().getTime() + ttl });
return value;
}, () => {
const error = new Error('Wikipedia page enrichment unavailable');
// A failed mint read says nothing new about display data; keep any cached value.
if (!fresh) this.remember(pageId, { error, expires: this.now().getTime() + 30_000 });
throw error;
}).finally(() => { if (fresh) this.mintActive--; else this.active--; this.pending.delete(pendingKey); });
this.pending.set(pendingKey, job);
return structuredClone(await job);
}
private remember(pageId: number, entry: CacheEntry): void {
this.cache.delete(pageId);
this.cache.set(pageId, entry);
while (this.cache.size > this.maxEntries) this.cache.delete(this.cache.keys().next().value!);
}
private async json(url: URL, lane: Lane, cache: CachePolicy | undefined, allowNotFound = false): Promise<unknown> {
const { response, signal } = await this.client.request(url, { lane, timeoutMs: 6000, cache });
if (allowNotFound && response.status === 404) { await response.body?.cancel(); return NOT_FOUND; }
if (!response.ok || response.redirected) {
await response.body?.cancel();
throw new UpstreamFailure(`Upstream HTTP ${response.status}`);
}
const length = response.headers.get('content-length');
if (
!/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') ||
(length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_BYTES)) || !response.body) {
await response.body?.cancel();
throw new UpstreamFailure('Invalid upstream response');
}
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let size = 0;
try {
for (;;) {
signal.throwIfAborted();
const { done, value } = await reader.read();
signal.throwIfAborted();
if (done) break;
size += value.byteLength;
if (size > MAX_BYTES) throw new Error('Upstream response too large');
chunks.push(value);
}
return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)));
} finally {
await reader.cancel();
reader.releaseLock();
}
}
private async load(pageId: number, now: Date, mode: Mode): Promise<PageEnrichment> {
const lane: Lane = mode === 'mint' ? 'mint' : mode === 'display' ? 'interactive' : 'background';
const fresh = mode === 'mint';
const url = new URL('https://en.wikipedia.org/w/api.php');
url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
pageids: String(pageId), prop: 'info|categories|pageprops|description|langlinks',
clshow: '!hidden', cllimit: '50', lllimit: '500', ppprop: 'wikibase_item' }).toString();
const metadata = metadataSchema.parse(await this.json(url, lane, fresh ? undefined : METADATA_CACHE));
const page = metadata.query.pages[0];
if (page.pageid !== pageId) throw new Error('Wikipedia page ID mismatch');
const months = monthWindow(now);
const latestMonth = months[5];
let pageviews: PageEnrichment['pageviews'] = { status: 'unavailable', months: [], latestMonth,
total: null, dailyAverage: null, trendPercent: null };
try {
const title = encodeURIComponent(page.title.replace(/ /g, '_'));
const stamp = (month: string) => `${month.replace('-', '')}0100`;
const endpoint = new URL(`https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/${title}/monthly/${stamp(months[0])}/${stamp(latestMonth)}`);
const raw = await this.json(endpoint, lane, fresh ? undefined : PAGEVIEWS_CACHE, true);
if (raw === NOT_FOUND) throw new Error('No pageview data');
const result = viewsSchema.parse(raw);
const byMonth = new Map<string, number>();
for (const item of result.items) {
const month = `${item.timestamp.slice(0, 4)}-${item.timestamp.slice(4, 6)}`;
if (!months.includes(month) || byMonth.has(month)) throw new Error('Invalid pageview month');
byMonth.set(month, item.views);
}
if (byMonth.size !== 6) throw new Error('Incomplete pageviews');
const ordered = months.map(month => ({ month, views: byMonth.get(month)! }));
const total = ordered.reduce((sum, item) => sum + item.views, 0);
if (!Number.isSafeInteger(total)) throw new Error('Pageview total overflow');
const latest = ordered[5].views, prior = ordered[4].views;
const days = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 0)).getUTCDate();
pageviews = { status: 'available', months: ordered, latestMonth, total,
dailyAverage: latest / days, trendPercent: prior > 0 ? (latest - prior) / prior * 100 : null };
} catch (error) {
// Mint: a throttled or failed upstream must not pin `attention: null` into a block forever.
if (fresh && transportFailure(error)) throw error;
/* Metadata remains useful; unknown views must not become fake zeroes. */
}
return { pageId, title: page.title, description: page.description ?? null,
wikidataId: page.pageprops?.wikibase_item ?? null, revisionId: page.lastrevid, pageBytes: page.length,
languageCount: page.langlinks?.length ?? 0, languagesTruncated: metadata.continue?.llcontinue !== undefined,
categories: (page.categories ?? []).map(item => item.title.replace(/^Category:/, '')),
categoriesTruncated: metadata.continue?.clcontinue !== undefined, fetchedAt: now.toISOString(), pageviews,
stats: deriveStats(page.length, page.langlinks?.length ?? 0,
pageviews.status === 'available' ? pageviews.months[5].views : null) };
}
}
export const wikipediaEnrichment = new WikipediaEnrichment();