forked from WikiDeck/wikideck
Night 2 (2026-11-17), from the `claude/wikideck` fork's `night/2026-11-17` branch. It follows Night 1 (#22).
## What's in it
- **Duel engine** (fork #72, #80): 40-card decks of cards you own, Notability/Citations, keywords, effects T1–T12, Connected, a hidden-information-safe `viewFor`/`logFor`, and replay. Balance v1 tuning (#84) gives the first player about 51%.
- **Card profiles** (#70, #82): Figure, Location, Event, Institution or Doctrine, plus stats, keywords and rules text, all derived from real article data. Templates only.
- **Design and story** (#66, #69, #83): DUEL_DESIGN.md, plus the match log and the encyclopedic "Battle of X (2026)" recap.
- **PvP** (#74, #77, #81, #76, #85):
- The server checks ownership, and 40-card eligibility is enforced.
- Modes: Classic (ranked), Draft and Trending.
- Invites, quick match, Elo, a ladder and recap pages. Online matches render on the duel board.
- **Duel board UI** (#79): MTG-style card faces and combat declaration, built for 360 px first.
- **Polite Wikimedia client** (#73, #75): throttled, cached and identified requests, the mint-path fix, and a prefetch pool.
- **Android mining** (#78): native RandomX, as a sideload build. The emulator mined a testnet block.
- **Testnet ops** (#67, #68).
PR numbers refer to `claude/wikideck`.
## Status
- **Tests:** 423, of which 422 pass, 0 fail and 1 is skipped. The build is OK (`node:24-alpine`, `--network none`).
- **Testnet:** running this commit (036489c) on r350.
- **Untouched:** no changes to `server/testnet/consensus.ts`, `randomx.ts`, `native/` or the mainnet compose.
- **Review:** night-review reviewed fork #66–#79. #80–#85 came from the agents' unfinished branches and were not reviewed.
- **Known gaps:**
- The PvP end-to-end test hasn't been run (it needs two eom.dev accounts).
- Bot matches aren't reachable from Play.
- Card identity pre-bake hasn't been done.
- The RandomX verifier latch (`server/testnet/randomx.ts:140`) is still open.
This PR is for Eric to merge.
🤖 Generated with [Claude Code](https://claude.com/claude-code)Reviewed-on: WikiDeck/wikideck#23
Co-authored-by: Claude <claude@eom.dev>
221 lines
12 KiB
TypeScript
221 lines
12 KiB
TypeScript
import { z } from 'zod';
|
|
import { WikimediaClient, WikimediaUnavailable, wikimediaClient, type CachePolicy, type Lane } from './wikimedia/client.ts';
|
|
import { deriveStats, type PageEnrichment } from '../shared/enrichment.ts';
|
|
|
|
const MAX_BYTES = 1024 * 1024;
|
|
const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
|
|
const text = z.string().max(4096);
|
|
const pageSchema = z.object({
|
|
pageid: integer.positive(), ns: z.literal(0), title: text.min(1),
|
|
missing: z.never().optional(), invalid: z.never().optional(),
|
|
lastrevid: integer.positive(), length: integer,
|
|
description: text.optional(),
|
|
pageprops: z.object({ wikibase_item: z.string().regex(/^Q[1-9]\d{0,19}$/).optional() }).optional(),
|
|
categories: z.array(z.object({ ns: z.literal(14), title: text.min(1) })).max(50).optional(),
|
|
langlinks: z.array(z.object({ lang: z.string().min(1).max(64) })).max(500).optional(),
|
|
});
|
|
const metadataSchema = z.object({
|
|
query: z.object({ pages: z.array(pageSchema).length(1) }),
|
|
continue: z.object({ clcontinue: text.optional(), llcontinue: text.optional() }).optional(),
|
|
});
|
|
const viewsSchema = z.object({ items: z.array(z.object({
|
|
timestamp: z.string().regex(/^\d{6}0100$/), views: integer,
|
|
project: z.literal('en.wikipedia'), access: z.literal('all-access'),
|
|
agent: z.literal('user'), granularity: z.literal('monthly'),
|
|
})).max(6) });
|
|
|
|
type CacheEntry = { expires: number; value: PageEnrichment } | { expires: number; error: Error };
|
|
type Options = { fetch?: typeof globalThis.fetch; client?: WikimediaClient; now?: () => Date; ttlMs?: number; maxEntries?: number };
|
|
type Mode = 'mint' | 'display' | 'background';
|
|
|
|
/** Disk-cache lifetimes (engineer review §4). Mint reads never use the cache. */
|
|
const METADATA_CACHE: CachePolicy = { ttlMs: 24 * 3600_000, staleWhileRevalidate: true, maxBytes: MAX_BYTES };
|
|
// The URL names its six completed months, which are immutable; a week bounds late corrections.
|
|
const PAGEVIEWS_CACHE: CachePolicy = { ttlMs: 7 * 24 * 3600_000, staleWhileRevalidate: true, maxBytes: MAX_BYTES };
|
|
const NOT_FOUND = Symbol('not found');
|
|
|
|
/** The upstream did not answer properly (throttled, 5xx, timeout, network, non-JSON). Not "no data". */
|
|
export class UpstreamFailure extends Error {
|
|
constructor(message: string) { super(message); this.name = 'UpstreamFailure'; }
|
|
}
|
|
function transportFailure(error: unknown): boolean {
|
|
return error instanceof WikimediaUnavailable || error instanceof UpstreamFailure ||
|
|
(error instanceof DOMException && (error.name === 'TimeoutError' || error.name === 'AbortError'));
|
|
}
|
|
|
|
function monthWindow(now: Date): string[] {
|
|
return Array.from({ length: 6 }, (_, index) =>
|
|
new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 6 + index, 1)).toISOString().slice(0, 7));
|
|
}
|
|
|
|
/** Fixed-source read-only enrichment; no consensus, persistence, URL or header inputs. */
|
|
export class WikipediaEnrichment {
|
|
private readonly client: WikimediaClient;
|
|
private readonly now: () => Date;
|
|
private readonly ttlMs: number;
|
|
private readonly maxEntries: number;
|
|
private readonly cache = new Map<number, CacheEntry>();
|
|
private readonly pending = new Map<string, Promise<PageEnrichment>>();
|
|
private active = 0;
|
|
private mintActive = 0;
|
|
|
|
constructor(options: Options = {}) {
|
|
this.client = options.client ?? (options.fetch ? WikimediaClient.unthrottled(options.fetch) : wikimediaClient);
|
|
this.now = options.now ?? (() => new Date());
|
|
this.ttlMs = options.ttlMs ?? 6 * 60 * 60 * 1000;
|
|
this.maxEntries = options.maxEntries ?? 128;
|
|
if (!Number.isSafeInteger(this.ttlMs) || this.ttlMs <= 0 ||
|
|
!Number.isSafeInteger(this.maxEntries) || this.maxEntries <= 0 || this.maxEntries > 128) {
|
|
throw new RangeError('Invalid enrichment cache options');
|
|
}
|
|
}
|
|
|
|
/** Card faces (`/api/wikipedia/:id`): memory, then the disk cache, then the interactive lane. */
|
|
async get(pageId: number): Promise<PageEnrichment> {
|
|
return this.loadCached(pageId, 'display');
|
|
}
|
|
|
|
/** Like get(), on the background lane: for trends and other work nobody is waiting on. */
|
|
async getBackground(pageId: number): Promise<PageEnrichment> {
|
|
return this.loadCached(pageId, 'background');
|
|
}
|
|
|
|
/**
|
|
* Mint data: bypasses every cache and uses the mint lane. If pageviews fail in transport
|
|
* (429, 5xx, timeout, throttled), this rejects so the template fails and the miner retries;
|
|
* `unavailable` is recorded only for a genuine 404 or missing data.
|
|
*/
|
|
async getFresh(pageId: number): Promise<PageEnrichment> {
|
|
return this.loadCached(pageId, 'mint');
|
|
}
|
|
|
|
private async loadCached(pageId: number, mode: Mode): Promise<PageEnrichment> {
|
|
const fresh = mode === 'mint';
|
|
if (!Number.isSafeInteger(pageId) || pageId <= 0) throw new RangeError('Invalid Wikipedia page ID');
|
|
const now = this.now();
|
|
const cached = this.cache.get(pageId);
|
|
if (!fresh && cached && cached.expires > now.getTime() &&
|
|
(!('value' in cached) || cached.value.pageviews.latestMonth === monthWindow(now)[5])) {
|
|
this.cache.delete(pageId);
|
|
this.cache.set(pageId, cached);
|
|
if ('error' in cached) throw cached.error;
|
|
return structuredClone(cached.value);
|
|
}
|
|
if (!fresh) this.cache.delete(pageId);
|
|
// Mint jobs never join a display job: that one may be served from the disk cache.
|
|
const pendingKey = `${fresh ? 'mint' : 'cached'}:${pageId}`;
|
|
const existing = this.pending.get(pendingKey);
|
|
if (existing) return structuredClone(await existing);
|
|
// Reject immediately: no unbounded waiting queue. Four active jobs also keep
|
|
// the pending-map ceiling of eight conservative (one upstream request/job).
|
|
// Mint jobs have their own small allowance so card faces can never starve a template.
|
|
if (fresh ? this.mintActive >= 2 : this.pending.size - this.mintActive >= 8 || this.active >= 4) {
|
|
throw new Error('Wikipedia enrichment busy');
|
|
}
|
|
if (fresh) this.mintActive++; else this.active++;
|
|
const job = this.load(pageId, now, mode).then(value => {
|
|
const ttl = value.pageviews.status === 'available' ? this.ttlMs : Math.min(this.ttlMs, 5 * 60 * 1000);
|
|
this.remember(pageId, { value, expires: this.now().getTime() + ttl });
|
|
return value;
|
|
}, () => {
|
|
const error = new Error('Wikipedia page enrichment unavailable');
|
|
// A failed mint read says nothing new about display data; keep any cached value.
|
|
if (!fresh) this.remember(pageId, { error, expires: this.now().getTime() + 30_000 });
|
|
throw error;
|
|
}).finally(() => { if (fresh) this.mintActive--; else this.active--; this.pending.delete(pendingKey); });
|
|
this.pending.set(pendingKey, job);
|
|
return structuredClone(await job);
|
|
}
|
|
|
|
private remember(pageId: number, entry: CacheEntry): void {
|
|
this.cache.delete(pageId);
|
|
this.cache.set(pageId, entry);
|
|
while (this.cache.size > this.maxEntries) this.cache.delete(this.cache.keys().next().value!);
|
|
}
|
|
|
|
private async json(url: URL, lane: Lane, cache: CachePolicy | undefined, allowNotFound = false): Promise<unknown> {
|
|
const { response, signal } = await this.client.request(url, { lane, timeoutMs: 6000, cache });
|
|
if (allowNotFound && response.status === 404) { await response.body?.cancel(); return NOT_FOUND; }
|
|
if (!response.ok || response.redirected) {
|
|
await response.body?.cancel();
|
|
throw new UpstreamFailure(`Upstream HTTP ${response.status}`);
|
|
}
|
|
const length = response.headers.get('content-length');
|
|
if (
|
|
!/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') ||
|
|
(length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_BYTES)) || !response.body) {
|
|
await response.body?.cancel();
|
|
throw new UpstreamFailure('Invalid upstream response');
|
|
}
|
|
const reader = response.body.getReader();
|
|
const chunks: Uint8Array[] = [];
|
|
let size = 0;
|
|
try {
|
|
for (;;) {
|
|
signal.throwIfAborted();
|
|
const { done, value } = await reader.read();
|
|
signal.throwIfAborted();
|
|
if (done) break;
|
|
size += value.byteLength;
|
|
if (size > MAX_BYTES) throw new Error('Upstream response too large');
|
|
chunks.push(value);
|
|
}
|
|
return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)));
|
|
} finally {
|
|
await reader.cancel();
|
|
reader.releaseLock();
|
|
}
|
|
}
|
|
|
|
private async load(pageId: number, now: Date, mode: Mode): Promise<PageEnrichment> {
|
|
const lane: Lane = mode === 'mint' ? 'mint' : mode === 'display' ? 'interactive' : 'background';
|
|
const fresh = mode === 'mint';
|
|
const url = new URL('https://en.wikipedia.org/w/api.php');
|
|
url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
|
|
pageids: String(pageId), prop: 'info|categories|pageprops|description|langlinks',
|
|
clshow: '!hidden', cllimit: '50', lllimit: '500', ppprop: 'wikibase_item' }).toString();
|
|
const metadata = metadataSchema.parse(await this.json(url, lane, fresh ? undefined : METADATA_CACHE));
|
|
const page = metadata.query.pages[0];
|
|
if (page.pageid !== pageId) throw new Error('Wikipedia page ID mismatch');
|
|
const months = monthWindow(now);
|
|
const latestMonth = months[5];
|
|
let pageviews: PageEnrichment['pageviews'] = { status: 'unavailable', months: [], latestMonth,
|
|
total: null, dailyAverage: null, trendPercent: null };
|
|
try {
|
|
const title = encodeURIComponent(page.title.replace(/ /g, '_'));
|
|
const stamp = (month: string) => `${month.replace('-', '')}0100`;
|
|
const endpoint = new URL(`https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/${title}/monthly/${stamp(months[0])}/${stamp(latestMonth)}`);
|
|
const raw = await this.json(endpoint, lane, fresh ? undefined : PAGEVIEWS_CACHE, true);
|
|
if (raw === NOT_FOUND) throw new Error('No pageview data');
|
|
const result = viewsSchema.parse(raw);
|
|
const byMonth = new Map<string, number>();
|
|
for (const item of result.items) {
|
|
const month = `${item.timestamp.slice(0, 4)}-${item.timestamp.slice(4, 6)}`;
|
|
if (!months.includes(month) || byMonth.has(month)) throw new Error('Invalid pageview month');
|
|
byMonth.set(month, item.views);
|
|
}
|
|
if (byMonth.size !== 6) throw new Error('Incomplete pageviews');
|
|
const ordered = months.map(month => ({ month, views: byMonth.get(month)! }));
|
|
const total = ordered.reduce((sum, item) => sum + item.views, 0);
|
|
if (!Number.isSafeInteger(total)) throw new Error('Pageview total overflow');
|
|
const latest = ordered[5].views, prior = ordered[4].views;
|
|
const days = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 0)).getUTCDate();
|
|
pageviews = { status: 'available', months: ordered, latestMonth, total,
|
|
dailyAverage: latest / days, trendPercent: prior > 0 ? (latest - prior) / prior * 100 : null };
|
|
} catch (error) {
|
|
// Mint: a throttled or failed upstream must not pin `attention: null` into a block forever.
|
|
if (fresh && transportFailure(error)) throw error;
|
|
/* Metadata remains useful; unknown views must not become fake zeroes. */
|
|
}
|
|
return { pageId, title: page.title, description: page.description ?? null,
|
|
wikidataId: page.pageprops?.wikibase_item ?? null, revisionId: page.lastrevid, pageBytes: page.length,
|
|
languageCount: page.langlinks?.length ?? 0, languagesTruncated: metadata.continue?.llcontinue !== undefined,
|
|
categories: (page.categories ?? []).map(item => item.title.replace(/^Category:/, '')),
|
|
categoriesTruncated: metadata.continue?.clcontinue !== undefined, fetchedAt: now.toISOString(), pageviews,
|
|
stats: deriveStats(page.length, page.langlinks?.length ?? 0,
|
|
pageviews.status === 'available' ? pageviews.months[5].views : null) };
|
|
}
|
|
}
|
|
|
|
export const wikipediaEnrichment = new WikipediaEnrichment();
|