diff --git a/server/game/identity-source.ts b/server/game/identity-source.ts new file mode 100644 index 0000000..a996ed0 --- /dev/null +++ b/server/game/identity-source.ts @@ -0,0 +1,112 @@ +// Live IdentitySource for server/game/identity.ts (engineer WS E, data §3). +// +// Every request goes through the process-wide polite Wikimedia client on the **background** lane: +// - the page enrichment (description, wikidataId, categories) via `enrichment.getBackground()`, +// which also feeds the disk cache card faces use; +// - one `wbgetentities&props=claims` per batch of at most 50 items; +// - one `wbgetentities&props=labels` per 50 class ids still missing a label. +// That is about 2 Wikidata requests per 50 new cards, plus one cached enrichment per card. +// Outlinks (prop=links, data §4) are not fetched tonight: they cost about 1.2 requests per card, +// and Connected falls back to its other tiers without them. +// +// The profile itself is built by duel-cards' pure `profileFromSources()` (shared/duel/profile.ts, #70), +// so the frozen identity and what derive() reads cannot drift apart. That module, and the client +// (server/wikimedia, #73), are injected structurally, so this file has no import from unmerged PRs. +// Wiring, once both have merged: +// wikimediaIdentitySource({ client: wikimediaClient, enrichment: wikipediaEnrichment, +// profile: await import('../../shared/duel/profile.ts') }) +// Any transport failure leaves the affected cards unknown (undefined), so they are retried later, +// never frozen half-empty. +import type { PageEnrichment } from '../../shared/enrichment.ts'; +import type { ChainArticle, IdentityFacts, IdentitySource, WikidataClass } from './identity.ts'; + +/** Structural slices of server/wikimedia/client.ts (#73) and shared/duel/profile.ts (#70). */ +export type BackgroundRequest = { lane: 'background'; timeoutMs: number; cache?: { ttlMs: number; maxBytes?: number } }; +export type WikidataEntity = { id?: string; claims?: Record }; +export type ProfileLike = { wikidataId: string | null; description: string | null; categories: readonly string[]; + instanceOf: readonly WikidataClass[]; subclassOf?: readonly WikidataClass[]; countries?: readonly string[]; + years?: readonly number[]; died?: boolean; links?: readonly number[]; cites?: readonly string[]; fetchedRevisionId?: number }; +export type ProfileBuilder = { + wikidataEntityParams(qids: readonly string[]): Record; + wikidataLabelParams(qids: readonly string[]): Record; + classIdsOf(entity: WikidataEntity | null | undefined): string[]; + profileFromSources(src: { enrichment: PageEnrichment; entity?: WikidataEntity | null; labels?: Readonly> }): ProfileLike; +}; +export type IdentitySourceDeps = { + enrichment: { getBackground(pageId: number): Promise }; + client: { request(url: URL, options: BackgroundRequest): Promise<{ response: Response }> }; + profile: ProfileBuilder; +}; + +const WIKIDATA_API = 'https://www.wikidata.org/w/api.php'; +const MAX_BYTES = 4 * 1024 * 1024; +const CACHE = { ttlMs: 7 * 24 * 60 * 60 * 1000, maxBytes: MAX_BYTES }; + +async function wikidata(client: IdentitySourceDeps['client'], params: Record): Promise> { + const url = new URL(WIKIDATA_API); + for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); + const { response } = await client.request(url, { lane: 'background', timeoutMs: 15000, cache: CACHE }); + if (!response.ok) { await response.body?.cancel(); throw new Error(`Wikidata HTTP ${response.status}`); } + const text = await response.text(); + if (text.length > MAX_BYTES) throw new Error('Wikidata response too large'); + const body = JSON.parse(text) as { entities?: Record | unknown[] }; + const entities = body.entities ?? {}; + // formatversion=2 returns an object keyed by id; tolerate an array too. + return Array.isArray(entities) + ? Object.fromEntries(entities.filter(e => e && typeof (e as { id?: unknown }).id === 'string').map(e => [(e as { id: string }).id, e])) + : entities as Record; +} + +const chunks = (xs: T[], n: number) => Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, i * n + n)); + +export function wikimediaIdentitySource(deps: IdentitySourceDeps): IdentitySource { + const { classIdsOf, profileFromSources, wikidataEntityParams, wikidataLabelParams } = deps.profile; + return { + kind: 'wikimedia', + async lookup(articles: ChainArticle[]) { + const out = new Map(articles.map(a => [a.pageId, undefined])); + // 1. Enrichment, one page at a time (the client paces it); failures stay unknown. + const enriched = new Map(); + for (const a of articles) { + try { enriched.set(a.pageId, await deps.enrichment.getBackground(a.pageId)); } catch { /* retry later */ } + } + // 2. Claims for every item we know, in batches of ≤ 50. + const qids = [...new Set([...enriched.values()].map(e => e.wikidataId).filter((q): q is string => !!q && /^Q\d+$/.test(q)))]; + const entities = new Map(); + let claimsFailed = false; + for (const batch of chunks(qids, 50)) { + try { for (const [id, e] of Object.entries(await wikidata(deps.client, wikidataEntityParams(batch)))) entities.set(id, e as WikidataEntity); } + catch { claimsFailed = true; } + } + // 3. English labels for the P31/P279 classes. + const classIds = [...new Set([...entities.values()].flatMap(e => classIdsOf(e)))]; + const labels: Record = {}; + let labelsFailed = false; + for (const batch of chunks(classIds, 50)) { + try { + for (const [id, e] of Object.entries(await wikidata(deps.client, wikidataLabelParams(batch)))) { + const v = (e as { labels?: { en?: { value?: unknown } } }).labels?.en?.value; + if (typeof v === 'string') labels[id] = v; + } + } catch { labelsFailed = true; } + } + // 4. Freeze only complete profiles: a card whose item couldn't be read stays unknown. + for (const a of articles) { + const e = enriched.get(a.pageId); + if (!e) continue; + const qid = e.wikidataId && /^Q\d+$/.test(e.wikidataId) ? e.wikidataId : null; + const entity = qid ? entities.get(qid) : null; + if (qid && !entity && claimsFailed) continue; // item not read: retry later + if (entity && labelsFailed) continue; // class labels not read: retry later + const p = profileFromSources({ enrichment: e, entity: entity ?? null, labels }); + out.set(a.pageId, { + fetchedRevisionId: p.fetchedRevisionId ?? null, wikidataId: p.wikidataId, description: p.description, + categories: [...p.categories], instanceOf: [...p.instanceOf], subclassOf: [...(p.subclassOf ?? [])], + countries: [...(p.countries ?? [])], years: [...(p.years ?? [])], died: p.died === true, + links: [...(p.links ?? [])], cites: [...(p.cites ?? [])], + }); + } + return out; + }, + }; +} diff --git a/tests/testnet-ops-identity-source.test.ts b/tests/testnet-ops-identity-source.test.ts new file mode 100644 index 0000000..0344ba1 --- /dev/null +++ b/tests/testnet-ops-identity-source.test.ts @@ -0,0 +1,82 @@ +// testnet-ops: live identity source over a FAKE Wikimedia client. Zero network calls. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { wikimediaIdentitySource } from '../server/game/identity-source.ts'; +import { IdentityStore, prebake, toProfile } from '../server/game/identity.ts'; +import type { BackgroundRequest, ProfileBuilder } from '../server/game/identity-source.ts'; + +// A minimal stand-in for shared/duel/profile.ts (#70), so this test needs no unmerged module. +const ids = (e: any, p: string): string[] => (e?.claims?.[p] ?? []).map((s: any) => s.mainsnak?.datavalue?.value?.id).filter(Boolean); +const yrs = (e: any, p: string): number[] => (e?.claims?.[p] ?? []).map((s: any) => Number(/^[+-]0*(\d+)-/.exec(s.mainsnak?.datavalue?.value?.time ?? '')?.[1])).filter(Boolean); +const profile: ProfileBuilder = { + wikidataEntityParams: q => ({ action: 'wbgetentities', ids: [...q].sort().join('|'), props: 'claims', format: 'json', formatversion: '2' }), + wikidataLabelParams: q => ({ action: 'wbgetentities', ids: [...q].sort().join('|'), props: 'labels', languages: 'en', format: 'json', formatversion: '2' }), + classIdsOf: e => [...new Set([...ids(e, 'P31'), ...ids(e, 'P279')])].sort(), + profileFromSources: ({ enrichment, entity, labels = {} }) => ({ + wikidataId: enrichment.wikidataId, description: enrichment.description, categories: enrichment.categories, + instanceOf: ids(entity, 'P31').map(id => ({ id, label: labels[id] ?? id })), countries: ids(entity, 'P27'), + years: [...yrs(entity, 'P569'), ...yrs(entity, 'P570')], died: ids(entity, 'P570').length > 0 || yrs(entity, 'P570').length > 0, + fetchedRevisionId: enrichment.revisionId }), +}; + +const enrichmentFor = (pageId: number, wikidataId: string | null) => ({ + pageId, title: `Page ${pageId}`, description: `desc ${pageId}`, wikidataId, revisionId: 1000 + pageId, pageBytes: 1, + languageCount: 1, languagesTruncated: false, categories: ['Living people', '1948 births'], categoriesTruncated: false, + fetchedAt: '2026-11-17T00:00:00Z', + pageviews: { status: 'unavailable', months: [], latestMonth: '2026-10', total: null, dailyAverage: null, trendPercent: null }, +}) as any; +const claim = (id: string) => ({ rank: 'normal', mainsnak: { snaktype: 'value', datavalue: { type: 'wikibase-entityid', value: { id } } } }); +const time = (t: string) => ({ rank: 'normal', mainsnak: { snaktype: 'value', datavalue: { type: 'time', value: { time: t } } } }); + +function fakeClient(opts: { failClaims?: boolean } = {}) { + const calls: { url: URL; options: BackgroundRequest }[] = []; + return { calls, async request(url: URL, options: BackgroundRequest) { + calls.push({ url, options }); + const props = url.searchParams.get('props'), ids = url.searchParams.get('ids')!.split('|'); + if (props === 'claims' && opts.failClaims) return { response: new Response('busy', { status: 503 }), signal: new AbortController().signal, fromCache: false }; + const entities = Object.fromEntries(ids.map(id => [id, props === 'claims' + ? { id, claims: { P31: [claim('Q5')], P27: [claim('Q30')], P569: [time('+1948-01-02T00:00:00Z')], P570: [time('+2001-01-01T00:00:00Z')] } } + : { id, labels: { en: { value: id === 'Q5' ? 'human' : `label ${id}` } } }])); + return { response: Response.json({ entities }), signal: new AbortController().signal, fromCache: false }; + } }; +} + +test('every request is on the background lane, batched, and the frozen identity is derive-ready', async () => { + const realFetch = globalThis.fetch; + globalThis.fetch = (() => { throw new Error('network call in test'); }) as typeof fetch; + const dir = mkdtempSync(join(tmpdir(), 'identity-src-')); + try { + const client = fakeClient(); + const source = wikimediaIdentitySource({ client, profile, enrichment: { async getBackground(id) { return enrichmentFor(id, `Q${id}`); } } }); + const store = new IdentityStore(join(dir, 'id.sqlite')); + const articles = [1, 2, 3].map(i => ({ tokenId: String(i).repeat(64), pageId: i, title: `Page ${i}`, revisionId: i })); + const r = await prebake(store, source, articles); + assert.deepEqual(r, { seen: 3, alreadyFrozen: 0, frozen: 3, unknown: 0 }); + assert.ok(client.calls.every(c => c.options.lane === 'background')); + assert.ok(client.calls.every(c => c.url.hostname === 'www.wikidata.org')); + assert.deepEqual(client.calls.map(c => c.url.searchParams.get('props')), ['claims', 'labels']); + assert.equal(client.calls[0].url.searchParams.get('ids'), 'Q1|Q2|Q3'); + const one = store.get('1'.repeat(64))!; + assert.equal(one.source, 'wikimedia'); + assert.deepEqual(one.instanceOf, [{ id: 'Q5', label: 'human' }]); + assert.deepEqual(one.countries, ['Q30']); + assert.deepEqual(one.years, [1948, 2001]); + assert.equal(one.died, true); + assert.equal(one.fetchedRevisionId, 1001); + assert.equal(toProfile(one).description, 'desc 1'); + store.close(); + } finally { globalThis.fetch = realFetch; rmSync(dir, { recursive: true, force: true }); } +}); + +test('a failed claims request leaves cards unknown so they are retried, never frozen half-empty', async () => { + const source = wikimediaIdentitySource({ client: fakeClient({ failClaims: true }), profile, + enrichment: { async getBackground(id) { if (id === 9) throw new Error('throttled'); return enrichmentFor(id, id === 8 ? null : `Q${id}`); } } }); + const got = await source.lookup([7, 8, 9].map(i => ({ tokenId: String(i).repeat(64), pageId: i, title: `P${i}` }))); + assert.equal(got.get(7), undefined); // has an item, claims failed → retry + assert.equal(got.get(9), undefined); // enrichment failed → retry + assert.equal(got.get(8)?.wikidataId, null); // no Wikidata item at all → enrichment-only profile is final + assert.deepEqual(got.get(8)?.instanceOf, []); +});