testnet-ops: live identity source (background lane only), deps injected structurally

wikimediaIdentitySource() takes the wikimedia client (#73), the page enrichment and
duel-cards' profile builder (#70) as structural deps, so it imports nothing unmerged.
About 2 Wikidata requests per 50 new cards; transport failures leave cards unknown.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-17 22:59:19 -05:00
co-authored by Claude Opus 5.5
parent 4f9cbfeaae
commit c9197e1a9f
2 changed files with 194 additions and 0 deletions
+112
View File
@@ -0,0 +1,112 @@
// Live IdentitySource for server/game/identity.ts (engineer WS E, data §3).
//
// Every request goes through the process-wide polite Wikimedia client on the **background** lane:
// - the page enrichment (description, wikidataId, categories) via `enrichment.getBackground()`,
// which also feeds the disk cache card faces use;
// - one `wbgetentities&props=claims` per batch of at most 50 items;
// - one `wbgetentities&props=labels` per 50 class ids still missing a label.
// That is about 2 Wikidata requests per 50 new cards, plus one cached enrichment per card.
// Outlinks (prop=links, data §4) are not fetched tonight: they cost about 1.2 requests per card,
// and Connected falls back to its other tiers without them.
//
// The profile itself is built by duel-cards' pure `profileFromSources()` (shared/duel/profile.ts, #70),
// so the frozen identity and what derive() reads cannot drift apart. That module, and the client
// (server/wikimedia, #73), are injected structurally, so this file has no import from unmerged PRs.
// Wiring, once both have merged:
// wikimediaIdentitySource({ client: wikimediaClient, enrichment: wikipediaEnrichment,
// profile: await import('../../shared/duel/profile.ts') })
// Any transport failure leaves the affected cards unknown (undefined), so they are retried later,
// never frozen half-empty.
import type { PageEnrichment } from '../../shared/enrichment.ts';
import type { ChainArticle, IdentityFacts, IdentitySource, WikidataClass } from './identity.ts';
/** Structural slices of server/wikimedia/client.ts (#73) and shared/duel/profile.ts (#70). */
export type BackgroundRequest = { lane: 'background'; timeoutMs: number; cache?: { ttlMs: number; maxBytes?: number } };
export type WikidataEntity = { id?: string; claims?: Record<string, readonly unknown[]> };
export type ProfileLike = { wikidataId: string | null; description: string | null; categories: readonly string[];
instanceOf: readonly WikidataClass[]; subclassOf?: readonly WikidataClass[]; countries?: readonly string[];
years?: readonly number[]; died?: boolean; links?: readonly number[]; cites?: readonly string[]; fetchedRevisionId?: number };
export type ProfileBuilder = {
wikidataEntityParams(qids: readonly string[]): Record<string, string>;
wikidataLabelParams(qids: readonly string[]): Record<string, string>;
classIdsOf(entity: WikidataEntity | null | undefined): string[];
profileFromSources(src: { enrichment: PageEnrichment; entity?: WikidataEntity | null; labels?: Readonly<Record<string, string>> }): ProfileLike;
};
export type IdentitySourceDeps = {
enrichment: { getBackground(pageId: number): Promise<PageEnrichment> };
client: { request(url: URL, options: BackgroundRequest): Promise<{ response: Response }> };
profile: ProfileBuilder;
};
const WIKIDATA_API = 'https://www.wikidata.org/w/api.php';
const MAX_BYTES = 4 * 1024 * 1024;
const CACHE = { ttlMs: 7 * 24 * 60 * 60 * 1000, maxBytes: MAX_BYTES };
async function wikidata(client: IdentitySourceDeps['client'], params: Record<string, string>): Promise<Record<string, unknown>> {
const url = new URL(WIKIDATA_API);
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
const { response } = await client.request(url, { lane: 'background', timeoutMs: 15000, cache: CACHE });
if (!response.ok) { await response.body?.cancel(); throw new Error(`Wikidata HTTP ${response.status}`); }
const text = await response.text();
if (text.length > MAX_BYTES) throw new Error('Wikidata response too large');
const body = JSON.parse(text) as { entities?: Record<string, unknown> | unknown[] };
const entities = body.entities ?? {};
// formatversion=2 returns an object keyed by id; tolerate an array too.
return Array.isArray(entities)
? Object.fromEntries(entities.filter(e => e && typeof (e as { id?: unknown }).id === 'string').map(e => [(e as { id: string }).id, e]))
: entities as Record<string, unknown>;
}
const chunks = <T>(xs: T[], n: number) => Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, i * n + n));
export function wikimediaIdentitySource(deps: IdentitySourceDeps): IdentitySource {
const { classIdsOf, profileFromSources, wikidataEntityParams, wikidataLabelParams } = deps.profile;
return {
kind: 'wikimedia',
async lookup(articles: ChainArticle[]) {
const out = new Map<number, IdentityFacts | undefined>(articles.map(a => [a.pageId, undefined]));
// 1. Enrichment, one page at a time (the client paces it); failures stay unknown.
const enriched = new Map<number, PageEnrichment>();
for (const a of articles) {
try { enriched.set(a.pageId, await deps.enrichment.getBackground(a.pageId)); } catch { /* retry later */ }
}
// 2. Claims for every item we know, in batches of ≤ 50.
const qids = [...new Set([...enriched.values()].map(e => e.wikidataId).filter((q): q is string => !!q && /^Q\d+$/.test(q)))];
const entities = new Map<string, WikidataEntity>();
let claimsFailed = false;
for (const batch of chunks(qids, 50)) {
try { for (const [id, e] of Object.entries(await wikidata(deps.client, wikidataEntityParams(batch)))) entities.set(id, e as WikidataEntity); }
catch { claimsFailed = true; }
}
// 3. English labels for the P31/P279 classes.
const classIds = [...new Set([...entities.values()].flatMap(e => classIdsOf(e)))];
const labels: Record<string, string> = {};
let labelsFailed = false;
for (const batch of chunks(classIds, 50)) {
try {
for (const [id, e] of Object.entries(await wikidata(deps.client, wikidataLabelParams(batch)))) {
const v = (e as { labels?: { en?: { value?: unknown } } }).labels?.en?.value;
if (typeof v === 'string') labels[id] = v;
}
} catch { labelsFailed = true; }
}
// 4. Freeze only complete profiles: a card whose item couldn't be read stays unknown.
for (const a of articles) {
const e = enriched.get(a.pageId);
if (!e) continue;
const qid = e.wikidataId && /^Q\d+$/.test(e.wikidataId) ? e.wikidataId : null;
const entity = qid ? entities.get(qid) : null;
if (qid && !entity && claimsFailed) continue; // item not read: retry later
if (entity && labelsFailed) continue; // class labels not read: retry later
const p = profileFromSources({ enrichment: e, entity: entity ?? null, labels });
out.set(a.pageId, {
fetchedRevisionId: p.fetchedRevisionId ?? null, wikidataId: p.wikidataId, description: p.description,
categories: [...p.categories], instanceOf: [...p.instanceOf], subclassOf: [...(p.subclassOf ?? [])],
countries: [...(p.countries ?? [])], years: [...(p.years ?? [])], died: p.died === true,
links: [...(p.links ?? [])], cites: [...(p.cites ?? [])],
});
}
return out;
},
};
}
+82
View File
@@ -0,0 +1,82 @@
// testnet-ops: live identity source over a FAKE Wikimedia client. Zero network calls.
import test from 'node:test';
import assert from 'node:assert/strict';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { wikimediaIdentitySource } from '../server/game/identity-source.ts';
import { IdentityStore, prebake, toProfile } from '../server/game/identity.ts';
import type { BackgroundRequest, ProfileBuilder } from '../server/game/identity-source.ts';
// A minimal stand-in for shared/duel/profile.ts (#70), so this test needs no unmerged module.
const ids = (e: any, p: string): string[] => (e?.claims?.[p] ?? []).map((s: any) => s.mainsnak?.datavalue?.value?.id).filter(Boolean);
const yrs = (e: any, p: string): number[] => (e?.claims?.[p] ?? []).map((s: any) => Number(/^[+-]0*(\d+)-/.exec(s.mainsnak?.datavalue?.value?.time ?? '')?.[1])).filter(Boolean);
const profile: ProfileBuilder = {
wikidataEntityParams: q => ({ action: 'wbgetentities', ids: [...q].sort().join('|'), props: 'claims', format: 'json', formatversion: '2' }),
wikidataLabelParams: q => ({ action: 'wbgetentities', ids: [...q].sort().join('|'), props: 'labels', languages: 'en', format: 'json', formatversion: '2' }),
classIdsOf: e => [...new Set([...ids(e, 'P31'), ...ids(e, 'P279')])].sort(),
profileFromSources: ({ enrichment, entity, labels = {} }) => ({
wikidataId: enrichment.wikidataId, description: enrichment.description, categories: enrichment.categories,
instanceOf: ids(entity, 'P31').map(id => ({ id, label: labels[id] ?? id })), countries: ids(entity, 'P27'),
years: [...yrs(entity, 'P569'), ...yrs(entity, 'P570')], died: ids(entity, 'P570').length > 0 || yrs(entity, 'P570').length > 0,
fetchedRevisionId: enrichment.revisionId }),
};
const enrichmentFor = (pageId: number, wikidataId: string | null) => ({
pageId, title: `Page ${pageId}`, description: `desc ${pageId}`, wikidataId, revisionId: 1000 + pageId, pageBytes: 1,
languageCount: 1, languagesTruncated: false, categories: ['Living people', '1948 births'], categoriesTruncated: false,
fetchedAt: '2026-11-17T00:00:00Z',
pageviews: { status: 'unavailable', months: [], latestMonth: '2026-10', total: null, dailyAverage: null, trendPercent: null },
}) as any;
const claim = (id: string) => ({ rank: 'normal', mainsnak: { snaktype: 'value', datavalue: { type: 'wikibase-entityid', value: { id } } } });
const time = (t: string) => ({ rank: 'normal', mainsnak: { snaktype: 'value', datavalue: { type: 'time', value: { time: t } } } });
function fakeClient(opts: { failClaims?: boolean } = {}) {
const calls: { url: URL; options: BackgroundRequest }[] = [];
return { calls, async request(url: URL, options: BackgroundRequest) {
calls.push({ url, options });
const props = url.searchParams.get('props'), ids = url.searchParams.get('ids')!.split('|');
if (props === 'claims' && opts.failClaims) return { response: new Response('busy', { status: 503 }), signal: new AbortController().signal, fromCache: false };
const entities = Object.fromEntries(ids.map(id => [id, props === 'claims'
? { id, claims: { P31: [claim('Q5')], P27: [claim('Q30')], P569: [time('+1948-01-02T00:00:00Z')], P570: [time('+2001-01-01T00:00:00Z')] } }
: { id, labels: { en: { value: id === 'Q5' ? 'human' : `label ${id}` } } }]));
return { response: Response.json({ entities }), signal: new AbortController().signal, fromCache: false };
} };
}
test('every request is on the background lane, batched, and the frozen identity is derive-ready', async () => {
const realFetch = globalThis.fetch;
globalThis.fetch = (() => { throw new Error('network call in test'); }) as typeof fetch;
const dir = mkdtempSync(join(tmpdir(), 'identity-src-'));
try {
const client = fakeClient();
const source = wikimediaIdentitySource({ client, profile, enrichment: { async getBackground(id) { return enrichmentFor(id, `Q${id}`); } } });
const store = new IdentityStore(join(dir, 'id.sqlite'));
const articles = [1, 2, 3].map(i => ({ tokenId: String(i).repeat(64), pageId: i, title: `Page ${i}`, revisionId: i }));
const r = await prebake(store, source, articles);
assert.deepEqual(r, { seen: 3, alreadyFrozen: 0, frozen: 3, unknown: 0 });
assert.ok(client.calls.every(c => c.options.lane === 'background'));
assert.ok(client.calls.every(c => c.url.hostname === 'www.wikidata.org'));
assert.deepEqual(client.calls.map(c => c.url.searchParams.get('props')), ['claims', 'labels']);
assert.equal(client.calls[0].url.searchParams.get('ids'), 'Q1|Q2|Q3');
const one = store.get('1'.repeat(64))!;
assert.equal(one.source, 'wikimedia');
assert.deepEqual(one.instanceOf, [{ id: 'Q5', label: 'human' }]);
assert.deepEqual(one.countries, ['Q30']);
assert.deepEqual(one.years, [1948, 2001]);
assert.equal(one.died, true);
assert.equal(one.fetchedRevisionId, 1001);
assert.equal(toProfile(one).description, 'desc 1');
store.close();
} finally { globalThis.fetch = realFetch; rmSync(dir, { recursive: true, force: true }); }
});
test('a failed claims request leaves cards unknown so they are retried, never frozen half-empty', async () => {
const source = wikimediaIdentitySource({ client: fakeClient({ failClaims: true }), profile,
enrichment: { async getBackground(id) { if (id === 9) throw new Error('throttled'); return enrichmentFor(id, id === 8 ? null : `Q${id}`); } } });
const got = await source.lookup([7, 8, 9].map(i => ({ tokenId: String(i).repeat(64), pageId: i, title: `P${i}` })));
assert.equal(got.get(7), undefined); // has an item, claims failed → retry
assert.equal(got.get(9), undefined); // enrichment failed → retry
assert.equal(got.get(8)?.wikidataId, null); // no Wikidata item at all → enrichment-only profile is final
assert.deepEqual(got.get(8)?.instanceOf, []);
});