forked from WikiDeck/wikideck
testnet-ops: live identity source (background lane only), deps injected structurally
wikimediaIdentitySource() takes the wikimedia client (#73), the page enrichment and duel-cards' profile builder (#70) as structural deps, so it imports nothing unmerged. About 2 Wikidata requests per 50 new cards; transport failures leave cards unknown. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,112 @@
|
||||
// Live IdentitySource for server/game/identity.ts (engineer WS E, data §3).
|
||||
//
|
||||
// Every request goes through the process-wide polite Wikimedia client on the **background** lane:
|
||||
// - the page enrichment (description, wikidataId, categories) via `enrichment.getBackground()`,
|
||||
// which also feeds the disk cache card faces use;
|
||||
// - one `wbgetentities&props=claims` per batch of at most 50 items;
|
||||
// - one `wbgetentities&props=labels` per 50 class ids still missing a label.
|
||||
// That is about 2 Wikidata requests per 50 new cards, plus one cached enrichment per card.
|
||||
// Outlinks (prop=links, data §4) are not fetched tonight: they cost about 1.2 requests per card,
|
||||
// and Connected falls back to its other tiers without them.
|
||||
//
|
||||
// The profile itself is built by duel-cards' pure `profileFromSources()` (shared/duel/profile.ts, #70),
|
||||
// so the frozen identity and what derive() reads cannot drift apart. That module, and the client
|
||||
// (server/wikimedia, #73), are injected structurally, so this file has no import from unmerged PRs.
|
||||
// Wiring, once both have merged:
|
||||
// wikimediaIdentitySource({ client: wikimediaClient, enrichment: wikipediaEnrichment,
|
||||
// profile: await import('../../shared/duel/profile.ts') })
|
||||
// Any transport failure leaves the affected cards unknown (undefined), so they are retried later,
|
||||
// never frozen half-empty.
|
||||
import type { PageEnrichment } from '../../shared/enrichment.ts';
|
||||
import type { ChainArticle, IdentityFacts, IdentitySource, WikidataClass } from './identity.ts';
|
||||
|
||||
/** Structural slices of server/wikimedia/client.ts (#73) and shared/duel/profile.ts (#70). */
|
||||
export type BackgroundRequest = { lane: 'background'; timeoutMs: number; cache?: { ttlMs: number; maxBytes?: number } };
|
||||
export type WikidataEntity = { id?: string; claims?: Record<string, readonly unknown[]> };
|
||||
export type ProfileLike = { wikidataId: string | null; description: string | null; categories: readonly string[];
|
||||
instanceOf: readonly WikidataClass[]; subclassOf?: readonly WikidataClass[]; countries?: readonly string[];
|
||||
years?: readonly number[]; died?: boolean; links?: readonly number[]; cites?: readonly string[]; fetchedRevisionId?: number };
|
||||
export type ProfileBuilder = {
|
||||
wikidataEntityParams(qids: readonly string[]): Record<string, string>;
|
||||
wikidataLabelParams(qids: readonly string[]): Record<string, string>;
|
||||
classIdsOf(entity: WikidataEntity | null | undefined): string[];
|
||||
profileFromSources(src: { enrichment: PageEnrichment; entity?: WikidataEntity | null; labels?: Readonly<Record<string, string>> }): ProfileLike;
|
||||
};
|
||||
export type IdentitySourceDeps = {
|
||||
enrichment: { getBackground(pageId: number): Promise<PageEnrichment> };
|
||||
client: { request(url: URL, options: BackgroundRequest): Promise<{ response: Response }> };
|
||||
profile: ProfileBuilder;
|
||||
};
|
||||
|
||||
const WIKIDATA_API = 'https://www.wikidata.org/w/api.php';
|
||||
const MAX_BYTES = 4 * 1024 * 1024;
|
||||
const CACHE = { ttlMs: 7 * 24 * 60 * 60 * 1000, maxBytes: MAX_BYTES };
|
||||
|
||||
async function wikidata(client: IdentitySourceDeps['client'], params: Record<string, string>): Promise<Record<string, unknown>> {
|
||||
const url = new URL(WIKIDATA_API);
|
||||
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
|
||||
const { response } = await client.request(url, { lane: 'background', timeoutMs: 15000, cache: CACHE });
|
||||
if (!response.ok) { await response.body?.cancel(); throw new Error(`Wikidata HTTP ${response.status}`); }
|
||||
const text = await response.text();
|
||||
if (text.length > MAX_BYTES) throw new Error('Wikidata response too large');
|
||||
const body = JSON.parse(text) as { entities?: Record<string, unknown> | unknown[] };
|
||||
const entities = body.entities ?? {};
|
||||
// formatversion=2 returns an object keyed by id; tolerate an array too.
|
||||
return Array.isArray(entities)
|
||||
? Object.fromEntries(entities.filter(e => e && typeof (e as { id?: unknown }).id === 'string').map(e => [(e as { id: string }).id, e]))
|
||||
: entities as Record<string, unknown>;
|
||||
}
|
||||
|
||||
const chunks = <T>(xs: T[], n: number) => Array.from({ length: Math.ceil(xs.length / n) }, (_, i) => xs.slice(i * n, i * n + n));
|
||||
|
||||
export function wikimediaIdentitySource(deps: IdentitySourceDeps): IdentitySource {
|
||||
const { classIdsOf, profileFromSources, wikidataEntityParams, wikidataLabelParams } = deps.profile;
|
||||
return {
|
||||
kind: 'wikimedia',
|
||||
async lookup(articles: ChainArticle[]) {
|
||||
const out = new Map<number, IdentityFacts | undefined>(articles.map(a => [a.pageId, undefined]));
|
||||
// 1. Enrichment, one page at a time (the client paces it); failures stay unknown.
|
||||
const enriched = new Map<number, PageEnrichment>();
|
||||
for (const a of articles) {
|
||||
try { enriched.set(a.pageId, await deps.enrichment.getBackground(a.pageId)); } catch { /* retry later */ }
|
||||
}
|
||||
// 2. Claims for every item we know, in batches of ≤ 50.
|
||||
const qids = [...new Set([...enriched.values()].map(e => e.wikidataId).filter((q): q is string => !!q && /^Q\d+$/.test(q)))];
|
||||
const entities = new Map<string, WikidataEntity>();
|
||||
let claimsFailed = false;
|
||||
for (const batch of chunks(qids, 50)) {
|
||||
try { for (const [id, e] of Object.entries(await wikidata(deps.client, wikidataEntityParams(batch)))) entities.set(id, e as WikidataEntity); }
|
||||
catch { claimsFailed = true; }
|
||||
}
|
||||
// 3. English labels for the P31/P279 classes.
|
||||
const classIds = [...new Set([...entities.values()].flatMap(e => classIdsOf(e)))];
|
||||
const labels: Record<string, string> = {};
|
||||
let labelsFailed = false;
|
||||
for (const batch of chunks(classIds, 50)) {
|
||||
try {
|
||||
for (const [id, e] of Object.entries(await wikidata(deps.client, wikidataLabelParams(batch)))) {
|
||||
const v = (e as { labels?: { en?: { value?: unknown } } }).labels?.en?.value;
|
||||
if (typeof v === 'string') labels[id] = v;
|
||||
}
|
||||
} catch { labelsFailed = true; }
|
||||
}
|
||||
// 4. Freeze only complete profiles: a card whose item couldn't be read stays unknown.
|
||||
for (const a of articles) {
|
||||
const e = enriched.get(a.pageId);
|
||||
if (!e) continue;
|
||||
const qid = e.wikidataId && /^Q\d+$/.test(e.wikidataId) ? e.wikidataId : null;
|
||||
const entity = qid ? entities.get(qid) : null;
|
||||
if (qid && !entity && claimsFailed) continue; // item not read: retry later
|
||||
if (entity && labelsFailed) continue; // class labels not read: retry later
|
||||
const p = profileFromSources({ enrichment: e, entity: entity ?? null, labels });
|
||||
out.set(a.pageId, {
|
||||
fetchedRevisionId: p.fetchedRevisionId ?? null, wikidataId: p.wikidataId, description: p.description,
|
||||
categories: [...p.categories], instanceOf: [...p.instanceOf], subclassOf: [...(p.subclassOf ?? [])],
|
||||
countries: [...(p.countries ?? [])], years: [...(p.years ?? [])], died: p.died === true,
|
||||
links: [...(p.links ?? [])], cites: [...(p.cites ?? [])],
|
||||
});
|
||||
}
|
||||
return out;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// testnet-ops: live identity source over a FAKE Wikimedia client. Zero network calls.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { wikimediaIdentitySource } from '../server/game/identity-source.ts';
|
||||
import { IdentityStore, prebake, toProfile } from '../server/game/identity.ts';
|
||||
import type { BackgroundRequest, ProfileBuilder } from '../server/game/identity-source.ts';
|
||||
|
||||
// A minimal stand-in for shared/duel/profile.ts (#70), so this test needs no unmerged module.
|
||||
const ids = (e: any, p: string): string[] => (e?.claims?.[p] ?? []).map((s: any) => s.mainsnak?.datavalue?.value?.id).filter(Boolean);
|
||||
const yrs = (e: any, p: string): number[] => (e?.claims?.[p] ?? []).map((s: any) => Number(/^[+-]0*(\d+)-/.exec(s.mainsnak?.datavalue?.value?.time ?? '')?.[1])).filter(Boolean);
|
||||
const profile: ProfileBuilder = {
|
||||
wikidataEntityParams: q => ({ action: 'wbgetentities', ids: [...q].sort().join('|'), props: 'claims', format: 'json', formatversion: '2' }),
|
||||
wikidataLabelParams: q => ({ action: 'wbgetentities', ids: [...q].sort().join('|'), props: 'labels', languages: 'en', format: 'json', formatversion: '2' }),
|
||||
classIdsOf: e => [...new Set([...ids(e, 'P31'), ...ids(e, 'P279')])].sort(),
|
||||
profileFromSources: ({ enrichment, entity, labels = {} }) => ({
|
||||
wikidataId: enrichment.wikidataId, description: enrichment.description, categories: enrichment.categories,
|
||||
instanceOf: ids(entity, 'P31').map(id => ({ id, label: labels[id] ?? id })), countries: ids(entity, 'P27'),
|
||||
years: [...yrs(entity, 'P569'), ...yrs(entity, 'P570')], died: ids(entity, 'P570').length > 0 || yrs(entity, 'P570').length > 0,
|
||||
fetchedRevisionId: enrichment.revisionId }),
|
||||
};
|
||||
|
||||
const enrichmentFor = (pageId: number, wikidataId: string | null) => ({
|
||||
pageId, title: `Page ${pageId}`, description: `desc ${pageId}`, wikidataId, revisionId: 1000 + pageId, pageBytes: 1,
|
||||
languageCount: 1, languagesTruncated: false, categories: ['Living people', '1948 births'], categoriesTruncated: false,
|
||||
fetchedAt: '2026-11-17T00:00:00Z',
|
||||
pageviews: { status: 'unavailable', months: [], latestMonth: '2026-10', total: null, dailyAverage: null, trendPercent: null },
|
||||
}) as any;
|
||||
const claim = (id: string) => ({ rank: 'normal', mainsnak: { snaktype: 'value', datavalue: { type: 'wikibase-entityid', value: { id } } } });
|
||||
const time = (t: string) => ({ rank: 'normal', mainsnak: { snaktype: 'value', datavalue: { type: 'time', value: { time: t } } } });
|
||||
|
||||
function fakeClient(opts: { failClaims?: boolean } = {}) {
|
||||
const calls: { url: URL; options: BackgroundRequest }[] = [];
|
||||
return { calls, async request(url: URL, options: BackgroundRequest) {
|
||||
calls.push({ url, options });
|
||||
const props = url.searchParams.get('props'), ids = url.searchParams.get('ids')!.split('|');
|
||||
if (props === 'claims' && opts.failClaims) return { response: new Response('busy', { status: 503 }), signal: new AbortController().signal, fromCache: false };
|
||||
const entities = Object.fromEntries(ids.map(id => [id, props === 'claims'
|
||||
? { id, claims: { P31: [claim('Q5')], P27: [claim('Q30')], P569: [time('+1948-01-02T00:00:00Z')], P570: [time('+2001-01-01T00:00:00Z')] } }
|
||||
: { id, labels: { en: { value: id === 'Q5' ? 'human' : `label ${id}` } } }]));
|
||||
return { response: Response.json({ entities }), signal: new AbortController().signal, fromCache: false };
|
||||
} };
|
||||
}
|
||||
|
||||
test('every request is on the background lane, batched, and the frozen identity is derive-ready', async () => {
|
||||
const realFetch = globalThis.fetch;
|
||||
globalThis.fetch = (() => { throw new Error('network call in test'); }) as typeof fetch;
|
||||
const dir = mkdtempSync(join(tmpdir(), 'identity-src-'));
|
||||
try {
|
||||
const client = fakeClient();
|
||||
const source = wikimediaIdentitySource({ client, profile, enrichment: { async getBackground(id) { return enrichmentFor(id, `Q${id}`); } } });
|
||||
const store = new IdentityStore(join(dir, 'id.sqlite'));
|
||||
const articles = [1, 2, 3].map(i => ({ tokenId: String(i).repeat(64), pageId: i, title: `Page ${i}`, revisionId: i }));
|
||||
const r = await prebake(store, source, articles);
|
||||
assert.deepEqual(r, { seen: 3, alreadyFrozen: 0, frozen: 3, unknown: 0 });
|
||||
assert.ok(client.calls.every(c => c.options.lane === 'background'));
|
||||
assert.ok(client.calls.every(c => c.url.hostname === 'www.wikidata.org'));
|
||||
assert.deepEqual(client.calls.map(c => c.url.searchParams.get('props')), ['claims', 'labels']);
|
||||
assert.equal(client.calls[0].url.searchParams.get('ids'), 'Q1|Q2|Q3');
|
||||
const one = store.get('1'.repeat(64))!;
|
||||
assert.equal(one.source, 'wikimedia');
|
||||
assert.deepEqual(one.instanceOf, [{ id: 'Q5', label: 'human' }]);
|
||||
assert.deepEqual(one.countries, ['Q30']);
|
||||
assert.deepEqual(one.years, [1948, 2001]);
|
||||
assert.equal(one.died, true);
|
||||
assert.equal(one.fetchedRevisionId, 1001);
|
||||
assert.equal(toProfile(one).description, 'desc 1');
|
||||
store.close();
|
||||
} finally { globalThis.fetch = realFetch; rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('a failed claims request leaves cards unknown so they are retried, never frozen half-empty', async () => {
|
||||
const source = wikimediaIdentitySource({ client: fakeClient({ failClaims: true }), profile,
|
||||
enrichment: { async getBackground(id) { if (id === 9) throw new Error('throttled'); return enrichmentFor(id, id === 8 ? null : `Q${id}`); } } });
|
||||
const got = await source.lookup([7, 8, 9].map(i => ({ tokenId: String(i).repeat(64), pageId: i, title: `P${i}` })));
|
||||
assert.equal(got.get(7), undefined); // has an item, claims failed → retry
|
||||
assert.equal(got.get(9), undefined); // enrichment failed → retry
|
||||
assert.equal(got.get(8)?.wikidataId, null); // no Wikidata item at all → enrichment-only profile is final
|
||||
assert.deepEqual(got.get(8)?.instanceOf, []);
|
||||
});
|
||||
Reference in New Issue
Block a user