Files
wikideck/tests/wikimedia-mint.test.ts
claudeandClaude Opus 5.5 f544019bfe Polite Wikimedia client, mint-path fix, disk cache and mining prefetch pool (#62)
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket
  (1 req/s, burst 3), 1 in flight per host, mint > interactive > background
  lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After
  and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and
  a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP).
- server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match,
  stale-while-revalidate on the background lane. Mint reads never use it.
- Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh()
  (template 503, miner retries) instead of minting attention: null forever.
- server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for
  /api/mining/template, about 3 requests per block regardless of miner count.
- User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or
  PUBLIC_ORIGIN) and a start-up warning when it is unset.
- Mint request parameters are byte-identical apart from maxlag (golden test).
  No consensus, block or protocol changes. Tests use fake fetch only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 21:51:20 -05:00

105 lines
6.0 KiB
TypeScript

import test from 'node:test';
import assert from 'node:assert/strict';
import { WikipediaEnrichment } from '../server/enrichment.ts';
import { WikipediaArticleSource, randomArticlesUrl } from '../server/wiki.ts';
import { WikimediaClient } from '../server/wikimedia/client.ts';
import { WikimediaCache } from '../server/wikimedia/cache.ts';
import { UNTHROTTLED } from '../server/wikimedia/client.ts';
import { json } from './wikimedia-helpers.ts';
const JANUARY = new Date('2025-01-15T23:59:59Z');
const MONTHS = ['2024-07', '2024-08', '2024-09', '2024-10', '2024-11', '2024-12'];
const metadata = (pageId = 736) => ({ query: { pages: [{ pageid: pageId, ns: 0, title: 'Albert Einstein', lastrevid: 1234,
length: 100_000, pageprops: { wikibase_item: 'Q937' }, categories: [], langlinks: [{ lang: 'de' }] }] } });
const views = (counts = [10, 20, 30, 40, 50, 100]) => ({ items: MONTHS.map((month, i) => ({ timestamp: `${month.replace('-', '')}0100`,
views: counts[i], project: 'en.wikipedia', access: 'all-access', agent: 'user', granularity: 'monthly' })) });
function service(viewsReply: () => Response | Promise<Response>, cache?: WikimediaCache) {
const urls: URL[] = [];
const fetcher: typeof fetch = async input => {
const url = new URL(String(input));
urls.push(url);
return url.hostname === 'en.wikipedia.org' ? json(metadata(Number(url.searchParams.get('pageids')))) : viewsReply();
};
const client = new WikimediaClient({ fetch: fetcher, config: UNTHROTTLED, cache });
return { urls, client, enrichment: new WikipediaEnrichment({ client, now: () => JANUARY }) };
}
test('mint fix: a 429, 5xx, timeout or network failure on pageviews fails the mint read instead of pinning attention: null', async () => {
const failures: (() => Response | Promise<Response>)[] = [
() => new Response('', { status: 429, headers: { 'Retry-After': '0' } }),
() => new Response('', { status: 503 }),
() => new Response('', { status: 500 }),
() => new Response('<html>busy</html>', { headers: { 'Content-Type': 'text/html' } }),
() => { throw new TypeError('fetch failed'); },
() => Promise.reject(new DOMException('The operation timed out.', 'TimeoutError')),
];
for (const failure of failures) {
const { enrichment } = service(failure);
await assert.rejects(enrichment.getFresh(736), /unavailable/);
// Display reads keep their old behaviour: metadata with unavailable views.
const shown = await enrichment.get(736);
assert.equal(shown.pageviews.status, 'unavailable');
assert.equal(shown.stats.attention, null);
}
});
test('mint records unavailable views only for a genuine 404 or missing data', async () => {
for (const reply of [() => json({ type: 'not found' }, {}, 404), () => json({ items: [] }), () => json({ items: views().items.slice(1) })]) {
const { enrichment } = service(reply);
const minted = await enrichment.getFresh(736);
assert.equal(minted.pageviews.status, 'unavailable');
assert.equal(minted.stats.attention, null);
}
const { enrichment } = service(() => json(views()));
assert.equal((await enrichment.getFresh(736)).stats.attention, 33);
});
test('mint reads never use the disk cache or join a display read', async () => {
const cache = new WikimediaCache(':memory:');
let count = 100;
const { enrichment, urls } = service(() => json(views([1, 1, 1, 1, 1, count++])), cache);
const shown = await enrichment.get(736);
assert.equal(cache.size(), 2);
const [display, minted] = await Promise.all([enrichment.get(736), enrichment.getFresh(736)]);
assert.equal(display.pageviews.months[5].views, shown.pageviews.months[5].views);
assert.equal(minted.pageviews.months[5].views, 101); // Fetched again, not the cached 100.
assert.equal(urls.length, 4);
// A later display read (memory cache cleared by a new service) is served from disk: no upstream call.
const second = new WikipediaEnrichment({ client: new WikimediaClient({ fetch: async () => { throw new Error('no network'); },
config: UNTHROTTLED, cache }), now: () => JANUARY });
assert.equal((await second.get(736)).pageviews.months[5].views, 100);
});
test('golden mint URLs: request parameters are byte-identical apart from the added maxlag', async () => {
assert.equal(randomArticlesUrl().toString(), 'https://en.wikipedia.org/w/api.php?action=query&format=json&formatversion=2'
+ '&generator=random&grnnamespace=0&grnlimit=10&prop=extracts%7Cpageimages%7Cinfo%7Ccategories&exintro=1&explaintext=1'
+ '&exchars=900&piprop=thumbnail&pithumbsize=800&cllimit=20');
const urls: string[] = [];
const fetcher: typeof fetch = async input => {
const url = new URL(String(input));
urls.push(url.toString());
if (url.searchParams.get('generator') === 'random') return json({ query: { pages: [{ pageid: 9, ns: 0, title: 'T',
extract: 'An extract that is comfortably longer than forty characters.', lastrevid: 2,
thumbnail: { source: 'https://upload.wikimedia.org/a.jpg' } }] } });
return url.hostname === 'en.wikipedia.org' ? json(metadata(9)) : json(views());
};
const client = new WikimediaClient({ fetch: fetcher, config: UNTHROTTLED });
const article = await new WikipediaArticleSource(client).random();
await new WikipediaEnrichment({ client, now: () => JANUARY }).getFresh(article.pageId);
assert.deepEqual(urls, [
`${randomArticlesUrl()}&maxlag=5`,
'https://en.wikipedia.org/w/api.php?action=query&format=json&formatversion=2&pageids=9'
+ '&prop=info%7Ccategories%7Cpageprops%7Cdescription%7Clanglinks&clshow=%21hidden&cllimit=50&lllimit=500'
+ '&ppprop=wikibase_item&maxlag=5',
'https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/Albert_Einstein/monthly/2024070100/2024120100',
]);
});
test('a throttled upstream stops random() at once instead of retrying three times', async () => {
let calls = 0;
const source = new WikipediaArticleSource(async () => { calls++; return new Response('', { status: 429 }); });
await assert.rejects(source.random(), /HTTP 429/);
assert.equal(calls, 1);
});