forked from WikiDeck/wikideck
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket (1 req/s, burst 3), 1 in flight per host, mint > interactive > background lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP). - server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match, stale-while-revalidate on the background lane. Mint reads never use it. - Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh() (template 503, miner retries) instead of minting attention: null forever. - server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for /api/mining/template, about 3 requests per block regardless of miner count. - User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or PUBLIC_ORIGIN) and a start-up warning when it is unset. - Mint request parameters are byte-identical apart from maxlag (golden test). No consensus, block or protocol changes. Tests use fake fetch only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
105 lines
6.0 KiB
TypeScript
105 lines
6.0 KiB
TypeScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { WikipediaEnrichment } from '../server/enrichment.ts';
|
|
import { WikipediaArticleSource, randomArticlesUrl } from '../server/wiki.ts';
|
|
import { WikimediaClient } from '../server/wikimedia/client.ts';
|
|
import { WikimediaCache } from '../server/wikimedia/cache.ts';
|
|
import { UNTHROTTLED } from '../server/wikimedia/client.ts';
|
|
import { json } from './wikimedia-helpers.ts';
|
|
|
|
const JANUARY = new Date('2025-01-15T23:59:59Z');
|
|
const MONTHS = ['2024-07', '2024-08', '2024-09', '2024-10', '2024-11', '2024-12'];
|
|
const metadata = (pageId = 736) => ({ query: { pages: [{ pageid: pageId, ns: 0, title: 'Albert Einstein', lastrevid: 1234,
|
|
length: 100_000, pageprops: { wikibase_item: 'Q937' }, categories: [], langlinks: [{ lang: 'de' }] }] } });
|
|
const views = (counts = [10, 20, 30, 40, 50, 100]) => ({ items: MONTHS.map((month, i) => ({ timestamp: `${month.replace('-', '')}0100`,
|
|
views: counts[i], project: 'en.wikipedia', access: 'all-access', agent: 'user', granularity: 'monthly' })) });
|
|
|
|
function service(viewsReply: () => Response | Promise<Response>, cache?: WikimediaCache) {
|
|
const urls: URL[] = [];
|
|
const fetcher: typeof fetch = async input => {
|
|
const url = new URL(String(input));
|
|
urls.push(url);
|
|
return url.hostname === 'en.wikipedia.org' ? json(metadata(Number(url.searchParams.get('pageids')))) : viewsReply();
|
|
};
|
|
const client = new WikimediaClient({ fetch: fetcher, config: UNTHROTTLED, cache });
|
|
return { urls, client, enrichment: new WikipediaEnrichment({ client, now: () => JANUARY }) };
|
|
}
|
|
|
|
test('mint fix: a 429, 5xx, timeout or network failure on pageviews fails the mint read instead of pinning attention: null', async () => {
|
|
const failures: (() => Response | Promise<Response>)[] = [
|
|
() => new Response('', { status: 429, headers: { 'Retry-After': '0' } }),
|
|
() => new Response('', { status: 503 }),
|
|
() => new Response('', { status: 500 }),
|
|
() => new Response('<html>busy</html>', { headers: { 'Content-Type': 'text/html' } }),
|
|
() => { throw new TypeError('fetch failed'); },
|
|
() => Promise.reject(new DOMException('The operation timed out.', 'TimeoutError')),
|
|
];
|
|
for (const failure of failures) {
|
|
const { enrichment } = service(failure);
|
|
await assert.rejects(enrichment.getFresh(736), /unavailable/);
|
|
// Display reads keep their old behaviour: metadata with unavailable views.
|
|
const shown = await enrichment.get(736);
|
|
assert.equal(shown.pageviews.status, 'unavailable');
|
|
assert.equal(shown.stats.attention, null);
|
|
}
|
|
});
|
|
|
|
test('mint records unavailable views only for a genuine 404 or missing data', async () => {
|
|
for (const reply of [() => json({ type: 'not found' }, {}, 404), () => json({ items: [] }), () => json({ items: views().items.slice(1) })]) {
|
|
const { enrichment } = service(reply);
|
|
const minted = await enrichment.getFresh(736);
|
|
assert.equal(minted.pageviews.status, 'unavailable');
|
|
assert.equal(minted.stats.attention, null);
|
|
}
|
|
const { enrichment } = service(() => json(views()));
|
|
assert.equal((await enrichment.getFresh(736)).stats.attention, 33);
|
|
});
|
|
|
|
test('mint reads never use the disk cache or join a display read', async () => {
|
|
const cache = new WikimediaCache(':memory:');
|
|
let count = 100;
|
|
const { enrichment, urls } = service(() => json(views([1, 1, 1, 1, 1, count++])), cache);
|
|
const shown = await enrichment.get(736);
|
|
assert.equal(cache.size(), 2);
|
|
const [display, minted] = await Promise.all([enrichment.get(736), enrichment.getFresh(736)]);
|
|
assert.equal(display.pageviews.months[5].views, shown.pageviews.months[5].views);
|
|
assert.equal(minted.pageviews.months[5].views, 101); // Fetched again, not the cached 100.
|
|
assert.equal(urls.length, 4);
|
|
// A later display read (memory cache cleared by a new service) is served from disk: no upstream call.
|
|
const second = new WikipediaEnrichment({ client: new WikimediaClient({ fetch: async () => { throw new Error('no network'); },
|
|
config: UNTHROTTLED, cache }), now: () => JANUARY });
|
|
assert.equal((await second.get(736)).pageviews.months[5].views, 100);
|
|
});
|
|
|
|
test('golden mint URLs: request parameters are byte-identical apart from the added maxlag', async () => {
|
|
assert.equal(randomArticlesUrl().toString(), 'https://en.wikipedia.org/w/api.php?action=query&format=json&formatversion=2'
|
|
+ '&generator=random&grnnamespace=0&grnlimit=10&prop=extracts%7Cpageimages%7Cinfo%7Ccategories&exintro=1&explaintext=1'
|
|
+ '&exchars=900&piprop=thumbnail&pithumbsize=800&cllimit=20');
|
|
const urls: string[] = [];
|
|
const fetcher: typeof fetch = async input => {
|
|
const url = new URL(String(input));
|
|
urls.push(url.toString());
|
|
if (url.searchParams.get('generator') === 'random') return json({ query: { pages: [{ pageid: 9, ns: 0, title: 'T',
|
|
extract: 'An extract that is comfortably longer than forty characters.', lastrevid: 2,
|
|
thumbnail: { source: 'https://upload.wikimedia.org/a.jpg' } }] } });
|
|
return url.hostname === 'en.wikipedia.org' ? json(metadata(9)) : json(views());
|
|
};
|
|
const client = new WikimediaClient({ fetch: fetcher, config: UNTHROTTLED });
|
|
const article = await new WikipediaArticleSource(client).random();
|
|
await new WikipediaEnrichment({ client, now: () => JANUARY }).getFresh(article.pageId);
|
|
assert.deepEqual(urls, [
|
|
`${randomArticlesUrl()}&maxlag=5`,
|
|
'https://en.wikipedia.org/w/api.php?action=query&format=json&formatversion=2&pageids=9'
|
|
+ '&prop=info%7Ccategories%7Cpageprops%7Cdescription%7Clanglinks&clshow=%21hidden&cllimit=50&lllimit=500'
|
|
+ '&ppprop=wikibase_item&maxlag=5',
|
|
'https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia.org/all-access/user/Albert_Einstein/monthly/2024070100/2024120100',
|
|
]);
|
|
});
|
|
|
|
test('a throttled upstream stops random() at once instead of retrying three times', async () => {
|
|
let calls = 0;
|
|
const source = new WikipediaArticleSource(async () => { calls++; return new Response('', { status: 429 }); });
|
|
await assert.rejects(source.random(), /HTTP 429/);
|
|
assert.equal(calls, 1);
|
|
});
|