Files
wikideck/tests/wikimedia-pool.test.ts
claudeandClaude Opus 5.5 f544019bfe Polite Wikimedia client, mint-path fix, disk cache and mining prefetch pool (#62)
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket
  (1 req/s, burst 3), 1 in flight per host, mint > interactive > background
  lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After
  and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and
  a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP).
- server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match,
  stale-while-revalidate on the background lane. Mint reads never use it.
- Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh()
  (template 503, miner retries) instead of minting attention: null forever.
- server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for
  /api/mining/template, about 3 requests per block regardless of miner count.
- User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or
  PUBLIC_ORIGIN) and a start-up warning when it is unset.
- Mint request parameters are byte-identical apart from maxlag (golden test).
  No consensus, block or protocol changes. Tests use fake fetch only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 21:51:20 -05:00

75 lines
3.8 KiB
TypeScript

import test from 'node:test';
import assert from 'node:assert/strict';
import { MintTemplatePool } from '../server/wikimedia/pool.ts';
import type { Article } from '../shared/protocol.ts';
import type { PageEnrichment } from '../shared/enrichment.ts';
function article(pageId: number): Article {
return { pageId, title: `Page ${pageId}`, extract: 'An extract that is comfortably longer than forty characters.',
image: 'https://upload.wikimedia.org/a.jpg', url: `https://en.wikipedia.org/?curid=${pageId}`, revisionId: pageId + 1,
source: 'live', category: 'Culture' };
}
function enrichment(pageId: number): PageEnrichment {
return { pageId, title: `Page ${pageId}`, description: null, wikidataId: null, revisionId: pageId + 1, pageBytes: 10,
languageCount: 1, languagesTruncated: false, categories: [], categoriesTruncated: false, fetchedAt: '2026-11-17T00:00:00Z',
pageviews: { status: 'available', months: [], latestMonth: '2026-10', total: 1, dailyAverage: 1, trendPercent: null },
stats: { attention: 1, depth: 1, reach: 1 } };
}
function fixture(options: { size?: number; maxAgeMs?: number } = {}) {
let next = 1, now = 0, fail = false;
const log: string[] = [];
const sources = {
articles: { random: async () => { log.push('random'); if (fail) throw new Error('Wikimedia unavailable: HTTP 429'); return article(next++); } },
enrichment: { getFresh: async (id: number) => { log.push(`fresh:${id}`); return enrichment(id); } },
};
const pool = new MintTemplatePool(sources, { now: () => now, ...options });
return { pool, log, advance: (ms: number) => { now += ms; }, setFail: (value: boolean) => { fail = value; } };
}
test('every template on one tip shares one article; a new tip gets a different one', async () => {
const { pool, log } = fixture({ size: 1 });
const first = await Promise.all(Array.from({ length: 20 }, () => pool.take('tip-a')));
assert.ok(first.every(pair => pair.article.pageId === first[0].article.pageId));
await pool.refill();
const requestsForTwentyMiners = log.length;
assert.equal(requestsForTwentyMiners, 4); // One direct pair, one prefetched pair: 2 calls each.
const second = await pool.take('tip-b');
assert.notEqual(second.article.pageId, first[0].article.pageId);
assert.equal(second.article.pageId, 2); // Served from the prefetched pair, no wait.
await pool.refill();
assert.equal(log.length, 6); // ...and that use alone triggers the next prefetch.
// The stats snapshot comes from a mint-path read of the same page.
assert.equal(second.statsSnapshot.revisionId, second.article.pageId + 1);
});
test('pairs older than maxAge are dropped, never served', async () => {
const { pool, advance } = fixture({ size: 1, maxAgeMs: 1000 });
const a = await pool.take('t1');
await pool.refill();
advance(1000);
const b = await pool.take('t1'); // Same tip, but its pair expired; the ready pair expired too.
assert.notEqual(b.article.pageId, a.article.pageId);
assert.equal(pool.status().expired, 1);
});
test('an idle pool makes no requests, and a failure rejects the template without a retry loop', async () => {
const { pool, log, setFail } = fixture({ size: 2 });
assert.equal(log.length, 0);
setFail(true);
await assert.rejects(pool.take('t1'), /429/);
await pool.refill();
assert.equal(log.filter(entry => entry === 'random').length, 2); // The direct attempt and one refill attempt.
setFail(false);
assert.ok((await pool.take('t1')).article.pageId > 0);
});
test('callers get copies they cannot mutate', async () => {
const { pool } = fixture();
const a = await pool.take('t');
a.article.title = 'changed';
assert.equal((await pool.take('t')).article.title.startsWith('Page'), true);
assert.throws(() => new MintTemplatePool({ articles: { random: async () => article(1) },
enrichment: { getFresh: async () => enrichment(1) } }, { size: 99 }), RangeError);
});