forked from WikiDeck/wikideck
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket (1 req/s, burst 3), 1 in flight per host, mint > interactive > background lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP). - server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match, stale-while-revalidate on the background lane. Mint reads never use it. - Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh() (template 503, miner retries) instead of minting attention: null forever. - server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for /api/mining/template, about 3 requests per block regardless of miner count. - User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or PUBLIC_ORIGIN) and a start-up warning when it is unset. - Mint request parameters are byte-identical apart from maxlag (golden test). No consensus, block or protocol changes. Tests use fake fetch only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
75 lines
3.8 KiB
TypeScript
75 lines
3.8 KiB
TypeScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { MintTemplatePool } from '../server/wikimedia/pool.ts';
|
|
import type { Article } from '../shared/protocol.ts';
|
|
import type { PageEnrichment } from '../shared/enrichment.ts';
|
|
|
|
function article(pageId: number): Article {
|
|
return { pageId, title: `Page ${pageId}`, extract: 'An extract that is comfortably longer than forty characters.',
|
|
image: 'https://upload.wikimedia.org/a.jpg', url: `https://en.wikipedia.org/?curid=${pageId}`, revisionId: pageId + 1,
|
|
source: 'live', category: 'Culture' };
|
|
}
|
|
function enrichment(pageId: number): PageEnrichment {
|
|
return { pageId, title: `Page ${pageId}`, description: null, wikidataId: null, revisionId: pageId + 1, pageBytes: 10,
|
|
languageCount: 1, languagesTruncated: false, categories: [], categoriesTruncated: false, fetchedAt: '2026-11-17T00:00:00Z',
|
|
pageviews: { status: 'available', months: [], latestMonth: '2026-10', total: 1, dailyAverage: 1, trendPercent: null },
|
|
stats: { attention: 1, depth: 1, reach: 1 } };
|
|
}
|
|
|
|
function fixture(options: { size?: number; maxAgeMs?: number } = {}) {
|
|
let next = 1, now = 0, fail = false;
|
|
const log: string[] = [];
|
|
const sources = {
|
|
articles: { random: async () => { log.push('random'); if (fail) throw new Error('Wikimedia unavailable: HTTP 429'); return article(next++); } },
|
|
enrichment: { getFresh: async (id: number) => { log.push(`fresh:${id}`); return enrichment(id); } },
|
|
};
|
|
const pool = new MintTemplatePool(sources, { now: () => now, ...options });
|
|
return { pool, log, advance: (ms: number) => { now += ms; }, setFail: (value: boolean) => { fail = value; } };
|
|
}
|
|
|
|
test('every template on one tip shares one article; a new tip gets a different one', async () => {
|
|
const { pool, log } = fixture({ size: 1 });
|
|
const first = await Promise.all(Array.from({ length: 20 }, () => pool.take('tip-a')));
|
|
assert.ok(first.every(pair => pair.article.pageId === first[0].article.pageId));
|
|
await pool.refill();
|
|
const requestsForTwentyMiners = log.length;
|
|
assert.equal(requestsForTwentyMiners, 4); // One direct pair, one prefetched pair: 2 calls each.
|
|
const second = await pool.take('tip-b');
|
|
assert.notEqual(second.article.pageId, first[0].article.pageId);
|
|
assert.equal(second.article.pageId, 2); // Served from the prefetched pair, no wait.
|
|
await pool.refill();
|
|
assert.equal(log.length, 6); // ...and that use alone triggers the next prefetch.
|
|
// The stats snapshot comes from a mint-path read of the same page.
|
|
assert.equal(second.statsSnapshot.revisionId, second.article.pageId + 1);
|
|
});
|
|
|
|
test('pairs older than maxAge are dropped, never served', async () => {
|
|
const { pool, advance } = fixture({ size: 1, maxAgeMs: 1000 });
|
|
const a = await pool.take('t1');
|
|
await pool.refill();
|
|
advance(1000);
|
|
const b = await pool.take('t1'); // Same tip, but its pair expired; the ready pair expired too.
|
|
assert.notEqual(b.article.pageId, a.article.pageId);
|
|
assert.equal(pool.status().expired, 1);
|
|
});
|
|
|
|
test('an idle pool makes no requests, and a failure rejects the template without a retry loop', async () => {
|
|
const { pool, log, setFail } = fixture({ size: 2 });
|
|
assert.equal(log.length, 0);
|
|
setFail(true);
|
|
await assert.rejects(pool.take('t1'), /429/);
|
|
await pool.refill();
|
|
assert.equal(log.filter(entry => entry === 'random').length, 2); // The direct attempt and one refill attempt.
|
|
setFail(false);
|
|
assert.ok((await pool.take('t1')).article.pageId > 0);
|
|
});
|
|
|
|
test('callers get copies they cannot mutate', async () => {
|
|
const { pool } = fixture();
|
|
const a = await pool.take('t');
|
|
a.article.title = 'changed';
|
|
assert.equal((await pool.take('t')).article.title.startsWith('Page'), true);
|
|
assert.throws(() => new MintTemplatePool({ articles: { random: async () => article(1) },
|
|
enrichment: { getFresh: async () => enrichment(1) } }, { size: 99 }), RangeError);
|
|
});
|