forked from WikiDeck/wikideck
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket (1 req/s, burst 3), 1 in flight per host, mint > interactive > background lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP). - server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match, stale-while-revalidate on the background lane. Mint reads never use it. - Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh() (template 503, miner retries) instead of minting attention: null forever. - server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for /api/mining/template, about 3 requests per block regardless of miner count. - User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or PUBLIC_ORIGIN) and a start-up warning when it is unset. - Mint request parameters are byte-identical apart from maxlag (golden test). No consensus, block or protocol changes. Tests use fake fetch only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
57 lines
3.4 KiB
TypeScript
57 lines
3.4 KiB
TypeScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { WikipediaArticleSource } from '../server/wiki.ts';
|
|
import { WikipediaEnrichment } from '../server/enrichment.ts';
|
|
import { PROJECT_URL, defaultUserAgent, wikimediaUserAgent } from '../server/wikimedia-user-agent.ts';
|
|
|
|
function capture() {
|
|
const agents: string[] = [];
|
|
const fetcher: typeof fetch = async (_input, init) => {
|
|
agents.push(new Headers(init?.headers).get('User-Agent') ?? '');
|
|
return new Response('rate limited', { status: 429, headers: { 'Content-Type': 'text/plain' } });
|
|
};
|
|
return { agents, fetcher };
|
|
}
|
|
async function agentsFor(env: string | undefined, contact?: string) {
|
|
const previous = process.env.WIKIDECK_USER_AGENT, previousContact = process.env.WIKIDECK_OPERATOR_CONTACT;
|
|
if (env === undefined) delete process.env.WIKIDECK_USER_AGENT; else process.env.WIKIDECK_USER_AGENT = env;
|
|
if (contact === undefined) delete process.env.WIKIDECK_OPERATOR_CONTACT; else process.env.WIKIDECK_OPERATOR_CONTACT = contact;
|
|
try {
|
|
const article = capture(), enrichment = capture();
|
|
await assert.rejects(new WikipediaArticleSource(article.fetcher).random());
|
|
await new WikipediaEnrichment({ fetch: enrichment.fetcher }).getFresh(12345).catch(() => {});
|
|
return { article: article.agents, enrichment: enrichment.agents };
|
|
} finally {
|
|
if (previous === undefined) delete process.env.WIKIDECK_USER_AGENT; else process.env.WIKIDECK_USER_AGENT = previous;
|
|
if (previousContact === undefined) delete process.env.WIKIDECK_OPERATOR_CONTACT;
|
|
else process.env.WIKIDECK_OPERATOR_CONTACT = previousContact;
|
|
}
|
|
}
|
|
|
|
test('default Wikimedia user agent names the project, the operator contact and node', async () => {
|
|
const { article, enrichment } = await agentsFor(undefined, 'ops@example.org');
|
|
const expected = `WikiDeck/0.1.0 (+${PROJECT_URL}; ops@example.org) node/${process.versions.node}`;
|
|
assert.equal(defaultUserAgent({ WIKIDECK_OPERATOR_CONTACT: 'ops@example.org' }), expected);
|
|
assert.ok(article.length > 0 && article.every(a => a === expected));
|
|
assert.ok(enrichment.length > 0 && enrichment.every(a => a === expected));
|
|
assert.equal(defaultUserAgent({ PUBLIC_ORIGIN: 'https://node.example' }),
|
|
`WikiDeck/0.1.0 (+${PROJECT_URL}; https://node.example) node/${process.versions.node}`);
|
|
assert.equal(defaultUserAgent({}), `WikiDeck/0.1.0 (+${PROJECT_URL}) node/${process.versions.node}`);
|
|
// Unsafe contacts are dropped rather than breaking the header's comment syntax.
|
|
assert.equal(defaultUserAgent({ WIKIDECK_OPERATOR_CONTACT: 'a); evil(' }), defaultUserAgent({}));
|
|
});
|
|
|
|
test('WIKIDECK_USER_AGENT overrides both Wikimedia clients', async () => {
|
|
const ua = 'WikiDeck-testnet/0.1 (https://gitea.eom.dev/claude/wikideck; private testnet)';
|
|
const { article, enrichment } = await agentsFor(ua);
|
|
assert.ok(article.length > 0 && article.every(a => a === ua));
|
|
assert.ok(enrichment.length > 0 && enrichment.every(a => a === ua));
|
|
});
|
|
|
|
test('blank, oversized or control-character overrides fall back to the default', () => {
|
|
for (const bad of ['', ' ', 'x'.repeat(257), 'bad\r\nX-Injected: 1', 'tab\there'])
|
|
assert.equal(wikimediaUserAgent('fallback', { WIKIDECK_USER_AGENT: bad }), 'fallback');
|
|
assert.equal(wikimediaUserAgent('fallback', {}), 'fallback');
|
|
assert.equal(wikimediaUserAgent('fallback', { WIKIDECK_USER_AGENT: ' Custom/1.0 (x) ' }), 'Custom/1.0 (x)');
|
|
});
|