forked from WikiDeck/wikideck
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket (1 req/s, burst 3), 1 in flight per host, mint > interactive > background lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP). - server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match, stale-while-revalidate on the background lane. Mint reads never use it. - Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh() (template 503, miner retries) instead of minting attention: null forever. - server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for /api/mining/template, about 3 requests per block regardless of miner count. - User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or PUBLIC_ORIGIN) and a start-up warning when it is unset. - Mint request parameters are byte-identical apart from maxlag (golden test). No consensus, block or protocol changes. Tests use fake fetch only. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
89 lines
4.1 KiB
TypeScript
89 lines
4.1 KiB
TypeScript
/**
|
|
* Mining-template prefetch pool (data review §5, "Mining load").
|
|
*
|
|
* Without it every miner costs 3+ Wikimedia requests per template, all in a burst at each new tip.
|
|
* The pool keeps a few fresh article + stats snapshots ready, and serves the SAME pair to every
|
|
* template built on the same tip (miners race for one block anyway). Wikimedia load is then about
|
|
* one pair (3 requests) per block, whatever the number of miners.
|
|
*
|
|
* Mint data stays transport-only: each pair comes from the mint-path reads (`random()` and
|
|
* `getFresh()`, byte-identical parameters, never the disk cache), and the snapshot is stamped when it
|
|
* is fetched, so `capturedAt` stays true. Pairs older than `maxAgeMs` are discarded, never served.
|
|
* The pool refills only after it is used, so an idle node makes no requests.
|
|
*/
|
|
import type { Article } from '../../shared/protocol.ts';
|
|
import { snapshotStats, type MintStatsSnapshot, type PageEnrichment } from '../../shared/enrichment.ts';
|
|
|
|
export type MintPair = { article: Article; statsSnapshot: MintStatsSnapshot };
|
|
type Entry = MintPair & { fetchedAt: number };
|
|
type Sources = {
|
|
articles: { random(): Promise<Article> };
|
|
enrichment: { getFresh(pageId: number): Promise<PageEnrichment> };
|
|
};
|
|
|
|
export class MintTemplatePool {
|
|
private readonly sources: Sources;
|
|
private readonly size: number;
|
|
private readonly maxAgeMs: number;
|
|
private readonly now: () => number;
|
|
private readonly ready: Entry[] = [];
|
|
private assigned: { tip: string; entry: Entry } | undefined;
|
|
private filling: Promise<void> | undefined;
|
|
private direct: Promise<Entry> | undefined;
|
|
readonly counters = { served: 0, shared: 0, fetched: 0, expired: 0, failed: 0 };
|
|
|
|
constructor(sources: Sources, options: { size?: number; maxAgeMs?: number; now?: () => number } = {}) {
|
|
this.sources = sources;
|
|
this.size = options.size ?? 2;
|
|
this.maxAgeMs = options.maxAgeMs ?? 5 * 60_000;
|
|
this.now = options.now ?? Date.now;
|
|
if (!Number.isSafeInteger(this.size) || this.size < 0 || this.size > 16 ||
|
|
!Number.isFinite(this.maxAgeMs) || this.maxAgeMs <= 0) throw new RangeError('Invalid mint pool options');
|
|
}
|
|
|
|
private fresh(entry: Entry): boolean { return this.now() - entry.fetchedAt < this.maxAgeMs; }
|
|
|
|
private async fetchOne(): Promise<Entry> {
|
|
const article = await this.sources.articles.random();
|
|
const metrics = await this.sources.enrichment.getFresh(article.pageId);
|
|
this.counters.fetched++;
|
|
return { article, statsSnapshot: snapshotStats(metrics), fetchedAt: this.now() };
|
|
}
|
|
|
|
/** The article and snapshot for a template on `tip`. Rejects when Wikimedia is unavailable. */
|
|
async take(tip: string): Promise<MintPair> {
|
|
let entry: Entry | undefined;
|
|
if (this.assigned && this.assigned.tip === tip && this.fresh(this.assigned.entry)) {
|
|
entry = this.assigned.entry;
|
|
this.counters.shared++;
|
|
} else {
|
|
while (this.ready.length && !this.fresh(this.ready[0])) { this.ready.shift(); this.counters.expired++; }
|
|
entry = this.ready.shift();
|
|
if (!entry) {
|
|
// Cold: fetch directly, and let concurrent templates share that one fetch.
|
|
this.direct ??= this.fetchOne().finally(() => { this.direct = undefined; });
|
|
try { entry = await this.direct; }
|
|
catch (error) { this.counters.failed++; throw error; }
|
|
if (this.assigned && this.assigned.tip === tip && this.fresh(this.assigned.entry)) entry = this.assigned.entry;
|
|
}
|
|
this.assigned = { tip, entry };
|
|
}
|
|
this.counters.served++;
|
|
this.refill();
|
|
return structuredClone({ article: entry.article, statsSnapshot: entry.statsSnapshot });
|
|
}
|
|
|
|
/** Top the pool up one pair at a time, in the background. Exposed for tests. */
|
|
refill(): Promise<void> {
|
|
this.filling ??= (async () => {
|
|
while (this.ready.filter(e => this.fresh(e)).length < this.size) {
|
|
try { this.ready.push(await this.fetchOne()); }
|
|
catch { this.counters.failed++; break; } // The next take() tries again; no retry loop.
|
|
}
|
|
})().finally(() => { this.filling = undefined; });
|
|
return this.filling;
|
|
}
|
|
|
|
status() { return { ready: this.ready.length, size: this.size, maxAgeMs: this.maxAgeMs, ...this.counters }; }
|
|
}
|