Files
wikideck/server/wikimedia/pool.ts
claudeandClaude Opus 5.5 f544019bfe Polite Wikimedia client, mint-path fix, disk cache and mining prefetch pool (#62)
- server/wikimedia/client.ts: one process-wide client. Per-host token bucket
  (1 req/s, burst 3), 1 in flight per host, mint > interactive > background
  lanes with a bounded queue, maxlag=5 on Action API requests, Retry-After
  and backoff, a circuit breaker after 3 consecutive 429s, a daily cap, and
  a per-egress-IP budget (WIKIDECK_WIKIMEDIA_NODES_PER_IP).
- server/wikimedia/cache.ts: node:sqlite HTTP cache with ETag/If-None-Match,
  stale-while-revalidate on the background lane. Mint reads never use it.
- Mint fix: a 429/5xx/timeout/non-JSON reply on pageviews now fails getFresh()
  (template 503, miner retries) instead of minting attention: null forever.
- server/wikimedia/pool.ts: per-tip article+snapshot prefetch pool for
  /api/mining/template, about 3 requests per block regardless of miner count.
- User-Agent with an operator contact (WIKIDECK_OPERATOR_CONTACT or
  PUBLIC_ORIGIN) and a start-up warning when it is unset.
- Mint request parameters are byte-identical apart from maxlag (golden test).
  No consensus, block or protocol changes. Tests use fake fetch only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 21:51:20 -05:00

89 lines
4.1 KiB
TypeScript

/**
* Mining-template prefetch pool (data review §5, "Mining load").
*
* Without it every miner costs 3+ Wikimedia requests per template, all in a burst at each new tip.
* The pool keeps a few fresh article + stats snapshots ready, and serves the SAME pair to every
* template built on the same tip (miners race for one block anyway). Wikimedia load is then about
* one pair (3 requests) per block, whatever the number of miners.
*
* Mint data stays transport-only: each pair comes from the mint-path reads (`random()` and
* `getFresh()`, byte-identical parameters, never the disk cache), and the snapshot is stamped when it
* is fetched, so `capturedAt` stays true. Pairs older than `maxAgeMs` are discarded, never served.
* The pool refills only after it is used, so an idle node makes no requests.
*/
import type { Article } from '../../shared/protocol.ts';
import { snapshotStats, type MintStatsSnapshot, type PageEnrichment } from '../../shared/enrichment.ts';
export type MintPair = { article: Article; statsSnapshot: MintStatsSnapshot };
type Entry = MintPair & { fetchedAt: number };
type Sources = {
articles: { random(): Promise<Article> };
enrichment: { getFresh(pageId: number): Promise<PageEnrichment> };
};
export class MintTemplatePool {
private readonly sources: Sources;
private readonly size: number;
private readonly maxAgeMs: number;
private readonly now: () => number;
private readonly ready: Entry[] = [];
private assigned: { tip: string; entry: Entry } | undefined;
private filling: Promise<void> | undefined;
private direct: Promise<Entry> | undefined;
readonly counters = { served: 0, shared: 0, fetched: 0, expired: 0, failed: 0 };
constructor(sources: Sources, options: { size?: number; maxAgeMs?: number; now?: () => number } = {}) {
this.sources = sources;
this.size = options.size ?? 2;
this.maxAgeMs = options.maxAgeMs ?? 5 * 60_000;
this.now = options.now ?? Date.now;
if (!Number.isSafeInteger(this.size) || this.size < 0 || this.size > 16 ||
!Number.isFinite(this.maxAgeMs) || this.maxAgeMs <= 0) throw new RangeError('Invalid mint pool options');
}
private fresh(entry: Entry): boolean { return this.now() - entry.fetchedAt < this.maxAgeMs; }
private async fetchOne(): Promise<Entry> {
const article = await this.sources.articles.random();
const metrics = await this.sources.enrichment.getFresh(article.pageId);
this.counters.fetched++;
return { article, statsSnapshot: snapshotStats(metrics), fetchedAt: this.now() };
}
/** The article and snapshot for a template on `tip`. Rejects when Wikimedia is unavailable. */
async take(tip: string): Promise<MintPair> {
let entry: Entry | undefined;
if (this.assigned && this.assigned.tip === tip && this.fresh(this.assigned.entry)) {
entry = this.assigned.entry;
this.counters.shared++;
} else {
while (this.ready.length && !this.fresh(this.ready[0])) { this.ready.shift(); this.counters.expired++; }
entry = this.ready.shift();
if (!entry) {
// Cold: fetch directly, and let concurrent templates share that one fetch.
this.direct ??= this.fetchOne().finally(() => { this.direct = undefined; });
try { entry = await this.direct; }
catch (error) { this.counters.failed++; throw error; }
if (this.assigned && this.assigned.tip === tip && this.fresh(this.assigned.entry)) entry = this.assigned.entry;
}
this.assigned = { tip, entry };
}
this.counters.served++;
this.refill();
return structuredClone({ article: entry.article, statsSnapshot: entry.statsSnapshot });
}
/** Top the pool up one pair at a time, in the background. Exposed for tests. */
refill(): Promise<void> {
this.filling ??= (async () => {
while (this.ready.filter(e => this.fresh(e)).length < this.size) {
try { this.ready.push(await this.fetchOne()); }
catch { this.counters.failed++; break; } // The next take() tries again; no retry loop.
}
})().finally(() => { this.filling = undefined; });
return this.filling;
}
status() { return { ready: this.ready.length, size: this.size, maxAgeMs: this.maxAgeMs, ...this.counters }; }
}