More cards
This commit is contained in:
+10
-1
@@ -53,10 +53,19 @@ export class WikipediaEnrichment {
|
||||
}
|
||||
|
||||
async get(pageId: number): Promise<PageEnrichment> {
|
||||
return this.loadCached(pageId, false);
|
||||
}
|
||||
|
||||
/** Bypass cached values when the result is being pinned into a newly minted block. */
|
||||
async getFresh(pageId: number): Promise<PageEnrichment> {
|
||||
return this.loadCached(pageId, true);
|
||||
}
|
||||
|
||||
private async loadCached(pageId: number, fresh: boolean): Promise<PageEnrichment> {
|
||||
if (!Number.isSafeInteger(pageId) || pageId <= 0) throw new RangeError('Invalid Wikipedia page ID');
|
||||
const now = this.now();
|
||||
const cached = this.cache.get(pageId);
|
||||
if (cached && cached.expires > now.getTime() &&
|
||||
if (!fresh && cached && cached.expires > now.getTime() &&
|
||||
(!('value' in cached) || cached.value.pageviews.latestMonth === monthWindow(now)[5])) {
|
||||
this.cache.delete(pageId);
|
||||
this.cache.set(pageId, cached);
|
||||
|
||||
+66
-19
@@ -1,16 +1,16 @@
|
||||
import { createHash, createPublicKey, verify } from 'node:crypto';
|
||||
import { z } from 'zod';
|
||||
import { STARTER_ARTICLES } from '../../shared/catalog.ts';
|
||||
import { NETWORK, CONSENSUS, articleTuple, headerMessage, blockClaimMessage, nftTransferMessage,
|
||||
import { NETWORK, CONSENSUS, headerMessage, blockClaimMessage, nftTransferMessage, statsSnapshotTuple,
|
||||
type BlockHeader, type TestnetBlock, type NftTransfer, type NftState, type MiningTemplate } from '../../shared/testnet.ts';
|
||||
import type { Article } from '../../shared/protocol.ts';
|
||||
import { deriveStats, type MintStatsSnapshot } from '../../shared/enrichment.ts';
|
||||
import type { PowEngine } from './randomx.ts';
|
||||
|
||||
export const sha256 = (data: string | Uint8Array) => createHash('sha256').update(data).digest('hex');
|
||||
export const MAX_TARGET = `0fff${'f'.repeat(60)}`;
|
||||
// Consensus manifest: an independently running node derives exactly the same genesis.
|
||||
export const GENESIS_TIME = 1_792_411_200;
|
||||
export const CHAIN_ID = sha256(JSON.stringify([NETWORK, CONSENSUS, GENESIS_TIME, MAX_TARGET,
|
||||
STARTER_ARTICLES.map(articleTuple)]));
|
||||
export const CHAIN_ID = sha256(JSON.stringify([NETWORK, CONSENSUS, GENESIS_TIME, MAX_TARGET]));
|
||||
export const GENESIS_HASH = sha256(`${NETWORK}:genesis:${CHAIN_ID}`);
|
||||
export const INITIAL_KEY = sha256(`${NETWORK}:initial-key:${CHAIN_ID}`);
|
||||
const hex = z.string().regex(/^[a-f0-9]{64}$/);
|
||||
@@ -19,11 +19,59 @@ const signature = z.string().regex(/^[A-Za-z0-9+/]+={0,2}$/).length(88);
|
||||
const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
|
||||
export const transferSchema = z.object({ network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
|
||||
cardHash: hex, from: hex, to: hex, sequence: integer, publicKey: key, signature }).strict();
|
||||
const articleSchema = z.object({ pageId: integer, title: z.string().max(200), extract: z.string().max(1000),
|
||||
image: z.string().max(1000).optional(), url: z.string().max(500), revisionId: integer,
|
||||
source: z.enum(['live', 'offline']), category: z.string().max(50) }).strict();
|
||||
export const blockSchema = z.object({ version: z.literal(2), network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
|
||||
height: integer, previousHash: hex, createdAt: integer, article: articleSchema, owner: hex,
|
||||
const articleSchema = z.object({ pageId: integer.positive(), title: z.string().trim().min(1).max(200),
|
||||
extract: z.string().trim().min(1).max(1000), image: z.string().url().max(1500),
|
||||
url: z.string().url().max(500), revisionId: integer.positive(),
|
||||
source: z.literal('live'), category: z.string().trim().min(1).max(50) }).strict().superRefine((article, ctx) => {
|
||||
if (article.url !== `https://en.wikipedia.org/?curid=${article.pageId}`) {
|
||||
ctx.addIssue({ code: 'custom', message: 'Article URL must identify its English Wikipedia page', path: ['url'] });
|
||||
}
|
||||
try {
|
||||
const image = new URL(article.image);
|
||||
if (image.protocol !== 'https:' || !['https://upload.wikimedia.org', 'https://thumb.wikimedia.org'].includes(image.origin) ||
|
||||
image.username || image.password || image.hash) {
|
||||
ctx.addIssue({ code: 'custom', message: 'Article image must use a Wikimedia HTTPS host', path: ['image'] });
|
||||
}
|
||||
} catch { ctx.addIssue({ code: 'custom', message: 'Invalid article image URL', path: ['image'] }); }
|
||||
});
|
||||
const monthSchema = z.string().regex(/^\d{4}-(0[1-9]|1[0-2])$/);
|
||||
const statsSnapshotSchema = z.object({ capturedAt: z.string().datetime(), revisionId: integer.positive(),
|
||||
pageBytes: integer, languageCount: integer, languagesTruncated: z.boolean(),
|
||||
pageviews: z.object({ status: z.enum(['available', 'unavailable']),
|
||||
months: z.array(z.object({ month: monthSchema, views: integer }).strict()).max(6),
|
||||
latestMonth: monthSchema, total: integer.nullable(), dailyAverage: z.number().min(0).nullable(),
|
||||
trendPercent: z.number().finite().nullable() }).strict(),
|
||||
stats: z.object({ attention: z.number().int().min(0).max(100).nullable(),
|
||||
depth: z.number().int().min(0).max(100), reach: z.number().int().min(0).max(100) }).strict() }).strict()
|
||||
.superRefine((snapshot, ctx) => {
|
||||
const views = snapshot.pageviews;
|
||||
if (views.status === 'available') {
|
||||
if (views.months.length !== 6 || views.months[5]?.month !== views.latestMonth) {
|
||||
ctx.addIssue({ code: 'custom', message: 'Available pageviews require six months ending at latestMonth', path: ['pageviews', 'months'] });
|
||||
} else {
|
||||
const total = views.months.reduce((sum, item) => sum + item.views, 0);
|
||||
const latest = views.months[5].views;
|
||||
const prior = views.months[4].views;
|
||||
const [year, month] = views.latestMonth.split('-').map(Number);
|
||||
const days = new Date(Date.UTC(year, month, 0)).getUTCDate();
|
||||
const trend = prior > 0 ? (latest - prior) / prior * 100 : null;
|
||||
if (!Number.isSafeInteger(total) || views.total !== total || views.dailyAverage !== latest / days ||
|
||||
views.trendPercent !== trend) {
|
||||
ctx.addIssue({ code: 'custom', message: 'Pageview summaries do not match the monthly values', path: ['pageviews'] });
|
||||
}
|
||||
}
|
||||
} else if (views.months.length !== 0 || views.total !== null || views.dailyAverage !== null || views.trendPercent !== null) {
|
||||
ctx.addIssue({ code: 'custom', message: 'Unavailable pageviews must not include invented counts', path: ['pageviews'] });
|
||||
}
|
||||
const monthlyViews = views.status === 'available' ? views.months[5]?.views ?? null : null;
|
||||
const expected = deriveStats(snapshot.pageBytes, snapshot.languageCount, monthlyViews);
|
||||
if (snapshot.stats.attention !== expected.attention || snapshot.stats.depth !== expected.depth ||
|
||||
snapshot.stats.reach !== expected.reach) {
|
||||
ctx.addIssue({ code: 'custom', message: 'Derived statistics do not match the page metrics', path: ['stats'] });
|
||||
}
|
||||
});
|
||||
export const blockSchema = z.object({ version: z.literal(4), network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
|
||||
height: integer, previousHash: hex, createdAt: integer, article: articleSchema, statsSnapshot: statsSnapshotSchema, owner: hex,
|
||||
publicKey: key, nonce: integer, target: hex, key: hex, transactionsRoot: hex, stateRoot: hex,
|
||||
hash: hex, tokenId: hex, powHash: hex, signature, transactions: z.array(transferSchema).max(CONSENSUS.maxTransactions) }).strict();
|
||||
|
||||
@@ -59,8 +107,9 @@ export function stateRoot(state: NftState): string {
|
||||
const n = state[id]; return sha256(JSON.stringify([id, n.owner, n.sequence, n.mintedHeight, n.pageId]));
|
||||
}));
|
||||
}
|
||||
export function mintId(h: Pick<BlockHeader, 'previousHash' | 'height' | 'owner' | 'article'>): string {
|
||||
return sha256(JSON.stringify([NETWORK, CHAIN_ID, 'mint', h.previousHash, h.height, h.owner, h.article.pageId]));
|
||||
export function mintId(h: Pick<BlockHeader, 'previousHash' | 'height' | 'owner' | 'article' | 'statsSnapshot'>): string {
|
||||
return sha256(JSON.stringify([NETWORK, CHAIN_ID, 'mint', h.previousHash, h.height, h.owner,
|
||||
h.article.pageId, statsSnapshotTuple(h.statsSnapshot)]));
|
||||
}
|
||||
export function miningBlob(header: BlockHeader): Buffer {
|
||||
const nonce = Buffer.alloc(8);
|
||||
@@ -68,10 +117,6 @@ export function miningBlob(header: BlockHeader): Buffer {
|
||||
return Buffer.concat([Buffer.from(sha256(headerMessage(header)), 'hex'), nonce]);
|
||||
}
|
||||
export function workFor(target: string): bigint { return (1n << 256n) / (BigInt(`0x${target}`) + 1n); }
|
||||
export function selectArticle(previousHash: string) {
|
||||
const index = Number(BigInt(`0x${sha256(`${CHAIN_ID}:article:${previousHash}`)}`) % BigInt(STARTER_ARTICLES.length));
|
||||
return { ...STARTER_ARTICLES[index] };
|
||||
}
|
||||
export function expectedKey(ancestors: TestnetBlock[], height: number): string {
|
||||
const keyHeight = Math.floor(Math.max(0, height - CONSENSUS.epochDelay) / CONSENSUS.epochBlocks) * CONSENSUS.epochBlocks;
|
||||
return keyHeight === 0 ? INITIAL_KEY : ancestors[keyHeight - 1].hash;
|
||||
@@ -112,12 +157,15 @@ export function transition(parent: NftState, header: BlockHeader, txs: NftTransf
|
||||
return next;
|
||||
}
|
||||
export function makeTemplate(ancestors: TestnetBlock[], state: NftState, owner: string,
|
||||
publicKey: string, transactions: NftTransfer[], now = Math.floor(Date.now()/1000)): MiningTemplate {
|
||||
publicKey: string, transactions: NftTransfer[], article: Article, statsSnapshot: MintStatsSnapshot,
|
||||
now = Math.floor(Date.now()/1000)): MiningTemplate {
|
||||
if (!hex.safeParse(owner).success || address(publicKey) !== owner) throw new Error('Invalid miner identity');
|
||||
const parsedArticle = articleSchema.parse(article);
|
||||
const parsedStats = statsSnapshotSchema.parse(statsSnapshot);
|
||||
const previousHash = ancestors.at(-1)?.hash ?? GENESIS_HASH;
|
||||
const header: BlockHeader = { version: 2, network: NETWORK, chainId: CHAIN_ID,
|
||||
const header: BlockHeader = { version: 4, network: NETWORK, chainId: CHAIN_ID,
|
||||
height: ancestors.length + 1, previousHash, createdAt: Math.max(now, medianTime(ancestors) + 1),
|
||||
article: selectArticle(previousHash), owner, publicKey, nonce: 0, target: expectedTarget(ancestors),
|
||||
article: parsedArticle, statsSnapshot: parsedStats, owner, publicKey, nonce: 0, target: expectedTarget(ancestors),
|
||||
key: expectedKey(ancestors, ancestors.length + 1), transactionsRoot: merkleRoot(transactions.map(transferId)), stateRoot: '' };
|
||||
header.stateRoot = stateRoot(transition(state, header, transactions));
|
||||
return { header, transactions, blob: miningBlob(header).toString('hex'), nonceOffset: 32 };
|
||||
@@ -129,7 +177,6 @@ export async function validateBlock(raw: unknown, ancestors: TestnetBlock[], par
|
||||
if (b.height !== ancestors.length + 1 || b.previousHash !== (ancestors.at(-1)?.hash ?? GENESIS_HASH)) throw new Error('Invalid parent/height');
|
||||
if (b.createdAt <= medianTime(ancestors) || b.createdAt > now + CONSENSUS.maxFutureSeconds) throw new Error('Invalid block timestamp');
|
||||
if (b.target !== expectedTarget(ancestors) || b.key !== expectedKey(ancestors, b.height)) throw new Error('Invalid difficulty or RandomX epoch key');
|
||||
if (JSON.stringify(articleTuple(b.article)) !== JSON.stringify(articleTuple(selectArticle(b.previousHash)))) throw new Error('Article is not in the consensus catalog');
|
||||
if (b.transactionsRoot !== merkleRoot(b.transactions.map(transferId))) throw new Error('Invalid transaction root');
|
||||
const next = transition(parentState, b, b.transactions);
|
||||
if (b.stateRoot !== stateRoot(next) || b.tokenId !== mintId(b)) throw new Error('Invalid NFT state commitment');
|
||||
|
||||
+13
-7
@@ -11,7 +11,8 @@ import { RandomX } from './randomx.ts';
|
||||
import { Ledger } from './ledger.ts';
|
||||
import { PeerNetwork } from './peers.ts';
|
||||
import { wikipediaEnrichment } from '../enrichment.ts';
|
||||
import { STARTER_ARTICLES } from '../../shared/catalog.ts';
|
||||
import { snapshotStats } from '../../shared/enrichment.ts';
|
||||
import { wikipediaArticles } from '../wiki.ts';
|
||||
|
||||
const root = fileURLToPath(new URL('../../', import.meta.url));
|
||||
const dataDir = process.env.WIKIDECK_DATA_DIR ?? resolve(root,'data');
|
||||
@@ -56,9 +57,6 @@ app.get('/api/peers',(_req,res)=>res.json({peers:peers.status()}));
|
||||
app.get('/api/wikipedia/:pageId', rateLimit({windowMs:60000,limit:60,standardHeaders:'draft-8',legacyHeaders:false}), async(req,res)=>{
|
||||
if(typeof req.params.pageId!=='string'||!/^[1-9]\d{0,9}$/.test(req.params.pageId)){res.status(400).json({error:'Invalid Wikipedia page ID'});return;}
|
||||
const pageId=Number(req.params.pageId);
|
||||
if(!STARTER_ARTICLES.some(a=>a.pageId===pageId)){
|
||||
res.status(404).json({error:'Page is not in the current consensus catalog'});return;
|
||||
}
|
||||
try {res.json(await wikipediaEnrichment.get(pageId));}
|
||||
catch {res.status(503).json({error:'Wikipedia enrichment is temporarily unavailable. NFT data is unaffected.'});}
|
||||
});
|
||||
@@ -76,11 +74,19 @@ app.get('/api/nfts/:id', (req,res)=>{
|
||||
let owner=block.owner,sequence=0;
|
||||
for(const tx of s.transfers)if(tx.cardHash===block.tokenId){owner=tx.to;sequence++;}
|
||||
res.json({network:NETWORK,chainId:CHAIN_ID,tokenId:block.tokenId,owner,sequence,blockHash:block.hash,
|
||||
confirmations:s.blocks.length-block.height+1,article:block.article,stateRoot:s.blocks.at(-1)?.stateRoot});
|
||||
confirmations:s.blocks.length-block.height+1,article:block.article,statsSnapshot:block.statsSnapshot,
|
||||
stateRoot:s.blocks.at(-1)?.stateRoot});
|
||||
});
|
||||
app.post('/api/mining/template', (req,res)=>{
|
||||
const templateLimit=rateLimit({windowMs:60000,limit:20,standardHeaders:'draft-8',legacyHeaders:false});
|
||||
app.post('/api/mining/template', templateLimit, async (req,res)=>{
|
||||
const body=z.object({owner:z.string().regex(/^[a-f0-9]{64}$/),publicKey:z.string().max(256)}).strict().parse(req.body);
|
||||
res.json(ledger.template(body.owner,body.publicKey));
|
||||
try {
|
||||
const article=await wikipediaArticles.random();
|
||||
const metrics=await wikipediaEnrichment.getFresh(article.pageId);
|
||||
res.json(ledger.template(body.owner,body.publicKey,article,snapshotStats(metrics)));
|
||||
} catch {
|
||||
res.status(503).json({error:'Could not prepare a Wikipedia article and current statistics. Retry shortly.'});
|
||||
}
|
||||
});
|
||||
const workLimit=rateLimit({windowMs:60000,limit:30,standardHeaders:'draft-8',legacyHeaders:false});
|
||||
app.post(['/api/blocks','/p2p/block'],workLimit,async(req,res)=>{
|
||||
|
||||
@@ -5,9 +5,11 @@ import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { z } from 'zod';
|
||||
import { NETWORK, CONSENSUS, type TestnetBlock, type NftTransfer, type NftState,
|
||||
type MiningTemplate, type TestnetSnapshot } from '../../shared/testnet.ts';
|
||||
import { CHAIN_ID, GENESIS_HASH, applyTransfer, blockSchema, makeTemplate, selectArticle,
|
||||
import { CHAIN_ID, GENESIS_HASH, applyTransfer, blockSchema, makeTemplate,
|
||||
transferId, transferSchema, validateBlock, workFor } from './consensus.ts';
|
||||
import type { PowEngine } from './randomx.ts';
|
||||
import type { Article } from '../../shared/protocol.ts';
|
||||
import type { MintStatsSnapshot } from '../../shared/enrichment.ts';
|
||||
|
||||
const MAX_BLOCKS = 10_000;
|
||||
const MAX_PENDING = 256;
|
||||
@@ -135,16 +137,17 @@ export class Ledger {
|
||||
});
|
||||
}
|
||||
|
||||
template(owner: string, publicKey: string): MiningTemplate {
|
||||
template(owner: string, publicKey: string, article: Article, statsSnapshot: MintStatsSnapshot): MiningTemplate {
|
||||
return makeTemplate(this.canonical(), this.tipState(), owner, publicKey,
|
||||
structuredClone(this.pending.slice(0, CONSENSUS.maxTransactions)));
|
||||
structuredClone(this.pending.slice(0, CONSENSUS.maxTransactions)), structuredClone(article),
|
||||
structuredClone(statsSnapshot));
|
||||
}
|
||||
snapshot(): TestnetSnapshot {
|
||||
const blocks = this.canonical();
|
||||
return structuredClone({ network: NETWORK, chainId: CHAIN_ID, genesis: GENESIS_HASH,
|
||||
tip: this.tip, cumulativeWork: this.tipWork().toString(), blocks,
|
||||
transfers: blocks.flatMap(b => b.transactions), pendingTransfers: this.pending,
|
||||
nextArticle: selectArticle(this.tip), peers: this.peers, algorithm: 'RandomX',
|
||||
peers: this.peers, algorithm: 'RandomX',
|
||||
maturity: CONSENSUS.coinbaseMaturity });
|
||||
}
|
||||
|
||||
|
||||
@@ -51,7 +51,7 @@ const stop = () => { stopping = true; engine.close(); };
|
||||
process.once('SIGINT',stop); process.once('SIGTERM',stop);
|
||||
async function api(path: string, body?: unknown) {
|
||||
const response = await fetch(new URL(path,endpoint), { method: body ? 'POST' : 'GET', redirect:'error',
|
||||
signal: AbortSignal.timeout(15000), headers: body ? {'Content-Type':'application/json'} : {}, body: body ? JSON.stringify(body) : undefined });
|
||||
signal: AbortSignal.timeout(30000), headers: body ? {'Content-Type':'application/json'} : {}, body: body ? JSON.stringify(body) : undefined });
|
||||
const text = await response.text();
|
||||
if (text.length > 131072) throw new Error('Node response too large');
|
||||
const data = JSON.parse(text);
|
||||
|
||||
+81
-26
@@ -1,13 +1,17 @@
|
||||
import { z } from 'zod';
|
||||
import type { Article } from '../shared/protocol.ts';
|
||||
import { STARTER_ARTICLES } from '../shared/catalog.ts';
|
||||
|
||||
const pageSchema = z.object({
|
||||
pageid: z.number().int().positive(), title: z.string(), extract: z.string().default(''),
|
||||
lastrevid: z.number().int().nonnegative().default(0),
|
||||
thumbnail: z.object({ source: z.string().url() }).optional(),
|
||||
categories: z.array(z.object({ title: z.string() })).optional(),
|
||||
pageid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), ns: z.literal(0),
|
||||
title: z.string().trim().min(1).max(200), extract: z.string().max(4096).optional(),
|
||||
lastrevid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
|
||||
thumbnail: z.object({ source: z.string().url().max(1500) }).optional(),
|
||||
categories: z.array(z.object({ title: z.string().min(1).max(300) })).max(50).optional(),
|
||||
});
|
||||
const responseSchema = z.object({ query: z.object({ pages: z.array(z.unknown()).max(10) }) });
|
||||
const MAX_RESPONSE_BYTES = 512 * 1024;
|
||||
const MAX_ATTEMPTS = 3;
|
||||
const imageOrigins = new Set(['https://upload.wikimedia.org', 'https://thumb.wikimedia.org']);
|
||||
|
||||
function category(title: string, categories: string[]): string {
|
||||
const text = `${title} ${categories.join(' ')}`.toLowerCase();
|
||||
@@ -18,27 +22,78 @@ function category(title: string, categories: string[]): string {
|
||||
if (/histor|ancient|archaeolog|empire/.test(text)) return 'History';
|
||||
return 'Culture';
|
||||
}
|
||||
export async function wikipediaArticle(): Promise<Article> {
|
||||
if (process.env.WIKIDECK_OFFLINE === '1') return { ...STARTER_ARTICLES[Math.floor(Math.random() * STARTER_ARTICLES.length)] };
|
||||
async function readJson(response: Response): Promise<unknown> {
|
||||
const length = response.headers.get('content-length');
|
||||
if (!response.ok || response.redirected ||
|
||||
!/^application\/json(?:\s*;|\s*$)/i.test(response.headers.get('content-type') ?? '') ||
|
||||
(length !== null && (!/^\d+$/.test(length) || Number(length) > MAX_RESPONSE_BYTES)) || !response.body) {
|
||||
await response.body?.cancel();
|
||||
throw new Error('Invalid Wikipedia API response');
|
||||
}
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let size = 0;
|
||||
try {
|
||||
const url = new URL('https://en.wikipedia.org/w/api.php');
|
||||
url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
|
||||
generator: 'random', grnnamespace: '0', grnlimit: '6', prop: 'extracts|pageimages|info|categories',
|
||||
exintro: '1', explaintext: '1', exchars: '700', piprop: 'thumbnail', pithumbsize: '900', cllimit: '8' }).toString();
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(6000), redirect: 'error',
|
||||
headers: { 'User-Agent': 'Wikideck/0.1 (local educational card-game prototype)', Accept: 'application/json' } });
|
||||
if (!response.ok) throw new Error('Wikipedia unavailable');
|
||||
const raw = await response.text();
|
||||
if (raw.length > 1_000_000) throw new Error('Wikipedia response too large');
|
||||
const result = z.object({ query: z.object({ pages: z.array(pageSchema) }) }).parse(JSON.parse(raw));
|
||||
const page = result.query.pages.find(p => p.extract.length > 120 && p.thumbnail) ?? result.query.pages.find(p => p.extract.length > 120);
|
||||
if (!page) throw new Error('No suitable article');
|
||||
const image = page.thumbnail?.source;
|
||||
return { pageId: page.pageid, title: page.title, extract: page.extract.slice(0, 700),
|
||||
image: image && ['https://upload.wikimedia.org', 'https://thumb.wikimedia.org'].includes(new URL(image).origin) ? image : undefined,
|
||||
url: `https://en.wikipedia.org/?curid=${page.pageid}`, revisionId: page.lastrevid,
|
||||
source: 'live', category: category(page.title, page.categories?.map(c => c.title) ?? []) };
|
||||
} catch {
|
||||
return { ...STARTER_ARTICLES[Math.floor(Math.random() * STARTER_ARTICLES.length)] };
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
size += value.byteLength;
|
||||
if (size > MAX_RESPONSE_BYTES) throw new Error('Wikipedia response too large');
|
||||
chunks.push(value);
|
||||
}
|
||||
return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)));
|
||||
} finally {
|
||||
await reader.cancel();
|
||||
reader.releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
function pictureUrl(value: string | undefined): string | undefined {
|
||||
if (!value) return undefined;
|
||||
try {
|
||||
const url = new URL(value);
|
||||
return url.protocol === 'https:' && imageOrigins.has(url.origin) && !url.username && !url.password && !url.hash
|
||||
? url.toString() : undefined;
|
||||
} catch { return undefined; }
|
||||
}
|
||||
|
||||
export class WikipediaArticleSource {
|
||||
private readonly fetcher: typeof globalThis.fetch;
|
||||
private active = 0;
|
||||
|
||||
constructor(fetcher: typeof globalThis.fetch = globalThis.fetch) { this.fetcher = fetcher; }
|
||||
|
||||
async random(): Promise<Article> {
|
||||
if (this.active >= 4) throw new Error('Wikipedia article service is busy');
|
||||
this.active++;
|
||||
try {
|
||||
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
const url = new URL('https://en.wikipedia.org/w/api.php');
|
||||
url.search = new URLSearchParams({ action: 'query', format: 'json', formatversion: '2',
|
||||
generator: 'random', grnnamespace: '0', grnlimit: '10', prop: 'extracts|pageimages|info|categories',
|
||||
exintro: '1', explaintext: '1', exchars: '900', piprop: 'thumbnail', pithumbsize: '800', cllimit: '20' }).toString();
|
||||
const response = await this.fetcher(url, { signal: AbortSignal.timeout(3500), redirect: 'error',
|
||||
headers: { User-Agent: 'Wikideck/0.1 (experimental Wikipedia article cards)', Accept: 'application/json' } });
|
||||
const data = responseSchema.parse(await readJson(response));
|
||||
for (const rawPage of data.query.pages) {
|
||||
const parsedPage = pageSchema.safeParse(rawPage);
|
||||
if (!parsedPage.success) continue;
|
||||
const page = parsedPage.data;
|
||||
const image = pictureUrl(page.thumbnail?.source);
|
||||
const extract = page.extract?.trim();
|
||||
if (!image || !extract || extract.length < 40) continue;
|
||||
return { pageId: page.pageid, title: page.title, extract: extract.slice(0, 900), image,
|
||||
url: `https://en.wikipedia.org/?curid=${page.pageid}`, revisionId: page.lastrevid,
|
||||
source: 'live', category: category(page.title, page.categories?.map(c => c.title) ?? []) };
|
||||
}
|
||||
} catch { /* Try another random API response, but keep the number of attempts bounded. */ }
|
||||
}
|
||||
throw new Error('Wikipedia did not return an article with a picture after three attempts');
|
||||
} finally {
|
||||
this.active--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const wikipediaArticles = new WikipediaArticleSource();
|
||||
|
||||
Reference in New Issue
Block a user