WikiDeck/wikideck
3
0
Fork 1
Code Issues 8 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

More cards

Browse Source
This commit is contained in:
eric
2026-10-19 18:52:49 -04:00
parent 716c9612cc
commit 0b9061372b
26 changed files with 549 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/testnet/consensus.ts
+66 -19
View File
@@ -1,16 +1,16 @@
import { createHash, createPublicKey, verify } from 'node:crypto';
import { z } from 'zod';
import { STARTER_ARTICLES } from '../../shared/catalog.ts';
import { NETWORK, CONSENSUS, articleTuple, headerMessage, blockClaimMessage, nftTransferMessage,
import { NETWORK, CONSENSUS, headerMessage, blockClaimMessage, nftTransferMessage, statsSnapshotTuple,
type BlockHeader, type TestnetBlock, type NftTransfer, type NftState, type MiningTemplate } from '../../shared/testnet.ts';
import type { Article } from '../../shared/protocol.ts';
import { deriveStats, type MintStatsSnapshot } from '../../shared/enrichment.ts';
import type { PowEngine } from './randomx.ts';
export const sha256 = (data: string | Uint8Array) => createHash('sha256').update(data).digest('hex');
export const MAX_TARGET = `0fff${'f'.repeat(60)}`;
// Consensus manifest: an independently running node derives exactly the same genesis.
export const GENESIS_TIME = 1_792_411_200;
export const CHAIN_ID = sha256(JSON.stringify([NETWORK, CONSENSUS, GENESIS_TIME, MAX_TARGET,
STARTER_ARTICLES.map(articleTuple)]));
export const CHAIN_ID = sha256(JSON.stringify([NETWORK, CONSENSUS, GENESIS_TIME, MAX_TARGET]));
export const GENESIS_HASH = sha256(`${NETWORK}:genesis:${CHAIN_ID}`);
export const INITIAL_KEY = sha256(`${NETWORK}:initial-key:${CHAIN_ID}`);
const hex = z.string().regex(/^[a-f0-9]{64}$/);
@@ -19,11 +19,59 @@ const signature = z.string().regex(/^[A-Za-z0-9+/]+={0,2}$/).length(88);
const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
export const transferSchema = z.object({ network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
cardHash: hex, from: hex, to: hex, sequence: integer, publicKey: key, signature }).strict();
const articleSchema = z.object({ pageId: integer, title: z.string().max(200), extract: z.string().max(1000),
image: z.string().max(1000).optional(), url: z.string().max(500), revisionId: integer,
source: z.enum(['live', 'offline']), category: z.string().max(50) }).strict();
export const blockSchema = z.object({ version: z.literal(2), network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
height: integer, previousHash: hex, createdAt: integer, article: articleSchema, owner: hex,
const articleSchema = z.object({ pageId: integer.positive(), title: z.string().trim().min(1).max(200),
extract: z.string().trim().min(1).max(1000), image: z.string().url().max(1500),
url: z.string().url().max(500), revisionId: integer.positive(),
source: z.literal('live'), category: z.string().trim().min(1).max(50) }).strict().superRefine((article, ctx) => {
if (article.url !== `https://en.wikipedia.org/?curid=${article.pageId}`) {
ctx.addIssue({ code: 'custom', message: 'Article URL must identify its English Wikipedia page', path: ['url'] });
}
try {
const image = new URL(article.image);
if (image.protocol !== 'https:' || !['https://upload.wikimedia.org', 'https://thumb.wikimedia.org'].includes(image.origin) ||
image.username || image.password || image.hash) {
ctx.addIssue({ code: 'custom', message: 'Article image must use a Wikimedia HTTPS host', path: ['image'] });
}
} catch { ctx.addIssue({ code: 'custom', message: 'Invalid article image URL', path: ['image'] }); }
});
const monthSchema = z.string().regex(/^\d{4}-(0[1-9]|1[0-2])$/);
const statsSnapshotSchema = z.object({ capturedAt: z.string().datetime(), revisionId: integer.positive(),
pageBytes: integer, languageCount: integer, languagesTruncated: z.boolean(),
pageviews: z.object({ status: z.enum(['available', 'unavailable']),
months: z.array(z.object({ month: monthSchema, views: integer }).strict()).max(6),
latestMonth: monthSchema, total: integer.nullable(), dailyAverage: z.number().min(0).nullable(),
trendPercent: z.number().finite().nullable() }).strict(),
stats: z.object({ attention: z.number().int().min(0).max(100).nullable(),
depth: z.number().int().min(0).max(100), reach: z.number().int().min(0).max(100) }).strict() }).strict()
.superRefine((snapshot, ctx) => {
const views = snapshot.pageviews;
if (views.status === 'available') {
if (views.months.length !== 6 || views.months[5]?.month !== views.latestMonth) {
ctx.addIssue({ code: 'custom', message: 'Available pageviews require six months ending at latestMonth', path: ['pageviews', 'months'] });
} else {
const total = views.months.reduce((sum, item) => sum + item.views, 0);
const latest = views.months[5].views;
const prior = views.months[4].views;
const [year, month] = views.latestMonth.split('-').map(Number);
const days = new Date(Date.UTC(year, month, 0)).getUTCDate();
const trend = prior > 0 ? (latest - prior) / prior * 100 : null;
if (!Number.isSafeInteger(total) || views.total !== total || views.dailyAverage !== latest / days ||
views.trendPercent !== trend) {
ctx.addIssue({ code: 'custom', message: 'Pageview summaries do not match the monthly values', path: ['pageviews'] });
}
}
} else if (views.months.length !== 0 || views.total !== null || views.dailyAverage !== null || views.trendPercent !== null) {
ctx.addIssue({ code: 'custom', message: 'Unavailable pageviews must not include invented counts', path: ['pageviews'] });
}
const monthlyViews = views.status === 'available' ? views.months[5]?.views ?? null : null;
const expected = deriveStats(snapshot.pageBytes, snapshot.languageCount, monthlyViews);
if (snapshot.stats.attention !== expected.attention || snapshot.stats.depth !== expected.depth ||
snapshot.stats.reach !== expected.reach) {
ctx.addIssue({ code: 'custom', message: 'Derived statistics do not match the page metrics', path: ['stats'] });
}
});
export const blockSchema = z.object({ version: z.literal(4), network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
height: integer, previousHash: hex, createdAt: integer, article: articleSchema, statsSnapshot: statsSnapshotSchema, owner: hex,
publicKey: key, nonce: integer, target: hex, key: hex, transactionsRoot: hex, stateRoot: hex,
hash: hex, tokenId: hex, powHash: hex, signature, transactions: z.array(transferSchema).max(CONSENSUS.maxTransactions) }).strict();
@@ -59,8 +107,9 @@ export function stateRoot(state: NftState): string {
const n = state[id]; return sha256(JSON.stringify([id, n.owner, n.sequence, n.mintedHeight, n.pageId]));
}));
}
export function mintId(h: Pick<BlockHeader, 'previousHash' | 'height' | 'owner' | 'article'>): string {
return sha256(JSON.stringify([NETWORK, CHAIN_ID, 'mint', h.previousHash, h.height, h.owner, h.article.pageId]));
export function mintId(h: Pick<BlockHeader, 'previousHash' | 'height' | 'owner' | 'article' | 'statsSnapshot'>): string {
return sha256(JSON.stringify([NETWORK, CHAIN_ID, 'mint', h.previousHash, h.height, h.owner,
h.article.pageId, statsSnapshotTuple(h.statsSnapshot)]));
}
export function miningBlob(header: BlockHeader): Buffer {
const nonce = Buffer.alloc(8);
@@ -68,10 +117,6 @@ export function miningBlob(header: BlockHeader): Buffer {
return Buffer.concat([Buffer.from(sha256(headerMessage(header)), 'hex'), nonce]);
}
export function workFor(target: string): bigint { return (1n << 256n) / (BigInt(`0x${target}`) + 1n); }
export function selectArticle(previousHash: string) {
const index = Number(BigInt(`0x${sha256(`${CHAIN_ID}:article:${previousHash}`)}`) % BigInt(STARTER_ARTICLES.length));
return { ...STARTER_ARTICLES[index] };
}
export function expectedKey(ancestors: TestnetBlock[], height: number): string {
const keyHeight = Math.floor(Math.max(0, height - CONSENSUS.epochDelay) / CONSENSUS.epochBlocks) * CONSENSUS.epochBlocks;
return keyHeight === 0 ? INITIAL_KEY : ancestors[keyHeight - 1].hash;
@@ -112,12 +157,15 @@ export function transition(parent: NftState, header: BlockHeader, txs: NftTransf
return next;
}
export function makeTemplate(ancestors: TestnetBlock[], state: NftState, owner: string,
publicKey: string, transactions: NftTransfer[], now = Math.floor(Date.now()/1000)): MiningTemplate {
publicKey: string, transactions: NftTransfer[], article: Article, statsSnapshot: MintStatsSnapshot,
now = Math.floor(Date.now()/1000)): MiningTemplate {
if (!hex.safeParse(owner).success || address(publicKey) !== owner) throw new Error('Invalid miner identity');
const parsedArticle = articleSchema.parse(article);
const parsedStats = statsSnapshotSchema.parse(statsSnapshot);
const previousHash = ancestors.at(-1)?.hash ?? GENESIS_HASH;
const header: BlockHeader = { version: 2, network: NETWORK, chainId: CHAIN_ID,
const header: BlockHeader = { version: 4, network: NETWORK, chainId: CHAIN_ID,
height: ancestors.length + 1, previousHash, createdAt: Math.max(now, medianTime(ancestors) + 1),
article: selectArticle(previousHash), owner, publicKey, nonce: 0, target: expectedTarget(ancestors),
article: parsedArticle, statsSnapshot: parsedStats, owner, publicKey, nonce: 0, target: expectedTarget(ancestors),
key: expectedKey(ancestors, ancestors.length + 1), transactionsRoot: merkleRoot(transactions.map(transferId)), stateRoot: '' };
header.stateRoot = stateRoot(transition(state, header, transactions));
return { header, transactions, blob: miningBlob(header).toString('hex'), nonceOffset: 32 };
@@ -129,7 +177,6 @@ export async function validateBlock(raw: unknown, ancestors: TestnetBlock[], par
if (b.height !== ancestors.length + 1 || b.previousHash !== (ancestors.at(-1)?.hash ?? GENESIS_HASH)) throw new Error('Invalid parent/height');
if (b.createdAt <= medianTime(ancestors) || b.createdAt > now + CONSENSUS.maxFutureSeconds) throw new Error('Invalid block timestamp');
if (b.target !== expectedTarget(ancestors) || b.key !== expectedKey(ancestors, b.height)) throw new Error('Invalid difficulty or RandomX epoch key');
if (JSON.stringify(articleTuple(b.article)) !== JSON.stringify(articleTuple(selectArticle(b.previousHash)))) throw new Error('Article is not in the consensus catalog');
if (b.transactionsRoot !== merkleRoot(b.transactions.map(transferId))) throw new Error('Invalid transaction root');
const next = transition(parentState, b, b.transactions);
if (b.stateRoot !== stateRoot(next) || b.tokenId !== mintId(b)) throw new Error('Invalid NFT state commitment');
server/testnet/http.ts
+13 -7
View File
@@ -11,7 +11,8 @@ import { RandomX } from './randomx.ts';
import { Ledger } from './ledger.ts';
import { PeerNetwork } from './peers.ts';
import { wikipediaEnrichment } from '../enrichment.ts';
import { STARTER_ARTICLES } from '../../shared/catalog.ts';
import { snapshotStats } from '../../shared/enrichment.ts';
import { wikipediaArticles } from '../wiki.ts';
const root = fileURLToPath(new URL('../../', import.meta.url));
const dataDir = process.env.WIKIDECK_DATA_DIR ?? resolve(root,'data');
@@ -56,9 +57,6 @@ app.get('/api/peers',(_req,res)=>res.json({peers:peers.status()}));
app.get('/api/wikipedia/:pageId', rateLimit({windowMs:60000,limit:60,standardHeaders:'draft-8',legacyHeaders:false}), async(req,res)=>{
if(typeof req.params.pageId!=='string'||!/^[1-9]\d{0,9}$/.test(req.params.pageId)){res.status(400).json({error:'Invalid Wikipedia page ID'});return;}
const pageId=Number(req.params.pageId);
if(!STARTER_ARTICLES.some(a=>a.pageId===pageId)){
res.status(404).json({error:'Page is not in the current consensus catalog'});return;
}
try {res.json(await wikipediaEnrichment.get(pageId));}
catch {res.status(503).json({error:'Wikipedia enrichment is temporarily unavailable. NFT data is unaffected.'});}
});
@@ -76,11 +74,19 @@ app.get('/api/nfts/:id', (req,res)=>{
let owner=block.owner,sequence=0;
for(const tx of s.transfers)if(tx.cardHash===block.tokenId){owner=tx.to;sequence++;}
res.json({network:NETWORK,chainId:CHAIN_ID,tokenId:block.tokenId,owner,sequence,blockHash:block.hash,
confirmations:s.blocks.length-block.height+1,article:block.article,stateRoot:s.blocks.at(-1)?.stateRoot});
confirmations:s.blocks.length-block.height+1,article:block.article,statsSnapshot:block.statsSnapshot,
stateRoot:s.blocks.at(-1)?.stateRoot});
});
app.post('/api/mining/template', (req,res)=>{
const templateLimit=rateLimit({windowMs:60000,limit:20,standardHeaders:'draft-8',legacyHeaders:false});
app.post('/api/mining/template', templateLimit, async (req,res)=>{
const body=z.object({owner:z.string().regex(/^[a-f0-9]{64}$/),publicKey:z.string().max(256)}).strict().parse(req.body);
res.json(ledger.template(body.owner,body.publicKey));
try {
const article=await wikipediaArticles.random();
const metrics=await wikipediaEnrichment.getFresh(article.pageId);
res.json(ledger.template(body.owner,body.publicKey,article,snapshotStats(metrics)));
} catch {
res.status(503).json({error:'Could not prepare a Wikipedia article and current statistics. Retry shortly.'});
}
});
const workLimit=rateLimit({windowMs:60000,limit:30,standardHeaders:'draft-8',legacyHeaders:false});
app.post(['/api/blocks','/p2p/block'],workLimit,async(req,res)=>{
server/testnet/ledger.ts
+7 -4
View File
@@ -5,9 +5,11 @@ import { setTimeout as delay } from 'node:timers/promises';
import { z } from 'zod';
import { NETWORK, CONSENSUS, type TestnetBlock, type NftTransfer, type NftState,
type MiningTemplate, type TestnetSnapshot } from '../../shared/testnet.ts';
import { CHAIN_ID, GENESIS_HASH, applyTransfer, blockSchema, makeTemplate, selectArticle,
import { CHAIN_ID, GENESIS_HASH, applyTransfer, blockSchema, makeTemplate,
transferId, transferSchema, validateBlock, workFor } from './consensus.ts';
import type { PowEngine } from './randomx.ts';
import type { Article } from '../../shared/protocol.ts';
import type { MintStatsSnapshot } from '../../shared/enrichment.ts';
const MAX_BLOCKS = 10_000;
const MAX_PENDING = 256;
@@ -135,16 +137,17 @@ export class Ledger {
});
}
template(owner: string, publicKey: string): MiningTemplate {
template(owner: string, publicKey: string, article: Article, statsSnapshot: MintStatsSnapshot): MiningTemplate {
return makeTemplate(this.canonical(), this.tipState(), owner, publicKey,
structuredClone(this.pending.slice(0, CONSENSUS.maxTransactions)));
structuredClone(this.pending.slice(0, CONSENSUS.maxTransactions)), structuredClone(article),
structuredClone(statsSnapshot));
}
snapshot(): TestnetSnapshot {
const blocks = this.canonical();
return structuredClone({ network: NETWORK, chainId: CHAIN_ID, genesis: GENESIS_HASH,
tip: this.tip, cumulativeWork: this.tipWork().toString(), blocks,
transfers: blocks.flatMap(b => b.transactions), pendingTransfers: this.pending,
nextArticle: selectArticle(this.tip), peers: this.peers, algorithm: 'RandomX',
peers: this.peers, algorithm: 'RandomX',
maturity: CONSENSUS.coinbaseMaturity });
}
server/testnet/miner.ts
+1 -1
View File
@@ -51,7 +51,7 @@ const stop = () => { stopping = true; engine.close(); };
process.once('SIGINT',stop); process.once('SIGTERM',stop);
async function api(path: string, body?: unknown) {
const response = await fetch(new URL(path,endpoint), { method: body ? 'POST' : 'GET', redirect:'error',
signal: AbortSignal.timeout(15000), headers: body ? {'Content-Type':'application/json'} : {}, body: body ? JSON.stringify(body) : undefined });
signal: AbortSignal.timeout(30000), headers: body ? {'Content-Type':'application/json'} : {}, body: body ? JSON.stringify(body) : undefined });
const text = await response.text();
if (text.length > 131072) throw new Error('Node response too large');
const data = JSON.parse(text);