Working version Reviewed-on: #21
This commit was merged in pull request #21.
This commit is contained in:
2026-11-01 22:07:51 +00:00
parent 1edf2e6200
commit 636f5b9c77
80 changed files with 4065 additions and 549 deletions
+186
View File
@@ -0,0 +1,186 @@
import type { DatabaseSync } from 'node:sqlite';
import { randomUUID } from 'node:crypto';
import { isNftTrade, transactionCardIds, type NftState, type TestnetBlock } from '../../shared/testnet.ts';
import { GENESIS_HASH, sha256, transactionId } from './consensus.ts';
import { RpcError } from '../resources.ts';
export type CanonicalEvent = { sequence: number; type: 'connect' | 'disconnect';
height: number; hash: string; previousHash: string; transactionIds: string[]; eventHash: string };
export type CanonicalEvents = { streamId: string; previousAnchor: string; anchor: string;
events: CanonicalEvent[]; cursor: number; head: number; caughtUp: boolean };
type NftRow = { token: string; owner: string; sequence: number; minted_height: number; block_hash: string };
export type IndexedNft = { block: TestnetBlock; owner: string; sequence: number; confirmations: number };
/** Local derived indexes, NOT consensus. Writes share the ledger's block/pending transaction. */
export class CanonicalIndex {
private readonly db: DatabaseSync;
private streamId = '';
constructor(db: DatabaseSync) { this.db = db; }
initialize(chain: TestnetBlock[], state: NftState): void {
this.db.exec('BEGIN IMMEDIATE');
try {
this.db.exec(`CREATE TABLE IF NOT EXISTS canonical_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS canonical_blocks(height INTEGER PRIMARY KEY,hash TEXT NOT NULL UNIQUE REFERENCES blocks(hash));
CREATE TABLE IF NOT EXISTS canonical_nfts(token TEXT PRIMARY KEY,owner TEXT NOT NULL,sequence INTEGER NOT NULL,
minted_height INTEGER NOT NULL,block_hash TEXT NOT NULL REFERENCES blocks(hash));
CREATE INDEX IF NOT EXISTS canonical_owner ON canonical_nfts(owner,minted_height,token);
CREATE TABLE IF NOT EXISTS canonical_transactions(id TEXT PRIMARY KEY,height INTEGER NOT NULL,block_hash TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS canonical_tx_height ON canonical_transactions(height);
CREATE TABLE IF NOT EXISTS canonical_undo(height INTEGER PRIMARY KEY,data TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS canonical_events(sequence INTEGER PRIMARY KEY AUTOINCREMENT,type TEXT NOT NULL,
height INTEGER NOT NULL,hash TEXT NOT NULL,previous_hash TEXT NOT NULL,transactions_json TEXT NOT NULL,
previous_event_hash TEXT NOT NULL,event_hash TEXT NOT NULL);
CREATE TRIGGER IF NOT EXISTS canonical_events_no_update BEFORE UPDATE ON canonical_events
BEGIN SELECT RAISE(ABORT,'Immutable canonical event'); END;
CREATE TRIGGER IF NOT EXISTS canonical_events_no_delete BEFORE DELETE ON canonical_events
BEGIN SELECT RAISE(ABORT,'Immutable canonical event'); END;`);
const version = this.db.prepare("SELECT value FROM canonical_meta WHERE key='version'").get() as { value: string } | undefined;
if (!version) {
// Additive backfill only after Ledger.open has independently validated all persisted proofs/state.
const count = this.db.prepare('SELECT COUNT(*) AS n FROM canonical_events').get() as { n: number };
if (count.n || (this.db.prepare('SELECT COUNT(*) AS n FROM canonical_blocks').get() as { n: number }).n) {
throw new Error('Partial canonical index; offline operator review required');
}
this.streamId = randomUUID();
this.db.prepare('INSERT INTO canonical_meta VALUES(?,?)').run('stream_id', this.streamId);
for (const block of chain) this.connect(block);
this.db.prepare('INSERT INTO canonical_meta VALUES(?,?)').run('version', '1');
} else if (version.value !== '1') throw new Error('Canonical index version mismatch');
else {
const row = this.db.prepare("SELECT value FROM canonical_meta WHERE key='stream_id'").get() as { value: string } | undefined;
if (!row || !/^[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}$/.test(row.value)) throw new Error('Canonical stream identity missing');
this.streamId = row.value;
}
this.verify(chain, state);
this.db.exec('COMMIT');
} catch (error) { this.db.exec('ROLLBACK'); throw error; }
}
private verify(chain: TestnetBlock[], state: NftState): void {
const blocks = this.db.prepare('SELECT height,hash FROM canonical_blocks ORDER BY height').all() as { height: number; hash: string }[];
if (blocks.length !== chain.length || blocks.some((b, i) => b.height !== chain[i].height || b.hash !== chain[i].hash)) {
throw new Error('Canonical block index mismatch');
}
const nfts = this.db.prepare('SELECT * FROM canonical_nfts').all() as NftRow[];
const expected = new Map(chain.map(b => [b.tokenId, b]));
if (nfts.length !== expected.size) throw new Error('Canonical NFT index mismatch');
for (const row of nfts) {
const nft = state[row.token], block = expected.get(row.token);
if (!nft || !block || row.owner !== nft.owner || row.sequence !== nft.sequence ||
row.minted_height !== block.height || row.block_hash !== block.hash) throw new Error('Canonical ownership index mismatch');
}
const transactions = this.db.prepare('SELECT id,height,block_hash FROM canonical_transactions').all() as { id: string; height: number; block_hash: string }[];
const expectedTxs = new Map(chain.flatMap(b => b.transactions.map(tx => [transactionId(tx), b] as const)));
if (transactions.length !== expectedTxs.size || transactions.some(row => {
const block = expectedTxs.get(row.id); return !block || row.height !== block.height || row.block_hash !== block.hash;
})) throw new Error('Canonical transaction index mismatch');
const undo = this.db.prepare('SELECT COUNT(*) AS n FROM canonical_undo').get() as { n: number };
if (undo.n !== chain.length) throw new Error('Canonical undo index mismatch');
// A cursor is only useful if history has no gaps and describes the same validated DAG transitions.
const stack: string[] = [];
let sequence = 0, anchor = GENESIS_HASH;
for (const raw of this.db.prepare(`SELECT e.*,b.block_json FROM canonical_events e
LEFT JOIN blocks b ON b.hash=e.hash ORDER BY e.sequence`).iterate() as Iterable<{
sequence: number; type: string; height: number; hash: string; previous_hash: string;
transactions_json: string; block_json: string | null; previous_event_hash: string; event_hash: string }>) {
const block = raw.block_json ? JSON.parse(raw.block_json) as TestnetBlock : undefined;
if (raw.sequence !== ++sequence || !block || block.height !== raw.height || block.previousHash !== raw.previous_hash ||
raw.transactions_json !== JSON.stringify(block.transactions.map(transactionId))) throw new Error('Canonical event integrity mismatch');
if (raw.previous_event_hash !== anchor || raw.event_hash !== this.eventHash(raw.sequence, raw.type,
block, anchor)) throw new Error('Canonical event anchor mismatch');
anchor = raw.event_hash;
if (raw.type === 'connect') {
if (raw.height !== stack.length + 1 || raw.previous_hash !== (stack.at(-1) ?? GENESIS_HASH)) throw new Error('Canonical event connect mismatch');
stack.push(raw.hash);
} else if (raw.type === 'disconnect') {
if (raw.height !== stack.length || stack.pop() !== raw.hash) throw new Error('Canonical event disconnect mismatch');
} else throw new Error('Unknown canonical event');
}
if (stack.length !== chain.length || stack.some((hash, i) => hash !== chain[i].hash)) throw new Error('Canonical event tip mismatch');
}
private event(type: CanonicalEvent['type'], block: TestnetBlock) {
const prior = this.db.prepare('SELECT sequence,event_hash FROM canonical_events ORDER BY sequence DESC LIMIT 1').get() as
{ sequence: number; event_hash: string } | undefined;
const sequence = (prior?.sequence ?? 0) + 1, anchor = prior?.event_hash ?? GENESIS_HASH;
this.db.prepare('INSERT INTO canonical_events(sequence,type,height,hash,previous_hash,transactions_json,previous_event_hash,event_hash) VALUES(?,?,?,?,?,?,?,?)')
.run(sequence, type, block.height, block.hash, block.previousHash, JSON.stringify(block.transactions.map(transactionId)),
anchor, this.eventHash(sequence, type, block, anchor));
}
private eventHash(sequence: number, type: string, block: TestnetBlock, previousAnchor: string): string {
return sha256(JSON.stringify(['wikideck-canonical-event-v1', this.streamId, sequence, type,
block.height, block.hash, block.previousHash, block.transactions.map(transactionId), previousAnchor]));
}
connect(block: TestnetBlock): void {
const tokens = new Set([block.tokenId, ...block.transactions.flatMap(transactionCardIds)]);
const undo = [...tokens].map(token => [token, this.db.prepare('SELECT * FROM canonical_nfts WHERE token=?').get(token) ?? null]);
this.db.prepare('INSERT INTO canonical_undo VALUES(?,?)').run(block.height, JSON.stringify(undo));
this.db.prepare('INSERT INTO canonical_blocks VALUES(?,?)').run(block.height, block.hash);
this.db.prepare('INSERT INTO canonical_nfts VALUES(?,?,?,?,?)').run(block.tokenId, block.owner, 0, block.height, block.hash);
const update = this.db.prepare('UPDATE canonical_nfts SET owner=?,sequence=sequence+1 WHERE token=?');
for (const tx of block.transactions) {
if (isNftTrade(tx)) {
for (const leg of tx.offered) update.run(tx.counterparty, leg.cardHash);
for (const leg of tx.requested) update.run(tx.proposer, leg.cardHash);
} else update.run(tx.to, tx.cardHash);
this.db.prepare('INSERT INTO canonical_transactions VALUES(?,?,?)').run(transactionId(tx), block.height, block.hash);
}
this.event('connect', block);
}
disconnect(block: TestnetBlock): void {
const row = this.db.prepare('SELECT data FROM canonical_undo WHERE height=?').get(block.height) as { data: string } | undefined;
if (!row) throw new Error('Missing canonical undo record');
for (const [token, prior] of JSON.parse(row.data) as [string, NftRow | null][]) {
this.db.prepare('DELETE FROM canonical_nfts WHERE token=?').run(token);
if (prior) this.db.prepare('INSERT INTO canonical_nfts VALUES(?,?,?,?,?)')
.run(prior.token, prior.owner, prior.sequence, prior.minted_height, prior.block_hash);
}
this.db.prepare('DELETE FROM canonical_transactions WHERE height=?').run(block.height);
this.db.prepare('DELETE FROM canonical_blocks WHERE height=?').run(block.height);
this.db.prepare('DELETE FROM canonical_undo WHERE height=?').run(block.height);
this.event('disconnect', block);
}
head(): number {
return (this.db.prepare('SELECT COALESCE(MAX(sequence),0) AS n FROM canonical_events').get() as { n: number }).n;
}
events(after: number, limit = 64): CanonicalEvents {
const head = this.head();
if (!Number.isSafeInteger(after) || after < 0 || after > head || !Number.isSafeInteger(limit) || limit < 1 || limit > 64) {
throw new RpcError(409, 'Invalid canonical event cursor; restore/reconcile local consumer state.');
}
const rows = this.db.prepare('SELECT * FROM canonical_events WHERE sequence>? ORDER BY sequence LIMIT ?').all(after, limit) as
{ sequence: number; type: CanonicalEvent['type']; height: number; hash: string; previous_hash: string;
transactions_json: string; event_hash: string }[];
if (rows.some((r, i) => r.sequence !== after + i + 1)) throw new Error('Canonical event history gap');
const events = rows.map(r => ({ sequence: r.sequence, type: r.type, height: r.height, hash: r.hash,
previousHash: r.previous_hash, transactionIds: JSON.parse(r.transactions_json) as string[], eventHash: r.event_hash }));
const cursor = events.at(-1)?.sequence ?? after;
if (cursor < head && !events.length) throw new Error('Missing canonical event history');
const previousAnchor = after === 0 ? GENESIS_HASH : (this.db.prepare('SELECT event_hash FROM canonical_events WHERE sequence=?').get(after) as { event_hash: string } | undefined)?.event_hash;
if (!previousAnchor) throw new Error('Missing canonical cursor anchor');
return { streamId: this.streamId, previousAnchor, anchor: events.at(-1)?.eventHash ?? previousAnchor,
events, cursor, head, caughtUp: cursor === head };
}
nft(token: string, height: number): IndexedNft | undefined {
const row = this.db.prepare(`SELECT n.owner,n.sequence,n.minted_height,b.block_json FROM canonical_nfts n
JOIN blocks b ON b.hash=n.block_hash WHERE n.token=?`).get(token) as
{ owner: string; sequence: number; minted_height: number; block_json: string } | undefined;
return row ? { block: JSON.parse(row.block_json), owner: row.owner, sequence: row.sequence,
confirmations: height - row.minted_height + 1 } : undefined;
}
transaction(id: string): { height: number; blockHash: string } | undefined {
const row = this.db.prepare('SELECT height,block_hash FROM canonical_transactions WHERE id=?').get(id) as
{ height: number; block_hash: string } | undefined;
return row ? { height: row.height, blockHash: row.block_hash } : undefined;
}
deck(owner: string, offset: number, limit: number, height: number) {
const total = (this.db.prepare('SELECT COUNT(*) AS n FROM canonical_nfts WHERE owner=?').get(owner) as { n: number }).n;
const rows = this.db.prepare(`SELECT n.owner,n.sequence,n.minted_height,b.block_json FROM canonical_nfts n
JOIN blocks b ON b.hash=n.block_hash WHERE n.owner=? ORDER BY n.minted_height,n.token LIMIT ? OFFSET ?`)
.all(owner, limit, offset) as { owner: string; sequence: number; minted_height: number; block_json: string }[];
return { total, nfts: rows.map(r => ({ block: JSON.parse(r.block_json) as TestnetBlock,
sequence: r.sequence, confirmations: height - r.minted_height + 1 })) };
}
tip(): string {
return (this.db.prepare('SELECT hash FROM canonical_blocks ORDER BY height DESC LIMIT 1').get() as { hash: string } | undefined)?.hash ?? GENESIS_HASH;
}
}
+122 -27
View File
@@ -1,15 +1,17 @@
import { createHash, createPublicKey, verify } from 'node:crypto';
import { z } from 'zod';
import { NETWORK, CONSENSUS, headerMessage, blockClaimMessage, nftTransferMessage, statsSnapshotTuple,
type BlockHeader, type TestnetBlock, type NftTransfer, type NftState, type MiningTemplate } from '../../shared/testnet.ts';
import { NETWORK, CONSENSUS, MAX_TARGET, headerMessage, blockClaimMessage, nftTransferMessage, nftTradeMessage, statsSnapshotTuple,
type BlockHeader, type TestnetBlock, type NftTransfer, type NftState, type NftStateLayer,
isNftTrade, type NftStateView, type NftTrade, type TestnetTransaction, type MiningTemplate } from '../../shared/testnet.ts';
import type { Article } from '../../shared/protocol.ts';
import { deriveStats, type MintStatsSnapshot } from '../../shared/enrichment.ts';
import type { PowEngine } from './randomx.ts';
import { NETWORK_GENESIS_TIME } from '../../shared/network.ts';
export const sha256 = (data: string | Uint8Array) => createHash('sha256').update(data).digest('hex');
export const MAX_TARGET = `0fff${'f'.repeat(60)}`;
export { MAX_TARGET };
// Consensus manifest: an independently running node derives exactly the same genesis.
export const GENESIS_TIME = 1_792_411_200;
export const GENESIS_TIME = NETWORK_GENESIS_TIME;
export const CHAIN_ID = sha256(JSON.stringify([NETWORK, CONSENSUS, GENESIS_TIME, MAX_TARGET]));
export const GENESIS_HASH = sha256(`${NETWORK}:genesis:${CHAIN_ID}`);
export const INITIAL_KEY = sha256(`${NETWORK}:initial-key:${CHAIN_ID}`);
@@ -19,6 +21,25 @@ const signature = z.string().regex(/^[A-Za-z0-9+/]+={0,2}$/).length(88);
const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER);
export const transferSchema = z.object({ network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
cardHash: hex, from: hex, to: hex, sequence: integer, publicKey: key, signature }).strict();
const tradeLegSchema = z.object({ cardHash: hex, sequence: integer }).strict();
export const tradeSchema = z.object({ type: z.literal('trade'), network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
proposer: hex, counterparty: hex, offered: z.array(tradeLegSchema).min(1).max(CONSENSUS.maxTradeCards),
requested: z.array(tradeLegSchema).min(1).max(CONSENSUS.maxTradeCards),
proposerPublicKey: key, proposerSignature: signature, counterpartyPublicKey: key,
counterpartySignature: signature }).strict().superRefine((trade, ctx) => {
const offered = new Set(trade.offered.map(leg => leg.cardHash));
const requested = new Set(trade.requested.map(leg => leg.cardHash));
if (trade.proposer === trade.counterparty) ctx.addIssue({ code: 'custom', message: 'Trade parties must differ' });
if (offered.size !== trade.offered.length || requested.size !== trade.requested.length ||
trade.requested.some(leg => offered.has(leg.cardHash))) {
ctx.addIssue({ code: 'custom', message: 'Trade cards must be unique and cannot appear on both sides' });
}
if (!verifySignature(trade.proposerPublicKey, trade.proposer, nftTradeMessage(trade), trade.proposerSignature) ||
!verifySignature(trade.counterpartyPublicKey, trade.counterparty, nftTradeMessage(trade), trade.counterpartySignature)) {
ctx.addIssue({ code: 'custom', message: 'Trade requires valid signatures from both parties' });
}
});
export const transactionSchema = z.union([transferSchema, tradeSchema]);
const articleSchema = z.object({ pageId: integer.positive(), title: z.string().trim().min(1).max(200),
extract: z.string().trim().min(1).max(1000), image: z.string().url().max(1500),
url: z.string().url().max(500), revisionId: integer.positive(),
@@ -70,10 +91,10 @@ const statsSnapshotSchema = z.object({ capturedAt: z.string().datetime(), revisi
ctx.addIssue({ code: 'custom', message: 'Derived statistics do not match the page metrics', path: ['stats'] });
}
});
export const blockSchema = z.object({ version: z.literal(4), network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
export const blockSchema = z.object({ version: z.literal(6), network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
height: integer, previousHash: hex, createdAt: integer, article: articleSchema, statsSnapshot: statsSnapshotSchema, owner: hex,
publicKey: key, nonce: integer, target: hex, key: hex, transactionsRoot: hex, stateRoot: hex,
hash: hex, tokenId: hex, powHash: hex, signature, transactions: z.array(transferSchema).max(CONSENSUS.maxTransactions) }).strict();
hash: hex, tokenId: hex, powHash: hex, signature, transactions: z.array(transactionSchema).max(CONSENSUS.maxTransactions) }).strict();
export function address(publicKey: string): string { return sha256(Buffer.from(publicKey, 'base64')); }
export function verifySignature(publicKey: string, owner: string, message: string, sig: string): boolean {
@@ -90,6 +111,10 @@ export function transferId(t: NftTransfer): string {
// Exclude malleable ECDSA signature bytes from transaction identity.
return sha256(nftTransferMessage(t));
}
export function transactionId(transaction: TestnetTransaction): string {
if (isNftTrade(transaction)) return sha256(nftTradeMessage(transaction));
return transferId(transaction);
}
export function merkleRoot(leaves: string[]): string {
if (!leaves.length) return sha256(`${NETWORK}:empty`);
let layer = leaves.map(leaf => sha256(Buffer.concat([Buffer.from([0]), Buffer.from(leaf,'hex')])));
@@ -102,9 +127,32 @@ export function merkleRoot(leaves: string[]): string {
// Commit the count as well: odd-leaf padding must not make [a,b,c] equal [a,b,c,c].
return sha256(JSON.stringify([NETWORK,'merkle',leaves.length,layer[0]]));
}
export function stateRoot(state: NftState): string {
return merkleRoot(Object.keys(state).sort().map(id => {
const n = state[id]; return sha256(JSON.stringify([id, n.owner, n.sequence, n.mintedHeight, n.pageId]));
export function materializeState(state: NftStateView): NftState {
const layers: NftState[] = [];
let cursor: NftStateView = state;
while (isStateLayer(cursor)) {
layers.push(cursor.changes);
cursor = cursor.parent;
}
const result: NftState = Object.assign(Object.create(null) as NftState, cursor as NftState);
for (let i = layers.length - 1; i >= 0; i--) Object.assign(result, layers[i]);
return result;
}
function stateEntry(state: NftStateView, id: string): NftState[string] | undefined {
let cursor: NftStateView = state;
while (isStateLayer(cursor)) {
if (Object.hasOwn(cursor.changes, id)) return cursor.changes[id];
cursor = cursor.parent;
}
return (cursor as NftState)[id];
}
function isStateLayer(state: NftStateView): state is NftStateLayer {
return (state as NftStateLayer).kind === 'layer';
}
export function stateRoot(state: NftStateView): string {
const materialized = materializeState(state);
return merkleRoot(Object.keys(materialized).sort().map(id => {
const n = materialized[id]; return sha256(JSON.stringify([id, n.owner, n.sequence, n.mintedHeight, n.pageId]));
}));
}
export function mintId(h: Pick<BlockHeader, 'previousHash' | 'height' | 'owner' | 'article' | 'statsSnapshot'>): string {
@@ -143,47 +191,94 @@ export function applyTransfer(state: NftState, tx: NftTransfer, height: number):
if (!verifySignature(tx.publicKey, tx.from, nftTransferMessage(tx), tx.signature)) throw new Error('Invalid transfer signature');
state[tx.cardHash] = { ...nft, owner: tx.to, sequence: tx.sequence + 1 };
}
export function transition(parent: NftState, header: BlockHeader, txs: NftTransfer[]): NftState {
const next = structuredClone(parent);
export function applyTransferOverlay(state: NftStateView, tx: NftTransfer, height: number): NftStateLayer {
if (!transferSchema.safeParse(tx).success) throw new Error('Invalid transfer encoding/network');
const nft = stateEntry(state, tx.cardHash);
if (!nft || nft.owner !== tx.from || tx.from === tx.to || nft.sequence !== tx.sequence) {
throw new Error('Invalid NFT owner/sequence');
}
if (height - nft.mintedHeight < CONSENSUS.coinbaseMaturity) throw new Error('NFT coinbase is immature');
if (!verifySignature(tx.publicKey, tx.from, nftTransferMessage(tx), tx.signature)) throw new Error('Invalid transfer signature');
const changes: NftState = Object.create(null) as NftState;
changes[tx.cardHash] = { ...nft, owner: tx.to, sequence: tx.sequence + 1 };
return { kind: 'layer', parent: state, changes };
}
export function applyTradeOverlay(state: NftStateView, trade: NftTrade, height: number): NftStateLayer {
const parsed = tradeSchema.parse(trade);
const allLegs = [...parsed.offered, ...parsed.requested];
const current = allLegs.map(leg => stateEntry(state, leg.cardHash));
if (current.some((entry, index) => !entry || entry.sequence !== allLegs[index].sequence ||
height - entry.mintedHeight < CONSENSUS.coinbaseMaturity)) {
throw new Error('Trade contains missing, stale, or immature NFTs');
}
if (current.slice(0, parsed.offered.length).some(entry => entry!.owner !== parsed.proposer) ||
current.slice(parsed.offered.length).some(entry => entry!.owner !== parsed.counterparty)) {
throw new Error('Trade NFT owners do not match the signed parties');
}
const changes: NftState = Object.create(null) as NftState;
parsed.offered.forEach((leg, index) => {
const nft = current[index]!;
changes[leg.cardHash] = { ...nft, owner: parsed.counterparty, sequence: nft.sequence + 1 };
});
parsed.requested.forEach((leg, index) => {
const nft = current[parsed.offered.length + index]!;
changes[leg.cardHash] = { ...nft, owner: parsed.proposer, sequence: nft.sequence + 1 };
});
return { kind: 'layer', parent: state, changes };
}
export function applyTransactionOverlay(state: NftStateView, transaction: TestnetTransaction, height: number): NftStateLayer {
return isNftTrade(transaction)
? applyTradeOverlay(state, transaction, height)
: applyTransferOverlay(state, transaction, height);
}
export function transition(parent: NftStateView, header: BlockHeader, txs: TestnetTransaction[]): NftStateLayer {
let next = parent;
const ids = new Set<string>();
for (const tx of txs) {
const id = transferId(tx);
for (const transaction of txs) {
const id = transactionId(transaction);
if (ids.has(id)) throw new Error('Duplicate transfer');
ids.add(id); applyTransfer(next, tx, header.height);
ids.add(id);
next = applyTransactionOverlay(next, transaction, header.height);
}
const id = mintId(header);
if (next[id]) throw new Error('Duplicate NFT issuance');
next[id] = { owner: header.owner, sequence: 0, mintedHeight: header.height, pageId: header.article.pageId };
return next;
if (stateEntry(next, id)) throw new Error('Duplicate NFT issuance');
const changes: NftState = Object.create(null) as NftState;
changes[id] = { owner: header.owner, sequence: 0, mintedHeight: header.height, pageId: header.article.pageId };
return { kind: 'layer', parent: next, changes };
}
export function makeTemplate(ancestors: TestnetBlock[], state: NftState, owner: string,
publicKey: string, transactions: NftTransfer[], article: Article, statsSnapshot: MintStatsSnapshot,
export function makeTemplate(ancestors: TestnetBlock[], state: NftStateView, owner: string,
publicKey: string, transactions: TestnetTransaction[], article: Article, statsSnapshot: MintStatsSnapshot,
now = Math.floor(Date.now()/1000)): MiningTemplate {
if (!hex.safeParse(owner).success || address(publicKey) !== owner) throw new Error('Invalid miner identity');
const parsedArticle = articleSchema.parse(article);
const parsedStats = statsSnapshotSchema.parse(statsSnapshot);
const previousHash = ancestors.at(-1)?.hash ?? GENESIS_HASH;
const header: BlockHeader = { version: 4, network: NETWORK, chainId: CHAIN_ID,
const header: BlockHeader = { version: 6, network: NETWORK, chainId: CHAIN_ID,
height: ancestors.length + 1, previousHash, createdAt: Math.max(now, medianTime(ancestors) + 1),
article: parsedArticle, statsSnapshot: parsedStats, owner, publicKey, nonce: 0, target: expectedTarget(ancestors),
key: expectedKey(ancestors, ancestors.length + 1), transactionsRoot: merkleRoot(transactions.map(transferId)), stateRoot: '' };
key: expectedKey(ancestors, ancestors.length + 1), transactionsRoot: merkleRoot(transactions.map(transactionId)), stateRoot: '' };
header.stateRoot = stateRoot(transition(state, header, transactions));
return { header, transactions, blob: miningBlob(header).toString('hex'), nonceOffset: 32 };
}
export async function validateBlock(raw: unknown, ancestors: TestnetBlock[], parentState: NftState,
pow: PowEngine, now = Math.floor(Date.now()/1000)): Promise<{block:TestnetBlock;state:NftState}> {
export async function validateBlock(raw: unknown, ancestors: TestnetBlock[], parentState: NftStateView,
pow: PowEngine, now = Math.floor(Date.now()/1000), allowHistoricalFutureTimestamp = false): Promise<{block:TestnetBlock;state:NftStateView}> {
if (Buffer.byteLength(JSON.stringify(raw)) > CONSENSUS.maxBlockBytes) throw new Error('Block size limit exceeded');
const b = blockSchema.parse(raw);
if (b.height !== ancestors.length + 1 || b.previousHash !== (ancestors.at(-1)?.hash ?? GENESIS_HASH)) throw new Error('Invalid parent/height');
if (b.createdAt <= medianTime(ancestors) || b.createdAt > now + CONSENSUS.maxFutureSeconds) throw new Error('Invalid block timestamp');
if (b.createdAt <= medianTime(ancestors) || (!allowHistoricalFutureTimestamp && b.createdAt > now + CONSENSUS.maxFutureSeconds)) {
throw new Error('Invalid block timestamp');
}
if (b.target !== expectedTarget(ancestors) || b.key !== expectedKey(ancestors, b.height)) throw new Error('Invalid difficulty or RandomX epoch key');
if (b.transactionsRoot !== merkleRoot(b.transactions.map(transferId))) throw new Error('Invalid transaction root');
const next = transition(parentState, b, b.transactions);
if (b.stateRoot !== stateRoot(next) || b.tokenId !== mintId(b)) throw new Error('Invalid NFT state commitment');
if (b.transactionsRoot !== merkleRoot(b.transactions.map(transactionId))) throw new Error('Invalid transaction root');
if (b.tokenId !== mintId(b)) throw new Error('Invalid NFT state commitment');
const blob = miningBlob(b);
if (b.hash !== sha256(blob)) throw new Error('Invalid block ID');
if (!verifySignature(b.publicKey, b.owner, blockClaimMessage(b.hash), b.signature)) throw new Error('Invalid miner signature');
if (BigInt(`0x${b.powHash}`) > BigInt(`0x${b.target}`)) throw new Error('Insufficient work');
if (await pow.hash(b.key, blob) !== b.powHash) throw new Error('Invalid RandomX proof');
// Verify native work before materializing/sorting the complete NFT state. A malformed
// low-work block must not force O(total NFTs) commitment work on the request path.
const next = transition(parentState, b, b.transactions);
if (b.stateRoot !== stateRoot(next)) throw new Error('Invalid NFT state commitment');
return { block: b, state: next };
}
+80 -22
View File
@@ -6,19 +6,29 @@ import { fileURLToPath } from 'node:url';
import { mkdir, open, unlink } from 'node:fs/promises';
import { z } from 'zod';
import { NETWORK } from '../../shared/testnet.ts';
import { CHAIN_ID, blockSchema, transferSchema } from './consensus.ts';
import { CHAIN_ID, blockSchema, tradeSchema, transferSchema, transactionSchema } from './consensus.ts';
import { RandomX } from './randomx.ts';
import { Ledger } from './ledger.ts';
import { PeerNetwork } from './peers.ts';
import { wikipediaEnrichment } from '../enrichment.ts';
import { snapshotStats } from '../../shared/enrichment.ts';
import { wikipediaArticles } from '../wiki.ts';
import { IS_MAINNET, PROFILE } from '../../shared/network.ts';
import { assertLaunchAllowed, launchManifest } from '../launch.ts';
import { resourceConfig, RpcBudget, RpcError } from '../resources.ts';
import { trustedProxies, operationsPort, startOperations } from '../operations.ts';
const root = fileURLToPath(new URL('../../', import.meta.url));
assertLaunchAllowed();
const limits = resourceConfig();
const proxyTrust=trustedProxies();
const opsPort=operationsPort();
const apiBudget = new RpcBudget(limits);
const peerBudget = new RpcBudget({ ...limits, requestsPerSecond: 40, writesPerSecond: 10, active: 16 });
const dataDir = process.env.WIKIDECK_DATA_DIR ?? resolve(root,'data');
await mkdir(dataDir,{recursive:true});
// Never permit two processes to rewrite the same ledger. A stale lock requires operator review.
const lockPath = resolve(dataDir,'testnet-node.lock');
const lockPath = resolve(dataDir,`${PROFILE}-node.lock`);
const lock = await open(lockPath,'wx',0o600).catch(()=>{throw new Error('Data directory locked. Stop its other node; remove stale testnet-node.lock only after verifying no node is running.');});
await lock.writeFile(String(process.pid));
let engine: RandomX;
@@ -30,15 +40,18 @@ catch(e) {engine.close();await lock.close();await unlink(lockPath);throw e;}
let peers: PeerNetwork;
try { peers = new PeerNetwork({ ledger, urls:(process.env.WIKIDECK_PEERS ?? '').split(',').map(s=>s.trim()).filter(Boolean),
allowPrivate:process.env.WIKIDECK_ALLOW_PRIVATE_PEERS === '1' }); }
catch(e) {engine.close();await lock.close();await unlink(lockPath);throw e;}
catch(e) {ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
const app = express();
app.disable('x-powered-by');
if(proxyTrust.length)app.set('trust proxy',proxyTrust);
app.use(helmet({contentSecurityPolicy:{directives:{defaultSrc:["'self'"],scriptSrc:["'self'"],
imgSrc:["'self'",'data:','https://upload.wikimedia.org','https://thumb.wikimedia.org'],styleSrc:["'self'","'unsafe-inline'"],
connectSrc:["'self'"],upgradeInsecureRequests:null}},crossOriginEmbedderPolicy:false,
strictTransportSecurity:process.env.NODE_ENV==='production'}));
app.get('/healthz',(_req,res)=>{res.setHeader('Cache-Control','no-store');res.json({status:'ok',network:NETWORK});});
app.use(express.json({limit:'68kb',strict:true}));
// Admission comes before body parsing and signature/schema work. Disable compressed request inflation.
app.use('/api',apiBudget.middleware());
app.use('/p2p',peerBudget.middleware());
app.use(['/api','/p2p'],rateLimit({windowMs:60000,limit:300,standardHeaders:'draft-8',legacyHeaders:false}));
app.use(['/api','/p2p'],(req,res,next)=>{
res.setHeader('Cache-Control','no-store');
@@ -50,9 +63,31 @@ app.use(['/api','/p2p'],(req,res,next)=>{
}
next();
});
app.get('/api/state',(_req,res)=>res.json(ledger.snapshot()));
app.get('/api/chain',(_req,res)=>res.json(ledger.snapshot()));
app.use(['/api','/p2p'],express.json({limit:'68kb',strict:true,inflate:false}));
app.get('/api/state',(_req,res)=>res.json(ledger.legacySnapshot(limits.legacyBlocks)));
app.get('/api/summary',(_req,res)=>res.json(ledger.summary()));
app.get('/api/canonical',(req,res)=>{
const q=z.object({tip:z.string().regex(/^[a-f0-9]{64}$/),offset:z.string().regex(/^\d{1,12}$/).optional(),
limit:z.string().regex(/^\d{1,2}$/).optional()}).strict().parse(req.query);
res.json(ledger.canonicalPage(q.tip,Number(q.offset??0),Number(q.limit??16)));
});
app.get('/api/network',(_req,res)=>res.json(launchManifest()));
app.get('/api/chain',(_req,res)=>res.json(ledger.legacySnapshot(limits.legacyBlocks)));
app.get('/api/peers',(_req,res)=>res.json({peers:peers.status()}));
app.get('/api/decks/:address', rateLimit({windowMs:60000,limit:60,standardHeaders:'draft-8',legacyHeaders:false}), (req,res)=>{
if(typeof req.params.address!=='string'||!/^[a-f0-9]{64}$/.test(req.params.address)){
res.status(400).json({error:'Invalid wallet address'});return;
}
const offset=Number(req.query.offset??0),limit=Number(req.query.limit??24);
if(!Number.isSafeInteger(offset)||offset<0||!Number.isSafeInteger(limit)||limit<1||limit>24){
res.status(400).json({error:'Invalid deck page'});return;
}
res.setHeader('Cache-Control','no-store');
if(req.query.tip!==undefined && req.query.tip!==ledger.summary().tip){
res.status(409).json({error:'Canonical tip changed; restart deck pagination.'});return;
}
res.json(ledger.deck(req.params.address,offset,limit));
});
// Optional live data is intentionally not used by genesis, block validation or NFT ownership.
app.get('/api/wikipedia/:pageId', rateLimit({windowMs:60000,limit:60,standardHeaders:'draft-8',legacyHeaders:false}), async(req,res)=>{
if(typeof req.params.pageId!=='string'||!/^[1-9]\d{0,9}$/.test(req.params.pageId)){res.status(400).json({error:'Invalid Wikipedia page ID'});return;}
@@ -60,22 +95,24 @@ app.get('/api/wikipedia/:pageId', rateLimit({windowMs:60000,limit:60,standardHea
try {res.json(await wikipediaEnrichment.get(pageId));}
catch {res.status(503).json({error:'Wikipedia enrichment is temporarily unavailable. NFT data is unaffected.'});}
});
app.get('/p2p/status',(_req,res)=>res.json({network:NETWORK,chainId:CHAIN_ID,size:ledger.size,tip:ledger.snapshot().tip}));
app.get('/p2p/status',(_req,res)=>res.json(ledger.peerStatus()));
app.get('/p2p/blocks',(req,res)=>{
const offset=Number(req.query.offset??0);
if(!Number.isSafeInteger(offset)||offset<0||offset>10000){res.status(400).json({error:'Invalid offset'});return;}
if(!Number.isSafeInteger(offset)||offset<0){res.status(400).json({error:'Invalid offset'});return;}
const blocks=ledger.inventory(offset,32);
res.json({blocks,next:offset+blocks.length<ledger.size?offset+blocks.length:null});
});
app.get('/p2p/mempool',(_req,res)=>res.json({transactions:ledger.snapshot().pendingTransfers}));
app.get('/p2p/mempool',(_req,res)=>res.json({transactions:ledger.pendingTransactions()}));
app.get('/api/nfts/:id', (req,res)=>{
const s=ledger.snapshot();const block=s.blocks.find(b=>b.tokenId===req.params.id);
if(!block){res.status(404).json({error:'NFT not found on the canonical chain'});return;}
let owner=block.owner,sequence=0;
for(const tx of s.transfers)if(tx.cardHash===block.tokenId){owner=tx.to;sequence++;}
res.json({network:NETWORK,chainId:CHAIN_ID,tokenId:block.tokenId,owner,sequence,blockHash:block.hash,
confirmations:s.blocks.length-block.height+1,article:block.article,statsSnapshot:block.statsSnapshot,
stateRoot:s.blocks.at(-1)?.stateRoot});
if(typeof req.params.id!=='string'||!/^[a-f0-9]{64}$/.test(req.params.id)){
res.status(400).json({error:'Invalid NFT identifier'});return;
}
const nft=ledger.nft(req.params.id);
if(!nft){res.status(404).json({error:'NFT not found on the canonical chain'});return;}
const block=nft.block;
res.json({network:NETWORK,chainId:CHAIN_ID,tokenId:block.tokenId,owner:nft.owner,sequence:nft.sequence,blockHash:block.hash,
confirmations:nft.confirmations,article:block.article,statsSnapshot:block.statsSnapshot,
stateRoot:ledger.block(ledger.summary().tip)?.stateRoot});
});
const templateLimit=rateLimit({windowMs:60000,limit:20,standardHeaders:'draft-8',legacyHeaders:false});
app.post('/api/mining/template', templateLimit, async (req,res)=>{
@@ -95,12 +132,23 @@ app.post(['/api/blocks','/p2p/block'],workLimit,async(req,res)=>{
res.status(result.accepted?201:200).json(result);
if(result.accepted)void peers.broadcastBlock(block).catch(()=>{});
});
app.post(['/api/transfer','/p2p/transaction'],workLimit,async(req,res)=>{
app.post('/api/transfer',workLimit,async(req,res)=>{
const tx=transferSchema.parse(req.body);
const id=await ledger.submitTransfer(tx);
res.status(202).json({id,status:'pending',message:'Ownership changes only when this transaction is mined.'});
void peers.broadcastTransfer(tx).catch(()=>{});
});
app.post('/api/trades',workLimit,async(req,res)=>{
const trade=tradeSchema.parse(req.body);
const id=await ledger.submitTransaction(trade);
res.status(202).json({id,status:'pending',message:'The complete multi-card trade is pending; ownership changes atomically when it is mined.'});
void peers.broadcastTransaction(trade).catch(()=>{});
});
app.post('/p2p/transaction',workLimit,async(req,res)=>{
const transaction=transactionSchema.parse(req.body);
const id=await ledger.submitTransaction(transaction);
res.status(202).json({id,status:'pending'});
});
app.use(['/api','/p2p'],(_req,res)=>{res.status(404).json({error:'Unknown testnet endpoint'});});
app.use(express.static(resolve(root,'dist')));
app.get('/{*path}',(_req,res)=>res.sendFile(resolve(root,'dist/index.html')));
@@ -108,17 +156,27 @@ app.use((err:Error & {status?:number},_req:express.Request,res:express.Response,
const input=err instanceof z.ZodError||err instanceof SyntaxError;
const message=input?'Invalid request encoding':err.message;
// No keys/passwords are accepted by any node endpoint.
res.status(err.status===413?413:400).json({error:message.slice(0,200)});
if(err instanceof RpcError&&err.status===503)res.setHeader('Retry-After','1');
res.status(err instanceof RpcError?err.status:err.status===413?413:err.status===415?415:400).json({error:message.slice(0,200)});
});
const port=Number(process.env.PORT??3001),host=process.env.HOST??'127.0.0.1';
const server=app.listen(port,host,()=>{
const a=server.address();console.log(`Wikideck RandomX testnet: http://${host}:${typeof a==='object'&&a?a.port:port} · chain ${CHAIN_ID} · ${ledger.snapshot().blocks.length} blocks`);
const a=server.address();console.log(`Wikideck RandomX ${IS_MAINNET?'mainnet candidate':'testnet'}: http://${host}:${typeof a==='object'&&a?a.port:port} · chain ${CHAIN_ID} · ${ledger.snapshot().blocks.length} blocks`);
peers.start();
});
const operations=startOperations(opsPort,()=>({network:NETWORK,
ledger:ledger.diagnostics(),native:engine.diagnostics(),
peers:{configured:peers.status().length,connected:peers.status().filter(p=>p.connected).length},
rpc:{api:apiBudget.status(),p2p:peerBudget.status()}}),resolve(dataDir,PROFILE));
server.maxConnections=256;
server.headersTimeout=10000;
server.requestTimeout=15000;
server.keepAliveTimeout=5000;
server.setTimeout(30000,socket=>socket.destroy());
let stopping=false;
for(const signal of ['SIGTERM','SIGINT']as const)process.once(signal,()=>{
if(stopping)return;stopping=true;peers.close();
if(stopping)return;stopping=true;peers.close();operations.close();
const timeout=setTimeout(()=>process.exit(1),25000);timeout.unref();
server.close(async()=>{engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
server.close(async()=>{ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
});
server.on('error',async()=>{peers.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});
server.on('error',async()=>{stopping=true;operations.close();peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});
+250 -100
View File
@@ -1,88 +1,191 @@
import { mkdir, open, rename, unlink } from 'node:fs/promises';
import { chmod, mkdir, readFile, stat } from 'node:fs/promises';
import { join } from 'node:path';
import { randomUUID } from 'node:crypto';
import { setTimeout as delay } from 'node:timers/promises';
import { DatabaseSync } from 'node:sqlite';
import { z } from 'zod';
import { NETWORK, CONSENSUS, type TestnetBlock, type NftTransfer, type NftState,
type MiningTemplate, type TestnetSnapshot } from '../../shared/testnet.ts';
import { CHAIN_ID, GENESIS_HASH, applyTransfer, blockSchema, makeTemplate,
transferId, transferSchema, validateBlock, workFor } from './consensus.ts';
import { NETWORK, CONSENSUS, type TestnetBlock,
type NftState, type NftStateView, type TestnetTransaction, type MiningTemplate,
type TestnetDeck, type TestnetSnapshot, type TestnetSummary, type CanonicalPage } from '../../shared/testnet.ts';
import { CHAIN_ID, GENESIS_HASH, applyTransactionOverlay, blockSchema, makeTemplate,
applyTransferOverlay, materializeState, transactionId, transactionSchema, transferSchema,
validateBlock, workFor } from './consensus.ts';
import type { PowEngine } from './randomx.ts';
import type { Article } from '../../shared/protocol.ts';
import type { MintStatsSnapshot } from '../../shared/enrichment.ts';
import { PROFILE } from '../../shared/network.ts';
import { RpcError } from '../resources.ts';
import { CanonicalIndex } from './canonical-index.ts';
const MAX_BLOCKS = 10_000;
const MAX_PENDING = 256;
const MAX_FILE = 128 * 1024 * 1024;
const MAX_DECK_PAGE = 24;
const MAX_LEGACY_FILE = 128 * 1024 * 1024;
const diskSchema = z.object({ version: z.literal(1), network: z.literal(NETWORK),
chainId: z.literal(CHAIN_ID), blocks: z.array(blockSchema).max(MAX_BLOCKS),
pendingTransfers: z.array(transferSchema).max(MAX_PENDING) }).strict();
type Node = { block: TestnetBlock; state: NftState; work: bigint };
chainId: z.literal(CHAIN_ID), blocks: z.array(blockSchema).max(10_000),
pendingTransfers: z.array(transactionSchema).max(MAX_PENDING) }).strict();
type Node = { block: TestnetBlock; state: NftStateView; work: bigint };
const DATABASE_SCHEMA_VERSION = '1';
/** Experimental, bounded DAG. The supplied engine is always used, including on restart.
* One process must own a data directory; there is no cross-process writer locking.
/** SQLite-backed DAG. Blocks append transactionally; pending rows are replaced transactionally.
* Each block state retains only its NFT delta, sharing the parent's state overlay.
*/
export class Ledger {
private readonly nodes = new Map<string, Node>();
private tip = GENESIS_HASH;
private pending: NftTransfer[] = [];
private pending: TestnetTransaction[] = [];
private pendingState: NftStateView = {};
private queue: Promise<unknown> = Promise.resolve();
private queued = 0;
/** Application-owned connectivity metadata; never part of consensus or persistence. */
peers = 0;
private readonly file: string;
private readonly db: DatabaseSync;
private readonly pow: PowEngine;
private constructor(file: string, pow: PowEngine) { this.file = file; this.pow = pow; }
private readonly index: CanonicalIndex;
private closed = false;
private revision = 0;
private reorgs = 0;
private readonly nowSeconds: () => number;
private canonicalCache: { tip: string; blocks: TestnetBlock[] } | undefined;
private constructor(file: string, db: DatabaseSync, pow: PowEngine, nowSeconds: () => number) {
this.file = file; this.db = db; this.pow = pow; this.nowSeconds = nowSeconds; this.index = new CanonicalIndex(db);
}
static async open(dir: string, pow: PowEngine): Promise<Ledger> {
const directory = join(dir, 'testnet');
static async open(dir: string, pow: PowEngine, nowSeconds: () => number = () => Math.floor(Date.now() / 1000)): Promise<Ledger> {
const directory = join(dir, PROFILE);
await mkdir(directory, { recursive: true, mode: 0o700 });
const ledger = new Ledger(join(directory, 'ledger.json'), pow);
let handle;
try { handle = await open(ledger.file, 'r'); }
catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
await ledger.persist([], []);
return ledger;
}
let text: string;
const file = join(directory, 'ledger.sqlite');
let databaseExists = false;
try { databaseExists = (await stat(file)).size > 0; }
catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
const db = new DatabaseSync(file);
try {
if ((await handle.stat()).size > MAX_FILE) throw new Error('Testnet ledger file exceeds 128 MiB');
// Bounded reads also protect against a file growing after stat().
const chunks: Buffer[] = [];
let size = 0;
for (;;) {
const buffer = Buffer.alloc(Math.min(64 * 1024, MAX_FILE + 1 - size));
const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
if (!bytesRead) break;
size += bytesRead;
if (size > MAX_FILE) throw new Error('Testnet ledger file exceeds 128 MiB');
chunks.push(buffer.subarray(0, bytesRead));
const tableExists = (name: string) => Boolean(db.prepare(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name));
if (databaseExists && !tableExists('metadata')) {
throw new Error('SQLite ledger metadata missing; refusing to initialize over unknown data');
}
text = Buffer.concat(chunks).toString('utf8');
} finally { await handle.close(); }
const disk = diskSchema.parse(JSON.parse(text)); // Mismatches/corruption fail closed, never reset.
for (const raw of disk.blocks) {
if (ledger.nodes.has(raw.hash)) throw new Error('Duplicate persisted block');
const node = await ledger.validate(raw);
ledger.nodes.set(node.block.hash, node);
if (node.work > ledger.tipWork()) ledger.tip = node.block.hash;
const readMeta = databaseExists ? db.prepare('SELECT value FROM metadata WHERE key = ?') : undefined;
if (!databaseExists) {
db.exec('BEGIN IMMEDIATE');
try {
db.exec(`CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
CREATE TABLE blocks (
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
hash TEXT NOT NULL UNIQUE,
previous_hash TEXT NOT NULL,
block_json TEXT NOT NULL
);
CREATE INDEX blocks_by_parent ON blocks(previous_hash);
CREATE TABLE pending_transfers (
position INTEGER PRIMARY KEY,
id TEXT NOT NULL UNIQUE,
transaction_json TEXT NOT NULL
);`);
const insertMeta = db.prepare('INSERT INTO metadata(key, value) VALUES(?, ?)');
insertMeta.run('schema_version', DATABASE_SCHEMA_VERSION);
insertMeta.run('network', NETWORK);
insertMeta.run('chain_id', CHAIN_ID);
db.exec('COMMIT');
} catch (error) { db.exec('ROLLBACK'); throw error; }
} else {
if (!tableExists('blocks') || !tableExists('pending_transfers')) {
throw new Error('SQLite ledger schema is incomplete; refusing to repair stored data automatically');
}
const schemaVersion = readMeta!.get('schema_version') as { value: string } | undefined;
const storedNetwork = readMeta!.get('network') as { value: string } | undefined;
const storedChain = readMeta!.get('chain_id') as { value: string } | undefined;
if (schemaVersion?.value !== DATABASE_SCHEMA_VERSION || storedNetwork?.value !== NETWORK || storedChain?.value !== CHAIN_ID) {
throw new Error('SQLite ledger schema or chain mismatch; refusing to reset stored data');
}
}
if (process.platform !== 'win32') await chmod(file, 0o600);
db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = FULL; PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;');
const migrated = db.prepare('SELECT value FROM metadata WHERE key = ?').get('legacy_json_migrated') as { value: string } | undefined;
if (!migrated) {
const legacyFile = join(directory, 'ledger.json');
let legacy: unknown;
try {
const info = await stat(legacyFile);
if (info.size > MAX_LEGACY_FILE) throw new Error('Legacy testnet ledger exceeds 128 MiB; preserve it and migrate offline');
legacy = JSON.parse(await readFile(legacyFile, 'utf8'));
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
}
const insertMeta = db.prepare('INSERT OR REPLACE INTO metadata(key, value) VALUES(?, ?)');
db.exec('BEGIN IMMEDIATE');
try {
if (legacy !== undefined) {
const oldDisk = diskSchema.parse(legacy);
const insertBlock = db.prepare('INSERT INTO blocks(hash, previous_hash, block_json) VALUES(?, ?, ?)');
for (const block of oldDisk.blocks) insertBlock.run(block.hash, block.previousHash, JSON.stringify(block));
const insertPending = db.prepare('INSERT INTO pending_transfers(position, id, transaction_json) VALUES(?, ?, ?)');
oldDisk.pendingTransfers.forEach((tx, index) => insertPending.run(index, transactionId(tx), JSON.stringify(tx)));
insertMeta.run('legacy_json_imported_at', String(nowSeconds()));
insertMeta.run('legacy_json_block_count', String(oldDisk.blocks.length));
}
insertMeta.run('legacy_json_migrated', '1');
db.exec('COMMIT');
} catch (error) { db.exec('ROLLBACK'); throw error; }
}
const ledger = new Ledger(file, db, pow, nowSeconds);
const importedAtText = (db.prepare('SELECT value FROM metadata WHERE key=?').get('legacy_json_imported_at') as { value: string } | undefined)?.value;
const importedCountText = (db.prepare('SELECT value FROM metadata WHERE key=?').get('legacy_json_block_count') as { value: string } | undefined)?.value;
const legacyImportedAt = importedAtText !== undefined && /^(0|[1-9]\d{0,10})$/.test(importedAtText)
? Number(importedAtText) : undefined;
const legacyImportedCount = importedCountText !== undefined && /^(0|[1-9]\d{0,9})$/.test(importedCountText)
? Number(importedCountText) : undefined;
if ((importedAtText !== undefined && (legacyImportedAt === undefined || !Number.isSafeInteger(legacyImportedAt))) ||
(importedCountText !== undefined && (legacyImportedCount === undefined || !Number.isSafeInteger(legacyImportedCount))) ||
(legacyImportedAt === undefined) !== (legacyImportedCount === undefined)) {
throw new Error('Legacy ledger import timestamp metadata is invalid');
}
for (const row of db.prepare('SELECT sequence, block_json FROM blocks ORDER BY sequence').iterate() as Iterable<{ sequence: number; block_json: string }>) {
const raw = blockSchema.parse(JSON.parse(row.block_json));
if (ledger.nodes.has(raw.hash)) throw new Error('Duplicate persisted block');
const importedLegacyBlock = legacyImportedCount !== undefined && row.sequence <= legacyImportedCount;
// Historical SQLite blocks already passed admission; wall-clock rollback must not
// strand them. Imported JSON blocks retain their original admission clock so an
// untrusted legacy file cannot inject arbitrarily future-dated history.
const node = importedLegacyBlock
? await ledger.validate(raw, false, legacyImportedAt!)
: await ledger.validate(raw, true);
ledger.nodes.set(node.block.hash, node);
if (node.work > ledger.tipWork()) ledger.tip = node.block.hash;
}
let state = ledger.tipState();
const seen = new Set(ledger.canonical().flatMap(b => b.transactions.map(transactionId)));
for (const row of db.prepare('SELECT id, transaction_json FROM pending_transfers ORDER BY position').iterate() as Iterable<{ id: string; transaction_json: string }>) {
const tx = transactionSchema.parse(JSON.parse(row.transaction_json));
const id = transactionId(tx);
if (row.id !== id || seen.has(id)) throw new Error('Duplicate or mismatched persisted transfer');
state = applyTransactionOverlay(state, tx, ledger.height() + 1);
seen.add(id);
ledger.pending.push(tx);
}
ledger.pendingState = state;
ledger.index.initialize(ledger.canonical(), materializeState(ledger.tipState()));
return ledger;
} catch (error) {
db.close();
throw error;
}
const state = structuredClone(ledger.tipState());
const seen = new Set(ledger.canonical().flatMap(b => b.transactions.map(transferId)));
for (const tx of disk.pendingTransfers) {
const id = transferId(tx);
if (seen.has(id)) throw new Error('Duplicate persisted transfer');
applyTransfer(state, tx, ledger.height() + 1);
seen.add(id);
}
ledger.pending = disk.pendingTransfers;
return ledger;
}
get size(): number { return this.nodes.size; }
get allBlocks(): TestnetBlock[] { return this.inventory(0, MAX_BLOCKS); }
diagnostics() { const createdAt=this.nodes.get(this.tip)?.block.createdAt??null;
return { height:this.height(), dagBlocks:this.size, pending:this.pending.length, mutationQueue:this.queued,
reorgsSinceStartup:this.reorgs,tipCreatedAt:createdAt,tipAgeSeconds:createdAt===null?null:Math.max(0,Date.now()/1000-createdAt) }; }
get allBlocks(): TestnetBlock[] {
const rows = this.db.prepare('SELECT block_json FROM blocks ORDER BY sequence').all() as { block_json: string }[];
return rows.map(row => blockSchema.parse(JSON.parse(row.block_json)));
}
close(): void {
if (this.closed) return;
this.closed = true;
this.db.close();
}
block(hash: string): TestnetBlock | undefined {
const block = this.nodes.get(hash)?.block;
return block ? structuredClone(block) : undefined;
@@ -91,10 +194,12 @@ export class Ledger {
* HTTP callers should request limit=32 and return next=offset+length or null.
*/
inventory(offset: number, limit: number): TestnetBlock[] {
if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > MAX_BLOCKS) {
if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > 1000) {
throw new Error('Invalid inventory range');
}
return structuredClone(Array.from(this.nodes.values()).slice(offset, offset + limit).map(n => n.block));
const rows = this.db.prepare('SELECT block_json FROM blocks ORDER BY sequence LIMIT ? OFFSET ?')
.all(limit, offset) as { block_json: string }[];
return rows.map(row => blockSchema.parse(JSON.parse(row.block_json)));
}
async submit(raw: unknown): Promise<{ accepted: boolean; reorg: boolean }> {
@@ -102,37 +207,47 @@ export class Ledger {
return this.serialize(async () => {
const parsed = blockSchema.parse(input);
if (this.nodes.has(parsed.hash)) return { accepted: false, reorg: false };
if (this.size >= MAX_BLOCKS) throw new Error('Experimental testnet limit: 10000 stored blocks');
const node = await this.validate(parsed);
const wins = node.work > this.tipWork();
const oldChain = this.canonical();
const chain = wins ? [...this.branch(node.block.previousHash), node.block] : oldChain;
const hashes = new Set(chain.map(b => b.hash));
const disconnected = wins ? oldChain.filter(b => !hashes.has(b.hash)) : [];
const pending = wins ? this.filterPending(node.state, chain,
[...disconnected.flatMap(b => b.transactions), ...this.pending]) : this.pending;
const filtered = wins ? this.filterPending(node.state, chain,
[...disconnected.flatMap(b => b.transactions), ...this.pending]) : undefined;
const pending = filtered?.pending ?? this.pending;
// Nothing visible changes until the atomic replacement succeeds.
await this.persist([...Array.from(this.nodes.values(), n => n.block), node.block], pending);
const oldHashes = new Set(oldChain.map(b => b.hash));
const connected = wins ? chain.filter(b => !oldHashes.has(b.hash)) : [];
this.persist(node.block, pending, disconnected, connected);
this.nodes.set(node.block.hash, node);
if (wins) this.tip = node.block.hash;
if (disconnected.length) this.reorgs++;
this.pending = pending;
if (filtered) this.pendingState = filtered.state;
this.revision++;
return { accepted: true, reorg: disconnected.length > 0 };
});
}
async submitTransfer(raw: unknown): Promise<string> {
const tx = transferSchema.parse(structuredClone(raw));
return this.submitTransaction(tx);
}
async submitTransaction(raw: unknown): Promise<string> {
const input = structuredClone(raw);
return this.serialize(async () => {
const tx = transferSchema.parse(input);
const id = transferId(tx);
if (this.pending.some(t => transferId(t) === id)) return id;
const tx = transactionSchema.parse(input);
const id = transactionId(tx);
if (this.pending.some(t => transactionId(t) === id)) return id;
if (this.pending.length >= MAX_PENDING) throw new Error('Testnet mempool limit: 256 transfers');
const state = structuredClone(this.tipState());
for (const pending of this.pending) applyTransfer(state, pending, this.height() + 1);
applyTransfer(state, tx, this.height() + 1); // No speculative issuance.
const nextPendingState = applyTransactionOverlay(this.pendingState, tx, this.height() + 1);
const pending = [...this.pending, tx];
await this.persist(Array.from(this.nodes.values(), n => n.block), pending);
this.persist(undefined, pending);
this.pending = pending;
this.pendingState = nextPendingState;
this.revision++;
return id;
});
}
@@ -150,9 +265,43 @@ export class Ledger {
peers: this.peers, algorithm: 'RandomX',
maturity: CONSENSUS.coinbaseMaturity });
}
/** Small metadata/mempool response. No full-chain traversal or clone. */
summary(): TestnetSummary {
return { network: NETWORK, chainId: CHAIN_ID, genesis: GENESIS_HASH, tip: this.tip,
height: this.height(), revision: this.revision, cumulativeWork: this.tipWork().toString(),
pendingTransfers: structuredClone(this.pending), peers: this.peers, algorithm: 'RandomX', maturity: CONSENSUS.coinbaseMaturity };
}
peerStatus() { return { network: NETWORK, chainId: CHAIN_ID, size: this.size, tip: this.tip }; }
pendingTransactions(): TestnetTransaction[] { return structuredClone(this.pending); }
canonicalEvents(after: number, limit = 64) { return this.index.events(after, limit); }
canonicalHead(): number { return this.index.head(); }
nft(token: string) { return this.index.nft(token, this.height()); }
canonicalTransaction(id: string) { return this.index.transaction(id); }
legacySnapshot(maxBlocks: number): TestnetSnapshot {
// Reject before any full-chain copy/serialization. Never truncate ownership history silently.
if (this.height() > maxBlocks) throw new RpcError(413, 'Full snapshot disabled at this height; use /api/summary and /api/canonical pages.');
return this.snapshot();
}
canonicalPage(tip: string, offset: number, limit = 16): CanonicalPage {
if (tip !== this.tip) throw new RpcError(409, 'Canonical tip changed; restart paged loading.');
if (!Number.isSafeInteger(offset) || offset < 0 || offset > this.height() ||
!Number.isSafeInteger(limit) || limit < 1 || limit > 16) throw new RpcError(400, 'Invalid canonical page');
const blocks = structuredClone(this.canonical().slice(offset, offset + limit));
return { network: NETWORK, chainId: CHAIN_ID, tip: this.tip, height: this.height(), offset, blocks,
next: offset + blocks.length < this.height() ? offset + blocks.length : null };
}
deck(address: string, offset = 0, limit = MAX_DECK_PAGE): TestnetDeck {
if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > MAX_DECK_PAGE) {
throw new Error('Invalid deck page');
}
const { total, nfts } = this.index.deck(address, offset, limit, this.height());
const next = offset + nfts.length < total ? offset + nfts.length : null;
return structuredClone({ network: NETWORK, chainId: CHAIN_ID, address, tip: this.tip,
offset, limit, total, next, nfts });
}
private height(): number { return this.nodes.get(this.tip)?.block.height ?? 0; }
private tipState(): NftState { return this.nodes.get(this.tip)?.state ?? {}; }
private tipState(): NftStateView { return this.nodes.get(this.tip)?.state ?? {}; }
private tipWork(): bigint { return this.nodes.get(this.tip)?.work ?? 0n; }
private branch(hash: string): TestnetBlock[] {
const result: TestnetBlock[] = [];
@@ -164,26 +313,31 @@ export class Ledger {
}
return result.reverse();
}
private canonical(): TestnetBlock[] { return this.branch(this.tip); }
private async validate(raw: TestnetBlock): Promise<Node> {
private canonical(): TestnetBlock[] {
if (this.canonicalCache?.tip !== this.tip) this.canonicalCache = { tip: this.tip, blocks: this.branch(this.tip) };
return this.canonicalCache.blocks; // Internal immutable block references only; public APIs clone their slice.
}
private async validate(raw: TestnetBlock, historicalReplay = false, now = this.nowSeconds()): Promise<Node> {
const ancestors = this.branch(raw.previousHash);
const parent = this.nodes.get(raw.previousHash);
const { block, state } = await validateBlock(raw, ancestors, parent?.state ?? {}, this.pow);
const { block, state } = await validateBlock(raw, ancestors, parent?.state ?? {}, this.pow,
now, historicalReplay);
return { block, state, work: (parent?.work ?? 0n) + workFor(block.target) };
}
private filterPending(parent: NftState, chain: TestnetBlock[], candidates: NftTransfer[]): NftTransfer[] {
const state = structuredClone(parent);
const seen = new Set(chain.flatMap(b => b.transactions.map(transferId)));
const pending: NftTransfer[] = [];
private filterPending(parent: NftStateView, chain: TestnetBlock[], candidates: TestnetTransaction[]):
{ pending: TestnetTransaction[]; state: NftStateView } {
let state = parent;
const seen = new Set(chain.flatMap(b => b.transactions.map(transactionId)));
const pending: TestnetTransaction[] = [];
for (const tx of candidates) {
if (pending.length === MAX_PENDING) break;
const id = transferId(tx);
const id = transactionId(tx);
if (seen.has(id)) continue;
try { applyTransfer(state, tx, chain.length + 1); }
try { state = applyTransactionOverlay(state, tx, chain.length + 1); }
catch { continue; } // Disconnected issuance, immature coins and conflicts are not replayable.
seen.add(id); pending.push(tx);
}
return pending;
return { pending, state };
}
private serialize<T>(operation: () => Promise<T>): Promise<T> {
if (this.queued >= 256) return Promise.reject(new Error('Testnet mutation queue full'));
@@ -192,25 +346,21 @@ export class Ledger {
this.queue = result.catch(() => undefined).finally(() => { this.queued--; });
return result;
}
private async persist(blocks: TestnetBlock[], pendingTransfers: NftTransfer[]): Promise<void> {
const data = JSON.stringify({ version: 1, network: NETWORK, chainId: CHAIN_ID, blocks, pendingTransfers });
if (Buffer.byteLength(data) > MAX_FILE) throw new Error('Testnet ledger file exceeds 128 MiB');
const temp = `${this.file}.${randomUUID()}.tmp`;
private persist(block: TestnetBlock | undefined, pendingTransfers: TestnetTransaction[],
disconnected: TestnetBlock[] = [], connected: TestnetBlock[] = []): void {
this.db.exec('BEGIN IMMEDIATE');
try {
const handle = await open(temp, 'wx', 0o600);
try { await handle.writeFile(data); await handle.sync(); }
finally { await handle.close(); }
// Windows scanners can briefly hold the destination open without delete sharing.
// Retry only transient sharing failures; never unlink the old durable ledger.
for (let attempt = 0; ; attempt++) {
try { await rename(temp, this.file); break; }
catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (process.platform !== 'win32' || attempt >= 5 ||
!['EPERM', 'EACCES', 'EBUSY'].includes(code ?? '')) throw error;
await delay(10 * 2 ** attempt);
}
}
} finally { await unlink(temp).catch(() => undefined); }
if (block) this.db.prepare('INSERT INTO blocks(hash, previous_hash, block_json) VALUES(?, ?, ?)')
.run(block.hash, block.previousHash, JSON.stringify(block));
this.db.exec('DELETE FROM pending_transfers');
const insertPending = this.db.prepare('INSERT INTO pending_transfers(position, id, transaction_json) VALUES(?, ?, ?)');
pendingTransfers.forEach((tx, index) => insertPending.run(index, transactionId(tx), JSON.stringify(tx)));
for (const old of [...disconnected].reverse()) this.index.disconnect(old);
for (const next of connected) this.index.connect(next);
this.db.exec('COMMIT');
} catch (error) {
this.db.exec('ROLLBACK');
throw error;
}
}
}
+5 -2
View File
@@ -3,9 +3,12 @@ import { resolve } from 'node:path';
import { stdin, stdout } from 'node:process';
import { randomInt } from 'node:crypto';
import { decryptWallet } from '../../shared/wallet-backup.ts';
import { blockClaimMessage, NETWORK, type MiningTemplate, type TestnetBlock } from '../../shared/testnet.ts';
import { blockClaimMessage, CONSENSUS, NETWORK, type MiningTemplate, type TestnetBlock } from '../../shared/testnet.ts';
import { CHAIN_ID, sha256, miningBlob, mintId, blockSchema } from './consensus.ts';
import { RandomX } from './randomx.ts';
import { assertLaunchAllowed } from '../launch.ts';
assertLaunchAllowed();
const args = process.argv.slice(2);
const arg = (key: string) => { const i = args.indexOf(key); return i < 0 ? undefined : args[i+1]; };
@@ -83,7 +86,7 @@ try {
header.nonce += 16;
stdout.write(`\r${(attempts / ((Date.now()-started)/1000)).toFixed(1)} H/s · searching block #${header.height} `);
const status = await api('/p2p/status');
if (status.tip !== header.previousHash || Date.now()/1000 - header.createdAt > 60) { console.log('\nRefreshing stale work'); break; }
if (status.tip !== header.previousHash || Date.now()/1000 - header.createdAt > CONSENSUS.targetSeconds * 2) { console.log('\nRefreshing stale work'); break; }
// Explicit idle interval; native miner never starts with the web page.
await new Promise(resolve => setTimeout(resolve,100));
}
+15 -12
View File
@@ -3,8 +3,8 @@ import { request as httpsRequest } from 'node:https';
import dns from 'node:dns/promises';
import { BlockList, isIP } from 'node:net';
import { z } from 'zod';
import { NETWORK, type TestnetBlock, type NftTransfer } from '../../shared/testnet.ts';
import { CHAIN_ID, blockSchema, transferId, transferSchema } from './consensus.ts';
import { NETWORK, type TestnetBlock, type TestnetTransaction } from '../../shared/testnet.ts';
import { CHAIN_ID, blockSchema, transactionId, transactionSchema } from './consensus.ts';
import type { Ledger } from './ledger.ts';
const MAX_BODY = 2 * 1024 * 1024;
@@ -24,14 +24,14 @@ function publicAddress(ip: string): boolean {
return family === 6 && globalV6.check(ip, 'ipv6') && !excluded.check(ip, 'ipv6');
}
const remoteStatus = z.object({ network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
size: z.number().int().min(0).max(10_000), tip: z.string().regex(/^[a-f0-9]{64}$/).optional() }).strict();
size: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), tip: z.string().regex(/^[a-f0-9]{64}$/).optional() }).strict();
const pageSchema = z.object({ blocks: z.array(blockSchema).max(32),
next: z.number().int().min(0).max(10_000).nullable() }).strict();
const poolSchema = z.object({ transactions: z.array(transferSchema).max(256) }).strict();
next: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).nullable() }).strict();
const poolSchema = z.object({ transactions: z.array(transactionSchema).max(256) }).strict();
type Peer = { url: string; connected: boolean; error?: string; offset: number; busy: boolean };
/** Outbound-only fixed allowlist. HTTP routes are deliberately left to the caller.
* allowPrivate is an explicit operator permission for local/Docker networks, not peer input.
* allowPrivate is an explicit operator permission for local/Docker/cluster networks, not peer input.
*/
export class PeerNetwork {
private readonly peers: Peer[];
@@ -90,10 +90,13 @@ export class PeerNetwork {
if (!known) throw new Error('Cannot broadcast an unaccepted block');
await this.broadcast('/p2p/block', known);
}
async broadcastTransfer(tx: NftTransfer): Promise<void> {
const id = transferId(transferSchema.parse(tx));
async broadcastTransfer(tx: TestnetTransaction): Promise<void> {
return this.broadcastTransaction(tx);
}
async broadcastTransaction(tx: TestnetTransaction): Promise<void> {
const id = transactionId(transactionSchema.parse(tx));
const snapshot = this.ledger.snapshot();
const known = [...snapshot.pendingTransfers, ...snapshot.transfers].find(t => transferId(t) === id);
const known = [...snapshot.pendingTransfers, ...snapshot.transfers].find(t => transactionId(t) === id);
if (!known) throw new Error('Cannot broadcast an unaccepted transfer');
await this.broadcast('/p2p/transaction', known);
}
@@ -121,8 +124,8 @@ export class PeerNetwork {
if (peer.offset < status.size) {
const page = pageSchema.parse(await this.request(peer, `/p2p/blocks?offset=${peer.offset}`));
const end = peer.offset + page.blocks.length;
if (!page.blocks.length || end > 10_000 ||
(page.next !== null && (page.next !== end || end >= 10_000)) ||
if (!page.blocks.length || !Number.isSafeInteger(end) ||
(page.next !== null && page.next !== end) ||
(page.next === null && end < status.size)) throw new Error('Invalid peer pagination');
for (const block of page.blocks) {
if (this.closed) throw new Error('Peer network closed');
@@ -134,7 +137,7 @@ export class PeerNetwork {
for (const tx of pool.transactions) {
if (this.closed) throw new Error('Peer network closed');
// Valid remote transactions may be immature/conflicting on our current tip.
try { await this.ledger.submitTransfer(tx); } catch { /* Independently reject; do not relay. */ }
try { await this.ledger.submitTransaction(tx); } catch { /* Independently reject; do not relay. */ }
}
}
+1
View File
@@ -87,6 +87,7 @@ export class RandomX implements PowEngine {
return line;
});
}
diagnostics() { return { available: !this.failure && !this.exited, active: !!this.active, queued: this.queue.length }; }
async mine(
key: string, blob: Uint8Array, target: string, nonceOffset: number, maxAttempts: number,