WikiDeck/wikideck
3
0
Fork 1
Code Issues 8 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

More scalable data storage

Browse Source
This commit is contained in:
eric
2026-10-19 20:17:10 -04:00
parent 0b9061372b
commit baad477ca5
16 changed files with 346 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/testnet/consensus.ts
+53 -15
View File
@@ -1,7 +1,8 @@
import { createHash, createPublicKey, verify } from 'node:crypto';
import { z } from 'zod';
import { NETWORK, CONSENSUS, headerMessage, blockClaimMessage, nftTransferMessage, statsSnapshotTuple,
type BlockHeader, type TestnetBlock, type NftTransfer, type NftState, type MiningTemplate } from '../../shared/testnet.ts';
type BlockHeader, type TestnetBlock, type NftTransfer, type NftState, type NftStateLayer,
type NftStateView, type MiningTemplate } from '../../shared/testnet.ts';
import type { Article } from '../../shared/protocol.ts';
import { deriveStats, type MintStatsSnapshot } from '../../shared/enrichment.ts';
import type { PowEngine } from './randomx.ts';
@@ -70,7 +71,7 @@ const statsSnapshotSchema = z.object({ capturedAt: z.string().datetime(), revisi
ctx.addIssue({ code: 'custom', message: 'Derived statistics do not match the page metrics', path: ['stats'] });
}
});
export const blockSchema = z.object({ version: z.literal(4), network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
export const blockSchema = z.object({ version: z.literal(5), network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
height: integer, previousHash: hex, createdAt: integer, article: articleSchema, statsSnapshot: statsSnapshotSchema, owner: hex,
publicKey: key, nonce: integer, target: hex, key: hex, transactionsRoot: hex, stateRoot: hex,
hash: hex, tokenId: hex, powHash: hex, signature, transactions: z.array(transferSchema).max(CONSENSUS.maxTransactions) }).strict();
@@ -102,9 +103,32 @@ export function merkleRoot(leaves: string[]): string {
// Commit the count as well: odd-leaf padding must not make [a,b,c] equal [a,b,c,c].
return sha256(JSON.stringify([NETWORK,'merkle',leaves.length,layer[0]]));
}
export function stateRoot(state: NftState): string {
return merkleRoot(Object.keys(state).sort().map(id => {
const n = state[id]; return sha256(JSON.stringify([id, n.owner, n.sequence, n.mintedHeight, n.pageId]));
export function materializeState(state: NftStateView): NftState {
const layers: NftState[] = [];
let cursor: NftStateView = state;
while (isStateLayer(cursor)) {
layers.push(cursor.changes);
cursor = cursor.parent;
}
const result: NftState = Object.assign(Object.create(null) as NftState, cursor as NftState);
for (let i = layers.length - 1; i >= 0; i--) Object.assign(result, layers[i]);
return result;
}
function stateEntry(state: NftStateView, id: string): NftState[string] | undefined {
let cursor: NftStateView = state;
while (isStateLayer(cursor)) {
if (Object.hasOwn(cursor.changes, id)) return cursor.changes[id];
cursor = cursor.parent;
}
return (cursor as NftState)[id];
}
function isStateLayer(state: NftStateView): state is NftStateLayer {
return (state as NftStateLayer).kind === 'layer';
}
export function stateRoot(state: NftStateView): string {
const materialized = materializeState(state);
return merkleRoot(Object.keys(materialized).sort().map(id => {
const n = materialized[id]; return sha256(JSON.stringify([id, n.owner, n.sequence, n.mintedHeight, n.pageId]));
}));
}
export function mintId(h: Pick<BlockHeader, 'previousHash' | 'height' | 'owner' | 'article' | 'statsSnapshot'>): string {
@@ -143,35 +167,49 @@ export function applyTransfer(state: NftState, tx: NftTransfer, height: number):
if (!verifySignature(tx.publicKey, tx.from, nftTransferMessage(tx), tx.signature)) throw new Error('Invalid transfer signature');
state[tx.cardHash] = { ...nft, owner: tx.to, sequence: tx.sequence + 1 };
}
export function transition(parent: NftState, header: BlockHeader, txs: NftTransfer[]): NftState {
const next = structuredClone(parent);
export function applyTransferOverlay(state: NftStateView, tx: NftTransfer, height: number): NftStateLayer {
if (!transferSchema.safeParse(tx).success) throw new Error('Invalid transfer encoding/network');
const nft = stateEntry(state, tx.cardHash);
if (!nft || nft.owner !== tx.from || tx.from === tx.to || nft.sequence !== tx.sequence) {
throw new Error('Invalid NFT owner/sequence');
}
if (height - nft.mintedHeight < CONSENSUS.coinbaseMaturity) throw new Error('NFT coinbase is immature');
if (!verifySignature(tx.publicKey, tx.from, nftTransferMessage(tx), tx.signature)) throw new Error('Invalid transfer signature');
const changes: NftState = Object.create(null) as NftState;
changes[tx.cardHash] = { ...nft, owner: tx.to, sequence: tx.sequence + 1 };
return { kind: 'layer', parent: state, changes };
}
export function transition(parent: NftStateView, header: BlockHeader, txs: NftTransfer[]): NftStateLayer {
let next = parent;
const ids = new Set<string>();
for (const tx of txs) {
const id = transferId(tx);
if (ids.has(id)) throw new Error('Duplicate transfer');
ids.add(id); applyTransfer(next, tx, header.height);
ids.add(id);
next = applyTransferOverlay(next, tx, header.height);
}
const id = mintId(header);
if (next[id]) throw new Error('Duplicate NFT issuance');
next[id] = { owner: header.owner, sequence: 0, mintedHeight: header.height, pageId: header.article.pageId };
return next;
if (stateEntry(next, id)) throw new Error('Duplicate NFT issuance');
const changes: NftState = Object.create(null) as NftState;
changes[id] = { owner: header.owner, sequence: 0, mintedHeight: header.height, pageId: header.article.pageId };
return { kind: 'layer', parent: next, changes };
}
export function makeTemplate(ancestors: TestnetBlock[], state: NftState, owner: string,
export function makeTemplate(ancestors: TestnetBlock[], state: NftStateView, owner: string,
publicKey: string, transactions: NftTransfer[], article: Article, statsSnapshot: MintStatsSnapshot,
now = Math.floor(Date.now()/1000)): MiningTemplate {
if (!hex.safeParse(owner).success || address(publicKey) !== owner) throw new Error('Invalid miner identity');
const parsedArticle = articleSchema.parse(article);
const parsedStats = statsSnapshotSchema.parse(statsSnapshot);
const previousHash = ancestors.at(-1)?.hash ?? GENESIS_HASH;
const header: BlockHeader = { version: 4, network: NETWORK, chainId: CHAIN_ID,
const header: BlockHeader = { version: 5, network: NETWORK, chainId: CHAIN_ID,
height: ancestors.length + 1, previousHash, createdAt: Math.max(now, medianTime(ancestors) + 1),
article: parsedArticle, statsSnapshot: parsedStats, owner, publicKey, nonce: 0, target: expectedTarget(ancestors),
key: expectedKey(ancestors, ancestors.length + 1), transactionsRoot: merkleRoot(transactions.map(transferId)), stateRoot: '' };
header.stateRoot = stateRoot(transition(state, header, transactions));
return { header, transactions, blob: miningBlob(header).toString('hex'), nonceOffset: 32 };
}
export async function validateBlock(raw: unknown, ancestors: TestnetBlock[], parentState: NftState,
pow: PowEngine, now = Math.floor(Date.now()/1000)): Promise<{block:TestnetBlock;state:NftState}> {
export async function validateBlock(raw: unknown, ancestors: TestnetBlock[], parentState: NftStateView,
pow: PowEngine, now = Math.floor(Date.now()/1000)): Promise<{block:TestnetBlock;state:NftStateView}> {
if (Buffer.byteLength(JSON.stringify(raw)) > CONSENSUS.maxBlockBytes) throw new Error('Block size limit exceeded');
const b = blockSchema.parse(raw);
if (b.height !== ancestors.length + 1 || b.previousHash !== (ancestors.at(-1)?.hash ?? GENESIS_HASH)) throw new Error('Invalid parent/height');