WikiDeck/wikideck
3
0
Fork 1
Code Issues 8 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

Canonical indexes and recoverable market observation

Browse Source
This commit is contained in:
eric
2026-10-20 14:35:45 -04:00
parent fc65e497a8
commit e48081f041
11 changed files with 583 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/market/store.ts
+74 -18
View File
@@ -11,8 +11,17 @@ import { isXmrListing, isXmrOrder, marketCommandV3Schema, marketMessageV3,
type MarketOrderRecord, type MarketOrderV3 } from '../../shared/market-v3.ts';
import { CHAIN_ID, transactionId, transferSchema, verifySignature } from '../testnet/consensus.ts';
import { mockQuote, mockXmrQuote, paymentMode } from './payments.ts';
import { RpcError } from '../resources.ts';
import type { CanonicalEvents, IndexedNft } from '../testnet/canonical-index.ts';
import { z } from 'zod';
type Chain = { snapshot(): TestnetSnapshot; summary?(): { tip: string; revision: number }; submitTransfer(raw: unknown): Promise<string> };
type Chain = { snapshot(): TestnetSnapshot; summary?(): { tip: string; revision: number; height?: number };
submitTransfer(raw: unknown): Promise<string>; nft?(token: string): IndexedNft | undefined;
canonicalTransaction?(id: string): { height: number; blockHash: string } | undefined;
canonicalHead?(): number; canonicalEvents?(after: number, limit?: number): CanonicalEvents;
pendingTransactions?(): TestnetSnapshot['pendingTransfers'] };
type ChainView = { height: number; pendingTransfers: TestnetSnapshot['pendingTransfers'];
nft(token: string): IndexedNft | undefined; transaction(id: string): { height: number; blockHash: string } | undefined };
type Config = { feeBps: number; confirmations: number };
export function marketConfig(env: NodeJS.ProcessEnv = process.env): Config {
paymentMode(env);
@@ -28,12 +37,17 @@ export function marketConfig(env: NodeJS.ProcessEnv = process.env): Config {
}
const PAYMENT_WINDOW = 30 * 60_000;
const DELIVERY_WINDOW = 48 * 60 * 60_000;
const cursorSchema = z.object({ version: z.literal(1), streamId: z.string().uuid(),
cursor: z.number().int().nonnegative().safe(), anchor: z.string().regex(/^[a-f0-9]{64}$/) }).strict();
/** Node-local marketplace, NOT consensus escrow. Its SQLite journal is backed up with the ledger. */
export class Market {
private queue: Promise<unknown> = Promise.resolve();
private queued = 0;
private reconciled: { tip: string; revision: number; at: number } | undefined;
private eventCursor = 0;
private eventCheckpoint: z.infer<typeof cursorSchema> | undefined;
private eventBacklog = false;
private readonly db: DatabaseSync;
private readonly chain: Chain;
readonly config: Config;
@@ -41,6 +55,11 @@ export class Market {
private readonly now: () => number;
private constructor(db: DatabaseSync, chain: Chain, config: Config, id: string, now: () => number) {
this.db = db; this.chain = chain; this.config = config; this.id = id; this.now = now;
const cursor = db.prepare("SELECT value FROM market_meta WHERE key='canonical_event_cursor'").get() as { value: string } | undefined;
if (cursor) {
this.eventCheckpoint = cursorSchema.parse(JSON.parse(cursor.value));
this.eventCursor = this.eventCheckpoint.cursor;
}
}
static async open(directory: string, chain: Chain, config = marketConfig(), now = Date.now): Promise<Market> {
@@ -77,7 +96,8 @@ export class Market {
close() { this.db.close(); }
diagnostics() {
const states=this.db.prepare("SELECT json_extract(data,'$.status') AS status,COUNT(*) AS count FROM orders GROUP BY json_extract(data,'$.status')").all();
return { queued:this.queued,lastReconciledAt:this.reconciled?.at??null,orderStates:states };
return { queued:this.queued,lastReconciledAt:this.reconciled?.at??null,orderStates:states,
canonicalEventCursor:this.eventCursor,canonicalEventHead:this.chain.canonicalHead?.()??null,eventBacklog:this.eventBacklog };
}
info(): MarketInfo {
return { network: NETWORK, chainId: CHAIN_ID, marketId: this.id, paymentMode: 'mock', marketVersion: 3, feeBps: this.config.feeBps,
@@ -110,11 +130,25 @@ export class Market {
try { const result = fn(); this.db.exec('COMMIT'); return result; }
catch (error) { this.db.exec('ROLLBACK'); throw error; }
}
private card(snapshot: TestnetSnapshot, hash: string, owner: string, sequence: number): boolean {
const block = snapshot.blocks.find(b => b.tokenId === hash);
return !!block && testnetOwnerOf(snapshot, hash) === owner &&
snapshot.transfers.filter(tx => transactionCardIds(tx).includes(hash)).length === sequence &&
snapshot.blocks.length + 1 - block.height >= CONSENSUS.coinbaseMaturity;
private view(): ChainView {
if (this.chain.nft && this.chain.canonicalTransaction && this.chain.pendingTransactions && this.chain.summary) {
const height = this.chain.summary().height;
if (height === undefined) throw new Error('Indexed chain height missing');
return { height, pendingTransfers: this.chain.pendingTransactions(), nft: token => this.chain.nft!(token),
transaction: id => this.chain.canonicalTransaction!(id) };
}
// Compatibility boundary for unit-test/legacy adapters only. The actual node always supplies indexed queries.
const s = this.chain.snapshot();
const txs = new Map(s.blocks.flatMap(b => b.transactions.map(tx => [transactionId(tx), { height: b.height, blockHash: b.hash }] as const)));
return { height: s.blocks.length, pendingTransfers: s.pendingTransfers, transaction: id => txs.get(id), nft: token => {
const block = s.blocks.find(b => b.tokenId === token);
return block ? { block, owner: testnetOwnerOf(s, token)!, sequence: s.transfers.filter(tx => transactionCardIds(tx).includes(token)).length,
confirmations: s.blocks.length + 1 - block.height } : undefined;
} };
}
private card(view: ChainView, hash: string, owner: string, sequence: number): boolean {
const nft = view.nft(hash);
return !!nft && nft.owner === owner && nft.sequence === sequence && nft.confirmations >= CONSENSUS.coinbaseMaturity;
}
private deliveryId(order: MarketOrderRecord): string {
// Identity excludes signature bytes. Recognize exact sale delivery through ANY submission route,
@@ -124,17 +158,34 @@ export class Market {
}
/** Reconcile from canonical transaction identity, never from a mempool acknowledgement. */
private reconcile() {
const s = this.chain.snapshot(), now = this.now();
const deliveries = new Map<string, TestnetSnapshot['blocks'][number]>();
for (const b of s.blocks) for (const tx of b.transactions) deliveries.set(transactionId(tx), b);
const events = this.chain.canonicalEvents?.(this.eventCursor, 64);
if (events && this.eventCheckpoint && (events.streamId !== this.eventCheckpoint.streamId ||
events.previousAnchor !== this.eventCheckpoint.anchor)) throw new RpcError(409, 'Canonical event history changed; restore matching ledger and market backups.');
const s = this.view(), now = this.now();
const pendingIds = new Set(s.pendingTransfers.map(transactionId));
this.atomic(() => {
// Process every disconnect, even if the same delivery reappeared before this consumer caught up.
// Never infer that a current tip erases an intermediate loss of previously settled evidence.
const disconnected = new Set(events?.events.filter(e => e.type === 'disconnect').flatMap(e => e.transactionIds) ?? []);
if (events) {
for (const o of this.rows<MarketOrderRecord>('orders')) {
if (o.status === 'complete' && disconnected.has(this.deliveryId(o))) {
o.status = 'review'; o.payout = 'review';
if (o.additionalFee) o.additionalFee.payout = 'review';
this.saveOrder(o);
}
}
this.db.prepare('INSERT INTO market_meta VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value')
.run('canonical_event_cursor', JSON.stringify({ version: 1, streamId: events.streamId, cursor: events.cursor, anchor: events.anchor }));
// Catch-up is bounded per pass. No new payout, timeout/refund decision or mutation while events remain.
if (!events.caughtUp) return;
}
for (const o of this.rows<MarketOrderRecord>('orders')) {
const l = this.listing(o.listingId);
if (o.status === 'refunded' || o.status === 'expired') continue;
const deliveryId = this.deliveryId(o);
const delivery = deliveries.get(deliveryId);
const confirmations = delivery ? s.blocks.length - delivery.height + 1 : 0;
const delivery = s.transaction(deliveryId);
const confirmations = delivery ? s.height - delivery.height + 1 : 0;
o.deliveryConfirmations = confirmations;
if (o.status === 'complete' || o.status === 'review') {
if (!delivery || confirmations < o.requiredConfirmations) {
@@ -174,8 +225,12 @@ export class Market {
}
this.db.prepare('DELETE FROM receipts WHERE expires < ?').run(now - 86400_000);
});
if (events) {
this.eventCursor = events.cursor; this.eventBacklog = !events.caughtUp;
this.eventCheckpoint = { version: 1, streamId: events.streamId, cursor: events.cursor, anchor: events.anchor };
}
const summary = this.chain.summary?.();
if (summary) this.reconciled = { ...summary, at: now };
if (summary && !this.eventBacklog) this.reconciled = { ...summary, at: now };
}
browse(offset = 0): { listings: MarketListingRecord[]; next: number | null; info: MarketInfo } {
if (!Number.isSafeInteger(offset) || offset < 0 || offset > 10000) throw new Error('Invalid marketplace page');
@@ -215,6 +270,7 @@ export class Market {
const summary = this.chain.summary?.(), last = this.reconciled;
if (a.type !== 'activity' || !summary || !last || summary.tip !== last.tip || summary.revision !== last.revision ||
this.now() - last.at >= 1000) this.reconcile();
if (this.eventBacklog && a.type !== 'activity') throw new RpcError(503, 'Market canonical reconciliation is catching up; retry shortly.');
if (a.type === 'activity') return {
orders: (this.db.prepare("SELECT data FROM orders WHERE json_extract(data,'$.seller')=? OR json_extract(data,'$.buyer')=? ORDER BY rowid DESC")
.all(c.actor,c.actor) as { data: string }[]).map(r=>JSON.parse(r.data) as MarketOrderRecord),
@@ -240,7 +296,7 @@ export class Market {
return value;
}
private mutateV2(c: MarketCommand): MarketActivityV3 | MarketListingRecord | MarketOrderRecord {
const a = c.action, now = this.now(), s = this.chain.snapshot();
const a = c.action, now = this.now(), s = this.view();
if (a.type === 'list') {
throw new Error('USD listing terms are retired; create a new native-XMR listing');
}
@@ -260,7 +316,7 @@ export class Market {
o.status = 'paid'; o.deliverBy = now + DELIVERY_WINDOW;
} else {
if (o.status !== 'refund-required' || s.pendingTransfers.some(tx => transactionCardIds(tx).includes(o.cardHash)) ||
s.blocks.some(b => b.transactions.some(tx => transactionId(tx) === this.deliveryId(o)))) throw new Error('Refund requires failed delivery and no pending/canonical sale transfer');
s.transaction(this.deliveryId(o))) throw new Error('Refund requires failed delivery and no pending/canonical sale transfer');
o.status = 'refunded'; const l = this.listing(o.listingId); l.status = 'cancelled'; this.saveListing(l);
}
this.saveOrder(o); return o;
@@ -275,7 +331,7 @@ export class Market {
}
private mutateV3(c: MarketCommandV3): MarketActivityV3 | MarketListingRecord | MarketOrderRecord {
const a = c.action, now = this.now(), s = this.chain.snapshot();
const a = c.action, now = this.now(), s = this.view();
if (a.type === 'list') {
if (a.feeBps !== this.config.feeBps) throw new Error('Market fee changed; review and sign again');
if (!this.card(s, a.cardHash, c.actor, a.sequence) || s.pendingTransfers.some(tx => transactionCardIds(tx).includes(a.cardHash))) throw new Error('Card is missing, stale, immature, or already pending');
@@ -283,7 +339,7 @@ export class Market {
if (this.rows<MarketListingRecord>('listings').filter(l => l.seller === c.actor && ['open','reserved'].includes(l.status)).length >= 100) throw new Error('Limit: 100 active listings per seller');
if ((this.db.prepare('SELECT COUNT(*) AS n FROM listings').get() as { n: number }).n >= 10000) throw new Error('Marketplace journal limit reached; operator archival required');
if (this.db.prepare("SELECT 1 FROM listings WHERE card=? AND status IN ('open','reserved')").get(a.cardHash)) throw new Error('Card already listed');
const mint = s.blocks.find(b => b.tokenId === a.cardHash)!;
const mint = s.nft(a.cardHash)!.block;
const l: MarketListingV3 = { id: randomUUID(), termsVersion: 3, cardHash: a.cardHash, sequence: a.sequence,
seller: c.actor, title: mint.article.title, image: mint.article.image!, kind: a.kind,
priceAtomicXmr: a.priceAtomicXmr, feeBps: a.feeBps, additionalFee: { ...ADDITIONAL_MARKET_FEE },
@@ -337,7 +393,7 @@ export class Market {
order.status = 'paid'; order.deliverBy = now + DELIVERY_WINDOW;
} else {
if (order.status !== 'refund-required' || s.pendingTransfers.some(tx => transactionCardIds(tx).includes(order.cardHash)) ||
s.blocks.some(b => b.transactions.some(tx => transactionId(tx) === this.deliveryId(order)))) throw new Error('Refund requires failed delivery and no pending/canonical sale transfer');
s.transaction(this.deliveryId(order))) throw new Error('Refund requires failed delivery and no pending/canonical sale transfer');
order.status = 'refunded'; const listing = this.listing(order.listingId); listing.status = 'cancelled'; this.saveListing(listing);
}
this.saveOrder(order); return order;
server/testnet/canonical-index.ts
Executable
+186
View File
@@ -0,0 +1,186 @@
import type { DatabaseSync } from 'node:sqlite';
import { randomUUID } from 'node:crypto';
import { isNftTrade, transactionCardIds, type NftState, type TestnetBlock } from '../../shared/testnet.ts';
import { GENESIS_HASH, sha256, transactionId } from './consensus.ts';
import { RpcError } from '../resources.ts';
export type CanonicalEvent = { sequence: number; type: 'connect' | 'disconnect';
height: number; hash: string; previousHash: string; transactionIds: string[]; eventHash: string };
export type CanonicalEvents = { streamId: string; previousAnchor: string; anchor: string;
events: CanonicalEvent[]; cursor: number; head: number; caughtUp: boolean };
type NftRow = { token: string; owner: string; sequence: number; minted_height: number; block_hash: string };
export type IndexedNft = { block: TestnetBlock; owner: string; sequence: number; confirmations: number };
/** Local derived indexes, NOT consensus. Writes share the ledger's block/pending transaction. */
export class CanonicalIndex {
private readonly db: DatabaseSync;
private streamId = '';
constructor(db: DatabaseSync) { this.db = db; }
initialize(chain: TestnetBlock[], state: NftState): void {
this.db.exec('BEGIN IMMEDIATE');
try {
this.db.exec(`CREATE TABLE IF NOT EXISTS canonical_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS canonical_blocks(height INTEGER PRIMARY KEY,hash TEXT NOT NULL UNIQUE REFERENCES blocks(hash));
CREATE TABLE IF NOT EXISTS canonical_nfts(token TEXT PRIMARY KEY,owner TEXT NOT NULL,sequence INTEGER NOT NULL,
minted_height INTEGER NOT NULL,block_hash TEXT NOT NULL REFERENCES blocks(hash));
CREATE INDEX IF NOT EXISTS canonical_owner ON canonical_nfts(owner,minted_height,token);
CREATE TABLE IF NOT EXISTS canonical_transactions(id TEXT PRIMARY KEY,height INTEGER NOT NULL,block_hash TEXT NOT NULL);
CREATE INDEX IF NOT EXISTS canonical_tx_height ON canonical_transactions(height);
CREATE TABLE IF NOT EXISTS canonical_undo(height INTEGER PRIMARY KEY,data TEXT NOT NULL);
CREATE TABLE IF NOT EXISTS canonical_events(sequence INTEGER PRIMARY KEY AUTOINCREMENT,type TEXT NOT NULL,
height INTEGER NOT NULL,hash TEXT NOT NULL,previous_hash TEXT NOT NULL,transactions_json TEXT NOT NULL,
previous_event_hash TEXT NOT NULL,event_hash TEXT NOT NULL);
CREATE TRIGGER IF NOT EXISTS canonical_events_no_update BEFORE UPDATE ON canonical_events
BEGIN SELECT RAISE(ABORT,'Immutable canonical event'); END;
CREATE TRIGGER IF NOT EXISTS canonical_events_no_delete BEFORE DELETE ON canonical_events
BEGIN SELECT RAISE(ABORT,'Immutable canonical event'); END;`);
const version = this.db.prepare("SELECT value FROM canonical_meta WHERE key='version'").get() as { value: string } | undefined;
if (!version) {
// Additive backfill only after Ledger.open has independently validated all persisted proofs/state.
const count = this.db.prepare('SELECT COUNT(*) AS n FROM canonical_events').get() as { n: number };
if (count.n || (this.db.prepare('SELECT COUNT(*) AS n FROM canonical_blocks').get() as { n: number }).n) {
throw new Error('Partial canonical index; offline operator review required');
}
this.streamId = randomUUID();
this.db.prepare('INSERT INTO canonical_meta VALUES(?,?)').run('stream_id', this.streamId);
for (const block of chain) this.connect(block);
this.db.prepare('INSERT INTO canonical_meta VALUES(?,?)').run('version', '1');
} else if (version.value !== '1') throw new Error('Canonical index version mismatch');
else {
const row = this.db.prepare("SELECT value FROM canonical_meta WHERE key='stream_id'").get() as { value: string } | undefined;
if (!row || !/^[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}$/.test(row.value)) throw new Error('Canonical stream identity missing');
this.streamId = row.value;
}
this.verify(chain, state);
this.db.exec('COMMIT');
} catch (error) { this.db.exec('ROLLBACK'); throw error; }
}
private verify(chain: TestnetBlock[], state: NftState): void {
const blocks = this.db.prepare('SELECT height,hash FROM canonical_blocks ORDER BY height').all() as { height: number; hash: string }[];
if (blocks.length !== chain.length || blocks.some((b, i) => b.height !== chain[i].height || b.hash !== chain[i].hash)) {
throw new Error('Canonical block index mismatch');
}
const nfts = this.db.prepare('SELECT * FROM canonical_nfts').all() as NftRow[];
const expected = new Map(chain.map(b => [b.tokenId, b]));
if (nfts.length !== expected.size) throw new Error('Canonical NFT index mismatch');
for (const row of nfts) {
const nft = state[row.token], block = expected.get(row.token);
if (!nft || !block || row.owner !== nft.owner || row.sequence !== nft.sequence ||
row.minted_height !== block.height || row.block_hash !== block.hash) throw new Error('Canonical ownership index mismatch');
}
const transactions = this.db.prepare('SELECT id,height,block_hash FROM canonical_transactions').all() as { id: string; height: number; block_hash: string }[];
const expectedTxs = new Map(chain.flatMap(b => b.transactions.map(tx => [transactionId(tx), b] as const)));
if (transactions.length !== expectedTxs.size || transactions.some(row => {
const block = expectedTxs.get(row.id); return !block || row.height !== block.height || row.block_hash !== block.hash;
})) throw new Error('Canonical transaction index mismatch');
const undo = this.db.prepare('SELECT COUNT(*) AS n FROM canonical_undo').get() as { n: number };
if (undo.n !== chain.length) throw new Error('Canonical undo index mismatch');
// A cursor is only useful if history has no gaps and describes the same validated DAG transitions.
const stack: string[] = [];
let sequence = 0, anchor = GENESIS_HASH;
for (const raw of this.db.prepare(`SELECT e.*,b.block_json FROM canonical_events e
LEFT JOIN blocks b ON b.hash=e.hash ORDER BY e.sequence`).iterate() as Iterable<{
sequence: number; type: string; height: number; hash: string; previous_hash: string;
transactions_json: string; block_json: string | null; previous_event_hash: string; event_hash: string }>) {
const block = raw.block_json ? JSON.parse(raw.block_json) as TestnetBlock : undefined;
if (raw.sequence !== ++sequence || !block || block.height !== raw.height || block.previousHash !== raw.previous_hash ||
raw.transactions_json !== JSON.stringify(block.transactions.map(transactionId))) throw new Error('Canonical event integrity mismatch');
if (raw.previous_event_hash !== anchor || raw.event_hash !== this.eventHash(raw.sequence, raw.type,
block, anchor)) throw new Error('Canonical event anchor mismatch');
anchor = raw.event_hash;
if (raw.type === 'connect') {
if (raw.height !== stack.length + 1 || raw.previous_hash !== (stack.at(-1) ?? GENESIS_HASH)) throw new Error('Canonical event connect mismatch');
stack.push(raw.hash);
} else if (raw.type === 'disconnect') {
if (raw.height !== stack.length || stack.pop() !== raw.hash) throw new Error('Canonical event disconnect mismatch');
} else throw new Error('Unknown canonical event');
}
if (stack.length !== chain.length || stack.some((hash, i) => hash !== chain[i].hash)) throw new Error('Canonical event tip mismatch');
}
private event(type: CanonicalEvent['type'], block: TestnetBlock) {
const prior = this.db.prepare('SELECT sequence,event_hash FROM canonical_events ORDER BY sequence DESC LIMIT 1').get() as
{ sequence: number; event_hash: string } | undefined;
const sequence = (prior?.sequence ?? 0) + 1, anchor = prior?.event_hash ?? GENESIS_HASH;
this.db.prepare('INSERT INTO canonical_events(sequence,type,height,hash,previous_hash,transactions_json,previous_event_hash,event_hash) VALUES(?,?,?,?,?,?,?,?)')
.run(sequence, type, block.height, block.hash, block.previousHash, JSON.stringify(block.transactions.map(transactionId)),
anchor, this.eventHash(sequence, type, block, anchor));
}
private eventHash(sequence: number, type: string, block: TestnetBlock, previousAnchor: string): string {
return sha256(JSON.stringify(['wikideck-canonical-event-v1', this.streamId, sequence, type,
block.height, block.hash, block.previousHash, block.transactions.map(transactionId), previousAnchor]));
}
connect(block: TestnetBlock): void {
const tokens = new Set([block.tokenId, ...block.transactions.flatMap(transactionCardIds)]);
const undo = [...tokens].map(token => [token, this.db.prepare('SELECT * FROM canonical_nfts WHERE token=?').get(token) ?? null]);
this.db.prepare('INSERT INTO canonical_undo VALUES(?,?)').run(block.height, JSON.stringify(undo));
this.db.prepare('INSERT INTO canonical_blocks VALUES(?,?)').run(block.height, block.hash);
this.db.prepare('INSERT INTO canonical_nfts VALUES(?,?,?,?,?)').run(block.tokenId, block.owner, 0, block.height, block.hash);
const update = this.db.prepare('UPDATE canonical_nfts SET owner=?,sequence=sequence+1 WHERE token=?');
for (const tx of block.transactions) {
if (isNftTrade(tx)) {
for (const leg of tx.offered) update.run(tx.counterparty, leg.cardHash);
for (const leg of tx.requested) update.run(tx.proposer, leg.cardHash);
} else update.run(tx.to, tx.cardHash);
this.db.prepare('INSERT INTO canonical_transactions VALUES(?,?,?)').run(transactionId(tx), block.height, block.hash);
}
this.event('connect', block);
}
disconnect(block: TestnetBlock): void {
const row = this.db.prepare('SELECT data FROM canonical_undo WHERE height=?').get(block.height) as { data: string } | undefined;
if (!row) throw new Error('Missing canonical undo record');
for (const [token, prior] of JSON.parse(row.data) as [string, NftRow | null][]) {
this.db.prepare('DELETE FROM canonical_nfts WHERE token=?').run(token);
if (prior) this.db.prepare('INSERT INTO canonical_nfts VALUES(?,?,?,?,?)')
.run(prior.token, prior.owner, prior.sequence, prior.minted_height, prior.block_hash);
}
this.db.prepare('DELETE FROM canonical_transactions WHERE height=?').run(block.height);
this.db.prepare('DELETE FROM canonical_blocks WHERE height=?').run(block.height);
this.db.prepare('DELETE FROM canonical_undo WHERE height=?').run(block.height);
this.event('disconnect', block);
}
head(): number {
return (this.db.prepare('SELECT COALESCE(MAX(sequence),0) AS n FROM canonical_events').get() as { n: number }).n;
}
events(after: number, limit = 64): CanonicalEvents {
const head = this.head();
if (!Number.isSafeInteger(after) || after < 0 || after > head || !Number.isSafeInteger(limit) || limit < 1 || limit > 64) {
throw new RpcError(409, 'Invalid canonical event cursor; restore/reconcile local consumer state.');
}
const rows = this.db.prepare('SELECT * FROM canonical_events WHERE sequence>? ORDER BY sequence LIMIT ?').all(after, limit) as
{ sequence: number; type: CanonicalEvent['type']; height: number; hash: string; previous_hash: string;
transactions_json: string; event_hash: string }[];
if (rows.some((r, i) => r.sequence !== after + i + 1)) throw new Error('Canonical event history gap');
const events = rows.map(r => ({ sequence: r.sequence, type: r.type, height: r.height, hash: r.hash,
previousHash: r.previous_hash, transactionIds: JSON.parse(r.transactions_json) as string[], eventHash: r.event_hash }));
const cursor = events.at(-1)?.sequence ?? after;
if (cursor < head && !events.length) throw new Error('Missing canonical event history');
const previousAnchor = after === 0 ? GENESIS_HASH : (this.db.prepare('SELECT event_hash FROM canonical_events WHERE sequence=?').get(after) as { event_hash: string } | undefined)?.event_hash;
if (!previousAnchor) throw new Error('Missing canonical cursor anchor');
return { streamId: this.streamId, previousAnchor, anchor: events.at(-1)?.eventHash ?? previousAnchor,
events, cursor, head, caughtUp: cursor === head };
}
nft(token: string, height: number): IndexedNft | undefined {
const row = this.db.prepare(`SELECT n.owner,n.sequence,n.minted_height,b.block_json FROM canonical_nfts n
JOIN blocks b ON b.hash=n.block_hash WHERE n.token=?`).get(token) as
{ owner: string; sequence: number; minted_height: number; block_json: string } | undefined;
return row ? { block: JSON.parse(row.block_json), owner: row.owner, sequence: row.sequence,
confirmations: height - row.minted_height + 1 } : undefined;
}
transaction(id: string): { height: number; blockHash: string } | undefined {
const row = this.db.prepare('SELECT height,block_hash FROM canonical_transactions WHERE id=?').get(id) as
{ height: number; block_hash: string } | undefined;
return row ? { height: row.height, blockHash: row.block_hash } : undefined;
}
deck(owner: string, offset: number, limit: number, height: number) {
const total = (this.db.prepare('SELECT COUNT(*) AS n FROM canonical_nfts WHERE owner=?').get(owner) as { n: number }).n;
const rows = this.db.prepare(`SELECT n.owner,n.sequence,n.minted_height,b.block_json FROM canonical_nfts n
JOIN blocks b ON b.hash=n.block_hash WHERE n.owner=? ORDER BY n.minted_height,n.token LIMIT ? OFFSET ?`)
.all(owner, limit, offset) as { owner: string; sequence: number; minted_height: number; block_json: string }[];
return { total, nfts: rows.map(r => ({ block: JSON.parse(r.block_json) as TestnetBlock,
sequence: r.sequence, confirmations: height - r.minted_height + 1 })) };
}
tip(): string {
return (this.db.prepare('SELECT hash FROM canonical_blocks ORDER BY height DESC LIMIT 1').get() as { hash: string } | undefined)?.hash ?? GENESIS_HASH;
}
}
server/testnet/http.ts
+17 -13
View File
@@ -6,7 +6,7 @@ import { fileURLToPath } from 'node:url';
import { mkdir, open, unlink } from 'node:fs/promises';
import { z } from 'zod';
import { NETWORK, CONSENSUS } from '../../shared/testnet.ts';
import { testnetOwnerOf, transactionCardIds } from '../../shared/testnet.ts';
import { transactionCardIds } from '../../shared/testnet.ts';
import { CHAIN_ID, blockSchema, tradeSchema, transferSchema, transactionSchema } from './consensus.ts';
import { RandomX } from './randomx.ts';
import { Ledger } from './ledger.ts';
@@ -56,11 +56,10 @@ let custody: CustodyMarket | undefined;
try {
if (custodyConfig === 'sandbox') custody = await CustodyMarket.open(dataDir, { feeBps: paymentConfig.feeBps,
card: (hash, owner, sequence) => {
const s = ledger.snapshot(), b = s.blocks.find(block => block.tokenId === hash);
if (!b || testnetOwnerOf(s, hash) !== owner || s.blocks.length + 1 - b.height < CONSENSUS.coinbaseMaturity ||
s.transfers.filter(tx => transactionCardIds(tx).includes(hash)).length !== sequence ||
s.pendingTransfers.some(tx => transactionCardIds(tx).includes(hash))) return undefined;
return { title: b.article.title };
const nft = ledger.nft(hash);
if (!nft || nft.owner !== owner || nft.confirmations < CONSENSUS.coinbaseMaturity || nft.sequence !== sequence ||
ledger.pendingTransactions().some(tx => transactionCardIds(tx).includes(hash))) return undefined;
return { title: nft.block.article.title };
} });
} catch(e) {market.close();peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
const app = express();
@@ -138,6 +137,9 @@ app.get('/api/decks/:address', rateLimit({windowMs:60000,limit:60,standardHeader
res.status(400).json({error:'Invalid deck page'});return;
}
res.setHeader('Cache-Control','no-store');
if(req.query.tip!==undefined && req.query.tip!==ledger.summary().tip){
res.status(409).json({error:'Canonical tip changed; restart deck pagination.'});return;
}
res.json(ledger.deck(req.params.address,offset,limit));
});
// Optional live data is intentionally not used by genesis, block validation or NFT ownership.
@@ -156,13 +158,15 @@ app.get('/p2p/blocks',(req,res)=>{
});
app.get('/p2p/mempool',(_req,res)=>res.json({transactions:ledger.pendingTransactions()}));
app.get('/api/nfts/:id', (req,res)=>{
const s=ledger.snapshot();const block=s.blocks.find(b=>b.tokenId===req.params.id);
if(!block){res.status(404).json({error:'NFT not found on the canonical chain'});return;}
const owner=testnetOwnerOf(s,block.tokenId)??block.owner;
const sequence=s.transfers.filter(tx=>transactionCardIds(tx).includes(block.tokenId)).length;
res.json({network:NETWORK,chainId:CHAIN_ID,tokenId:block.tokenId,owner,sequence,blockHash:block.hash,
confirmations:s.blocks.length-block.height+1,article:block.article,statsSnapshot:block.statsSnapshot,
stateRoot:s.blocks.at(-1)?.stateRoot});
if(typeof req.params.id!=='string'||!/^[a-f0-9]{64}$/.test(req.params.id)){
res.status(400).json({error:'Invalid NFT identifier'});return;
}
const nft=ledger.nft(req.params.id);
if(!nft){res.status(404).json({error:'NFT not found on the canonical chain'});return;}
const block=nft.block;
res.json({network:NETWORK,chainId:CHAIN_ID,tokenId:block.tokenId,owner:nft.owner,sequence:nft.sequence,blockHash:block.hash,
confirmations:nft.confirmations,article:block.article,statsSnapshot:block.statsSnapshot,
stateRoot:ledger.block(ledger.summary().tip)?.stateRoot});
});
const templateLimit=rateLimit({windowMs:60000,limit:20,standardHeaders:'draft-8',legacyHeaders:false});
app.post('/api/mining/template', templateLimit, async (req,res)=>{
server/testnet/ledger.ts
+21 -15
View File
@@ -13,6 +13,7 @@ import type { Article } from '../../shared/protocol.ts';
import type { MintStatsSnapshot } from '../../shared/enrichment.ts';
import { PROFILE } from '../../shared/network.ts';
import { RpcError } from '../resources.ts';
import { CanonicalIndex } from './canonical-index.ts';
const MAX_PENDING = 256;
const MAX_DECK_PAGE = 24;
@@ -39,11 +40,14 @@ export class Ledger {
private readonly file: string;
private readonly db: DatabaseSync;
private readonly pow: PowEngine;
private readonly index: CanonicalIndex;
private closed = false;
private revision = 0;
private reorgs = 0;
private canonicalCache: { tip: string; blocks: TestnetBlock[] } | undefined;
private constructor(file: string, db: DatabaseSync, pow: PowEngine) { this.file = file; this.db = db; this.pow = pow; }
private constructor(file: string, db: DatabaseSync, pow: PowEngine) {
this.file = file; this.db = db; this.pow = pow; this.index = new CanonicalIndex(db);
}
static async open(dir: string, pow: PowEngine): Promise<Ledger> {
const directory = join(dir, PROFILE);
@@ -141,6 +145,7 @@ export class Ledger {
ledger.pending.push(tx);
}
ledger.pendingState = state;
ledger.index.initialize(ledger.canonical(), materializeState(ledger.tipState()));
return ledger;
} catch (error) {
db.close();
@@ -192,7 +197,9 @@ export class Ledger {
[...disconnected.flatMap(b => b.transactions), ...this.pending]) : undefined;
const pending = filtered?.pending ?? this.pending;
// Nothing visible changes until the atomic replacement succeeds.
await this.persist(node.block, pending);
const oldHashes = new Set(oldChain.map(b => b.hash));
const connected = wins ? chain.filter(b => !oldHashes.has(b.hash)) : [];
this.persist(node.block, pending, disconnected, connected);
this.nodes.set(node.block.hash, node);
if (wins) this.tip = node.block.hash;
if (disconnected.length) this.reorgs++;
@@ -217,7 +224,7 @@ export class Ledger {
if (this.pending.length >= MAX_PENDING) throw new Error('Testnet mempool limit: 256 transfers');
const nextPendingState = applyTransactionOverlay(this.pendingState, tx, this.height() + 1);
const pending = [...this.pending, tx];
await this.persist(undefined, pending);
this.persist(undefined, pending);
this.pending = pending;
this.pendingState = nextPendingState;
this.revision++;
@@ -246,6 +253,10 @@ export class Ledger {
}
peerStatus() { return { network: NETWORK, chainId: CHAIN_ID, size: this.size, tip: this.tip }; }
pendingTransactions(): TestnetTransaction[] { return structuredClone(this.pending); }
canonicalEvents(after: number, limit = 64) { return this.index.events(after, limit); }
canonicalHead(): number { return this.index.head(); }
nft(token: string) { return this.index.nft(token, this.height()); }
canonicalTransaction(id: string) { return this.index.transaction(id); }
legacySnapshot(maxBlocks: number): TestnetSnapshot {
// Reject before any full-chain copy/serialization. Never truncate ownership history silently.
if (this.height() > maxBlocks) throw new RpcError(413, 'Full snapshot disabled at this height; use /api/summary and /api/canonical pages.');
@@ -263,18 +274,10 @@ export class Ledger {
if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > MAX_DECK_PAGE) {
throw new Error('Invalid deck page');
}
const blocks = this.canonical();
const byToken = new Map(blocks.map(block => [block.tokenId, block]));
const state = materializeState(this.tipState());
const owned = Object.entries(state).filter(([, nft]) => nft.owner === address)
.flatMap(([tokenId, nft]) => {
const block = byToken.get(tokenId);
return block ? [{ block, sequence: nft.sequence, confirmations: blocks.length - block.height + 1 }] : [];
}).sort((a, b) => a.block.height - b.block.height || a.block.tokenId.localeCompare(b.block.tokenId));
const nfts = owned.slice(offset, offset + limit);
const next = offset + nfts.length < owned.length ? offset + nfts.length : null;
const { total, nfts } = this.index.deck(address, offset, limit, this.height());
const next = offset + nfts.length < total ? offset + nfts.length : null;
return structuredClone({ network: NETWORK, chainId: CHAIN_ID, address, tip: this.tip,
offset, limit, total: owned.length, next, nfts });
offset, limit, total, next, nfts });
}
private height(): number { return this.nodes.get(this.tip)?.block.height ?? 0; }
@@ -322,7 +325,8 @@ export class Ledger {
this.queue = result.catch(() => undefined).finally(() => { this.queued--; });
return result;
}
private async persist(block: TestnetBlock | undefined, pendingTransfers: TestnetTransaction[]): Promise<void> {
private persist(block: TestnetBlock | undefined, pendingTransfers: TestnetTransaction[],
disconnected: TestnetBlock[] = [], connected: TestnetBlock[] = []): void {
this.db.exec('BEGIN IMMEDIATE');
try {
if (block) this.db.prepare('INSERT INTO blocks(hash, previous_hash, block_json) VALUES(?, ?, ?)')
@@ -330,6 +334,8 @@ export class Ledger {
this.db.exec('DELETE FROM pending_transfers');
const insertPending = this.db.prepare('INSERT INTO pending_transfers(position, id, transaction_json) VALUES(?, ?, ?)');
pendingTransfers.forEach((tx, index) => insertPending.run(index, transactionId(tx), JSON.stringify(tx)));
for (const old of [...disconnected].reverse()) this.index.disconnect(old);
for (const next of connected) this.index.connect(next);
this.db.exec('COMMIT');
} catch (error) {
this.db.exec('ROLLBACK');