WikiDeck/wikideck
3
0
Fork 1
Code Issues 8 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

Bounded reconciliation increment

Browse Source
This commit is contained in:
eric
2026-10-21 09:57:50 -04:00
parent 8b0e2287a5
commit 1f11210b5b
5 changed files with 260 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/market/store.ts
+142 -67
View File
@@ -38,8 +38,22 @@ export function marketConfig(env: NodeJS.ProcessEnv = process.env): Config {
}
const PAYMENT_WINDOW = 30 * 60_000;
const DELIVERY_WINDOW = 48 * 60 * 60_000;
const RECONCILIATION_FRESH_MS = 15_000;
const cursorSchema = z.object({ version: z.literal(1), streamId: z.string().uuid(),
cursor: z.number().int().nonnegative().safe(), anchor: z.string().regex(/^[a-f0-9]{64}$/) }).strict();
const canonicalEventSchema = z.object({ sequence: z.number().int().positive().safe(), type: z.enum(['connect', 'disconnect']),
height: z.number().int().positive().safe(), hash: z.string().regex(/^[a-f0-9]{64}$/),
previousHash: z.string().regex(/^[a-f0-9]{64}$/), transactionIds: z.array(z.string().regex(/^[a-f0-9]{64}$/)).max(64),
eventHash: z.string().regex(/^[a-f0-9]{64}$/) }).strict();
const canonicalEventPageSchema = z.object({ streamId: z.string().uuid(), previousAnchor: z.string().regex(/^[a-f0-9]{64}$/),
anchor: z.string().regex(/^[a-f0-9]{64}$/), events: z.array(canonicalEventSchema).max(64),
cursor: z.number().int().nonnegative().safe(), head: z.number().int().nonnegative().safe(), caughtUp: z.boolean() }).strict();
const reconciliationJobSchema = z.object({ version: z.literal(1), stage: z.enum(['orders', 'listings']),
afterId: z.string().nullable(), observedAt: z.number().int().nonnegative().safe(),
tip: z.string().regex(/^[a-f0-9]{64}$/).nullable(), revision: z.number().int().nonnegative().safe().nullable(),
eventPage: canonicalEventPageSchema.nullable() }).strict();
const RECONCILE_BATCH_SIZE = 128;
const RECONCILIATION_JOB_KEY = 'canonical_reconciliation_job';
/** Node-local marketplace, NOT consensus escrow. Its SQLite journal is backed up with the ledger. */
export class Market {
@@ -49,6 +63,7 @@ export class Market {
private eventCursor = 0;
private eventCheckpoint: z.infer<typeof cursorSchema> | undefined;
private eventBacklog = false;
private reconciliationJob: z.infer<typeof reconciliationJobSchema> | undefined;
private readonly db: DatabaseSync;
private readonly chain: Chain;
readonly config: Config;
@@ -61,6 +76,9 @@ export class Market {
this.eventCheckpoint = cursorSchema.parse(JSON.parse(cursor.value));
this.eventCursor = this.eventCheckpoint.cursor;
}
const job = db.prepare('SELECT value FROM market_meta WHERE key=?').get(RECONCILIATION_JOB_KEY) as { value: string } | undefined;
if (job) { this.reconciliationJob = reconciliationJobSchema.parse(JSON.parse(job.value)); this.eventBacklog = true; }
else this.eventBacklog = !!chain.canonicalHead && this.eventCursor < chain.canonicalHead();
}
static async open(directory: string, chain: Chain, config = marketConfig(), now = Date.now): Promise<Market> {
@@ -101,15 +119,16 @@ export class Market {
return { queued:this.queued,lastReconciledAt:this.reconciled?.at??null,orderStates:states,
canonicalEventCursor:this.eventCursor,canonicalEventHead:this.chain.canonicalHead?.()??null,eventBacklog:this.eventBacklog };
}
hasReconciliationBacklog(): boolean { return this.eventBacklog || !!this.reconciliationJob; }
info(): MarketInfo {
return { network: NETWORK, chainId: CHAIN_ID, marketId: this.id, paymentMode: 'mock', marketVersion: 3, feeBps: this.config.feeBps,
...resolvedMarketPolicy(this.config),
additionalFee: { ...ADDITIONAL_MARKET_FEE },
deliveryConfirmations: this.config.confirmations, serverTime: this.now() };
}
private rows<T>(table: 'listings' | 'orders'): T[] {
return (this.db.prepare(`SELECT data FROM ${table} ORDER BY rowid DESC LIMIT 10000`).all() as { data: string }[])
.map(row => JSON.parse(row.data) as T);
private nextReconciliationRows(table: 'listings' | 'orders', afterId: string | null): { id: string; data: string }[] {
return this.db.prepare(`SELECT id,data FROM ${table} WHERE (? IS NULL OR id>?) ORDER BY id LIMIT ?`)
.all(afterId, afterId, RECONCILE_BATCH_SIZE) as { id: string; data: string }[];
}
private listing(id: string): MarketListingRecord {
const row = this.db.prepare('SELECT data FROM listings WHERE id=?').get(id) as { data: string } | undefined;
@@ -159,88 +178,142 @@ export class Market {
return transactionId({ network: NETWORK, chainId: CHAIN_ID, cardHash: order.cardHash,
from: order.seller, to: order.buyer, sequence: order.sequence, publicKey: '', signature: '' });
}
/** Reconcile from canonical transaction identity, never from a mempool acknowledgement. */
/** Reconcile one durable keyset batch from canonical transaction identity, never mempool acknowledgement. */
private reconcile() {
const events = this.chain.canonicalEvents?.(this.eventCursor, 64);
if (events && this.eventCheckpoint && (events.streamId !== this.eventCheckpoint.streamId ||
events.previousAnchor !== this.eventCheckpoint.anchor)) throw new RpcError(409, 'Canonical event history changed; restore matching ledger and market backups.');
const s = this.view(), now = this.now();
const pendingIds = new Set(s.pendingTransfers.map(transactionId));
const now = this.now();
const summary = this.chain.summary?.();
if (!this.reconciliationJob) {
const events = this.chain.canonicalEvents?.(this.eventCursor, 64);
if (events && this.eventCheckpoint && (events.streamId !== this.eventCheckpoint.streamId ||
events.previousAnchor !== this.eventCheckpoint.anchor)) throw new RpcError(409, 'Canonical event history changed; restore matching ledger and market backups.');
const job = reconciliationJobSchema.parse({ version: 1, stage: 'orders', afterId: null, observedAt: now,
tip: summary?.tip ?? null, revision: summary?.revision ?? null, eventPage: events ?? null });
this.atomic(() => this.db.prepare('INSERT INTO market_meta(key,value) VALUES(?,?)')
.run(RECONCILIATION_JOB_KEY, JSON.stringify(job)));
this.reconciliationJob = job; this.eventBacklog = true;
}
let job = this.reconciliationJob;
if (summary && (job.tip !== summary.tip || job.revision !== summary.revision)) {
// Canonical state changed while a multi-batch scan was running. Rewind the
// keyset and bind a fresh event page so every row is checked against one
// observed chain revision before the canonical cursor can advance.
const events = this.chain.canonicalEvents?.(this.eventCursor, 64);
if (events && this.eventCheckpoint && (events.streamId !== this.eventCheckpoint.streamId ||
events.previousAnchor !== this.eventCheckpoint.anchor)) throw new RpcError(409, 'Canonical event history changed; restore matching ledger and market backups.');
job = reconciliationJobSchema.parse({ version: 1, stage: 'orders', afterId: null, observedAt: now,
tip: summary.tip, revision: summary.revision, eventPage: events ?? null });
this.atomic(() => this.db.prepare('UPDATE market_meta SET value=? WHERE key=?')
.run(JSON.stringify(job), RECONCILIATION_JOB_KEY));
this.reconciliationJob = job;
this.eventBacklog = true;
}
const s = this.view(), pendingIds = new Set(s.pendingTransfers.map(transactionId));
const eventPage = job.eventPage as CanonicalEvents | null;
const disconnected = new Set(eventPage?.events.filter(e => e.type === 'disconnect').flatMap(e => e.transactionIds) ?? []);
const rows = this.nextReconciliationRows(job.stage, job.afterId);
let nextJob: z.infer<typeof reconciliationJobSchema> | undefined;
let completed = false;
this.atomic(() => {
// Process every disconnect, even if the same delivery reappeared before this consumer caught up.
// Never infer that a current tip erases an intermediate loss of previously settled evidence.
const disconnected = new Set(events?.events.filter(e => e.type === 'disconnect').flatMap(e => e.transactionIds) ?? []);
if (events) {
for (const o of this.rows<MarketOrderRecord>('orders')) {
if (job.stage === 'orders' && rows.length) {
for (const row of rows) {
const o = JSON.parse(row.data) as MarketOrderRecord;
const l = this.listing(o.listingId);
// A disconnect of completed delivery is sticky even if the same transaction has reappeared.
if (o.status === 'complete' && disconnected.has(this.deliveryId(o))) {
o.status = 'review'; o.payout = 'review';
if (o.additionalFee) o.additionalFee.payout = 'review';
this.saveOrder(o);
}
}
this.db.prepare('INSERT INTO market_meta VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value')
.run('canonical_event_cursor', JSON.stringify({ version: 1, streamId: events.streamId, cursor: events.cursor, anchor: events.anchor }));
// Catch-up is bounded per pass. No new payout, timeout/refund decision or mutation while events remain.
if (!events.caughtUp) return;
}
for (const o of this.rows<MarketOrderRecord>('orders')) {
const l = this.listing(o.listingId);
if (o.status === 'refunded' || o.status === 'expired') continue;
const deliveryId = this.deliveryId(o);
const delivery = s.transaction(deliveryId);
const confirmations = delivery ? s.height - delivery.height + 1 : 0;
o.deliveryConfirmations = confirmations;
if (o.status === 'complete' || o.status === 'review') {
if (!delivery || confirmations < o.requiredConfirmations) {
if (o.status === 'refunded' || o.status === 'expired') continue;
const deliveryId = this.deliveryId(o);
const delivery = s.transaction(deliveryId);
const confirmations = delivery ? s.height - delivery.height + 1 : 0;
o.deliveryConfirmations = confirmations;
if (o.status === 'complete' || o.status === 'review') {
if (!delivery || confirmations < o.requiredConfirmations) {
o.status = 'review'; o.payout = 'review';
if (o.additionalFee) o.additionalFee.payout = 'review';
}
this.saveOrder(o); continue; // Never automatically pay a second time after a reorganization.
}
if (delivery && o.status === 'awaiting-payment') {
o.status = 'review'; o.payout = 'review';
if (o.additionalFee) o.additionalFee.payout = 'review';
} else if (delivery) {
o.transferId = deliveryId; o.status = 'delivery-pending';
if (confirmations >= o.requiredConfirmations) {
o.status = 'complete'; o.payout = 'mock-paid'; l.status = 'sold';
if (o.additionalFee) o.additionalFee.payout = 'mock-paid';
}
} else if (o.status === 'awaiting-payment' && job.observedAt >= o.payBy) {
o.status = 'expired'; l.status = l.kind === 'fixed' ? 'open' : 'expired';
} else if (o.status !== 'awaiting-payment') {
const pending = pendingIds.has(deliveryId);
const conflict = s.pendingTransfers.some(tx => transactionCardIds(tx).includes(o.cardHash) && transactionId(tx) !== deliveryId);
if (!this.card(s, o.cardHash, o.seller, o.sequence) || conflict || (job.observedAt >= o.deliverBy && !pending)) o.status = 'refund-required';
else if (o.status === 'delivery-pending' && !pending) o.status = 'paid';
}
this.saveOrder(o); continue; // Never automatically pay a second time after a reorganization.
this.saveOrder(o); this.saveListing(l);
}
if (delivery && o.status === 'awaiting-payment') {
// A gift/direct delivery is not evidence that payment was received. Never pay an unpaid order.
o.status = 'review'; o.payout = 'review';
if (o.additionalFee) o.additionalFee.payout = 'review';
} else if (delivery) {
o.transferId = deliveryId;
o.status = 'delivery-pending';
if (confirmations >= o.requiredConfirmations) {
o.status = 'complete'; o.payout = 'mock-paid'; l.status = 'sold';
if (o.additionalFee) o.additionalFee.payout = 'mock-paid';
} else if (job.stage === 'listings' && rows.length) {
for (const row of rows) {
const l = JSON.parse(row.data) as MarketListingRecord;
if (l.status !== 'open') continue;
if (l.additionalFee?.bps !== ADDITIONAL_MARKET_FEE.bps || l.additionalFee?.address !== ADDITIONAL_MARKET_FEE.address ||
!this.card(s, l.cardHash, l.seller, l.sequence) || s.pendingTransfers.some(tx => transactionCardIds(tx).includes(l.cardHash))) l.status = 'invalid';
else if (l.kind === 'auction' && job.observedAt >= l.endsAt! + PAYMENT_WINDOW) l.status = 'expired';
else if (l.kind === 'auction' && job.observedAt >= l.endsAt! && !l.highestBid) l.status = 'expired';
this.saveListing(l);
}
}
if (job.stage === 'orders') {
if (rows.length < RECONCILE_BATCH_SIZE) nextJob = { ...job, stage: 'listings', afterId: null };
else nextJob = { ...job, afterId: rows.at(-1)!.id };
} else {
if (rows.length < RECONCILE_BATCH_SIZE) {
this.db.prepare('DELETE FROM receipts WHERE expires < ?').run(job.observedAt - 86400_000);
if (eventPage) {
this.db.prepare('INSERT INTO market_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value')
.run('canonical_event_cursor', JSON.stringify({ version: 1, streamId: eventPage.streamId,
cursor: eventPage.cursor, anchor: eventPage.anchor }));
}
} else if (o.status === 'awaiting-payment' && now >= o.payBy) {
o.status = 'expired'; l.status = l.kind === 'fixed' ? 'open' : 'expired';
} else if (o.status !== 'awaiting-payment') {
const pending = pendingIds.has(deliveryId);
const conflict = s.pendingTransfers.some(tx => transactionCardIds(tx).includes(o.cardHash) && transactionId(tx) !== deliveryId);
if (!this.card(s, o.cardHash, o.seller, o.sequence) || conflict || (now >= o.deliverBy && !pending)) o.status = 'refund-required';
else if (o.status === 'delivery-pending' && !pending) o.status = 'paid';
}
this.saveOrder(o); this.saveListing(l);
this.db.prepare('DELETE FROM market_meta WHERE key=?').run(RECONCILIATION_JOB_KEY);
completed = true;
} else nextJob = { ...job, afterId: rows.at(-1)!.id };
}
for (const l of this.rows<MarketListingRecord>('listings')) {
if (l.status !== 'open') continue;
if (l.additionalFee?.bps !== ADDITIONAL_MARKET_FEE.bps || l.additionalFee?.address !== ADDITIONAL_MARKET_FEE.address ||
!this.card(s, l.cardHash, l.seller, l.sequence) || s.pendingTransfers.some(tx => transactionCardIds(tx).includes(l.cardHash))) l.status = 'invalid';
else if (l.kind === 'auction' && now >= l.endsAt! + PAYMENT_WINDOW) l.status = 'expired';
else if (l.kind === 'auction' && now >= l.endsAt! && !l.highestBid) l.status = 'expired';
this.saveListing(l);
}
this.db.prepare('DELETE FROM receipts WHERE expires < ?').run(now - 86400_000);
if (nextJob) this.db.prepare('UPDATE market_meta SET value=? WHERE key=?').run(JSON.stringify(nextJob), RECONCILIATION_JOB_KEY);
});
if (events) {
this.eventCursor = events.cursor; this.eventBacklog = !events.caughtUp;
this.eventCheckpoint = { version: 1, streamId: events.streamId, cursor: events.cursor, anchor: events.anchor };
if (completed) {
this.reconciliationJob = undefined;
if (eventPage) {
this.eventCursor = eventPage.cursor;
this.eventCheckpoint = { version: 1, streamId: eventPage.streamId, cursor: eventPage.cursor, anchor: eventPage.anchor };
// The ledger can append events while large journals are processed over many bounded calls.
// Do not mark the market reconciled to a newer tip until those post-page events are consumed.
const currentHead = this.chain.canonicalHead?.() ?? eventPage.head;
this.eventBacklog = !eventPage.caughtUp || currentHead > eventPage.cursor;
} else this.eventBacklog = false;
const current = this.chain.summary?.();
if (current && !this.eventBacklog) this.reconciled = { ...current, at: now };
} else {
this.reconciliationJob = nextJob;
this.eventBacklog = true;
}
const summary = this.chain.summary?.();
if (summary && !this.eventBacklog) this.reconciled = { ...summary, at: now };
// Empty/short order pages can advance to the listing stage in the same bounded call. This
// avoids rejecting the first command on an empty market while still capping work at 2×128 rows.
if (!completed && job.stage === 'orders' && rows.length < RECONCILE_BATCH_SIZE) this.reconcile();
}
browse(offset = 0): { listings: MarketListingRecord[]; next: number | null; info: MarketInfo } {
if (!Number.isSafeInteger(offset) || offset < 0 || offset > 10000) throw new Error('Invalid marketplace page');
// Reconciliation must not interleave with delivery submission while its chain await is outstanding.
const summary = this.chain.summary?.();
const last = this.reconciled;
if (!this.queued && (!summary || !last || summary.tip !== last.tip || summary.revision !== last.revision || this.now() - last.at >= 1000)) this.reconcile();
if (!this.queued && (this.hasReconciliationBacklog() || !summary || !last || summary.tip !== last.tip ||
summary.revision !== last.revision || this.now() - last.at >= RECONCILIATION_FRESH_MS)) this.reconcile();
if (!resolvedMarketPolicy(this.config).marketEnabled) return { listings: [], next: null, info: this.info() };
const rows = this.db.prepare("SELECT data FROM listings WHERE status IN ('open','reserved') ORDER BY rowid DESC LIMIT 25 OFFSET ?")
.all(offset) as { data: string }[];
@@ -273,8 +346,8 @@ export class Market {
if (!permitsMarketAction(this.config, c.action.type)) throw new Error('New market trading is disabled by this host; existing activity and resolution remain available');
const a = c.action;
const summary = this.chain.summary?.(), last = this.reconciled;
if (a.type !== 'activity' || !summary || !last || summary.tip !== last.tip || summary.revision !== last.revision ||
this.now() - last.at >= 1000) this.reconcile();
if (this.hasReconciliationBacklog() || a.type !== 'activity' || !summary || !last || summary.tip !== last.tip || summary.revision !== last.revision ||
this.now() - last.at >= RECONCILIATION_FRESH_MS) this.reconcile();
if (this.eventBacklog && a.type !== 'activity') throw new RpcError(503, 'Market canonical reconciliation is catching up; retry shortly.');
if (a.type === 'activity') return {
orders: (this.db.prepare("SELECT data FROM orders WHERE json_extract(data,'$.seller')=? OR json_extract(data,'$.buyer')=? ORDER BY rowid DESC")
@@ -343,7 +416,9 @@ export class Market {
if ((a.hostFeeAddress ?? ADDITIONAL_MARKET_FEE.address) !== hostFeeAddress) throw new Error('Host fee recipient changed; review and sign again');
if (!this.card(s, a.cardHash, c.actor, a.sequence) || s.pendingTransfers.some(tx => transactionCardIds(tx).includes(a.cardHash))) throw new Error('Card is missing, stale, immature, or already pending');
if (a.kind === 'fixed' ? a.endsAt !== null : a.endsAt === null || a.endsAt < now + 60_000 || a.endsAt > now + 30 * 86400_000) throw new Error('Auction duration must be 1 minute to 30 days; fixed listings have no end');
if (this.rows<MarketListingRecord>('listings').filter(l => l.seller === c.actor && ['open','reserved'].includes(l.status)).length >= 100) throw new Error('Limit: 100 active listings per seller');
const activeListings = this.db.prepare(`SELECT COUNT(*) AS n FROM listings
WHERE json_extract(data,'$.seller')=? AND status IN ('open','reserved')`).get(c.actor) as { n: number };
if (activeListings.n >= 100) throw new Error('Limit: 100 active listings per seller');
if ((this.db.prepare('SELECT COUNT(*) AS n FROM listings').get() as { n: number }).n >= 10000) throw new Error('Marketplace journal limit reached; operator archival required');
if (this.db.prepare("SELECT 1 FROM listings WHERE card=? AND status IN ('open','reserved')").get(a.cardHash)) throw new Error('Card already listed');
const mint = s.nft(a.cardHash)!.block;
server/testnet/http.ts
+8 -2
View File
@@ -251,11 +251,17 @@ server.setTimeout(30000,socket=>socket.destroy());
// Advance simulated payout/expiry state even when no browser is open. No real funds are sent.
const marketTimer=setInterval(()=>{try {market.browse();}catch{console.error('Marketplace reconciliation failed; operator review required.');}},15000);
marketTimer.unref();
// When durable reconciliation is behind, advance bounded keyset pages without requiring a browser request.
const marketCatchupTimer=setInterval(()=>{
if(!market.hasReconciliationBacklog())return;
try {market.browse();}catch{console.error('Marketplace canonical catch-up failed; operator review required.');}
},250);
marketCatchupTimer.unref();
let stopping=false;
for(const signal of ['SIGTERM','SIGINT']as const)process.once(signal,()=>{
if(stopping)return;stopping=true;peers.close();operations.close();clearInterval(marketTimer);
if(stopping)return;stopping=true;peers.close();operations.close();clearInterval(marketTimer);clearInterval(marketCatchupTimer);
const observerDrain=custodyObserver?.close()??Promise.resolve();
const timeout=setTimeout(()=>process.exit(1),25000);timeout.unref();
server.close(async()=>{await observerDrain;await custody?.close();market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
});
server.on('error',async()=>{clearInterval(marketTimer);operations.close();peers.close();await custodyObserver?.close();await custody?.close();market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});
server.on('error',async()=>{clearInterval(marketTimer);clearInterval(marketCatchupTimer);operations.close();peers.close();await custodyObserver?.close();await custody?.close();market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});