WikiDeck/wikideck
3
0
Fork 1
Code Issues 8 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

Worker-owned reconciliation commits

Browse Source
This commit is contained in:
eric
2026-10-21 11:07:36 -04:00
parent a127a40293
commit 7b6873921a
10 changed files with 322 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/market/reconciliation-commit-worker.js
Executable
+52
View File
@@ -0,0 +1,52 @@
import { DatabaseSync } from 'node:sqlite';
import { parentPort, workerData } from 'node:worker_threads';
import { isDeepStrictEqual } from 'node:util';
const db = new DatabaseSync(workerData.marketFile);
db.exec('PRAGMA busy_timeout=5000; PRAGMA synchronous=FULL;');
const port = parentPort;
if (!port) throw new Error('Reconciliation commit worker requires a parent port');
function atomic(fn) {
db.exec('BEGIN IMMEDIATE');
try { const result = fn(); db.exec('COMMIT'); return result; }
catch (error) { db.exec('ROLLBACK'); throw error; }
}
function assertCurrentJob(expected) {
const row = db.prepare("SELECT value FROM market_meta WHERE key='canonical_reconciliation_job'").get();
if (!row || !isDeepStrictEqual(JSON.parse(row.value), expected)) {
throw new Error('Reconciliation job changed before worker commit');
}
}
port.on('message', request => {
try {
if (request.operation === 'persist-batch') {
atomic(() => {
assertCurrentJob(request.expectedJob);
const order = db.prepare(`INSERT INTO orders(id,data) VALUES(?,?)
ON CONFLICT(id) DO UPDATE SET data=excluded.data WHERE orders.data<>excluded.data`);
for (const row of request.orders) order.run(row.id, row.data);
const listing = db.prepare(`INSERT INTO listings(id,card,status,data) VALUES(?,?,?,?)
ON CONFLICT(id) DO UPDATE SET status=excluded.status,data=excluded.data WHERE listings.data<>excluded.data`);
for (const row of request.listings) listing.run(row.id, row.card, row.status, row.data);
db.prepare("UPDATE market_meta SET value=? WHERE key='canonical_reconciliation_job'")
.run(JSON.stringify(request.nextJob));
});
} else {
atomic(() => {
assertCurrentJob(request.expectedJob);
db.prepare('DELETE FROM receipts WHERE expires < ?').run(request.observedAt - 86400_000);
if (request.cursor) {
db.prepare(`INSERT INTO market_meta(key,value) VALUES('canonical_event_cursor',?)
ON CONFLICT(key) DO UPDATE SET value=excluded.value`).run(request.cursor);
}
db.prepare("DELETE FROM market_meta WHERE key='canonical_reconciliation_job'").run();
});
}
port.postMessage({ id: request.id, ok: true });
} catch (error) {
port.postMessage({ id: request.id, ok: false, error: error instanceof Error ? error.message : 'Worker batch failed' });
}
});
port.on('close', () => db.close());
server/market/reconciliation-commit-worker.ts
Executable
+1
View File
@@ -0,0 +1 @@
// The runtime worker is reconciliation-commit-worker.js so it is independent of TS loaders.
server/market/store.ts
+168 -19
View File
@@ -2,6 +2,7 @@ import { DatabaseSync } from 'node:sqlite';
import { randomUUID } from 'node:crypto';
import { mkdir, chmod } from 'node:fs/promises';
import { join } from 'node:path';
import { Worker } from 'node:worker_threads';
import { NETWORK, CONSENSUS, nftTransferMessage, testnetOwnerOf, transactionCardIds,
type TestnetSnapshot, type NftTransfer } from '../../shared/testnet.ts';
import { ADDITIONAL_MARKET_FEE, marketCommandSchema, marketMessage, type MarketCommand, type MarketListing,
@@ -48,7 +49,7 @@ const canonicalEventSchema = z.object({ sequence: z.number().int().positive().sa
const canonicalEventPageSchema = z.object({ streamId: z.string().uuid(), previousAnchor: z.string().regex(/^[a-f0-9]{64}$/),
anchor: z.string().regex(/^[a-f0-9]{64}$/), events: z.array(canonicalEventSchema).max(64),
cursor: z.number().int().nonnegative().safe(), head: z.number().int().nonnegative().safe(), caughtUp: z.boolean() }).strict();
const reconciliationJobSchema = z.object({ version: z.literal(1), stage: z.enum(['orders', 'listings']),
const reconciliationJobSchema = z.object({ version: z.literal(1), stage: z.enum(['orders', 'listings', 'finalize']),
afterId: z.string().nullable(), observedAt: z.number().int().nonnegative().safe(),
tip: z.string().regex(/^[a-f0-9]{64}$/).nullable(), revision: z.number().int().nonnegative().safe().nullable(),
eventPage: canonicalEventPageSchema.nullable() }).strict();
@@ -65,13 +66,39 @@ export class Market {
private eventCheckpoint: z.infer<typeof cursorSchema> | undefined;
private eventBacklog = false;
private reconciliationJob: z.infer<typeof reconciliationJobSchema> | undefined;
private readonly reconciliationWorker?: Worker;
private workerRequestId = 0;
private readonly workerRequests = new Map<number, { resolve(): void; reject(error: Error): void }>();
private reconciliationWorkerBusy = false;
private reconciliationWorkerError: string | undefined;
private stagedReconciliationWrites: { orders: Map<string, MarketOrderRecord>;
listings: Map<string, MarketListingRecord> } | undefined;
private readonly db: DatabaseSync;
private readonly chain: Chain;
readonly config: Config;
readonly id: string;
private readonly now: () => number;
private constructor(db: DatabaseSync, chain: Chain, config: Config, id: string, now: () => number) {
private constructor(db: DatabaseSync, chain: Chain, config: Config, id: string, now: () => number, worker?: Worker) {
this.db = db; this.chain = chain; this.config = config; this.id = id; this.now = now;
this.reconciliationWorker = worker;
worker?.on('message', (message: { id: number; ok: boolean; error?: string }) => {
const pending = this.workerRequests.get(message.id);
if (!pending) return;
this.workerRequests.delete(message.id);
if (message.ok) pending.resolve(); else pending.reject(new Error(message.error ?? 'Reconciliation worker failed'));
});
worker?.on('error', error => {
this.reconciliationWorkerError = 'Reconciliation worker failed';
for (const pending of this.workerRequests.values()) pending.reject(error);
this.workerRequests.clear();
});
worker?.on('exit', code => {
if (!this.workerRequests.size) return;
const error = new Error(`Reconciliation worker exited with code ${code}`);
this.reconciliationWorkerError = 'Reconciliation worker stopped unexpectedly';
for (const pending of this.workerRequests.values()) pending.reject(error);
this.workerRequests.clear();
});
const cursor = db.prepare("SELECT value FROM market_meta WHERE key='canonical_event_cursor'").get() as { value: string } | undefined;
if (cursor) {
this.eventCheckpoint = cursorSchema.parse(JSON.parse(cursor.value));
@@ -82,12 +109,14 @@ export class Market {
else this.eventBacklog = !!chain.canonicalHead && this.eventCursor < chain.canonicalHead();
}
static async open(directory: string, chain: Chain, config = marketConfig(), now = Date.now): Promise<Market> {
static async open(directory: string, chain: Chain, config = marketConfig(), now = Date.now,
reconciliation: { worker?: boolean } = {}): Promise<Market> {
await validateMarketPolicy(config);
const folder = join(directory, NETWORK.includes('mainnet') ? 'mainnet' : 'testnet');
await mkdir(folder, { recursive: true, mode: 0o700 });
const file = join(folder, 'market.sqlite');
const db = new DatabaseSync(file);
let worker: Worker | undefined;
try {
db.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA busy_timeout=5000;
CREATE TABLE IF NOT EXISTS market_meta(key TEXT PRIMARY KEY, value TEXT NOT NULL);
@@ -111,14 +140,22 @@ export class Market {
db.prepare('INSERT INTO market_meta VALUES(?,?)').run('identity', JSON.stringify({ version: 1, network: NETWORK, chainId: CHAIN_ID, id }));
}
if (process.platform !== 'win32') await chmod(file, 0o600);
return new Market(db, chain, config, id, now);
} catch (error) { db.close(); throw error; }
worker = reconciliation.worker ? new Worker(new URL('./reconciliation-commit-worker.js', import.meta.url),
{ workerData: { marketFile: file }, execArgv: [] }) : undefined;
worker?.unref();
return new Market(db, chain, config, id, now, worker);
} catch (error) { if (worker) await worker.terminate(); db.close(); throw error; }
}
close() { this.db.close(); }
async close(): Promise<void> {
if (this.reconciliationWorker) await this.reconciliationWorker.terminate();
this.db.close();
}
isReconciliationWorkerBusy(): boolean { return this.reconciliationWorkerBusy; }
diagnostics() {
const states=this.db.prepare("SELECT json_extract(data,'$.status') AS status,COUNT(*) AS count FROM orders GROUP BY json_extract(data,'$.status')").all();
return { queued:this.queued,lastReconciledAt:this.reconciled?.at??null,orderStates:states,
canonicalEventCursor:this.eventCursor,canonicalEventHead:this.chain.canonicalHead?.()??null,eventBacklog:this.eventBacklog };
canonicalEventCursor:this.eventCursor,canonicalEventHead:this.chain.canonicalHead?.()??null,eventBacklog:this.eventBacklog,
reconciliationWorkerBusy:this.reconciliationWorkerBusy,reconciliationWorkerFailed:!!this.reconciliationWorkerError };
}
hasReconciliationBacklog(): boolean { return this.eventBacklog || !!this.reconciliationJob; }
info(): MarketInfo {
@@ -142,10 +179,18 @@ export class Market {
return JSON.parse(row.data) as MarketOrderRecord;
}
private saveListing(l: MarketListingRecord) {
if (this.stagedReconciliationWrites) {
this.stagedReconciliationWrites.listings.set(l.id, l);
return;
}
this.db.prepare('INSERT INTO listings VALUES(?,?,?,?) ON CONFLICT(id) DO UPDATE SET status=excluded.status,data=excluded.data WHERE listings.data<>excluded.data')
.run(l.id, l.cardHash, l.status, JSON.stringify(l));
}
private saveOrder(o: MarketOrderRecord) {
if (this.stagedReconciliationWrites) {
this.stagedReconciliationWrites.orders.set(o.id, o);
return;
}
this.db.prepare('INSERT INTO orders VALUES(?,?) ON CONFLICT(id) DO UPDATE SET data=excluded.data WHERE orders.data<>excluded.data').run(o.id, JSON.stringify(o));
}
private atomic<T>(fn: () => T): T {
@@ -179,8 +224,77 @@ export class Market {
return transactionId({ network: NETWORK, chainId: CHAIN_ID, cardHash: order.cardHash,
from: order.seller, to: order.buyer, sequence: order.sequence, publicKey: '', signature: '' });
}
private workerRequest(request: Record<string, unknown>): Promise<void> {
const worker = this.reconciliationWorker;
if (!worker) return Promise.reject(new Error('Reconciliation worker is unavailable'));
const id = ++this.workerRequestId;
return new Promise((resolve, reject) => {
this.workerRequests.set(id, { resolve, reject });
try { worker.postMessage({ ...request, id }); }
catch (error) { this.workerRequests.delete(id); reject(error instanceof Error ? error : new Error('Worker request failed')); }
});
}
private scheduleWorkerContinuation(): void {
setImmediate(() => { if (!this.reconciliationWorkerBusy && this.hasReconciliationBacklog()) this.reconcile(); }).unref();
}
private persistWorkerBatch(job: z.infer<typeof reconciliationJobSchema>, nextJob: z.infer<typeof reconciliationJobSchema>,
writes: { orders: Map<string, MarketOrderRecord>; listings: Map<string, MarketListingRecord> }): void {
if (this.reconciliationWorkerBusy) return;
this.reconciliationWorkerBusy = true;
void this.workerRequest({ operation: 'persist-batch', expectedJob: job, nextJob,
orders: [...writes.orders.values()].map(order => ({ id: order.id, data: JSON.stringify(order) })),
listings: [...writes.listings.values()].map(listing => ({ id: listing.id, card: listing.cardHash,
status: listing.status, data: JSON.stringify(listing) })) })
.then(() => { this.reconciliationJob = nextJob; this.reconciliationWorkerError = undefined; this.eventBacklog = true; })
.catch(error => { this.reconciliationWorkerError = error instanceof Error ? error.message : 'Worker commit failed'; this.eventBacklog = true; })
.finally(() => {
this.reconciliationWorkerBusy = false;
if (!this.reconciliationWorkerError) this.scheduleWorkerContinuation();
});
}
private finalizeWorkerJob(job: z.infer<typeof reconciliationJobSchema>): void {
const summary = this.chain.summary?.(), eventPage = job.eventPage as CanonicalEvents | null;
const currentHead = this.chain.canonicalHead?.() ?? eventPage?.head ?? 0;
if ((summary && (job.tip !== summary.tip || job.revision !== summary.revision)) ||
(eventPage && currentHead > eventPage.head)) {
const events = this.chain.canonicalEvents?.(this.eventCursor, 64);
if (events && this.eventCheckpoint && (events.streamId !== this.eventCheckpoint.streamId ||
events.previousAnchor !== this.eventCheckpoint.anchor)) throw new RpcError(409, 'Canonical event history changed; restore matching ledger and market backups.');
const restarted = reconciliationJobSchema.parse({ version: 1, stage: 'orders', afterId: null, observedAt: this.now(),
tip: summary?.tip ?? null, revision: summary?.revision ?? null, eventPage: events ?? null });
this.atomic(() => this.db.prepare('UPDATE market_meta SET value=? WHERE key=?')
.run(JSON.stringify(restarted), RECONCILIATION_JOB_KEY));
this.reconciliationJob = restarted; this.eventBacklog = true; this.reconciliationWorkerError = undefined;
this.scheduleWorkerContinuation();
return;
}
if (this.reconciliationWorkerBusy) return;
this.reconciliationWorkerBusy = true;
const cursor = eventPage ? JSON.stringify({ version: 1, streamId: eventPage.streamId,
cursor: eventPage.cursor, anchor: eventPage.anchor }) : null;
void this.workerRequest({ operation: 'finalize', expectedJob: job, cursor, observedAt: job.observedAt })
.then(() => {
this.reconciliationJob = undefined;
if (eventPage) {
this.eventCursor = eventPage.cursor;
this.eventCheckpoint = { version: 1, streamId: eventPage.streamId, cursor: eventPage.cursor, anchor: eventPage.anchor };
}
const current = this.chain.summary?.();
const head = this.chain.canonicalHead?.() ?? eventPage?.head ?? 0;
this.eventBacklog = (!!eventPage && (!eventPage.caughtUp || head > eventPage.cursor)) ||
(!!current && (current.tip !== job.tip || current.revision !== job.revision));
if (current && !this.eventBacklog) this.reconciled = { ...current, at: this.now() };
this.reconciliationWorkerError = undefined;
})
.catch(error => { this.reconciliationWorkerError = error instanceof Error ? error.message : 'Worker finalize failed'; this.eventBacklog = true; })
.finally(() => {
this.reconciliationWorkerBusy = false;
if (!this.reconciliationWorkerError && this.hasReconciliationBacklog()) this.scheduleWorkerContinuation();
});
}
/** Reconcile one durable keyset batch from canonical transaction identity, never mempool acknowledgement. */
private reconcile() {
if (this.reconciliationWorkerBusy) return;
const now = this.now();
const summary = this.chain.summary?.();
if (!this.reconciliationJob) {
@@ -209,14 +323,38 @@ export class Market {
this.reconciliationJob = job;
this.eventBacklog = true;
}
if (job.stage === 'finalize') {
if (this.reconciliationWorker) { this.finalizeWorkerJob(job); return; }
const eventPage = job.eventPage as CanonicalEvents | null;
this.atomic(() => {
this.db.prepare('DELETE FROM receipts WHERE expires < ?').run(job.observedAt - 86400_000);
if (eventPage) this.db.prepare('INSERT INTO market_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value')
.run('canonical_event_cursor', JSON.stringify({ version: 1, streamId: eventPage.streamId,
cursor: eventPage.cursor, anchor: eventPage.anchor }));
this.db.prepare('DELETE FROM market_meta WHERE key=?').run(RECONCILIATION_JOB_KEY);
});
this.reconciliationJob = undefined;
if (eventPage) {
this.eventCursor = eventPage.cursor;
this.eventCheckpoint = { version: 1, streamId: eventPage.streamId, cursor: eventPage.cursor, anchor: eventPage.anchor };
}
const head = this.chain.canonicalHead?.() ?? eventPage?.head ?? 0;
this.eventBacklog = !!eventPage && (!eventPage.caughtUp || head > eventPage.cursor);
const current = this.chain.summary?.();
if (current && !this.eventBacklog) this.reconciled = { ...current, at: now };
return;
}
const s = this.view(), pendingIds = new Set(s.pendingTransfers.map(transactionId));
const eventPage = job.eventPage as CanonicalEvents | null;
const disconnected = new Set(eventPage?.events.filter(e => e.type === 'disconnect').flatMap(e => e.transactionIds) ?? []);
const rows = this.nextReconciliationRows(job.stage, job.afterId);
let nextJob: z.infer<typeof reconciliationJobSchema> | undefined;
let completed = false;
const workerWrites = this.reconciliationWorker ? { orders: new Map<string, MarketOrderRecord>(),
listings: new Map<string, MarketListingRecord>() } : undefined;
this.stagedReconciliationWrites = workerWrites;
this.atomic(() => {
try { this.atomic(() => {
if (job.stage === 'orders' && rows.length) {
for (const row of rows) {
const o = JSON.parse(row.data) as MarketOrderRecord;
@@ -275,18 +413,28 @@ export class Market {
else nextJob = { ...job, afterId: rows.at(-1)!.id };
} else {
if (rows.length < MARKET_RECONCILIATION_BATCH_SIZE) {
this.db.prepare('DELETE FROM receipts WHERE expires < ?').run(job.observedAt - 86400_000);
if (eventPage) {
this.db.prepare('INSERT INTO market_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value')
.run('canonical_event_cursor', JSON.stringify({ version: 1, streamId: eventPage.streamId,
cursor: eventPage.cursor, anchor: eventPage.anchor }));
if (this.reconciliationWorker) nextJob = { ...job, stage: 'finalize', afterId: null };
else {
this.db.prepare('DELETE FROM receipts WHERE expires < ?').run(job.observedAt - 86400_000);
if (eventPage) {
this.db.prepare('INSERT INTO market_meta(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value')
.run('canonical_event_cursor', JSON.stringify({ version: 1, streamId: eventPage.streamId,
cursor: eventPage.cursor, anchor: eventPage.anchor }));
}
this.db.prepare('DELETE FROM market_meta WHERE key=?').run(RECONCILIATION_JOB_KEY);
completed = true;
}
this.db.prepare('DELETE FROM market_meta WHERE key=?').run(RECONCILIATION_JOB_KEY);
completed = true;
} else nextJob = { ...job, afterId: rows.at(-1)!.id };
}
if (nextJob) this.db.prepare('UPDATE market_meta SET value=? WHERE key=?').run(JSON.stringify(nextJob), RECONCILIATION_JOB_KEY);
});
if (nextJob && !this.reconciliationWorker) this.db.prepare('UPDATE market_meta SET value=? WHERE key=?').run(JSON.stringify(nextJob), RECONCILIATION_JOB_KEY);
}); } finally { this.stagedReconciliationWrites = undefined; }
if (this.reconciliationWorker) {
if (!nextJob) throw new Error('Worker reconciliation batch did not produce a continuation');
if (!workerWrites) throw new Error('Worker reconciliation writes were not staged');
this.persistWorkerBatch(job, nextJob, workerWrites);
return;
}
if (completed) {
this.reconciliationJob = undefined;
@@ -347,8 +495,9 @@ export class Market {
if (!permitsMarketAction(this.config, c.action.type)) throw new Error('New market trading is disabled by this host; existing activity and resolution remain available');
const a = c.action;
const summary = this.chain.summary?.(), last = this.reconciled;
if (this.hasReconciliationBacklog() || a.type !== 'activity' || !summary || !last || summary.tip !== last.tip || summary.revision !== last.revision ||
this.now() - last.at >= RECONCILIATION_FRESH_MS) this.reconcile();
const stale = !summary || !last || summary.tip !== last.tip || summary.revision !== last.revision ||
this.now() - last.at >= RECONCILIATION_FRESH_MS;
if (this.hasReconciliationBacklog() || stale || (!this.reconciliationWorker && a.type !== 'activity')) this.reconcile();
if (this.eventBacklog && a.type !== 'activity') throw new RpcError(503, 'Market canonical reconciliation is catching up; retry shortly.');
if (a.type === 'activity') return {
orders: (this.db.prepare("SELECT data FROM orders WHERE json_extract(data,'$.seller')=? OR json_extract(data,'$.buyer')=? ORDER BY rowid DESC")
server/testnet/http.ts
+8 -6
View File
@@ -63,7 +63,7 @@ try { peers = new PeerNetwork({ ledger, urls:(process.env.WIKIDECK_PEERS ?? '').
allowPrivate:process.env.WIKIDECK_ALLOW_PRIVATE_PEERS === '1' }); }
catch(e) {ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
let market: Market;
try { market = await Market.open(dataDir, ledger, paymentConfig); }
try { market = await Market.open(dataDir, ledger, paymentConfig, Date.now, { worker: true }); }
catch(e) {peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
let custody: CustodyMarket | PostgresCustody | undefined;
let custodyObserver: CanonicalObserver | undefined;
@@ -80,7 +80,7 @@ try {
custodyObserver = new CanonicalObserver(custody, ledger);
if ((await custodyObserver.start()).state !== 'running') throw new Error('Canonical observer rehearsal failed to start; inspect custody diagnostics');
}
} catch(e) {await custodyObserver?.close();await custody?.close();market.close();peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
} catch(e) {await custodyObserver?.close();await custody?.close();await market.close();peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
const app = express();
app.disable('x-powered-by');
if(proxyTrust.length)app.set('trust proxy',proxyTrust);
@@ -258,10 +258,12 @@ let catchupImmediate:NodeJS.Immediate|undefined;
// SQLite slice, then setImmediate yields so HTTP and peer work can run between passes.
const advanceMarketCatchup=()=>{
catchupImmediate=undefined;
if(stopping||catchupPaused||!market.hasReconciliationBacklog())return;
if(stopping||catchupPaused||market.isReconciliationWorkerBusy()||!market.hasReconciliationBacklog())return;
try {market.browse();}
catch {catchupPaused=true;console.error('Marketplace canonical catch-up paused; operator review required.');return;}
if(market.hasReconciliationBacklog()){catchupImmediate=setImmediate(advanceMarketCatchup);catchupImmediate.unref();}
if(market.hasReconciliationBacklog()&&!market.isReconciliationWorkerBusy()){
catchupImmediate=setImmediate(advanceMarketCatchup);catchupImmediate.unref();
}
};
const marketCatchupTimer=setInterval(()=>{
if(!stopping&&!catchupPaused&&!catchupImmediate&&market.hasReconciliationBacklog()){
@@ -274,6 +276,6 @@ for(const signal of ['SIGTERM','SIGINT']as const)process.once(signal,()=>{
if(catchupImmediate)clearImmediate(catchupImmediate);
const observerDrain=custodyObserver?.close()??Promise.resolve();
const timeout=setTimeout(()=>process.exit(1),25000);timeout.unref();
server.close(async()=>{await observerDrain;await custody?.close();market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
server.close(async()=>{await observerDrain;await custody?.close();await market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
});
server.on('error',async()=>{stopping=true;clearInterval(marketTimer);clearInterval(marketCatchupTimer);if(catchupImmediate)clearImmediate(catchupImmediate);operations.close();peers.close();await custodyObserver?.close();await custody?.close();market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});
server.on('error',async()=>{stopping=true;clearInterval(marketTimer);clearInterval(marketCatchupTimer);if(catchupImmediate)clearImmediate(catchupImmediate);operations.close();peers.close();await custodyObserver?.close();await custody?.close();await market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});