From c07f3a737be1a360c2307b3d7183382332afae99 Mon Sep 17 00:00:00 2001 From: Eric Meehan Date: Wed, 21 Oct 2026 12:40:00 -0400 Subject: [PATCH] Loopback HTTP benchmark --- docs/canonical-indexes.md | 4 +- docs/production-plan.md | 2 +- scripts/benchmark-market-reconciliation.ts | 48 ++++++++++++++++++++-- tests/api.test.ts | 2 + 4 files changed, 50 insertions(+), 6 deletions(-) diff --git a/docs/canonical-indexes.md b/docs/canonical-indexes.md index 187267a..444faf6 100755 --- a/docs/canonical-indexes.md +++ b/docs/canonical-indexes.md @@ -49,10 +49,10 @@ Observed in the development Docker environment: These numbers are a reproducible **baseline**, not acceptance SLOs: host CPU/storage and production concurrency were not selected, outputs include synthetic article payloads, warm caches apply, one transaction amortizes insertion fsync, and real validation/startup/HTTP/market workloads are excluded. They do not establish a supported production envelope. -The synthetic market-journal harness [benchmark](../scripts/benchmark-market-reconciliation.ts), available as `npm run benchmark:market -- 10000 [terminal|confirmed-delivery] [worker]`, measures terminal and confirmed-delivery reconciliation in its own temporary SQLite directory. Both modes use the same Ledger-shaped fixture; worker mode reads it through a separate read-only SQLite connection and performs market row selection, canonical transaction/NFT fact reads, status evaluation and market commits on the worker. In the latest development Docker run, synchronous 32-row batches processed 10,000 terminal orders plus 10,000 listings in 4,100 ms (about 4,878 records/s; sampled event-loop-delay p95/max 18.43/21.95 ms), and confirmed delivery in 5,853 ms (about 3,417 records/s; 21.64/55.67 ms). Worker-owned selection/evaluation/fact reads/commits took 4,508 ms for terminal rows (about 4,437 records/s; delay p95/max 10.93/13.40 ms) and 4,088 ms for confirmed delivery (about 4,893 records/s; 10.80/12.48 ms). Thus worker execution reduced sampled delay, improved this delivery fixture, and slightly slowed terminal-only work. Timings vary; synchronous max pass was 47.30 ms for confirmed delivery, worker dispatch max 14.45 ms. The SQLite fixture is synthetic, not validated Ledger evidence. These are not capacity claims and exclude concurrent HTTP traffic, PostgreSQL, actual chain validation and adversarial workloads. +The synthetic market-journal harness [benchmark](../scripts/benchmark-market-reconciliation.ts), available as `npm run benchmark:market -- 10000 [terminal|confirmed-delivery] [worker] [http]`, measures terminal and confirmed-delivery reconciliation in its own temporary SQLite directory. Both modes use the same Ledger-shaped fixture; worker mode reads it through a separate read-only SQLite connection and performs market row selection, canonical transaction/NFT fact reads, status evaluation and market commits on the worker. In the latest development Docker run, synchronous 32-row batches processed 10,000 terminal orders plus 10,000 listings in 4,100 ms (about 4,878 records/s; sampled event-loop-delay p95/max 18.43/21.95 ms), and confirmed delivery in 5,853 ms (about 3,417 records/s; 21.64/55.67 ms). Worker-owned selection/evaluation/fact reads/commits took 4,508 ms for terminal rows (about 4,437 records/s; delay p95/max 10.93/13.40 ms) and 4,088 ms for confirmed delivery (about 4,893 records/s; 10.80/12.48 ms). Thus worker execution reduced sampled delay, improved this delivery fixture, and slightly slowed terminal-only work. A new optional loopback HTTP mode issues 160 health/market requests during catch-up: worker run request p50/p95/max was 285/322/324 ms, versus 1,416/2,844/2,971 ms for synchronous reconciliation. These two runs are preliminary, local, synthetic, and lack repeated confidence intervals or a representative arrival distribution; the latency is not an SLO. SQLite fixture data is not validated Ledger evidence. All measurements exclude real concurrent users, PostgreSQL and adversarial workloads. ## Remaining gates -M2 still needs target-hardware/data/SLO decisions, populated long-journal/large-event benchmarks, concurrent HTTP tests during worker activity, production financial database/schema and measured recovery/retention/storage growth. The worker removes row selection, canonical fact reads and per-row evaluation from the request event loop, but event-page acquisition and post-batch consistency fencing remain on-loop; queue isolation, multi-process fencing and a demonstrated production throughput envelope remain open. Ledger PoW replay and DAG state memory still scale with retained history; indexed queries do not solve those costs. Startup validation checks undo-record counts but not every possible future undo mutation against a second reconstructed undo model; comprehensive corruption/fuzz coverage and independent review remain required. +M2 still needs target-hardware/data/SLO decisions, repeated concurrent HTTP tests with realistic request mix/rates, multi-instance/restart/in-flight-worker fencing evidence, production financial database/schema and measured recovery/retention/storage growth. The worker removes row selection, canonical fact reads and per-row evaluation from the request event loop, but event-page acquisition and post-batch consistency fencing remain on-loop; horizontally shared job ownership has not been implemented. Ledger PoW replay and DAG state memory still scale with retained history; indexed queries do not solve those costs. Startup validation checks undo-record counts but not every possible future undo mutation against a second reconstructed undo model; comprehensive corruption/fuzz coverage and independent review remain required. Custodial reservations still stop at `awaiting-lock`. This increment does **not** consume ordinary transfers as custodial settlement evidence: doing that would bypass the reviewed sale-lock/cancel protocol. Real Monero deposit/withdrawal integration, production treasury keys, host attestations and a reviewed versioned NFT lock/completion/cancellation specification remain blocked. No real funds, credentials, external processor or genesis change were used. diff --git a/docs/production-plan.md b/docs/production-plan.md index 95a5c69..c560f82 100755 --- a/docs/production-plan.md +++ b/docs/production-plan.md @@ -63,7 +63,7 @@ Acceptance: exact-amount property/boundary tests, versioned signature tests, zer Can run alongside P0/P1. Addresses SEC-01/SEC-06 rather than adding more fixed caps. -**Implemented M2 increments, 2026-10-20–21:** [canonical indexes and durable market observation](canonical-indexes.md). Canonical ownership/transaction SQL indexes now serve NFT, deck, custody availability and mock-market queries; per-block undo and anchored connect/disconnect events commit with ledger updates. Market checkpoint/reorg holds persist atomically and bounded catch-up blocks new mutations. Order/listing reconciliation advances in durable, stable-ID keyset batches of at most 32 records per table, resumes after restart, and re-fences when the chain tip/revision changes. The HTTP process now uses a single worker thread to select/evaluate bounded row batches, read canonical facts using its own read-only Ledger connection, and atomically persist changes with cursor progress. Final TypeScript/Vite build and **178 offline tests passed**; focused market tests passed 27/27; the disposable PostgreSQL suite passed 31/31. A main-loop timer regression confirms a timer can run while a large worker backlog is still processing. In a same-fixture synthetic 10,000 confirmed-delivery comparison, worker processing took 4,088 ms with sampled event-loop-delay max 12.48 ms versus 5,853 ms and 55.67 ms synchronously on the development host. This is local evidence, not an SLO. Main-thread event-page acquisition/post-batch fencing, concurrent HTTP load, multi-process fencing, target hardware/data, production financial storage/recovery, retained-DAG capacity and operator SLO decisions remain open. +**Implemented M2 increments, 2026-10-20–21:** [canonical indexes and durable market observation](canonical-indexes.md). Canonical ownership/transaction SQL indexes now serve NFT, deck, custody availability and mock-market queries; per-block undo and anchored connect/disconnect events commit with ledger updates. Market checkpoint/reorg holds persist atomically and bounded catch-up blocks new mutations. Order/listing reconciliation advances in durable, stable-ID keyset batches of at most 32 records per table, resumes after restart, and re-fences when the chain tip/revision changes. The HTTP process now uses a single worker thread to select/evaluate bounded row batches, read canonical facts using its own read-only Ledger connection, and atomically persist changes with cursor progress. Final TypeScript/Vite build and **178 offline tests passed**; focused market tests passed 27/27; the disposable PostgreSQL suite passed 31/31. Regressions verify a main-loop timer runs during worker backlog and that a second server process cannot start against the same data directory. A synthetic 10,000 confirmed-delivery run with 160 loopback health/market requests measured worker HTTP p50/p95/max of 285/322/324 ms versus 1,416/2,844/2,971 ms synchronously. This single local run is preliminary (no repeated confidence intervals or realistic request arrivals), not an SLO. Main-thread event-page acquisition/post-batch fencing, realistic concurrent HTTP load, multi-process fencing beyond the existing exclusive directory lock, target hardware/data, production financial storage/recovery, retained-DAG capacity and operator SLO decisions remain open. **Next financial-storage increment:** [ADR 003 — PostgreSQL](postgres-financial-storage.md) selects and integrates the custody SQL backend with existing HTTP callers, exact constrained accounting, explicit migration/runtime roles, source-freezing SQLite import and dump/restore rehearsal. The adapter remains on the fictitious sandbox domain; it is not a production deposit/withdrawal worker. SQLite node-only backup deliberately refuses PostgreSQL-bound financial state. Production hardware/SLO tests, coordinated backup/PITR/failover, real payment schema/worker controls and independent review remain open, so this does not close M2/M4 or authorize funds. diff --git a/scripts/benchmark-market-reconciliation.ts b/scripts/benchmark-market-reconciliation.ts index dce15ca..9d056a3 100755 --- a/scripts/benchmark-market-reconciliation.ts +++ b/scripts/benchmark-market-reconciliation.ts @@ -5,6 +5,7 @@ import { join } from 'node:path'; import { monitorEventLoopDelay, performance } from 'node:perf_hooks'; import { randomUUID } from 'node:crypto'; import { setImmediate as yieldToEventLoop } from 'node:timers/promises'; +import { createServer } from 'node:http'; import { MARKET_RECONCILIATION_BATCH_SIZE, Market } from '../server/market/store.ts'; import { CHAIN_ID, GENESIS_HASH, transactionId } from '../server/testnet/consensus.ts'; import { NETWORK } from '../shared/testnet.ts'; @@ -15,13 +16,17 @@ const text = process.argv[2] ?? '10000'; if (!/^[1-9]\d{0,5}$/.test(text) || Number(text) > 100000) throw new Error('Fixture size must be 1..100000 orders'); const count = Number(text), scenario = process.argv[3] ?? 'terminal'; if (!['terminal', 'confirmed-delivery'].includes(scenario)) throw new Error('Scenario must be terminal or confirmed-delivery'); -const workerEnabled = process.argv[4] === 'worker'; +const executionMode = process.argv[4] ?? 'sync'; +if (!['sync', 'worker'].includes(executionMode)) throw new Error('Execution mode must be sync or worker'); +const workerEnabled = executionMode === 'worker'; +const httpLoadEnabled = process.argv[5] === 'http'; const directory = await mkdtemp(join(tmpdir(), 'wikideck-market-reconciliation-')); const streamId = randomUUID(), tip = 'a'.repeat(64); const ledgerFile = join(directory, NETWORK.includes('mainnet') ? 'mainnet' : 'testnet', 'ledger.sqlite'); const seller = '1'.repeat(64), buyer = '2'.repeat(64); let market: Market | undefined; let ledgerDb: DatabaseSync | undefined; +let loadServer: ReturnType | undefined; try { // Indexed-only adapter: the benchmark must not fall back to materializing chain snapshots. await mkdir(join(directory, NETWORK.includes('mainnet') ? 'mainnet' : 'testnet'), { recursive: true }); @@ -90,6 +95,37 @@ try { db.exec('COMMIT'); } catch (error) { db.exec('ROLLBACK'); throw error; } } finally { db.close(); } + let httpLatencies: number[] = [], httpRequests = 0; + let httpLoadPromise: Promise | undefined; + if (httpLoadEnabled) { + loadServer = createServer((request, response) => { + try { + if (request.url === '/health') { + response.writeHead(200, { 'content-type': 'application/json' }); response.end('{"ok":true}'); return; + } + if (request.url === '/market') { + const result = market!.browse(); + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ listingCount: result.listings.length, backlog: market!.hasReconciliationBacklog() })); + return; + } + response.writeHead(404); response.end(); + } catch { + response.writeHead(503, { 'retry-after': '1' }); response.end('{"error":"reconciling"}'); + } + }); + await new Promise(resolve => loadServer!.listen(0, '127.0.0.1', resolve)); + const address = loadServer.address(); + if (!address || typeof address === 'string') throw new Error('Benchmark HTTP listener did not bind TCP'); + const baseUrl = `http://127.0.0.1:${address.port}`; + httpLoadPromise = Promise.all(Array.from({ length: 160 }, (_, i) => (async () => { + const before = performance.now(); + const response = await fetch(`${baseUrl}${i % 4 === 0 ? '/market' : '/health'}`); + if (response.status !== 200) throw new Error(`Loopback benchmark endpoint returned ${response.status}`); + await response.arrayBuffer(); + httpLatencies.push(performance.now() - before); + })())).then(() => { httpLatencies.sort((a, b) => a - b); httpRequests = httpLatencies.length; }); + } const latencies: number[] = []; const deadline = Date.now() + Math.max(30_000, count * 20); const eventLoopDelay = monitorEventLoopDelay({ resolution: 10 }); @@ -97,7 +133,8 @@ try { const start = performance.now(); let started = false; try { - while (Date.now() < deadline && (!started || market.hasReconciliationBacklog() || market.isReconciliationWorkerBusy())) { + while (Date.now() < deadline && (!started || market.hasReconciliationBacklog() || market.isReconciliationWorkerBusy() || + (!!httpLoadPromise && httpRequests === 0))) { if (!workerEnabled || !market.isReconciliationWorkerBusy()) { const before = performance.now(); market.browse(); @@ -110,19 +147,24 @@ try { } finally { eventLoopDelay.disable(); } const elapsedMs = performance.now() - start; if (!started || market.hasReconciliationBacklog() || market.isReconciliationWorkerBusy()) throw new Error('Reconciliation did not finish before benchmark deadline'); + await httpLoadPromise; await market.close(); market = undefined; ledgerDb.close(); ledgerDb = undefined; const bytes = (await stat(join(directory, NETWORK.includes('mainnet') ? 'mainnet' : 'testnet', 'market.sqlite'))).size; latencies.sort((a, b) => a - b); const percentile = (p: number) => latencies[Math.min(latencies.length - 1, Math.ceil(latencies.length * p) - 1)]; - console.log(JSON.stringify({ fixture: `synthetic-${scenario}-orders-and-listings`, workerEnabled, orders: count, listings: count, + const httpPercentile = (p: number) => httpLatencies.length ? httpLatencies[Math.min(httpLatencies.length - 1, Math.ceil(httpLatencies.length * p) - 1)] : null; + console.log(JSON.stringify({ fixture: `synthetic-${scenario}-orders-and-listings`, workerEnabled, httpLoadEnabled, orders: count, listings: count, batchSizePerTable: MARKET_RECONCILIATION_BATCH_SIZE, reconciliationPasses: latencies.length, elapsedMs, rowsPerSecond: Math.round((count * 2 * 1000) / elapsedMs), passLatencyP50Ms: percentile(0.50), passLatencyP95Ms: percentile(0.95), passLatencyMaxMs: latencies.at(-1), sqliteBytes: bytes, eventLoopDelayResolutionMs: 10, eventLoopDelayP95Ms: eventLoopDelay.percentile(95) / 1e6, eventLoopDelayMaxMs: eventLoopDelay.max / 1e6, + httpRequests, httpLatencyP50Ms: httpPercentile(0.5), httpLatencyP95Ms: httpPercentile(0.95), + httpLatencyMaxMs: httpLatencies.at(-1) ?? null, warning: 'Synthetic fixture on this development host only; SQLite Ledger-shaped facts are not mined or validated chain evidence. Excludes concurrent HTTP load, PostgreSQL, and production SLO certification.' }, null, 2)); } finally { + if (loadServer) await new Promise(resolve => loadServer!.close(() => resolve())); if (market) await market.close(); ledgerDb?.close(); await rm(directory, { recursive: true, force: true }); diff --git a/tests/api.test.ts b/tests/api.test.ts index fc7f8d1..5dadbdc 100644 --- a/tests/api.test.ts +++ b/tests/api.test.ts @@ -219,6 +219,8 @@ test('native RandomX HTTP: three blocks, mined transfer, peer gossip/sync and du const alice = wallet(), bob = wallet(); assert.notEqual(alice.owner, bob.owner); let first = await start(firstDir); + await assert.rejects(start(firstDir), /Data directory locked/, + 'a second server process must fail closed rather than run another marketplace worker on the same data directory'); const custody = await json(first.base, '/api/custody'); assert.equal(custody.info.mode, 'sandbox'); assert.equal(custody.info.liveDeposits, false); const custodyTerms: Omit = { version: 1, mode: 'sandbox', policy: CUSTODY_POLICY,