WikiDeck/wikideck
3
0
Fork 1
Code Issues 8 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

Remove markets

Browse Source
This commit is contained in:
eric
2026-10-28 18:33:52 -04:00
parent eea2deb1fb
commit b3a1179703
141 changed files with 554 additions and 17365 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/testnet/http.ts
+8 -245
View File
@@ -1,13 +1,11 @@
import express from 'express';
import { Pool } from 'pg';
import helmet from 'helmet';
import rateLimit from 'express-rate-limit';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { mkdir, open, unlink } from 'node:fs/promises';
import { z } from 'zod';
import { NETWORK, CONSENSUS } from '../../shared/testnet.ts';
import { transactionCardIds } from '../../shared/testnet.ts';
import { NETWORK } from '../../shared/testnet.ts';
import { CHAIN_ID, blockSchema, tradeSchema, transferSchema, transactionSchema } from './consensus.ts';
import { RandomX } from './randomx.ts';
import { Ledger } from './ledger.ts';
@@ -15,66 +13,13 @@ import { PeerNetwork } from './peers.ts';
import { wikipediaEnrichment } from '../enrichment.ts';
import { snapshotStats } from '../../shared/enrichment.ts';
import { wikipediaArticles } from '../wiki.ts';
import { Market, marketConfig } from '../market/store.ts';
import { paymentMode } from '../market/payments.ts';
import { marketCommandSchema } from '../../shared/market.ts';
import { marketCommandV3Schema } from '../../shared/market-v3.ts';
import { custodyCommandSchema } from '../../shared/custody.ts';
import { moneroDepositAddressRequestSchema } from '../../shared/monero-deposit.ts';
import { mainnetCustodyBalanceRequestSchema, mainnetWithdrawalRequestSchema,
mainnetWithdrawalCancellationSchema } from '../../shared/monero-withdrawal.ts';
import { CustodyMarket, CustodyError, custodyMode } from '../market/custody.ts';
import { openCustodyStorage } from '../market/custody-storage.ts';
import { PostgresCustody } from '../market/postgres-custody.ts';
import { CanonicalObserver } from '../market/canonical-observer.ts';
import { enrollMoneroDepositAddress, type MainnetEnrollmentContext } from '../market/monero-deposit-enrollment.ts';
import { cancelMainnetWithdrawal, readMainnetCustodyBalance, reserveMainnetWithdrawal } from '../market/mainnet-withdrawal.ts';
import { verifyMainnetCustodyIdentity } from '../market/mainnet-custody-identity.ts';
import { createMoneroDepositAddressRpc, loadMoneroObserverConfig } from '../market/monero-observer.ts';
import { loadMoneroMtlsTransport, loadMoneroMtlsTransportConfig } from '../market/monero-mtls-transport.ts';
import { postgresConfig } from '../market/postgres-config.ts';
import { validateMarketPolicy } from '../market/policy.ts';
import { IS_MAINNET, IS_PILOT, PROFILE } from '../../shared/network.ts';
import { IS_MAINNET, PROFILE } from '../../shared/network.ts';
import { assertLaunchAllowed, launchManifest } from '../launch.ts';
import { resourceConfig, RpcBudget, RpcError } from '../resources.ts';
import { trustedProxies, operationsPort, startOperations } from '../operations.ts';
const root = fileURLToPath(new URL('../../', import.meta.url));
assertLaunchAllowed();
const marketRuntimeSetting = process.env.WIKIDECK_MARKET_RUNTIME_ENABLED ?? '0';
if (marketRuntimeSetting !== '0' && marketRuntimeSetting !== '1') {
throw new Error('WIKIDECK_MARKET_RUNTIME_ENABLED must be exactly 0 or 1');
}
const marketRuntimeEnabled = marketRuntimeSetting === '1';
const custodyConfig = custodyMode(); // Explicit opt-in sandbox only. Unknown/live modes fail before opening resources.
paymentMode(); // Keep real-payment modes fail-closed even when the optional market runtime is off.
let paymentConfig: ReturnType<typeof marketConfig> | undefined;
if (marketRuntimeEnabled || custodyConfig === 'sandbox') {
paymentConfig = marketConfig();
await validateMarketPolicy(paymentConfig); // Reject invalid recipient checksum/network before opening node resources.
}
const canonicalObserverMode = process.env.WIKIDECK_CUSTODY_CANONICAL_REHEARSAL;
if (canonicalObserverMode !== undefined && canonicalObserverMode !== '0' && canonicalObserverMode !== '1') {
throw new Error('WIKIDECK_CUSTODY_CANONICAL_REHEARSAL must be exactly 0 or 1');
}
const depositEnrollmentMode = process.env.WIKIDECK_MONERO_DEPOSIT_ENROLLMENT ?? '0';
if (depositEnrollmentMode !== '0' && depositEnrollmentMode !== '1') {
throw new Error('WIKIDECK_MONERO_DEPOSIT_ENROLLMENT must be exactly 0 or 1');
}
if (depositEnrollmentMode === '1' && !IS_MAINNET) {
throw new Error('Monero deposit enrollment requires the mainnet profile');
}
const withdrawalRequestMode = process.env.WIKIDECK_MONERO_WITHDRAWAL_REQUESTS ?? '0';
if (withdrawalRequestMode !== '0' && withdrawalRequestMode !== '1') {
throw new Error('WIKIDECK_MONERO_WITHDRAWAL_REQUESTS must be exactly 0 or 1');
}
if (withdrawalRequestMode === '1' && !IS_MAINNET) {
throw new Error('Monero withdrawal requests require the mainnet profile');
}
if (canonicalObserverMode === '1' && (process.env.NODE_ENV === 'production' || custodyConfig !== 'sandbox' ||
process.env.WIKIDECK_CUSTODY_STORAGE !== 'postgres' || process.env.WIKIDECK_PG_DISPOSABLE_TEST !== '1')) {
throw new Error('Canonical observer rehearsal requires sandbox custody, PostgreSQL, and disposable test isolation');
}
const limits = resourceConfig();
const proxyTrust=trustedProxies();
const opsPort=operationsPort();
@@ -96,68 +41,6 @@ let peers: PeerNetwork;
try { peers = new PeerNetwork({ ledger, urls:(process.env.WIKIDECK_PEERS ?? '').split(',').map(s=>s.trim()).filter(Boolean),
allowPrivate:process.env.WIKIDECK_ALLOW_PRIVATE_PEERS === '1' }); }
catch(e) {ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
let market: Market | undefined;
try { if (marketRuntimeEnabled) market = await Market.open(dataDir, ledger, paymentConfig!, Date.now, { worker: true }); }
catch(e) {peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
let custody: CustodyMarket | PostgresCustody | undefined;
let custodyObserver: CanonicalObserver | undefined;
let moneroDepositPool: Pool | undefined;
let moneroDepositRpc: ReturnType<typeof createMoneroDepositAddressRpc> | undefined;
let moneroDepositContext: MainnetEnrollmentContext | undefined;
let moneroWithdrawalPool: Pool | undefined;
let moneroWithdrawalCustodianId: string | undefined;
try {
if (custodyConfig === 'sandbox') custody = await openCustodyStorage(dataDir, { ...paymentConfig!,
card: (hash, owner, sequence) => {
const nft = ledger.nft(hash);
if (!nft || nft.owner !== owner || nft.confirmations < CONSENSUS.coinbaseMaturity || nft.sequence !== sequence ||
ledger.pendingTransactions().some(tx => transactionCardIds(tx).includes(hash))) return undefined;
return { title: nft.block.article.title };
} });
if (canonicalObserverMode === '1') {
if (!(custody instanceof PostgresCustody)) throw new Error('Canonical observer rehearsal requires PostgreSQL custody');
custodyObserver = new CanonicalObserver(custody, ledger);
if ((await custodyObserver.start()).state !== 'running') throw new Error('Canonical observer rehearsal failed to start; inspect custody diagnostics');
}
if (depositEnrollmentMode === '1') {
const custodianId = z.string().uuid().parse(process.env.WIKIDECK_MONERO_CUSTODIAN_ID);
const walletGenerationId = z.string().uuid().parse(process.env.WIKIDECK_MONERO_WALLET_GENERATION_ID);
const expectedFingerprint = z.string().regex(/^[a-f0-9]{64}$/).parse(process.env.WIKIDECK_MONERO_WALLET_FINGERPRINT);
const accountText = process.env.WIKIDECK_MONERO_DEPOSIT_ACCOUNT_INDEX;
if (!accountText || !/^(0|[1-9]\d{0,9})$/.test(accountText) || Number(accountText) > 0xffff_ffff) {
throw new Error('Invalid configured Monero deposit account index');
}
const observerConfig = await loadMoneroObserverConfig({ ...process.env, WIKIDECK_MONERO_OBSERVER: '1' });
const mtlsConfig = await loadMoneroMtlsTransportConfig();
if (new URL(observerConfig.url).href !== new URL(mtlsConfig.url).href) {
throw new Error('Monero observer and mTLS gateway endpoints do not match');
}
const transport = await loadMoneroMtlsTransport(mtlsConfig);
moneroDepositRpc = createMoneroDepositAddressRpc(observerConfig, transport);
const actualFingerprint = await moneroDepositRpc.verifyWallet();
if (actualFingerprint !== expectedFingerprint) throw new Error('Configured Monero wallet fingerprint does not match the protected wallet');
const allocatorUrlFile = process.env.WIKIDECK_MONERO_ALLOCATOR_PG_URL_FILE;
const allocatorCaFile = process.env.WIKIDECK_MONERO_ALLOCATOR_PG_CA_FILE;
if (!allocatorUrlFile || !allocatorCaFile) throw new Error('Monero deposit allocator PostgreSQL URL and CA files are required');
moneroDepositPool = new Pool(await postgresConfig({ WIKIDECK_CUSTODY_PG_URL_FILE: allocatorUrlFile,
WIKIDECK_CUSTODY_PG_CA_FILE: allocatorCaFile, WIKIDECK_CUSTODY_PG_TRANSPORT: 'tls' }));
moneroDepositPool.on('error', () => console.error('Monero deposit allocator database unavailable.'));
await moneroDepositPool.query('SELECT 1 FROM wk_custody.monero_deposit_allocations LIMIT 0');
moneroDepositContext = { custodianId, chainId: CHAIN_ID, walletGenerationId, walletFingerprint: actualFingerprint,
accountIndex: Number(accountText) };
}
if (withdrawalRequestMode === '1') {
moneroWithdrawalCustodianId = z.string().uuid().parse(process.env.WIKIDECK_MONERO_WITHDRAWAL_CUSTODIAN_ID);
const urlFile = process.env.WIKIDECK_MONERO_WITHDRAWAL_PG_URL_FILE;
const caFile = process.env.WIKIDECK_MONERO_WITHDRAWAL_PG_CA_FILE;
if (!urlFile || !caFile) throw new Error('Monero withdrawal request PostgreSQL URL and CA files are required');
moneroWithdrawalPool = new Pool(await postgresConfig({ WIKIDECK_CUSTODY_PG_URL_FILE: urlFile,
WIKIDECK_CUSTODY_PG_CA_FILE: caFile, WIKIDECK_CUSTODY_PG_TRANSPORT: 'tls' }));
moneroWithdrawalPool.on('error', () => console.error('Monero withdrawal request database unavailable.'));
await verifyMainnetCustodyIdentity(moneroWithdrawalPool, moneroWithdrawalCustodianId);
await moneroWithdrawalPool.query('SELECT 1 FROM wk_custody.monero_withdrawal_intents LIMIT 0');
}
} catch(e) {await moneroWithdrawalPool?.end();await moneroDepositPool?.end();await custodyObserver?.close();await custody?.close();await market?.close();peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
const app = express();
app.disable('x-powered-by');
if(proxyTrust.length)app.set('trust proxy',proxyTrust);
@@ -191,93 +74,6 @@ app.get('/api/canonical',(req,res)=>{
app.get('/api/network',(_req,res)=>res.json(launchManifest()));
app.get('/api/chain',(_req,res)=>res.json(ledger.legacySnapshot(limits.legacyBlocks)));
app.get('/api/peers',(_req,res)=>res.json({peers:peers.status()}));
if (market) {
app.get('/api/market', (req,res)=>res.json(market!.browse(Number(req.query.offset ?? 0))));
app.post('/api/market/command',rateLimit({windowMs:60000,limit:60,standardHeaders:'draft-8',legacyHeaders:false}),async(req,res)=>{
const current = marketCommandV3Schema.safeParse(req.body);
const command = current.success ? current.data : marketCommandSchema.parse(req.body);
try {
const result = await market!.execute(command);
res.json(result);
if(command.action.type==='deliver')void peers.broadcastTransfer(command.action.transfer).catch(()=>{});
} catch(error) {
if(error instanceof RpcError)throw error;
// Marketplace storage/payment internals must never leak through the generic node error handler.
if(error instanceof Error && !/SQLITE|constraint|database/i.test(error.message))res.status(400).json({error:error.message.slice(0,200)});
else res.status(409).json({error:'Marketplace state conflict. Refresh and retry.'});
}
});
}
app.get('/api/custody', async (req,res)=> {
if (!custody) { res.json({ enabled: false }); return; }
try { res.json(await custody.browse(Number(req.query.offset ?? 0))); }
catch (error) {
if (error instanceof CustodyError) res.status(400).json({ error: error.message });
else res.status(503).json({ error: 'Custody sandbox unavailable; operator review required.' });
}
});
app.post('/api/custody/command', rateLimit({windowMs:60000,limit:30,standardHeaders:'draft-8',legacyHeaders:false}), async (req,res)=> {
if (!custody) { res.status(503).json({ error: 'Custody sandbox is disabled on this host.' }); return; }
const parsed = custodyCommandSchema.safeParse(req.body);
if (!parsed.success) { res.status(400).json({ error: 'Invalid custody command schema.' }); return; }
try { res.json(await custody.execute(parsed.data)); }
catch (error) {
if (error instanceof CustodyError) res.status(400).json({ error: error.message });
else res.status(503).json({ error: 'Custody state conflict; refresh activity before retrying. Operator review may be required.' });
}
});
if (depositEnrollmentMode === '1' && moneroDepositContext && moneroDepositRpc && moneroDepositPool) {
app.get('/api/monero/deposit-config', (_req,res) => {
res.json({ network: 'wikideck-randomx-mainnet-1', chainId: moneroDepositContext!.chainId,
custodianId: moneroDepositContext!.custodianId, walletGenerationId: moneroDepositContext!.walletGenerationId,
walletFingerprint: moneroDepositContext!.walletFingerprint, accountIndex: moneroDepositContext!.accountIndex,
addressEnrollmentEnabled: true, depositCreditEnabled: false, withdrawalsEnabled: false });
});
app.post('/api/monero/deposit-address', rateLimit({windowMs:60000,limit:5,standardHeaders:'draft-8',legacyHeaders:false}), async (req,res) => {
const parsed = moneroDepositAddressRequestSchema.safeParse(req.body);
if (!parsed.success) { res.status(400).json({ error: 'Invalid signed Monero deposit-address request.' }); return; }
try {
const result = await enrollMoneroDepositAddress(moneroDepositPool!, moneroDepositRpc!, moneroDepositContext!, parsed.data);
res.json({ ...result, status: 'address-enrolled-not-credited', depositCreditEnabled: false, withdrawalsEnabled: false });
} catch {
res.status(409).json({ error: 'Deposit address enrollment failed; verify signed terms and retry with the same request ID.' });
}
});
}
if (withdrawalRequestMode === '1' && moneroWithdrawalPool && moneroWithdrawalCustodianId) {
app.get('/api/monero/withdrawal-config', (_req,res) => {
res.json({ network: 'wikideck-randomx-mainnet-1', chainId: CHAIN_ID,
custodianId: moneroWithdrawalCustodianId, withdrawalRequestsEnabled: true,
transactionSigningEnabled: false, transactionRelayEnabled: false });
});
app.post('/api/monero/custody-balance', rateLimit({windowMs:60000,limit:20,standardHeaders:'draft-8',legacyHeaders:false}), async (req,res) => {
const parsed = mainnetCustodyBalanceRequestSchema.safeParse(req.body);
if (!parsed.success) { res.status(400).json({ error: 'Invalid signed mainnet custody balance request.' }); return; }
try { res.json(await readMainnetCustodyBalance(moneroWithdrawalPool!, parsed.data)); }
catch { res.status(403).json({ error: 'Mainnet custody balance is unavailable for this signed request.' }); }
});
app.post('/api/monero/withdrawals', rateLimit({windowMs:60000,limit:5,standardHeaders:'draft-8',legacyHeaders:false}), async (req,res) => {
const parsed = mainnetWithdrawalRequestSchema.safeParse(req.body);
if (!parsed.success) { res.status(400).json({ error: 'Invalid signed mainnet withdrawal request.' }); return; }
try {
const result = await reserveMainnetWithdrawal(moneroWithdrawalPool!, parsed.data);
res.status(202).json({ ...result, status: result.status === 'reserved' ? 'reserved-not-sent' : result.status,
transactionSigningEnabled: false, transactionRelayEnabled: false });
} catch {
res.status(409).json({ error: 'Withdrawal request could not be reserved; verify signed terms and available balance.' });
}
});
app.post('/api/monero/withdrawals/cancel', rateLimit({windowMs:60000,limit:5,standardHeaders:'draft-8',legacyHeaders:false}), async (req,res) => {
const parsed = mainnetWithdrawalCancellationSchema.safeParse(req.body);
if (!parsed.success) { res.status(400).json({ error: 'Invalid signed withdrawal cancellation.' }); return; }
try {
const result = await cancelMainnetWithdrawal(moneroWithdrawalPool!, parsed.data);
res.json({ ...result, transactionSigningEnabled: false, transactionRelayEnabled: false });
} catch {
res.status(409).json({ error: 'Withdrawal cancellation could not be applied; it may already be cancelled or under operator processing.' });
}
});
}
app.get('/api/decks/:address', rateLimit({windowMs:60000,limit:60,standardHeaders:'draft-8',legacyHeaders:false}), (req,res)=>{
if(typeof req.params.address!=='string'||!/^[a-f0-9]{64}$/.test(req.params.address)){
res.status(400).json({error:'Invalid wallet address'});return;
@@ -365,55 +161,22 @@ app.use((err:Error & {status?:number},_req:express.Request,res:express.Response,
});
const port=Number(process.env.PORT??3001),host=process.env.HOST??'127.0.0.1';
const server=app.listen(port,host,()=>{
const a=server.address();console.log(`Wikideck RandomX ${IS_MAINNET?'mainnet candidate':IS_PILOT?'private pilot':'testnet'}: http://${host}:${typeof a==='object'&&a?a.port:port} · chain ${CHAIN_ID} · ${ledger.snapshot().blocks.length} blocks`);
const a=server.address();console.log(`Wikideck RandomX ${IS_MAINNET?'mainnet candidate':'testnet'}: http://${host}:${typeof a==='object'&&a?a.port:port} · chain ${CHAIN_ID} · ${ledger.snapshot().blocks.length} blocks`);
peers.start();
});
const operations=startOperations(opsPort,()=>({network:NETWORK,paymentMode:'mock',
const operations=startOperations(opsPort,()=>({network:NETWORK,
ledger:ledger.diagnostics(),native:engine.diagnostics(),
peers:{configured:peers.status().length,connected:peers.status().filter(p=>p.connected).length},
rpc:{api:apiBudget.status(),p2p:peerBudget.status()},market:market?.diagnostics() ?? { enabled:false },
canonicalObserver:custodyObserver?.status(),
custody: custody && 'diagnostics' in custody ? custody.diagnostics() : { backend: custody ? 'sqlite-sandbox' : 'off' }}),resolve(dataDir,PROFILE));
rpc:{api:apiBudget.status(),p2p:peerBudget.status()}}),resolve(dataDir,PROFILE));
server.maxConnections=256;
server.headersTimeout=10000;
server.requestTimeout=15000;
server.keepAliveTimeout=5000;
server.setTimeout(30000,socket=>socket.destroy());
// Advance simulated payout/expiry state even when no browser is open. No real funds are sent.
let stopping=false;
let catchupPaused=false;
let catchupImmediate:NodeJS.Immediate|undefined;
let marketTimer:NodeJS.Timeout|undefined;
let marketCatchupTimer:NodeJS.Timeout|undefined;
// Drain durable catch-up without page-load dependence. Each pass is a smaller synchronous
// SQLite slice, then setImmediate yields so HTTP and peer work can run between passes.
const advanceMarketCatchup=()=>{
catchupImmediate=undefined;
if(!market||stopping||catchupPaused||market.isReconciliationWorkerBusy()||!market.hasReconciliationBacklog())return;
if(market.isReconciliationWorkerFailed()){
catchupPaused=true;console.error('Marketplace reconciliation worker failed; operator review required.');return;
}
try {market.browse();}
catch {catchupPaused=true;console.error('Marketplace canonical catch-up paused; operator review required.');return;}
if(market.hasReconciliationBacklog()&&!market.isReconciliationWorkerBusy()){
catchupImmediate=setImmediate(advanceMarketCatchup);catchupImmediate.unref();
}
};
if (market) {
marketTimer=setInterval(()=>{try {market!.browse();}catch{console.error('Marketplace reconciliation failed; operator review required.');}},15000);
marketTimer.unref();
marketCatchupTimer=setInterval(()=>{
if(!stopping&&!catchupPaused&&!catchupImmediate&&market!.hasReconciliationBacklog()){
catchupImmediate=setImmediate(advanceMarketCatchup);catchupImmediate.unref();
}
},250);
marketCatchupTimer.unref();
}
for(const signal of ['SIGTERM','SIGINT']as const)process.once(signal,()=>{
if(stopping)return;stopping=true;peers.close();operations.close();if(marketTimer)clearInterval(marketTimer);if(marketCatchupTimer)clearInterval(marketCatchupTimer);
if(catchupImmediate)clearImmediate(catchupImmediate);
const observerDrain=custodyObserver?.close()??Promise.resolve();
if(stopping)return;stopping=true;peers.close();operations.close();
const timeout=setTimeout(()=>process.exit(1),25000);timeout.unref();
server.close(async()=>{await observerDrain;await moneroWithdrawalPool?.end();await moneroDepositPool?.end();await custody?.close();await market?.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
server.close(async()=>{ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
});
server.on('error',async()=>{stopping=true;if(marketTimer)clearInterval(marketTimer);if(marketCatchupTimer)clearInterval(marketCatchupTimer);if(catchupImmediate)clearImmediate(catchupImmediate);operations.close();peers.close();await custodyObserver?.close();await moneroWithdrawalPool?.end();await moneroDepositPool?.end();await custody?.close();await market?.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});
server.on('error',async()=>{stopping=true;operations.close();peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});