wikideck/server/testnet/http.ts

import express from 'express';
import helmet from 'helmet';
import rateLimit from 'express-rate-limit';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { mkdir, open, unlink } from 'node:fs/promises';
import { z } from 'zod';
import { NETWORK, CONSENSUS } from '../../shared/testnet.ts';
import { transactionCardIds } from '../../shared/testnet.ts';
import { CHAIN_ID, blockSchema, tradeSchema, transferSchema, transactionSchema } from './consensus.ts';
import { RandomX } from './randomx.ts';
import { Ledger } from './ledger.ts';
import { PeerNetwork } from './peers.ts';
import { wikipediaEnrichment } from '../enrichment.ts';
import { snapshotStats } from '../../shared/enrichment.ts';
import { wikipediaArticles } from '../wiki.ts';
import { Market, marketConfig } from '../market/store.ts';
import { marketCommandSchema } from '../../shared/market.ts';
import { marketCommandV3Schema } from '../../shared/market-v3.ts';
import { custodyCommandSchema } from '../../shared/custody.ts';
import { CustodyMarket, CustodyError, custodyMode } from '../market/custody.ts';
import { openCustodyStorage } from '../market/custody-storage.ts';
import { PostgresCustody } from '../market/postgres-custody.ts';
import { CanonicalObserver } from '../market/canonical-observer.ts';
import { validateMarketPolicy } from '../market/policy.ts';
import { IS_MAINNET, PROFILE } from '../../shared/network.ts';
import { assertLaunchAllowed, launchManifest } from '../launch.ts';
import { resourceConfig, RpcBudget, RpcError } from '../resources.ts';
import { trustedProxies, operationsPort, startOperations } from '../operations.ts';

const root = fileURLToPath(new URL('../../', import.meta.url));
assertLaunchAllowed();
const paymentConfig = marketConfig(); // Fail before opening node resources if live payments are requested.
await validateMarketPolicy(paymentConfig); // Reject invalid recipient checksum/network before opening node resources.
const custodyConfig = custodyMode(); // Explicit opt-in sandbox only. Unknown/live modes fail before opening resources.
const canonicalObserverMode = process.env.WIKIDECK_CUSTODY_CANONICAL_REHEARSAL;
if (canonicalObserverMode !== undefined && canonicalObserverMode !== '0' && canonicalObserverMode !== '1') {
  throw new Error('WIKIDECK_CUSTODY_CANONICAL_REHEARSAL must be exactly 0 or 1');
}
if (canonicalObserverMode === '1' && (process.env.NODE_ENV === 'production' || custodyConfig !== 'sandbox' ||
  process.env.WIKIDECK_CUSTODY_STORAGE !== 'postgres' || process.env.WIKIDECK_PG_DISPOSABLE_TEST !== '1')) {
  throw new Error('Canonical observer rehearsal requires sandbox custody, PostgreSQL, and disposable test isolation');
}
const limits = resourceConfig();
const proxyTrust=trustedProxies();
const opsPort=operationsPort();
const apiBudget = new RpcBudget(limits);
const peerBudget = new RpcBudget({ ...limits, requestsPerSecond: 40, writesPerSecond: 10, active: 16 });
const dataDir = process.env.WIKIDECK_DATA_DIR ?? resolve(root,'data');
await mkdir(dataDir,{recursive:true});
// Never permit two processes to rewrite the same ledger. A stale lock requires operator review.
const lockPath = resolve(dataDir,`${PROFILE}-node.lock`);
const lock = await open(lockPath,'wx',0o600).catch(()=>{throw new Error('Data directory locked. Stop its other node; remove stale testnet-node.lock only after verifying no node is running.');});
await lock.writeFile(String(process.pid));
let engine: RandomX;
try { engine = new RandomX(); }
catch(e) { await lock.close();await unlink(lockPath);throw e; }
let ledger: Ledger;
try { ledger = await Ledger.open(dataDir,engine); }
catch(e) {engine.close();await lock.close();await unlink(lockPath);throw e;}
let peers: PeerNetwork;
try { peers = new PeerNetwork({ ledger, urls:(process.env.WIKIDECK_PEERS ?? '').split(',').map(s=>s.trim()).filter(Boolean),
  allowPrivate:process.env.WIKIDECK_ALLOW_PRIVATE_PEERS === '1' }); }
catch(e) {ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
let market: Market;
try { market = await Market.open(dataDir, ledger, paymentConfig, Date.now, { worker: true }); }
catch(e) {peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
let custody: CustodyMarket | PostgresCustody | undefined;
let custodyObserver: CanonicalObserver | undefined;
try {
  if (custodyConfig === 'sandbox') custody = await openCustodyStorage(dataDir, { ...paymentConfig,
    card: (hash, owner, sequence) => {
      const nft = ledger.nft(hash);
      if (!nft || nft.owner !== owner || nft.confirmations < CONSENSUS.coinbaseMaturity || nft.sequence !== sequence ||
        ledger.pendingTransactions().some(tx => transactionCardIds(tx).includes(hash))) return undefined;
      return { title: nft.block.article.title };
    } });
  if (canonicalObserverMode === '1') {
    if (!(custody instanceof PostgresCustody)) throw new Error('Canonical observer rehearsal requires PostgreSQL custody');
    custodyObserver = new CanonicalObserver(custody, ledger);
    if ((await custodyObserver.start()).state !== 'running') throw new Error('Canonical observer rehearsal failed to start; inspect custody diagnostics');
  }
} catch(e) {await custodyObserver?.close();await custody?.close();await market.close();peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
const app = express();
app.disable('x-powered-by');
if(proxyTrust.length)app.set('trust proxy',proxyTrust);
app.use(helmet({contentSecurityPolicy:{directives:{defaultSrc:["'self'"],scriptSrc:["'self'"],
  imgSrc:["'self'",'data:','https://upload.wikimedia.org','https://thumb.wikimedia.org'],styleSrc:["'self'","'unsafe-inline'"],
  connectSrc:["'self'"],upgradeInsecureRequests:null}},crossOriginEmbedderPolicy:false,
  strictTransportSecurity:process.env.NODE_ENV==='production'}));
app.get('/healthz',(_req,res)=>{res.setHeader('Cache-Control','no-store');res.json({status:'ok',network:NETWORK});});
// Admission comes before body parsing and signature/schema work. Disable compressed request inflation.
app.use('/api',apiBudget.middleware());
app.use('/p2p',peerBudget.middleware());
app.use(['/api','/p2p'],rateLimit({windowMs:60000,limit:300,standardHeaders:'draft-8',legacyHeaders:false}));
app.use(['/api','/p2p'],(req,res,next)=>{
  res.setHeader('Cache-Control','no-store');
  if(req.method==='POST') {
    if(!req.is('application/json')){res.status(415).json({error:'JSON required'});return;}
    const origin=req.get('origin');
    const allowed=new Set(['http://127.0.0.1:5173','http://localhost:5173','http://127.0.0.1:3001','http://localhost:3001',process.env.PUBLIC_ORIGIN]);
    if(origin&&!allowed.has(origin)){res.status(403).json({error:'Origin not allowed'});return;}
  }
  next();
});
app.use(['/api','/p2p'],express.json({limit:'68kb',strict:true,inflate:false}));
app.get('/api/state',(_req,res)=>res.json(ledger.legacySnapshot(limits.legacyBlocks)));
app.get('/api/summary',(_req,res)=>res.json(ledger.summary()));
app.get('/api/canonical',(req,res)=>{
  const q=z.object({tip:z.string().regex(/^[a-f0-9]{64}$/),offset:z.string().regex(/^\d{1,12}$/).optional(),
    limit:z.string().regex(/^\d{1,2}$/).optional()}).strict().parse(req.query);
  res.json(ledger.canonicalPage(q.tip,Number(q.offset??0),Number(q.limit??16)));
});
app.get('/api/network',(_req,res)=>res.json(launchManifest()));
app.get('/api/chain',(_req,res)=>res.json(ledger.legacySnapshot(limits.legacyBlocks)));
app.get('/api/peers',(_req,res)=>res.json({peers:peers.status()}));
app.get('/api/market', (req,res)=>res.json(market.browse(Number(req.query.offset ?? 0))));
app.get('/api/custody', async (req,res)=> {
  if (!custody) { res.json({ enabled: false }); return; }
  try { res.json(await custody.browse(Number(req.query.offset ?? 0))); }
  catch (error) {
    if (error instanceof CustodyError) res.status(400).json({ error: error.message });
    else res.status(503).json({ error: 'Custody sandbox unavailable; operator review required.' });
  }
});
app.post('/api/custody/command', rateLimit({windowMs:60000,limit:30,standardHeaders:'draft-8',legacyHeaders:false}), async (req,res)=> {
  if (!custody) { res.status(503).json({ error: 'Custody sandbox is disabled on this host.' }); return; }
  const parsed = custodyCommandSchema.safeParse(req.body);
  if (!parsed.success) { res.status(400).json({ error: 'Invalid custody command schema.' }); return; }
  try { res.json(await custody.execute(parsed.data)); }
  catch (error) {
    if (error instanceof CustodyError) res.status(400).json({ error: error.message });
    else res.status(503).json({ error: 'Custody state conflict; refresh activity before retrying. Operator review may be required.' });
  }
});
app.post('/api/market/command',rateLimit({windowMs:60000,limit:60,standardHeaders:'draft-8',legacyHeaders:false}),async(req,res)=>{
  const current = marketCommandV3Schema.safeParse(req.body);
  const command = current.success ? current.data : marketCommandSchema.parse(req.body);
  try {
    const result = await market.execute(command);
    res.json(result);
    if(command.action.type==='deliver')void peers.broadcastTransfer(command.action.transfer).catch(()=>{});
  } catch(error) {
    if(error instanceof RpcError)throw error;
    // Marketplace storage/payment internals must never leak through the generic node error handler.
    if(error instanceof Error && !/SQLITE|constraint|database/i.test(error.message))res.status(400).json({error:error.message.slice(0,200)});
    else res.status(409).json({error:'Marketplace state conflict. Refresh and retry.'});
  }
});
app.get('/api/decks/:address', rateLimit({windowMs:60000,limit:60,standardHeaders:'draft-8',legacyHeaders:false}), (req,res)=>{
  if(typeof req.params.address!=='string'||!/^[a-f0-9]{64}$/.test(req.params.address)){
    res.status(400).json({error:'Invalid wallet address'});return;
  }
  const offset=Number(req.query.offset??0),limit=Number(req.query.limit??24);
  if(!Number.isSafeInteger(offset)||offset<0||!Number.isSafeInteger(limit)||limit<1||limit>24){
    res.status(400).json({error:'Invalid deck page'});return;
  }
  res.setHeader('Cache-Control','no-store');
  if(req.query.tip!==undefined && req.query.tip!==ledger.summary().tip){
    res.status(409).json({error:'Canonical tip changed; restart deck pagination.'});return;
  }
  res.json(ledger.deck(req.params.address,offset,limit));
});
// Optional live data is intentionally not used by genesis, block validation or NFT ownership.
app.get('/api/wikipedia/:pageId', rateLimit({windowMs:60000,limit:60,standardHeaders:'draft-8',legacyHeaders:false}), async(req,res)=>{
  if(typeof req.params.pageId!=='string'||!/^[1-9]\d{0,9}$/.test(req.params.pageId)){res.status(400).json({error:'Invalid Wikipedia page ID'});return;}
  const pageId=Number(req.params.pageId);
  try {res.json(await wikipediaEnrichment.get(pageId));}
  catch {res.status(503).json({error:'Wikipedia enrichment is temporarily unavailable. NFT data is unaffected.'});}
});
app.get('/p2p/status',(_req,res)=>res.json(ledger.peerStatus()));
app.get('/p2p/blocks',(req,res)=>{
  const offset=Number(req.query.offset??0);
  if(!Number.isSafeInteger(offset)||offset<0){res.status(400).json({error:'Invalid offset'});return;}
  const blocks=ledger.inventory(offset,32);
  res.json({blocks,next:offset+blocks.length<ledger.size?offset+blocks.length:null});
});
app.get('/p2p/mempool',(_req,res)=>res.json({transactions:ledger.pendingTransactions()}));
app.get('/api/nfts/:id', (req,res)=>{
  if(typeof req.params.id!=='string'||!/^[a-f0-9]{64}$/.test(req.params.id)){
    res.status(400).json({error:'Invalid NFT identifier'});return;
  }
  const nft=ledger.nft(req.params.id);
  if(!nft){res.status(404).json({error:'NFT not found on the canonical chain'});return;}
  const block=nft.block;
  res.json({network:NETWORK,chainId:CHAIN_ID,tokenId:block.tokenId,owner:nft.owner,sequence:nft.sequence,blockHash:block.hash,
    confirmations:nft.confirmations,article:block.article,statsSnapshot:block.statsSnapshot,
    stateRoot:ledger.block(ledger.summary().tip)?.stateRoot});
});
const templateLimit=rateLimit({windowMs:60000,limit:20,standardHeaders:'draft-8',legacyHeaders:false});
app.post('/api/mining/template', templateLimit, async (req,res)=>{
  const body=z.object({owner:z.string().regex(/^[a-f0-9]{64}$/),publicKey:z.string().max(256)}).strict().parse(req.body);
  try {
    const article=await wikipediaArticles.random();
    const metrics=await wikipediaEnrichment.getFresh(article.pageId);
    res.json(ledger.template(body.owner,body.publicKey,article,snapshotStats(metrics)));
  } catch {
    res.status(503).json({error:'Could not prepare a Wikipedia article and current statistics. Retry shortly.'});
  }
});
const workLimit=rateLimit({windowMs:60000,limit:30,standardHeaders:'draft-8',legacyHeaders:false});
app.post(['/api/blocks','/p2p/block'],workLimit,async(req,res)=>{
  const block=blockSchema.parse(req.body);
  const result=await ledger.submit(block);
  res.status(result.accepted?201:200).json(result);
  if(result.accepted)void peers.broadcastBlock(block).catch(()=>{});
});
app.post('/api/transfer',workLimit,async(req,res)=>{
  const tx=transferSchema.parse(req.body);
  const id=await ledger.submitTransfer(tx);
  res.status(202).json({id,status:'pending',message:'Ownership changes only when this transaction is mined.'});
  void peers.broadcastTransfer(tx).catch(()=>{});
});
app.post('/api/trades',workLimit,async(req,res)=>{
  const trade=tradeSchema.parse(req.body);
  const id=await ledger.submitTransaction(trade);
  res.status(202).json({id,status:'pending',message:'The complete multi-card trade is pending; ownership changes atomically when it is mined.'});
  void peers.broadcastTransaction(trade).catch(()=>{});
});
app.post('/p2p/transaction',workLimit,async(req,res)=>{
  const transaction=transactionSchema.parse(req.body);
  const id=await ledger.submitTransaction(transaction);
  res.status(202).json({id,status:'pending'});
});
app.use(['/api','/p2p'],(_req,res)=>{res.status(404).json({error:'Unknown testnet endpoint'});});
app.use(express.static(resolve(root,'dist')));
app.get('/{*path}',(_req,res)=>res.sendFile(resolve(root,'dist/index.html')));
app.use((err:Error & {status?:number},_req:express.Request,res:express.Response,_next:express.NextFunction)=>{
  const input=err instanceof z.ZodError||err instanceof SyntaxError;
  const message=input?'Invalid request encoding':err.message;
  // No keys/passwords are accepted by any node endpoint.
  if(err instanceof RpcError&&err.status===503)res.setHeader('Retry-After','1');
  res.status(err instanceof RpcError?err.status:err.status===413?413:err.status===415?415:400).json({error:message.slice(0,200)});
});
const port=Number(process.env.PORT??3001),host=process.env.HOST??'127.0.0.1';
const server=app.listen(port,host,()=>{
  const a=server.address();console.log(`Wikideck RandomX ${IS_MAINNET?'mainnet candidate':'testnet'}: http://${host}:${typeof a==='object'&&a?a.port:port} · chain ${CHAIN_ID} · ${ledger.snapshot().blocks.length} blocks`);
  peers.start();
});
const operations=startOperations(opsPort,()=>({network:NETWORK,paymentMode:'mock',
  ledger:ledger.diagnostics(),native:engine.diagnostics(),
  peers:{configured:peers.status().length,connected:peers.status().filter(p=>p.connected).length},
  rpc:{api:apiBudget.status(),p2p:peerBudget.status()},market:market.diagnostics(),
  canonicalObserver:custodyObserver?.status(),
  custody: custody && 'diagnostics' in custody ? custody.diagnostics() : { backend: custody ? 'sqlite-sandbox' : 'off' }}),resolve(dataDir,PROFILE));
server.maxConnections=256;
server.headersTimeout=10000;
server.requestTimeout=15000;
server.keepAliveTimeout=5000;
server.setTimeout(30000,socket=>socket.destroy());
// Advance simulated payout/expiry state even when no browser is open. No real funds are sent.
const marketTimer=setInterval(()=>{try {market.browse();}catch{console.error('Marketplace reconciliation failed; operator review required.');}},15000);
marketTimer.unref();
let stopping=false;
let catchupPaused=false;
let catchupImmediate:NodeJS.Immediate|undefined;
// Drain durable catch-up without page-load dependence. Each pass is a smaller synchronous
// SQLite slice, then setImmediate yields so HTTP and peer work can run between passes.
const advanceMarketCatchup=()=>{
  catchupImmediate=undefined;
  if(stopping||catchupPaused||market.isReconciliationWorkerBusy()||!market.hasReconciliationBacklog())return;
  try {market.browse();}
  catch {catchupPaused=true;console.error('Marketplace canonical catch-up paused; operator review required.');return;}
  if(market.hasReconciliationBacklog()&&!market.isReconciliationWorkerBusy()){
    catchupImmediate=setImmediate(advanceMarketCatchup);catchupImmediate.unref();
  }
};
const marketCatchupTimer=setInterval(()=>{
  if(!stopping&&!catchupPaused&&!catchupImmediate&&market.hasReconciliationBacklog()){
    catchupImmediate=setImmediate(advanceMarketCatchup);catchupImmediate.unref();
  }
},250);
marketCatchupTimer.unref();
for(const signal of ['SIGTERM','SIGINT']as const)process.once(signal,()=>{
  if(stopping)return;stopping=true;peers.close();operations.close();clearInterval(marketTimer);clearInterval(marketCatchupTimer);
  if(catchupImmediate)clearImmediate(catchupImmediate);
  const observerDrain=custodyObserver?.close()??Promise.resolve();
  const timeout=setTimeout(()=>process.exit(1),25000);timeout.unref();
  server.close(async()=>{await observerDrain;await custody?.close();await market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
});
server.on('error',async()=>{stopping=true;clearInterval(marketTimer);clearInterval(marketCatchupTimer);if(catchupImmediate)clearImmediate(catchupImmediate);operations.close();peers.close();await custodyObserver?.close();await custody?.close();await market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});