Deployment monitoring and recovery tooling

server/testnet/http.ts

@@ -19,11 +19,14 @@ import { marketCommandSchema } from '../../shared/market.ts';
 import { IS_MAINNET, PROFILE } from '../../shared/network.ts';
 import { assertLaunchAllowed, launchManifest } from '../launch.ts';
 import { resourceConfig, RpcBudget, RpcError } from '../resources.ts';
+import { trustedProxies, operationsPort, startOperations } from '../operations.ts';
 
 const root = fileURLToPath(new URL('../../', import.meta.url));
 assertLaunchAllowed();
 const paymentConfig = marketConfig(); // Fail before opening node resources if live payments are requested.
 const limits = resourceConfig();
+const proxyTrust=trustedProxies();
+const opsPort=operationsPort();
 const apiBudget = new RpcBudget(limits);
 const peerBudget = new RpcBudget({ ...limits, requestsPerSecond: 40, writesPerSecond: 10, active: 16 });
 const dataDir = process.env.WIKIDECK_DATA_DIR ?? resolve(root,'data');
@@ -47,6 +50,7 @@ try { market = await Market.open(dataDir, ledger, paymentConfig); }
 catch(e) {peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
 const app = express();
 app.disable('x-powered-by');
+if(proxyTrust.length)app.set('trust proxy',proxyTrust);
 app.use(helmet({contentSecurityPolicy:{directives:{defaultSrc:["'self'"],scriptSrc:["'self'"],
   imgSrc:["'self'",'data:','https://upload.wikimedia.org','https://thumb.wikimedia.org'],styleSrc:["'self'","'unsafe-inline'"],
   connectSrc:["'self'"],upgradeInsecureRequests:null}},crossOriginEmbedderPolicy:false,
@@ -176,6 +180,10 @@ const server=app.listen(port,host,()=>{
   const a=server.address();console.log(`Wikideck RandomX ${IS_MAINNET?'mainnet candidate':'testnet'}: http://${host}:${typeof a==='object'&&a?a.port:port} · chain ${CHAIN_ID} · ${ledger.snapshot().blocks.length} blocks`);
   peers.start();
 });
+const operations=startOperations(opsPort,()=>({network:NETWORK,paymentMode:'mock',
+  ledger:ledger.diagnostics(),native:engine.diagnostics(),
+  peers:{configured:peers.status().length,connected:peers.status().filter(p=>p.connected).length},
+  rpc:{api:apiBudget.status(),p2p:peerBudget.status()},market:market.diagnostics()}),resolve(dataDir,PROFILE));
 server.maxConnections=256;
 server.headersTimeout=10000;
 server.requestTimeout=15000;
@@ -186,8 +194,8 @@ const marketTimer=setInterval(()=>{try {market.browse();}catch{console.error('Ma
 marketTimer.unref();
 let stopping=false;
 for(const signal of ['SIGTERM','SIGINT']as const)process.once(signal,()=>{
-  if(stopping)return;stopping=true;peers.close();clearInterval(marketTimer);
+  if(stopping)return;stopping=true;peers.close();operations.close();clearInterval(marketTimer);
   const timeout=setTimeout(()=>process.exit(1),25000);timeout.unref();
   server.close(async()=>{market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
 });
-server.on('error',async()=>{clearInterval(marketTimer);peers.close();market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});
+server.on('error',async()=>{clearInterval(marketTimer);operations.close();peers.close();market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});