Deployment monitoring and recovery tooling

server/testnet/http.ts

@@ -19,11 +19,14 @@ import { marketCommandSchema } from '../../shared/market.ts';
 import { IS_MAINNET, PROFILE } from '../../shared/network.ts';
 import { assertLaunchAllowed, launchManifest } from '../launch.ts';
 import { resourceConfig, RpcBudget, RpcError } from '../resources.ts';
+import { trustedProxies, operationsPort, startOperations } from '../operations.ts';
 
 const root = fileURLToPath(new URL('../../', import.meta.url));
 assertLaunchAllowed();
 const paymentConfig = marketConfig(); // Fail before opening node resources if live payments are requested.
 const limits = resourceConfig();
+const proxyTrust=trustedProxies();
+const opsPort=operationsPort();
 const apiBudget = new RpcBudget(limits);
 const peerBudget = new RpcBudget({ ...limits, requestsPerSecond: 40, writesPerSecond: 10, active: 16 });
 const dataDir = process.env.WIKIDECK_DATA_DIR ?? resolve(root,'data');
@@ -47,6 +50,7 @@ try { market = await Market.open(dataDir, ledger, paymentConfig); }
 catch(e) {peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
 const app = express();
 app.disable('x-powered-by');
+if(proxyTrust.length)app.set('trust proxy',proxyTrust);
 app.use(helmet({contentSecurityPolicy:{directives:{defaultSrc:["'self'"],scriptSrc:["'self'"],
   imgSrc:["'self'",'data:','https://upload.wikimedia.org','https://thumb.wikimedia.org'],styleSrc:["'self'","'unsafe-inline'"],
   connectSrc:["'self'"],upgradeInsecureRequests:null}},crossOriginEmbedderPolicy:false,
@@ -176,6 +180,10 @@ const server=app.listen(port,host,()=>{
   const a=server.address();console.log(`Wikideck RandomX ${IS_MAINNET?'mainnet candidate':'testnet'}: http://${host}:${typeof a==='object'&&a?a.port:port} · chain ${CHAIN_ID} · ${ledger.snapshot().blocks.length} blocks`);
   peers.start();
 });
+const operations=startOperations(opsPort,()=>({network:NETWORK,paymentMode:'mock',
+  ledger:ledger.diagnostics(),native:engine.diagnostics(),
+  peers:{configured:peers.status().length,connected:peers.status().filter(p=>p.connected).length},
+  rpc:{api:apiBudget.status(),p2p:peerBudget.status()},market:market.diagnostics()}),resolve(dataDir,PROFILE));
 server.maxConnections=256;
 server.headersTimeout=10000;
 server.requestTimeout=15000;
@@ -186,8 +194,8 @@ const marketTimer=setInterval(()=>{try {market.browse();}catch{console.error('Ma
 marketTimer.unref();
 let stopping=false;
 for(const signal of ['SIGTERM','SIGINT']as const)process.once(signal,()=>{
-  if(stopping)return;stopping=true;peers.close();clearInterval(marketTimer);
+  if(stopping)return;stopping=true;peers.close();operations.close();clearInterval(marketTimer);
   const timeout=setTimeout(()=>process.exit(1),25000);timeout.unref();
   server.close(async()=>{market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
 });
-server.on('error',async()=>{clearInterval(marketTimer);peers.close();market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});
+server.on('error',async()=>{clearInterval(marketTimer);operations.close();peers.close();market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});

server/testnet/ledger.ts

@@ -41,6 +41,7 @@ export class Ledger {
   private readonly pow: PowEngine;
   private closed = false;
   private revision = 0;
+  private reorgs = 0;
   private canonicalCache: { tip: string; blocks: TestnetBlock[] } | undefined;
   private constructor(file: string, db: DatabaseSync, pow: PowEngine) { this.file = file; this.db = db; this.pow = pow; }
 
@@ -148,6 +149,9 @@ export class Ledger {
   }
 
   get size(): number { return this.nodes.size; }
+  diagnostics() { const createdAt=this.nodes.get(this.tip)?.block.createdAt??null;
+    return { height:this.height(), dagBlocks:this.size, pending:this.pending.length, mutationQueue:this.queued,
+      reorgsSinceStartup:this.reorgs,tipCreatedAt:createdAt,tipAgeSeconds:createdAt===null?null:Math.max(0,Date.now()/1000-createdAt) }; }
   get allBlocks(): TestnetBlock[] {
     const rows = this.db.prepare('SELECT block_json FROM blocks ORDER BY sequence').all() as { block_json: string }[];
     return rows.map(row => blockSchema.parse(JSON.parse(row.block_json)));
@@ -191,6 +195,7 @@ export class Ledger {
       await this.persist(node.block, pending);
       this.nodes.set(node.block.hash, node);
       if (wins) this.tip = node.block.hash;
+      if (disconnected.length) this.reorgs++;
       this.pending = pending;
       if (filtered) this.pendingState = filtered.state;
       this.revision++;

server/testnet/randomx.ts

@@ -87,6 +87,7 @@ export class RandomX implements PowEngine {
       return line;
     });
   }
+  diagnostics() { return { available: !this.failure && !this.exited, active: !!this.active, queued: this.queue.length }; }
 
   async mine(
     key: string, blob: Uint8Array, target: string, nonceOffset: number, maxAttempts: number,