Deployment monitoring and recovery tooling

server/analysis/calibration.ts

@@ -63,4 +63,4 @@ export function calibrationReport() {
     knownDeficitCatchup:[.1,.25,.4,.49].map(share=>({attackerShare:share,probability:idealCatchup(share,10),currentDeficit:10})),
     candidateTargets:[1,10,100,1000,10000].map(hps=>({assumedNetworkHashesPerSecond:hps,target:targetForHashrate(hps)})),
     decision:'NO-GO: measurement, independent analysis, multi-host rehearsal and explicit operator approval still required.' };
-}
+}

server/market/store.ts

@@ -73,6 +73,10 @@ export class Market {
     } catch (error) { db.close(); throw error; }
   }
   close() { this.db.close(); }
+  diagnostics() {
+    const states=this.db.prepare("SELECT json_extract(data,'$.status') AS status,COUNT(*) AS count FROM orders GROUP BY json_extract(data,'$.status')").all();
+    return { queued:this.queued,lastReconciledAt:this.reconciled?.at??null,orderStates:states };
+  }
   info(): MarketInfo {
     return { network: NETWORK, chainId: CHAIN_ID, marketId: this.id, paymentMode: 'mock', feeBps: this.config.feeBps,
       additionalFee: { ...ADDITIONAL_MARKET_FEE },

server/operations-check.ts

@@ -0,0 +1,26 @@
+import { z } from 'zod';
+const nonnegative=z.number().finite().nonnegative();
+const schema=z.object({schema:z.literal('wikideck-operations-v1'),observedAt:z.string().datetime(),
+  native:z.object({available:z.boolean(),queued:nonnegative}),
+  ledger:z.object({height:nonnegative,mutationQueue:nonnegative,tipAgeSeconds:nonnegative.nullable()}),
+  peers:z.object({configured:nonnegative,connected:nonnegative}),
+  eventLoop:z.object({meanMs:nonnegative}),disk:z.unknown(),diskSampledAt:nonnegative.nullable(),
+  market:z.object({queued:nonnegative,lastReconciledAt:nonnegative.nullable(),
+    orderStates:z.array(z.object({status:z.string(),count:nonnegative})).max(20)})});
+export function checkOperations(raw:unknown,now=Date.now()){
+  const s=schema.parse(raw),warnings:string[]=[];
+  const observation=Date.parse(s.observedAt);
+  if(Math.abs(now-observation)>60000)warnings.push('diagnostics-clock-or-freshness');
+  if(!s.native.available)warnings.push('native-engine-unavailable');
+  if(s.native.queued>=48||s.ledger.mutationQueue>=192||s.market.queued>=48)warnings.push('queue-pressure');
+  if(!s.peers.configured||!s.peers.connected)warnings.push('peer-isolation');
+  if(s.ledger.height===0||s.ledger.tipAgeSeconds===null||s.ledger.tipAgeSeconds>3600)warnings.push('empty-or-stale-chain');
+  if(s.eventLoop.meanMs>250)warnings.push('event-loop-delay');
+  if(!s.market.lastReconciledAt||now-s.market.lastReconciledAt>60000)warnings.push('market-reconciliation-stale');
+  if(s.market.orderStates.some(r=>(r.status==='review'||r.status==='refund-required')&&r.count>0))warnings.push('market-operator-review');
+  const disk=z.object({availableBytes:nonnegative}).safeParse(s.disk);
+  if(!disk.success||!s.diskSampledAt||now-s.diskSampledAt>90000)warnings.push('disk-sample-unavailable-or-stale');
+  else if(disk.data.availableBytes<512*1024*1024)warnings.push('disk-low');
+  return {healthy:warnings.length===0,warnings,
+    note:'Local threshold hints only, not consensus readiness or launch approval. Reorg/rejection counters and native RSS need external time-series monitoring.'};
+}

server/operations.ts

@@ -0,0 +1,63 @@
+import { createServer } from 'node:http';
+import { isIP } from 'node:net';
+import { monitorEventLoopDelay } from 'node:perf_hooks';
+import { stat,statfs } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { AddressInfo } from 'node:net';
+
+/** No hop-count/boolean trust. Match only explicit proxy addresses/subnets. The edge must
+ * overwrite forwarded headers and prevent direct backend access from trusted addresses. */
+export function trustedProxies(env: NodeJS.ProcessEnv = process.env): string[] {
+  const raw = env.WIKIDECK_TRUSTED_PROXIES;
+  if (!raw) return [];
+  const values=raw.split(',').map(s=>s.trim());
+  if(values.length>16)throw new Error('At most 16 trusted proxy addresses/subnets');
+  for(const value of values){
+    const [ip,prefix,...extra]=value.split('/'),family=isIP(ip);
+    if(!family||extra.length||(prefix!==undefined&&(!/^\d{1,3}$/.test(prefix)||Number(prefix)<(family===4?8:32)||Number(prefix)>(family===4?32:128))))
+      throw new Error('Trusted proxies must be literal IPs or bounded CIDRs, never booleans, names or hop counts');
+  }
+  return [...new Set(values)];
+}
+export function operationsPort(env: NodeJS.ProcessEnv = process.env): number | undefined {
+  if(env.WIKIDECK_OPS_PORT===undefined||env.WIKIDECK_OPS_PORT==='')return undefined;
+  const n=Number(env.WIKIDECK_OPS_PORT);
+  if(!/^\d+$/.test(env.WIKIDECK_OPS_PORT)||!Number.isInteger(n)||n<1||n>65535)throw new Error('Invalid WIKIDECK_OPS_PORT');
+  return n;
+}
+/** Separate loopback listener, disabled by default. No bearer tokens or details on public RPC.
+ * Local users/processes share this trust boundary; never forward/publish this port. */
+export function startOperations(port: number | undefined, snapshot: ()=>unknown, directory?:string) {
+  if(port===undefined)return {close() {}};
+  const delay=monitorEventLoopDelay({resolution:20});delay.enable();
+  let disk:unknown=null,lastDiskSample=0;
+  const sampleDisk=async()=>{
+    if(!directory)return;
+    try {
+      const fs=await statfs(directory);
+      const sizes=await Promise.all(['ledger.sqlite','ledger.sqlite-wal','market.sqlite','market.sqlite-wal'].map(async name=>{
+        try{return {name,bytes:(await stat(join(directory,name))).size};}
+        catch(error){if((error as NodeJS.ErrnoException).code==='ENOENT')return {name,bytes:0};throw error;}
+      }));
+      disk={availableBytes:fs.bavail*fs.bsize,files:sizes};lastDiskSample=Date.now();
+    }catch{disk={available:false,error:'Disk sample failed; inspect local filesystem and permissions.'};lastDiskSample=Date.now();}
+  };
+  void sampleDisk();const diskTimer=setInterval(()=>{void sampleDisk();},30000);diskTimer.unref();
+  const server=createServer((req,res)=>{
+    res.setHeader('Cache-Control','no-store');res.setHeader('Content-Type','application/json');
+    if(req.method!=='GET'||req.url!=='/status'){res.writeHead(404).end('{"error":"Not found"}');return;}
+    try {
+      const data=JSON.stringify({schema:'wikideck-operations-v1',observedAt:new Date().toISOString(),
+        uptimeSeconds:process.uptime(),nodeRssBytes:process.memoryUsage().rss,
+        eventLoop:{meanMs:Number.isFinite(delay.mean)?delay.mean/1e6:0,maxMs:delay.max/1e6},
+        disk,diskSampledAt:lastDiskSample||null,...snapshot() as object});
+      if(Buffer.byteLength(data)>64*1024)throw new Error('Diagnostics size');
+      res.end(data);
+    }catch{res.writeHead(503).end('{"error":"Diagnostics unavailable"}');}
+  });
+  server.maxConnections=4;server.headersTimeout=5000;server.requestTimeout=5000;server.keepAliveTimeout=1000;
+  server.setTimeout(5000,socket=>socket.destroy());
+  server.on('error',()=>{clearInterval(diskTimer);delay.disable();console.error('Private operations listener failed; check local monitoring configuration.');});
+  server.listen(port,'127.0.0.1');
+  return {close(){clearInterval(diskTimer);delay.disable();server.closeAllConnections();server.close();},address:()=>server.address() as AddressInfo|null};
+}

server/preflight.ts

@@ -29,7 +29,7 @@ export function launchPreflight(env: NodeJS.ProcessEnv = process.env, now = Date
     { id: 'wallet-key-protection', status: 'blocked', detail: 'Stage-2 encrypted vault storage, explicit migration/unlock and lock-aware signing exist. Legacy plaintext copies, unlocked-page compromise, migration/lifecycle coverage and independent production wallet/recovery review remain unresolved.' },
     { id: 'live-market-settlement', status: 'blocked', detail: 'No live SHKeeper adapter or reviewed payout/refund/reorg system exists. Mock payment cannot constitute real payment for valuable cards.' },
     { id: 'release-security-review', status: 'unverified', detail: 'Independent consensus/application review and release-specific dependency/native/container scans require external evidence; an operator environment flag is not evidence.' },
-    { id: 'deployment-rehearsal', status: 'unverified', detail: 'Verify real HTTPS endpoints, proxy/IP limits, independent peers and miners, alerts, incident response and backup/restore on the intended hosts.' },
+    { id: 'deployment-rehearsal', status: 'unverified', detail: 'Stage-4 explicit proxy trust, private diagnostics/checker and stopped-node paired backup/restore with local native replay tests exist. Actual HTTPS/proxy topology, independent hosts/miners, alert delivery, off-host recovery and on-call procedures remain unverified.' },
     { id: 'fee-recipient-control', status: 'unverified', detail: 'Confirm the fixed public XMR fee address, checksum and recipient control out of band before any future live payout. No keys or proof are requested here.' },
   );
   return { ...launchManifest(), assessedAt: new Date(now).toISOString(), productionReady: false,

server/recovery.ts

@@ -0,0 +1,104 @@
+import { createHash } from 'node:crypto';
+import { createReadStream } from 'node:fs';
+import { mkdir, open, unlink, lstat, readFile, writeFile, chmod, copyFile, realpath } from 'node:fs/promises';
+import { join, resolve, relative, isAbsolute } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { z } from 'zod';
+import { NETWORK, PROFILE } from '../shared/network.ts';
+import { CHAIN_ID } from './testnet/consensus.ts';
+
+const files=['ledger.sqlite','market.sqlite'] as const;
+const manifestSchema=z.object({format:z.literal('wikideck-stopped-backup'),version:z.literal(1),
+  network:z.literal(NETWORK),chainId:z.literal(CHAIN_ID),profile:z.literal(PROFILE),createdAt:z.string().datetime(),
+  files:z.array(z.object({name:z.enum(files),bytes:z.number().int().positive().max(64*1024**3),sha256:z.string().regex(/^[a-f0-9]{64}$/)}).strict()).length(2)}).strict();
+async function regular(path:string){const s=await lstat(path);if(!s.isFile()||s.isSymbolicLink())throw new Error('Expected regular non-symlink database file');return s;}
+async function digest(path:string){const hash=createHash('sha256');for await(const chunk of createReadStream(path))hash.update(chunk);return hash.digest('hex');}
+async function flush(path:string){const handle=await open(path,'r+');try{await handle.sync();}finally{await handle.close();}}
+async function flushDirectory(path:string){
+  if(process.platform==='win32')return; // Windows directory durability requires filesystem-specific operational validation.
+  const handle=await open(path,'r');try{await handle.sync();}finally{await handle.close();}
+}
+function inspect(file:string,name:typeof files[number]) {
+  const db=new DatabaseSync(file,{readOnly:true});
+  try {
+    const result=db.prepare('PRAGMA quick_check').all();
+    if(result.length!==1||Object.values(result[0])[0]!=='ok')throw new Error('SQLite integrity check failed');
+    if(name==='ledger.sqlite'){
+      const meta=(key:string)=>(db.prepare('SELECT value FROM metadata WHERE key=?').get(key) as {value:string}|undefined)?.value;
+      if(meta('schema_version')!=='1'||meta('network')!==NETWORK||meta('chain_id')!==CHAIN_ID)throw new Error('Ledger network/schema mismatch');
+    }else{
+      const row=db.prepare("SELECT value FROM market_meta WHERE key='identity'").get() as {value:string}|undefined;
+      const identity=row?JSON.parse(row.value):undefined;
+      if(identity?.version!==1||identity.network!==NETWORK||identity.chainId!==CHAIN_ID||!z.string().uuid().safeParse(identity.id).success)throw new Error('Market identity mismatch');
+    }
+  }finally{db.close();}
+}
+/** Share the exact node startup lock. A stale lock is never removed automatically. */
+async function offlineLock<T>(dataDir:string,fn:()=>Promise<T>):Promise<T>{
+  const lockPath=join(dataDir,`${PROFILE}-node.lock`);
+  const handle=await open(lockPath,'wx',0o600).catch(()=>{throw new Error('Node directory is locked; stop the node and review stale locks manually.');});
+  try{await handle.writeFile(`maintenance:${process.pid}`);return await fn();}
+  finally{await handle.close();await unlink(lockPath);}
+}
+export async function createBackup(dataDir:string,destination:string){
+  const source=await realpath(dataDir),dest=resolve(destination);
+  const rel=relative(source,dest);
+  if(!rel||(!rel.startsWith('..')&&!isAbsolute(rel)))throw new Error('Backup destination must be outside the node data directory');
+  return offlineLock(source,async()=>{
+    if(await realpath(join(source,PROFILE))!==join(source,PROFILE))throw new Error('Network data directory must not be symlinked');
+    await mkdir(dest,{mode:0o700}); // Exclusive: never overwrite or recursively create a surprising destination.
+    const entries=[];
+    for(const name of files){
+      const src=join(source,PROFILE,name),target=join(dest,name);
+      await regular(src);inspect(src,name);
+      const db=new DatabaseSync(src,{readOnly:true});
+      try{db.prepare('VACUUM INTO ?').run(target);}finally{db.close();}
+      if(process.platform!=='win32')await chmod(target,0o600);
+      await flush(target);
+      inspect(target,name);const info=await regular(target);
+      entries.push({name,bytes:info.size,sha256:await digest(target)});
+    }
+    const manifest=manifestSchema.parse({format:'wikideck-stopped-backup',version:1,network:NETWORK,chainId:CHAIN_ID,profile:PROFILE,
+      createdAt:new Date().toISOString(),files:entries});
+    // Completion marker written last. Partial output is preserved for operator inspection, never treated as usable.
+    await writeFile(join(dest,'manifest.json'),JSON.stringify(manifest,null,2),{flag:'wx',mode:0o600});
+    await flush(join(dest,'manifest.json'));
+    await flushDirectory(dest);
+    return manifest;
+  });
+}
+export async function verifyBackup(directory:string){
+  const root=await realpath(directory),manifestFile=join(root,'manifest.json');
+  if((await regular(manifestFile)).size>16384)throw new Error('Backup manifest too large');
+  const manifest=manifestSchema.parse(JSON.parse(await readFile(manifestFile,'utf8')));
+  if(new Set(manifest.files.map(f=>f.name)).size!==2)throw new Error('Duplicate backup database');
+  for(const file of manifest.files){
+    const path=join(root,file.name),info=await regular(path);
+    if(info.size!==file.bytes||await digest(path)!==file.sha256)throw new Error('Backup database hash/size mismatch');
+    // A completed package has standalone DBs. Sidecars would change what SQLite reads versus the hashed file.
+    for(const suffix of ['-wal','-shm','-journal']){
+      try{await lstat(path+suffix);}catch(e){if((e as NodeJS.ErrnoException).code==='ENOENT')continue;throw e;}
+      throw new Error('Backup database has unexpected SQLite sidecar');
+    }
+    inspect(path,file.name);
+  }
+  return manifest;
+}
+export async function restoreBackup(directory:string,destination:string){
+  const manifest=await verifyBackup(directory),dest=resolve(destination);
+  await mkdir(dest,{mode:0o700}); // Must not exist, even if empty. Never reset a node in place.
+  return offlineLock(dest,async()=>{
+    const folder=join(dest,PROFILE);await mkdir(folder,{mode:0o700});
+    for(const file of manifest.files){
+      const target=join(folder,file.name);
+      await copyFile(join(directory,file.name),target,1); // COPYFILE_EXCL
+      if(process.platform!=='win32')await chmod(target,0o600);
+      await flush(target);
+      if(await digest(target)!==file.sha256)throw new Error('Restored copy hash mismatch');
+      inspect(target,file.name);
+    }
+    await flushDirectory(folder);await flushDirectory(dest);
+    return {network:NETWORK,chainId:CHAIN_ID,restored:true,proofsRevalidated:false,
+      warning:'SQLite/hash checks only. Replay with the reference engine in an isolated reviewed rehearsal before production use.'};
+  });
+}

server/testnet/http.ts

@@ -19,11 +19,14 @@ import { marketCommandSchema } from '../../shared/market.ts';
 import { IS_MAINNET, PROFILE } from '../../shared/network.ts';
 import { assertLaunchAllowed, launchManifest } from '../launch.ts';
 import { resourceConfig, RpcBudget, RpcError } from '../resources.ts';
+import { trustedProxies, operationsPort, startOperations } from '../operations.ts';
 
 const root = fileURLToPath(new URL('../../', import.meta.url));
 assertLaunchAllowed();
 const paymentConfig = marketConfig(); // Fail before opening node resources if live payments are requested.
 const limits = resourceConfig();
+const proxyTrust=trustedProxies();
+const opsPort=operationsPort();
 const apiBudget = new RpcBudget(limits);
 const peerBudget = new RpcBudget({ ...limits, requestsPerSecond: 40, writesPerSecond: 10, active: 16 });
 const dataDir = process.env.WIKIDECK_DATA_DIR ?? resolve(root,'data');
@@ -47,6 +50,7 @@ try { market = await Market.open(dataDir, ledger, paymentConfig); }
 catch(e) {peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
 const app = express();
 app.disable('x-powered-by');
+if(proxyTrust.length)app.set('trust proxy',proxyTrust);
 app.use(helmet({contentSecurityPolicy:{directives:{defaultSrc:["'self'"],scriptSrc:["'self'"],
   imgSrc:["'self'",'data:','https://upload.wikimedia.org','https://thumb.wikimedia.org'],styleSrc:["'self'","'unsafe-inline'"],
   connectSrc:["'self'"],upgradeInsecureRequests:null}},crossOriginEmbedderPolicy:false,
@@ -176,6 +180,10 @@ const server=app.listen(port,host,()=>{
   const a=server.address();console.log(`Wikideck RandomX ${IS_MAINNET?'mainnet candidate':'testnet'}: http://${host}:${typeof a==='object'&&a?a.port:port} · chain ${CHAIN_ID} · ${ledger.snapshot().blocks.length} blocks`);
   peers.start();
 });
+const operations=startOperations(opsPort,()=>({network:NETWORK,paymentMode:'mock',
+  ledger:ledger.diagnostics(),native:engine.diagnostics(),
+  peers:{configured:peers.status().length,connected:peers.status().filter(p=>p.connected).length},
+  rpc:{api:apiBudget.status(),p2p:peerBudget.status()},market:market.diagnostics()}),resolve(dataDir,PROFILE));
 server.maxConnections=256;
 server.headersTimeout=10000;
 server.requestTimeout=15000;
@@ -186,8 +194,8 @@ const marketTimer=setInterval(()=>{try {market.browse();}catch{console.error('Ma
 marketTimer.unref();
 let stopping=false;
 for(const signal of ['SIGTERM','SIGINT']as const)process.once(signal,()=>{
-  if(stopping)return;stopping=true;peers.close();clearInterval(marketTimer);
+  if(stopping)return;stopping=true;peers.close();operations.close();clearInterval(marketTimer);
   const timeout=setTimeout(()=>process.exit(1),25000);timeout.unref();
   server.close(async()=>{market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
 });
-server.on('error',async()=>{clearInterval(marketTimer);peers.close();market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});
+server.on('error',async()=>{clearInterval(marketTimer);operations.close();peers.close();market.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});

server/testnet/ledger.ts

@@ -41,6 +41,7 @@ export class Ledger {
   private readonly pow: PowEngine;
   private closed = false;
   private revision = 0;
+  private reorgs = 0;
   private canonicalCache: { tip: string; blocks: TestnetBlock[] } | undefined;
   private constructor(file: string, db: DatabaseSync, pow: PowEngine) { this.file = file; this.db = db; this.pow = pow; }
 
@@ -148,6 +149,9 @@ export class Ledger {
   }
 
   get size(): number { return this.nodes.size; }
+  diagnostics() { const createdAt=this.nodes.get(this.tip)?.block.createdAt??null;
+    return { height:this.height(), dagBlocks:this.size, pending:this.pending.length, mutationQueue:this.queued,
+      reorgsSinceStartup:this.reorgs,tipCreatedAt:createdAt,tipAgeSeconds:createdAt===null?null:Math.max(0,Date.now()/1000-createdAt) }; }
   get allBlocks(): TestnetBlock[] {
     const rows = this.db.prepare('SELECT block_json FROM blocks ORDER BY sequence').all() as { block_json: string }[];
     return rows.map(row => blockSchema.parse(JSON.parse(row.block_json)));
@@ -191,6 +195,7 @@ export class Ledger {
       await this.persist(node.block, pending);
       this.nodes.set(node.block.hash, node);
       if (wins) this.tip = node.block.hash;
+      if (disconnected.length) this.reorgs++;
       this.pending = pending;
       if (filtered) this.pendingState = filtered.state;
       this.revision++;

server/testnet/randomx.ts

@@ -87,6 +87,7 @@ export class RandomX implements PowEngine {
       return line;
     });
   }
+  diagnostics() { return { available: !this.failure && !this.exited, active: !!this.active, queued: this.queue.length }; }
 
   async mine(
     key: string, blob: Uint8Array, target: string, nonceOffset: number, maxAttempts: number,