Deployment monitoring and recovery tooling

tests/api.test.ts

@@ -17,6 +17,9 @@ import { NETWORK, CONSENSUS, blockClaimMessage, nftTransferMessage,
 import { marketMessage, type MarketAction, type MarketInfo, type MarketListing, type MarketOrder,
   type MarketActivity } from '../shared/market.ts';
 import { StateClient } from '../shared/rpc-client.ts';
+import { createBackup,restoreBackup } from '../server/recovery.ts';
+import { Ledger } from '../server/testnet/ledger.ts';
+import { Market } from '../server/market/store.ts';
 
 const root = fileURLToPath(new URL('../', import.meta.url));
 const wikipediaMock = resolve(root, 'tests', 'mock-wikipedia.mjs');
@@ -361,6 +364,20 @@ test('native RandomX HTTP: three blocks, mined transfer, peer gossip/sync and du
         assert.equal(first.child.exitCode, 0, first.output());
         assert.equal(existsSync(join(firstDir, 'testnet-node.lock')), false);
       }
+      const backupDir=join(directory,'stopped-pair-backup'),drillDir=join(directory,'isolated-restore-drill');
+      await createBackup(firstDir,backupDir);await restoreBackup(backupDir,drillDir);
+      const drillLedger=await Ledger.open(drillDir,engine!); // Revalidate real RandomX proofs from the copied backup.
+      try {
+        assert.deepEqual(drillLedger.snapshot(),{...canonical,peers:0});
+        const drillMarket=await Market.open(drillDir,drillLedger);
+        try {assert.equal(drillMarket.id,market.info.marketId);}finally{drillMarket.close();}
+        const db=new DatabaseSync(join(drillDir,'testnet','market.sqlite'),{readOnly:true});
+        try {
+          const row=db.prepare('SELECT data FROM orders WHERE id=?').get(order.id) as {data:string};
+          assert.equal(JSON.parse(row.data).buyer,bob.owner);
+          assert.ok((db.prepare('SELECT COUNT(*) AS n FROM receipts').get() as {n:number}).n>0);
+        }finally{db.close();}
+      }finally{drillLedger.close();}
       first = await start(firstDir); // Ledger.open revalidates every persisted block using native RandomX.
       const restored = await snapshot(first.base);
       assert.deepEqual(await pagedClient.load(),restored);

tests/calibration.test.ts

@@ -53,4 +53,4 @@ test('analysis CLI is offline and benchmark refuses CPU work without consent',()
   const bad=spawnSync(process.execPath,['--experimental-strip-types','scripts/benchmark-randomx.ts','--consent','--hashes','1000000'],{
     encoding:'utf8',timeout:30000,env:{...process.env,RANDOMX_BINARY:'/nonexistent'}});
   assert.notEqual(bad.status,0);assert.match(bad.stderr,/Invalid --hashes/);
-});
+});

tests/operations.test.ts

@@ -0,0 +1,49 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { trustedProxies, operationsPort, startOperations } from '../server/operations.ts';
+import express from 'express';
+import { once } from 'node:events';
+import { createServer } from 'node:net';
+import { checkOperations } from '../server/operations-check.ts';
+
+test('proxy trust is explicit literal IP/CIDR only; defaults disabled',()=>{
+  assert.deepEqual(trustedProxies({}),[]);
+  assert.deepEqual(trustedProxies({WIKIDECK_TRUSTED_PROXIES:'127.0.0.1,::1,172.20.1.0/24'}),['127.0.0.1','::1','172.20.1.0/24']);
+  for(const value of ['true','1','loopback','localhost','0.0.0.0/0','::/0','127.0.0.1/33','127.0.0.1,','http://proxy'])assert.throws(()=>trustedProxies({WIKIDECK_TRUSTED_PROXIES:value}));
+  assert.equal(operationsPort({}),undefined);
+  for(const value of ['0','-1','65536','x'])assert.throws(()=>operationsPort({WIKIDECK_OPS_PORT:value}));
+});
+test('untrusted forwarded headers do not alter client IP; explicit loopback proxy is recognized',async t=>{
+  const app=express();app.get('/ip',(req,res)=>{res.json({ip:req.ip});});
+  const server=app.listen(0,'127.0.0.1');await once(server,'listening');
+  t.after(()=>{server.closeAllConnections();server.close();});
+  const a=server.address();assert.ok(a&&typeof a==='object');const url=`http://127.0.0.1:${a.port}/ip`;
+  const request=()=>fetch(url,{headers:{'X-Forwarded-For':'203.0.113.12'}}).then(r=>r.json());
+  assert.equal((await request()).ip,'127.0.0.1');
+  app.set('trust proxy',trustedProxies({WIKIDECK_TRUSTED_PROXIES:'127.0.0.1'}));
+  assert.equal((await request()).ip,'203.0.113.12');
+});
+test('private operations listener exposes bounded aggregates only on separate loopback port',async t=>{
+  const reserve=createServer();reserve.listen(0,'127.0.0.1');await once(reserve,'listening');
+  const a=reserve.address();assert.ok(a&&typeof a==='object');await new Promise<void>(resolve=>reserve.close(()=>resolve()));
+  const ops=startOperations(a.port,()=>({network:'test',native:{available:true}}));t.after(()=>ops.close());
+  const r=await fetch(`http://127.0.0.1:${a.port}/status`,{signal:AbortSignal.timeout(5000)});
+  assert.equal(r.status,200);assert.equal(r.headers.get('cache-control'),'no-store');
+  const data=await r.json();assert.equal(data.network,'test');assert.equal(data.native.available,true);
+  assert.ok(data.nodeRssBytes>0);assert.equal('privateKey' in data,false);
+  assert.equal((await fetch(`http://127.0.0.1:${a.port}/metrics`)).status,404);
+  assert.equal(ops.address?.()?.address,'127.0.0.1');
+});
+test('operations checks identify isolation, stale reconciliation, disk pressure and market reviews',()=>{
+  const now=Date.now();
+  const status={schema:'wikideck-operations-v1',observedAt:new Date(now).toISOString(),
+    native:{available:true,queued:0},ledger:{height:10,mutationQueue:0,tipAgeSeconds:60},
+    peers:{configured:2,connected:2},eventLoop:{meanMs:20},disk:{availableBytes:1024**3},diskSampledAt:now,
+    market:{queued:0,lastReconciledAt:now,orderStates:[] as {status:string;count:number}[]}};
+  assert.equal(checkOperations(status,now).healthy,true);
+  const bad={...status,native:{available:false,queued:50},peers:{configured:2,connected:0},disk:null,
+    market:{queued:0,lastReconciledAt:now-120000,orderStates:[{status:'review',count:1}]}};
+  const report=checkOperations(bad,now);
+  for(const warning of ['native-engine-unavailable','queue-pressure','peer-isolation','market-reconciliation-stale','market-operator-review','disk-sample-unavailable-or-stale'])assert.ok(report.warnings.includes(warning));
+  assert.throws(()=>checkOperations({}));
+});

tests/recovery.test.ts

@@ -0,0 +1,44 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp,rm,writeFile,readFile,mkdir,access } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { DatabaseSync } from 'node:sqlite';
+import { Ledger } from '../server/testnet/ledger.ts';
+import { Market } from '../server/market/store.ts';
+import { createBackup,verifyBackup,restoreBackup } from '../server/recovery.ts';
+
+test('stopped-node pair backup restores journal identity and replays ledger in isolated destination',async t=>{
+  const root=await mkdtemp(join(tmpdir(),'wikideck-recovery-'));t.after(()=>rm(root,{recursive:true,force:true}));
+  const source=join(root,'source');
+  const pow={hash:async()=>{throw new Error('Empty ledger should require no proof work');},close(){}};
+  const ledger=await Ledger.open(source,pow),market=await Market.open(source,ledger);
+  const id=market.id,snapshot=ledger.snapshot();market.close();ledger.close();
+  const backup=join(root,'backup');const manifest=await createBackup(source,backup);
+  assert.equal(manifest.files.length,2);assert.deepEqual(await verifyBackup(backup),manifest);
+  const dest=join(root,'restore');assert.equal((await restoreBackup(backup,dest)).proofsRevalidated,false);
+  const restored=await Ledger.open(dest,pow),journal=await Market.open(dest,restored);
+  assert.deepEqual(restored.snapshot(),snapshot);assert.equal(journal.id,id);
+  journal.close();restored.close();
+  await assert.rejects(restoreBackup(backup,dest),/EEXIST/);
+  await assert.rejects(createBackup(source,backup),/EEXIST/);
+  await assert.rejects(createBackup(source,join(source,'inside')),/outside/);
+  await writeFile(join(source,'testnet-node.lock'),'simulated-running');
+  await assert.rejects(createBackup(source,join(root,'locked-backup')),/locked/);
+  assert.equal(await readFile(join(source,'testnet-node.lock'),'utf8'),'simulated-running');
+});
+
+test('backup verifier rejects tampering, sidecars, network mismatch and incomplete packages',async t=>{
+  const root=await mkdtemp(join(tmpdir(),'wikideck-recovery-bad-'));t.after(()=>rm(root,{recursive:true,force:true}));
+  const source=join(root,'source'),ledger=await Ledger.open(source,{hash:async()=>'',close(){}}),market=await Market.open(source,ledger);
+  market.close();ledger.close();const backup=join(root,'backup');await createBackup(source,backup);
+  const original=await readFile(join(backup,'manifest.json'),'utf8');
+  await writeFile(join(backup,'ledger.sqlite-wal'),'unexpected');await assert.rejects(verifyBackup(backup),/sidecar/);
+  await rm(join(backup,'ledger.sqlite-wal'));
+  await writeFile(join(backup,'manifest.json'),original.replace('wikideck-randomx-testnet-6','wrong-network'));await assert.rejects(verifyBackup(backup));
+  await writeFile(join(backup,'manifest.json'),original);
+  const db=new DatabaseSync(join(backup,'market.sqlite'));db.exec("UPDATE market_meta SET value='tampered' WHERE key='identity'");db.close();
+  await assert.rejects(verifyBackup(backup),/hash\/size/);
+  const missing=join(root,'incomplete');await mkdir(missing);await assert.rejects(verifyBackup(missing));
+  await assert.rejects(access(join(source,'testnet-node.lock'))); // maintenance lock released, not silently left behind
+});