Deployment monitoring and recovery tooling

tests/api.test.ts

@@ -17,6 +17,9 @@ import { NETWORK, CONSENSUS, blockClaimMessage, nftTransferMessage,
 import { marketMessage, type MarketAction, type MarketInfo, type MarketListing, type MarketOrder,
   type MarketActivity } from '../shared/market.ts';
 import { StateClient } from '../shared/rpc-client.ts';
+import { createBackup,restoreBackup } from '../server/recovery.ts';
+import { Ledger } from '../server/testnet/ledger.ts';
+import { Market } from '../server/market/store.ts';
 
 const root = fileURLToPath(new URL('../', import.meta.url));
 const wikipediaMock = resolve(root, 'tests', 'mock-wikipedia.mjs');
@@ -361,6 +364,20 @@ test('native RandomX HTTP: three blocks, mined transfer, peer gossip/sync and du
         assert.equal(first.child.exitCode, 0, first.output());
         assert.equal(existsSync(join(firstDir, 'testnet-node.lock')), false);
       }
+      const backupDir=join(directory,'stopped-pair-backup'),drillDir=join(directory,'isolated-restore-drill');
+      await createBackup(firstDir,backupDir);await restoreBackup(backupDir,drillDir);
+      const drillLedger=await Ledger.open(drillDir,engine!); // Revalidate real RandomX proofs from the copied backup.
+      try {
+        assert.deepEqual(drillLedger.snapshot(),{...canonical,peers:0});
+        const drillMarket=await Market.open(drillDir,drillLedger);
+        try {assert.equal(drillMarket.id,market.info.marketId);}finally{drillMarket.close();}
+        const db=new DatabaseSync(join(drillDir,'testnet','market.sqlite'),{readOnly:true});
+        try {
+          const row=db.prepare('SELECT data FROM orders WHERE id=?').get(order.id) as {data:string};
+          assert.equal(JSON.parse(row.data).buyer,bob.owner);
+          assert.ok((db.prepare('SELECT COUNT(*) AS n FROM receipts').get() as {n:number}).n>0);
+        }finally{db.close();}
+      }finally{drillLedger.close();}
       first = await start(firstDir); // Ledger.open revalidates every persisted block using native RandomX.
       const restored = await snapshot(first.base);
       assert.deepEqual(await pagedClient.load(),restored);