Improvements

2026-10-20 04:07:37 -04:00
parent 0cc5032857
commit 3cee89d023
20 changed files with 511 additions and 23 deletions
@@ -18,10 +18,14 @@ import { Market, marketConfig } from '../market/store.ts';
 import { marketCommandSchema } from '../../shared/market.ts';
 import { IS_MAINNET, PROFILE } from '../../shared/network.ts';
 import { assertLaunchAllowed, launchManifest } from '../launch.ts';
+import { resourceConfig, RpcBudget, RpcError } from '../resources.ts';

 const root = fileURLToPath(new URL('../../', import.meta.url));
 assertLaunchAllowed();
 const paymentConfig = marketConfig(); // Fail before opening node resources if live payments are requested.
+const limits = resourceConfig();
+const apiBudget = new RpcBudget(limits);
+const peerBudget = new RpcBudget({ ...limits, requestsPerSecond: 40, writesPerSecond: 10, active: 16 });
 const dataDir = process.env.WIKIDECK_DATA_DIR ?? resolve(root,'data');
 await mkdir(dataDir,{recursive:true});
 // Never permit two processes to rewrite the same ledger. A stale lock requires operator review.
@@ -48,7 +52,9 @@ app.use(helmet({contentSecurityPolicy:{directives:{defaultSrc:["'self'"],scriptS
  connectSrc:["'self'"],upgradeInsecureRequests:null}},crossOriginEmbedderPolicy:false,
  strictTransportSecurity:process.env.NODE_ENV==='production'}));
 app.get('/healthz',(_req,res)=>{res.setHeader('Cache-Control','no-store');res.json({status:'ok',network:NETWORK});});
-app.use(express.json({limit:'68kb',strict:true}));
+// Admission comes before body parsing and signature/schema work. Disable compressed request inflation.
+app.use('/api',apiBudget.middleware());
+app.use('/p2p',peerBudget.middleware());
 app.use(['/api','/p2p'],rateLimit({windowMs:60000,limit:300,standardHeaders:'draft-8',legacyHeaders:false}));
 app.use(['/api','/p2p'],(req,res,next)=>{
  res.setHeader('Cache-Control','no-store');
@@ -60,9 +66,16 @@ app.use(['/api','/p2p'],(req,res,next)=>{
  }
  next();
 });
-app.get('/api/state',(_req,res)=>res.json(ledger.snapshot()));
+app.use(['/api','/p2p'],express.json({limit:'68kb',strict:true,inflate:false}));
+app.get('/api/state',(_req,res)=>res.json(ledger.legacySnapshot(limits.legacyBlocks)));
+app.get('/api/summary',(_req,res)=>res.json(ledger.summary()));
+app.get('/api/canonical',(req,res)=>{
+  const q=z.object({tip:z.string().regex(/^[a-f0-9]{64}$/),offset:z.string().regex(/^\d{1,12}$/).optional(),
+    limit:z.string().regex(/^\d{1,2}$/).optional()}).strict().parse(req.query);
+  res.json(ledger.canonicalPage(q.tip,Number(q.offset??0),Number(q.limit??16)));
+});
 app.get('/api/network',(_req,res)=>res.json(launchManifest()));
-app.get('/api/chain',(_req,res)=>res.json(ledger.snapshot()));
+app.get('/api/chain',(_req,res)=>res.json(ledger.legacySnapshot(limits.legacyBlocks)));
 app.get('/api/peers',(_req,res)=>res.json({peers:peers.status()}));
 app.get('/api/market', (req,res)=>res.json(market.browse(Number(req.query.offset ?? 0))));
 app.post('/api/market/command',rateLimit({windowMs:60000,limit:60,standardHeaders:'draft-8',legacyHeaders:false}),async(req,res)=>{
@@ -72,6 +85,7 @@ app.post('/api/market/command',rateLimit({windowMs:60000,limit:60,standardHeader
    res.json(result);
    if(command.action.type==='deliver')void peers.broadcastTransfer(command.action.transfer).catch(()=>{});
  } catch(error) {
+    if(error instanceof RpcError)throw error;
    // Marketplace storage/payment internals must never leak through the generic node error handler.
    if(error instanceof Error && !/SQLITE|constraint|database/i.test(error.message))res.status(400).json({error:error.message.slice(0,200)});
    else res.status(409).json({error:'Marketplace state conflict. Refresh and retry.'});
@@ -95,14 +109,14 @@ app.get('/api/wikipedia/:pageId', rateLimit({windowMs:60000,limit:60,standardHea
  try {res.json(await wikipediaEnrichment.get(pageId));}
  catch {res.status(503).json({error:'Wikipedia enrichment is temporarily unavailable. NFT data is unaffected.'});}
 });
-app.get('/p2p/status',(_req,res)=>res.json({network:NETWORK,chainId:CHAIN_ID,size:ledger.size,tip:ledger.snapshot().tip}));
+app.get('/p2p/status',(_req,res)=>res.json(ledger.peerStatus()));
 app.get('/p2p/blocks',(req,res)=>{
  const offset=Number(req.query.offset??0);
  if(!Number.isSafeInteger(offset)||offset<0){res.status(400).json({error:'Invalid offset'});return;}
  const blocks=ledger.inventory(offset,32);
  res.json({blocks,next:offset+blocks.length<ledger.size?offset+blocks.length:null});
 });
-app.get('/p2p/mempool',(_req,res)=>res.json({transactions:ledger.snapshot().pendingTransfers}));
+app.get('/p2p/mempool',(_req,res)=>res.json({transactions:ledger.pendingTransactions()}));
 app.get('/api/nfts/:id', (req,res)=>{
  const s=ledger.snapshot();const block=s.blocks.find(b=>b.tokenId===req.params.id);
  if(!block){res.status(404).json({error:'NFT not found on the canonical chain'});return;}
@@ -154,13 +168,19 @@ app.use((err:Error & {status?:number},_req:express.Request,res:express.Response,
  const input=err instanceof z.ZodError||err instanceof SyntaxError;
  const message=input?'Invalid request encoding':err.message;
  // No keys/passwords are accepted by any node endpoint.
-  res.status(err.status===413?413:400).json({error:message.slice(0,200)});
+  if(err instanceof RpcError&&err.status===503)res.setHeader('Retry-After','1');
+  res.status(err instanceof RpcError?err.status:err.status===413?413:err.status===415?415:400).json({error:message.slice(0,200)});
 });
 const port=Number(process.env.PORT??3001),host=process.env.HOST??'127.0.0.1';
 const server=app.listen(port,host,()=>{
  const a=server.address();console.log(`Wikideck RandomX ${IS_MAINNET?'mainnet candidate':'testnet'}: http://${host}:${typeof a==='object'&&a?a.port:port} · chain ${CHAIN_ID} · ${ledger.snapshot().blocks.length} blocks`);
  peers.start();
 });
+server.maxConnections=256;
+server.headersTimeout=10000;
+server.requestTimeout=15000;
+server.keepAliveTimeout=5000;
+server.setTimeout(30000,socket=>socket.destroy());
 // Advance simulated payout/expiry state even when no browser is open. No real funds are sent.
 const marketTimer=setInterval(()=>{try {market.browse();}catch{console.error('Marketplace reconciliation failed; operator review required.');}},15000);
 marketTimer.unref();
@@ -4,7 +4,7 @@ import { DatabaseSync } from 'node:sqlite';
 import { z } from 'zod';
 import { NETWORK, CONSENSUS, type TestnetBlock,
  type NftState, type NftStateView, type TestnetTransaction, type MiningTemplate,
-  type TestnetDeck, type TestnetSnapshot } from '../../shared/testnet.ts';
+  type TestnetDeck, type TestnetSnapshot, type TestnetSummary, type CanonicalPage } from '../../shared/testnet.ts';
 import { CHAIN_ID, GENESIS_HASH, applyTransactionOverlay, blockSchema, makeTemplate,
  applyTransferOverlay, materializeState, transactionId, transactionSchema, transferSchema,
  validateBlock, workFor } from './consensus.ts';
@@ -12,6 +12,7 @@ import type { PowEngine } from './randomx.ts';
 import type { Article } from '../../shared/protocol.ts';
 import type { MintStatsSnapshot } from '../../shared/enrichment.ts';
 import { PROFILE } from '../../shared/network.ts';
+import { RpcError } from '../resources.ts';

 const MAX_PENDING = 256;
 const MAX_DECK_PAGE = 24;
@@ -39,6 +40,8 @@ export class Ledger {
  private readonly db: DatabaseSync;
  private readonly pow: PowEngine;
  private closed = false;
+  private revision = 0;
+  private canonicalCache: { tip: string; blocks: TestnetBlock[] } | undefined;
  private constructor(file: string, db: DatabaseSync, pow: PowEngine) { this.file = file; this.db = db; this.pow = pow; }

  static async open(dir: string, pow: PowEngine): Promise<Ledger> {
@@ -190,6 +193,7 @@ export class Ledger {
      if (wins) this.tip = node.block.hash;
      this.pending = pending;
      if (filtered) this.pendingState = filtered.state;
+      this.revision++;
      return { accepted: true, reorg: disconnected.length > 0 };
    });
  }
@@ -211,6 +215,7 @@ export class Ledger {
      await this.persist(undefined, pending);
      this.pending = pending;
      this.pendingState = nextPendingState;
+      this.revision++;
      return id;
    });
  }
@@ -228,6 +233,27 @@ export class Ledger {
      peers: this.peers, algorithm: 'RandomX',
      maturity: CONSENSUS.coinbaseMaturity });
  }
+  /** Small metadata/mempool response. No full-chain traversal or clone. */
+  summary(): TestnetSummary {
+    return { network: NETWORK, chainId: CHAIN_ID, genesis: GENESIS_HASH, tip: this.tip,
+      height: this.height(), revision: this.revision, cumulativeWork: this.tipWork().toString(),
+      pendingTransfers: structuredClone(this.pending), peers: this.peers, algorithm: 'RandomX', maturity: CONSENSUS.coinbaseMaturity };
+  }
+  peerStatus() { return { network: NETWORK, chainId: CHAIN_ID, size: this.size, tip: this.tip }; }
+  pendingTransactions(): TestnetTransaction[] { return structuredClone(this.pending); }
+  legacySnapshot(maxBlocks: number): TestnetSnapshot {
+    // Reject before any full-chain copy/serialization. Never truncate ownership history silently.
+    if (this.height() > maxBlocks) throw new RpcError(413, 'Full snapshot disabled at this height; use /api/summary and /api/canonical pages.');
+    return this.snapshot();
+  }
+  canonicalPage(tip: string, offset: number, limit = 16): CanonicalPage {
+    if (tip !== this.tip) throw new RpcError(409, 'Canonical tip changed; restart paged loading.');
+    if (!Number.isSafeInteger(offset) || offset < 0 || offset > this.height() ||
+        !Number.isSafeInteger(limit) || limit < 1 || limit > 16) throw new RpcError(400, 'Invalid canonical page');
+    const blocks = structuredClone(this.canonical().slice(offset, offset + limit));
+    return { network: NETWORK, chainId: CHAIN_ID, tip: this.tip, height: this.height(), offset, blocks,
+      next: offset + blocks.length < this.height() ? offset + blocks.length : null };
+  }
  deck(address: string, offset = 0, limit = MAX_DECK_PAGE): TestnetDeck {
    if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > MAX_DECK_PAGE) {
      throw new Error('Invalid deck page');
@@ -259,7 +285,10 @@ export class Ledger {
    }
    return result.reverse();
  }
-  private canonical(): TestnetBlock[] { return this.branch(this.tip); }
+  private canonical(): TestnetBlock[] {
+    if (this.canonicalCache?.tip !== this.tip) this.canonicalCache = { tip: this.tip, blocks: this.branch(this.tip) };
+    return this.canonicalCache.blocks; // Internal immutable block references only; public APIs clone their slice.
+  }
  private async validate(raw: TestnetBlock): Promise<Node> {
    const ancestors = this.branch(raw.previousHash);
    const parent = this.nodes.get(raw.previousHash);