383 lines
22 KiB
TypeScript
383 lines
22 KiB
TypeScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { spawn, type ChildProcess } from 'node:child_process';
|
|
import { access, mkdtemp, readFile, rm, unlink } from 'node:fs/promises';
|
|
import { constants, existsSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { delimiter, join, resolve } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { generateKeyPairSync, sign } from 'node:crypto';
|
|
import { DatabaseSync } from 'node:sqlite';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { RandomX } from '../server/testnet/randomx.ts';
|
|
import { address, CHAIN_ID, GENESIS_HASH, MAX_TARGET, miningBlob, mintId, sha256,
|
|
transferId, workFor } from '../server/testnet/consensus.ts';
|
|
import { NETWORK, CONSENSUS, blockClaimMessage, nftTransferMessage,
|
|
type MiningTemplate, type NftTransfer, type TestnetBlock, type TestnetSnapshot } from '../shared/testnet.ts';
|
|
import { StateClient } from '../shared/rpc-client.ts';
|
|
import { createBackup,restoreBackup } from '../server/recovery.ts';
|
|
import { Ledger } from '../server/testnet/ledger.ts';
|
|
|
|
const root = fileURLToPath(new URL('../', import.meta.url));
|
|
const wikipediaMock = resolve(root, 'tests', 'mock-wikipedia.mjs');
|
|
const binary = resolve(process.env.RANDOMX_BINARY ?? join(root, 'native', 'build',
|
|
`wikideck-randomx${process.platform === 'win32' ? '.exe' : ''}`));
|
|
|
|
function wallet() {
|
|
// Ephemeral, independent P-256 keys; no credentials or private keys go to the nodes.
|
|
const keys = generateKeyPairSync('ec', { namedCurve: 'prime256v1' });
|
|
const publicKey = keys.publicKey.export({ type: 'spki', format: 'der' }).toString('base64');
|
|
return { publicKey, owner: address(publicKey), signed: (message: string) =>
|
|
sign('sha256', Buffer.from(message), { key: keys.privateKey, dsaEncoding: 'ieee-p1363' }).toString('base64') };
|
|
}
|
|
|
|
function tamperSignature(signature: string): string {
|
|
const bytes = Buffer.from(signature, 'base64');
|
|
bytes[0] ^= 1;
|
|
return bytes.toString('base64');
|
|
}
|
|
|
|
async function bounded<T>(promise: Promise<T>, milliseconds: number, label: string): Promise<T> {
|
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
try {
|
|
return await Promise.race([promise, new Promise<never>((_resolve, reject) => {
|
|
timer = setTimeout(() => reject(new Error(`${label} timed out`)), milliseconds);
|
|
})]);
|
|
} finally { clearTimeout(timer); }
|
|
}
|
|
|
|
test('native RandomX HTTP: three blocks, mined transfer, peer gossip/sync and durable restart',
|
|
{ timeout: 180_000 }, async t => {
|
|
try { await access(binary, constants.X_OK); }
|
|
catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
|
t.skip(`Native RandomX binary absent: ${binary}. Build the reference engine as documented in docs/randomx-native.md, or set RANDOMX_BINARY. No fake PoW fallback.`);
|
|
return;
|
|
}
|
|
|
|
const directory = await mkdtemp(join(tmpdir(), 'wikideck-randomx-http-'));
|
|
const firstDir = join(directory, 'node-1');
|
|
const secondDir = join(directory, 'node-2');
|
|
const originalPath = process.env.PATH;
|
|
// The local MinGW build depends on these DLLs. Inherit the caller's PATH elsewhere.
|
|
const mingw = `${process.env.SystemDrive ?? 'C:'}/Strawberry/c/bin`;
|
|
if (process.platform === 'win32' && existsSync(mingw)) {
|
|
process.env.PATH = `${mingw}${delimiter}${originalPath ?? ''}`;
|
|
}
|
|
type RunningNode = {
|
|
child: ChildProcess; closed: Promise<void>; output: () => string; base: string;
|
|
};
|
|
const nodes: RunningNode[] = [];
|
|
let engine: RandomX | undefined;
|
|
const abort = () => {
|
|
engine?.close();
|
|
for (const { child } of nodes) {
|
|
if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL');
|
|
}
|
|
};
|
|
t.signal.addEventListener('abort', abort, { once: true });
|
|
|
|
async function start(dataDir: string, peer?: string): Promise<RunningNode> {
|
|
t.signal.throwIfAborted();
|
|
const child = spawn(process.execPath, ['--import', wikipediaMock, '--experimental-strip-types', 'server/index.ts'], {
|
|
cwd: root, windowsHide: true, shell: false,
|
|
env: { ...process.env, HOST: '127.0.0.1', PORT: '0', NODE_ENV: 'production',
|
|
WIKIDECK_DATA_DIR: dataDir, RANDOMX_BINARY: binary,
|
|
WIKIDECK_PEERS: peer ?? '', WIKIDECK_ALLOW_PRIVATE_PEERS: peer ? '1' : '0',
|
|
PUBLIC_ORIGIN: 'http://127.0.0.1:3001',
|
|
WIKIDECK_RPC_WRITES_PER_SECOND: '50' },
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
});
|
|
let output = '';
|
|
const node: RunningNode = { child, base: '', output: () => output,
|
|
closed: new Promise(resolve => child.once('close', () => resolve())) };
|
|
nodes.push(node); // Register before startup can fail, so finally owns every child.
|
|
const ready = new Promise<string>((resolve, reject) => {
|
|
const log = (chunk: Buffer) => {
|
|
output = (output + chunk.toString()).slice(-16_384);
|
|
const match = /Wikideck RandomX testnet: (http:\/\/127\.0\.0\.1:\d+)/.exec(output);
|
|
if (match) resolve(match[1]);
|
|
};
|
|
child.stdout!.on('data', log);
|
|
child.stderr!.on('data', chunk => { output = (output + chunk.toString()).slice(-16_384); });
|
|
child.once('error', reject);
|
|
child.once('exit', (code, signal) => reject(new Error(`Node exited (${signal ?? code}): ${output}`)));
|
|
});
|
|
try { node.base = await bounded(ready, 30_000, 'Node startup'); }
|
|
catch (error) { throw new Error(`${String(error)}\n${output}`); }
|
|
return node;
|
|
}
|
|
|
|
async function stop(node: RunningNode): Promise<void> {
|
|
if (node.child.exitCode === null && node.child.signalCode === null) node.child.kill('SIGTERM');
|
|
try { await bounded(node.closed, 5000, 'Node shutdown'); }
|
|
catch {
|
|
node.child.kill('SIGKILL');
|
|
await bounded(node.closed, 5000, 'Forced node shutdown');
|
|
}
|
|
assert.ok(node.child.exitCode !== null || node.child.signalCode !== null, node.output());
|
|
}
|
|
|
|
async function request(base: string, path: string, expected = 200, body?: unknown,
|
|
headers: Record<string, string> = {}): Promise<Response> {
|
|
const response = await fetch(`${base}${path}`, {
|
|
method: body === undefined ? 'GET' : 'POST', redirect: 'error',
|
|
signal: AbortSignal.any([t.signal, AbortSignal.timeout(30_000)]),
|
|
headers: { ...(body === undefined ? {} : { 'Content-Type': 'application/json' }), ...headers },
|
|
body: body === undefined ? undefined : JSON.stringify(body),
|
|
});
|
|
if (response.status !== expected) {
|
|
assert.fail(`${path}: expected ${expected}, got ${response.status}: ${await response.text()}`);
|
|
}
|
|
return response;
|
|
}
|
|
async function json<T>(base: string, path: string, expected = 200, body?: unknown): Promise<T> {
|
|
return (await request(base, path, expected, body)).json() as Promise<T>;
|
|
}
|
|
const snapshot = (base: string) => json<TestnetSnapshot>(base, '/api/chain');
|
|
async function waitFor<T>(label: string, read: () => Promise<T>, predicate: (value: T) => boolean): Promise<T> {
|
|
const until = Date.now() + 25_000;
|
|
let value: T;
|
|
do {
|
|
value = await read();
|
|
if (predicate(value)) return value;
|
|
await delay(500, undefined, { signal: t.signal });
|
|
} while (Date.now() < until);
|
|
assert.fail(`${label} did not converge: ${JSON.stringify(value)}`);
|
|
}
|
|
async function mine(base: string, miner: ReturnType<typeof wallet>): Promise<TestnetBlock> {
|
|
const template = await json<MiningTemplate>(base, '/api/mining/template', 200,
|
|
{ owner: miner.owner, publicKey: miner.publicKey });
|
|
const header = { ...template.header };
|
|
assert.equal(header.network, NETWORK);
|
|
assert.equal(header.chainId, CHAIN_ID);
|
|
assert.equal(header.target, MAX_TARGET); // Initial target is approximately one success per 16 hashes.
|
|
assert.equal(template.nonceOffset, 32);
|
|
assert.equal(template.blob, miningBlob(header).toString('hex'));
|
|
const until = Date.now() + 45_000;
|
|
for (let batch = 0; batch < 16 && Date.now() < until; batch++) {
|
|
t.signal.throwIfAborted();
|
|
const result = await bounded(engine!.mine(header.key, miningBlob(header), header.target,
|
|
template.nonceOffset, 64), Math.min(45_000, until - Date.now()), 'Native mining batch');
|
|
if (result) {
|
|
header.nonce = result.nonce;
|
|
const blob = miningBlob(header);
|
|
assert.equal(await engine!.hash(header.key, blob), result.hash);
|
|
assert.ok(BigInt(`0x${result.hash}`) <= BigInt(`0x${header.target}`));
|
|
const hash = sha256(blob);
|
|
return { ...header, hash, tokenId: mintId(header), powHash: result.hash,
|
|
signature: miner.signed(blockClaimMessage(hash)), transactions: template.transactions };
|
|
}
|
|
header.nonce += 64;
|
|
}
|
|
assert.fail('Native RandomX mining exhausted its bounded search (1024 nonces / 45s)');
|
|
}
|
|
async function submit(base: string, block: TestnetBlock) {
|
|
assert.deepEqual(await json(base, '/api/blocks', 201, block), { accepted: true, reorg: false });
|
|
assert.deepEqual(await json(base, '/api/blocks', 200, block), { accepted: false, reorg: false });
|
|
}
|
|
const persisted = (dir: string) => {
|
|
const db = new DatabaseSync(join(dir, 'testnet', 'ledger.sqlite'));
|
|
try {
|
|
return JSON.stringify({
|
|
blocks: db.prepare('SELECT hash, previous_hash, block_json FROM blocks ORDER BY sequence').all(),
|
|
pending: db.prepare('SELECT position, id, transaction_json FROM pending_transfers ORDER BY position').all(),
|
|
});
|
|
} finally { db.close(); }
|
|
};
|
|
|
|
try {
|
|
engine = new RandomX({ binary }); // Always the actual reference RandomX engine, never a test double.
|
|
const alice = wallet(), bob = wallet();
|
|
assert.notEqual(alice.owner, bob.owner);
|
|
let first = await start(firstDir);
|
|
await assert.rejects(start(firstDir), /Data directory locked/,
|
|
'a second server process must fail closed rather than run a second writer on the same data directory');
|
|
const health = await request(first.base, '/healthz');
|
|
assert.deepEqual(await health.json(), { status: 'ok', network: NETWORK });
|
|
assert.equal(health.headers.get('cache-control'), 'no-store');
|
|
assert.equal(health.headers.has('ratelimit'), false);
|
|
await (await request(first.base, '/api/wikipedia/0', 400)).text();
|
|
await (await request(first.base, '/api/wikipedia/not-a-page', 400)).text();
|
|
await (await request(first.base, '/api/wikipedia/9999999999', 503)).text();
|
|
const initial = await snapshot(first.base);
|
|
const pagedClient=new StateClient(((path: string,init?: RequestInit)=>fetch(new URL(path,first.base),init)) as typeof fetch);
|
|
assert.deepEqual(await pagedClient.load(),initial);
|
|
await (await request(first.base,'/api/canonical?tip=bad',400)).text();
|
|
await (await request(first.base,`/api/canonical?tip=${GENESIS_HASH}&limit=17`,400)).text();
|
|
assert.equal(initial.network, NETWORK);
|
|
assert.equal(initial.chainId, CHAIN_ID);
|
|
assert.equal(initial.genesis, GENESIS_HASH);
|
|
assert.equal(initial.tip, GENESIS_HASH);
|
|
assert.equal(initial.cumulativeWork, '0');
|
|
assert.equal(initial.algorithm, 'RandomX');
|
|
assert.equal(initial.maturity, CONSENSUS.coinbaseMaturity);
|
|
assert.deepEqual(initial.blocks, []);
|
|
assert.deepEqual(await json(first.base, '/p2p/status'),
|
|
{ network: NETWORK, chainId: CHAIN_ID, size: 0, tip: GENESIS_HASH });
|
|
await (await request(first.base, '/api/mining/template', 400, {})).text();
|
|
await (await request(first.base, '/api/blocks', 400, {})).text();
|
|
await (await request(first.base, '/api/transfer', 400, {})).text();
|
|
await (await request(first.base, '/api/trades', 400, {})).text();
|
|
await (await request(first.base, `/api/decks/${'x'.repeat(64)}`, 400)).text();
|
|
await (await request(first.base, '/api/mining/template', 403, {}, { Origin: 'https://evil.example' })).text();
|
|
await (await request(first.base, '/api/mining/template', 415, {}, { 'Content-Type': 'text/plain' })).text();
|
|
const malformed = await fetch(`${first.base}/api/blocks`, { method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' }, body: '{',
|
|
signal: AbortSignal.any([t.signal, AbortSignal.timeout(5000)]) });
|
|
assert.equal(malformed.status, 400);
|
|
assert.deepEqual(await malformed.json(), { error: 'Invalid request encoding' });
|
|
for (const path of ['/api/missing', '/api/market', '/api/custody', '/api/monero/deposit-config',
|
|
'/p2p/missing', `/api/nfts/${'0'.repeat(64)}`]) {
|
|
await (await request(first.base, path, 404)).text();
|
|
}
|
|
for (const path of ['/api/market/command', '/api/custody/command', '/api/monero/deposit-address']) {
|
|
await (await request(first.base, path, 404, {})).text();
|
|
}
|
|
for (const offset of ['-1', '1.5', '9007199254740992', 'nope']) {
|
|
await (await request(first.base, `/p2p/blocks?offset=${offset}`, 400)).text();
|
|
}
|
|
|
|
const block1 = await mine(first.base, alice);
|
|
assert.equal(block1.height, 1);
|
|
assert.equal(block1.article.source, 'live');
|
|
assert.ok(block1.article.image);
|
|
assert.ok(['https://upload.wikimedia.org', 'https://thumb.wikimedia.org'].includes(new URL(block1.article.image).origin));
|
|
assert.equal(block1.statsSnapshot.pageviews.status, 'available');
|
|
assert.equal(block1.statsSnapshot.pageviews.total, 600);
|
|
assert.equal(block1.statsSnapshot.revisionId, block1.article.pageId + 1);
|
|
await submit(first.base, block1);
|
|
const unsigned = { network: NETWORK, chainId: CHAIN_ID, cardHash: block1.tokenId,
|
|
from: alice.owner, to: bob.owner, sequence: 0 };
|
|
const transfer: NftTransfer = { ...unsigned, publicKey: alice.publicKey,
|
|
signature: alice.signed(nftTransferMessage(unsigned)) };
|
|
const immature = await json<{ error: string }>(first.base, '/api/transfer', 400, transfer);
|
|
assert.match(immature.error, /immature/i);
|
|
const block2 = await mine(first.base, alice);
|
|
assert.equal(block2.height, 2);
|
|
await submit(first.base, block2);
|
|
const aliceDeck = await json<{ address: string; tip: string; total: number; nfts: { block: TestnetBlock }[] }>(
|
|
first.base, `/api/decks/${alice.owner}`);
|
|
assert.equal(aliceDeck.address, alice.owner);
|
|
assert.equal(aliceDeck.tip, block2.hash);
|
|
assert.equal(aliceDeck.total, 2);
|
|
assert.deepEqual(aliceDeck.nfts.map(item => item.block.tokenId), [block1.tokenId, block2.tokenId]);
|
|
assert.equal((await json<{ owner: string; sequence: number }>(first.base, `/api/nfts/${block1.tokenId}`)).owner, alice.owner);
|
|
await request(first.base, '/api/nfts/not-a-token', 400);
|
|
await request(first.base, `/api/decks/${alice.owner}?tip=${block1.hash}`, 409);
|
|
assert.equal((await json<{ total: number }>(first.base, `/api/decks/${alice.owner}?tip=${block2.hash}&limit=1`)).total, 2);
|
|
await (await request(first.base, `/api/decks/${'z'.repeat(64)}`, 400)).text();
|
|
|
|
// A truly independent process reconstructs the canonical chain over HTTP and checks real work.
|
|
const second = await start(secondDir, first.base);
|
|
const synced = await waitFor('Second node canonical sync', () => snapshot(second.base),
|
|
state => state.tip === block2.hash && state.peers === 1);
|
|
assert.deepEqual(synced.blocks, [block1, block2]);
|
|
assert.equal(synced.cumulativeWork, (2n * workFor(MAX_TARGET)).toString());
|
|
const peerStatus = await json<{ peers: { url: string; connected: boolean }[] }>(second.base, '/api/peers');
|
|
assert.deepEqual(peerStatus.peers, [{ url: first.base, connected: true }]);
|
|
|
|
const badTransfer = await json<{ error: string }>(second.base, '/api/transfer', 400,
|
|
{ ...transfer, signature: tamperSignature(transfer.signature) });
|
|
assert.match(badTransfer.error, /signature/i);
|
|
const pending = { id: transferId(transfer), status: 'pending',
|
|
message: 'Ownership changes only when this transaction is mined.' };
|
|
assert.deepEqual(await json(second.base, '/api/transfer', 202, transfer), pending);
|
|
assert.deepEqual(await json(second.base, '/api/transfer', 202, transfer), pending); // Pending duplicates are idempotent.
|
|
await waitFor('Transaction gossip to first node', () => snapshot(first.base),
|
|
state => state.pendingTransfers.length === 1);
|
|
for (const node of [first, second]) {
|
|
const state = await snapshot(node.base);
|
|
assert.deepEqual(state.pendingTransfers, [transfer]);
|
|
assert.deepEqual(state.transfers, []);
|
|
const metadata = await json<Record<string, unknown>>(node.base, `/api/nfts/${block1.tokenId}`);
|
|
assert.equal(metadata.owner, alice.owner); // Mempool never changes canonical ownership.
|
|
assert.equal(metadata.sequence, 0);
|
|
assert.equal(metadata.confirmations, 2);
|
|
}
|
|
|
|
const block3 = await mine(second.base, bob);
|
|
assert.equal(block3.height, 3);
|
|
assert.deepEqual(block3.transactions, [transfer]);
|
|
// Reject validly encoded signature and PoW tampering before accepting this same block ID.
|
|
const beforeInvalid = await persisted(secondDir);
|
|
const badSignature = await json<{ error: string }>(second.base, '/api/blocks', 400,
|
|
{ ...block3, signature: tamperSignature(block3.signature) });
|
|
assert.match(badSignature.error, /miner signature/i);
|
|
const badProof = await json<{ error: string }>(second.base, '/api/blocks', 400,
|
|
{ ...block3, powHash: block3.powHash === '0'.repeat(64) ? '0'.repeat(63) + '1' : '0'.repeat(64) });
|
|
assert.match(badProof.error, /RandomX proof/i);
|
|
assert.equal(await persisted(secondDir), beforeInvalid, 'Invalid blocks must not touch durable state');
|
|
assert.equal((await snapshot(second.base)).tip, block2.hash);
|
|
await submit(second.base, block3);
|
|
const gossiped = await waitFor('Block gossip to first node', () => snapshot(first.base),
|
|
state => state.tip === block3.hash);
|
|
const canonical = await snapshot(second.base);
|
|
assert.deepEqual(gossiped.blocks, canonical.blocks);
|
|
assert.deepEqual(canonical.blocks, [block1, block2, block3]);
|
|
assert.deepEqual(canonical.transfers, [transfer]);
|
|
assert.deepEqual(canonical.pendingTransfers, []);
|
|
assert.equal(canonical.cumulativeWork, (3n * workFor(MAX_TARGET)).toString());
|
|
for (const node of [first, second]) {
|
|
const replay = await json<{ error: string }>(node.base, '/api/transfer', 400, transfer);
|
|
assert.match(replay.error, /owner\/sequence/i);
|
|
assert.deepEqual(await json(node.base, `/api/nfts/${block1.tokenId}`), {
|
|
network: NETWORK, chainId: CHAIN_ID, tokenId: block1.tokenId, owner: bob.owner, sequence: 1,
|
|
blockHash: block1.hash, confirmations: 3, article: block1.article,
|
|
statsSnapshot: block1.statsSnapshot, stateRoot: block3.stateRoot,
|
|
});
|
|
const inventory = await json<{ blocks: TestnetBlock[]; next: null }>(node.base, '/p2p/blocks?offset=0');
|
|
assert.deepEqual(inventory, { blocks: canonical.blocks, next: null });
|
|
assert.deepEqual(await json(node.base, '/p2p/blocks?offset=3'), { blocks: [], next: null });
|
|
}
|
|
|
|
await stop(second); // Stop its outbound sync before restarting its peer.
|
|
await stop(first);
|
|
if (process.platform === 'win32') {
|
|
// Windows kill(SIGTERM) bypasses the JS handler. Explicit crash-recovery simulation:
|
|
// only our isolated lock, only after stop() confirmed process exit/stdio closure.
|
|
const lock = join(firstDir, 'testnet-node.lock');
|
|
try {
|
|
assert.equal(await readFile(lock, 'utf8'), String(first.child.pid));
|
|
await unlink(lock);
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
|
}
|
|
} else {
|
|
assert.equal(first.child.exitCode, 0, first.output());
|
|
assert.equal(existsSync(join(firstDir, 'testnet-node.lock')), false);
|
|
}
|
|
const backupDir=join(directory,'stopped-pair-backup'),drillDir=join(directory,'isolated-restore-drill');
|
|
const backupManifest = await createBackup(firstDir,backupDir);await restoreBackup(backupDir,drillDir);
|
|
const drillLedger=await Ledger.open(drillDir,engine!); // Revalidate real RandomX proofs from the copied backup.
|
|
try {
|
|
assert.deepEqual(drillLedger.snapshot(),{...canonical,peers:0});
|
|
}finally{drillLedger.close();}
|
|
first = await start(firstDir); // Ledger.open revalidates every persisted block using native RandomX.
|
|
const restored = await snapshot(first.base);
|
|
assert.deepEqual(await pagedClient.load(),restored);
|
|
assert.deepEqual(restored, { ...canonical, peers: 0 });
|
|
assert.deepEqual(await json(first.base, `/api/nfts/${block1.tokenId}`), {
|
|
network: NETWORK, chainId: CHAIN_ID, tokenId: block1.tokenId, owner: bob.owner, sequence: 1,
|
|
blockHash: block1.hash, confirmations: 3, article: block1.article,
|
|
statsSnapshot: block1.statsSnapshot, stateRoot: block3.stateRoot,
|
|
});
|
|
assert.deepEqual(await json(first.base, '/api/blocks', 200, block3), { accepted: false, reorg: false });
|
|
await (await request(first.base, '/api/transfer', 400, transfer)).text();
|
|
t.diagnostic('Real native RandomX: 3 mined blocks, 2 independent HTTP nodes, transfer gossip/confirmation, rejection and restart checks passed.');
|
|
} finally {
|
|
engine?.close();
|
|
const stopped = await Promise.allSettled(nodes.map(stop));
|
|
t.signal.removeEventListener('abort', abort);
|
|
if (originalPath === undefined) delete process.env.PATH;
|
|
else process.env.PATH = originalPath;
|
|
// Do not remove a directory if a child could still be writing to it.
|
|
if (stopped.every(result => result.status === 'fulfilled')) {
|
|
await rm(directory, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
} else {
|
|
throw new AggregateError(stopped.filter(result => result.status === 'rejected')
|
|
.map(result => result.reason), `Child cleanup failed; isolated data retained at ${directory}`);
|
|
}
|
|
}
|
|
});
|